6 Commits

35 changed files with 2182 additions and 3130 deletions

View File

@@ -211,29 +211,18 @@ func removePrunedStateRecords(ctx context.Context, backend storage.Backend, stat
if len(paths) == 0 {
return false, nil
}
if document.SingleOwner != nil {
next, changed := state.RemoveMissingOutputs(*document.SingleOwner, paths)
if document.Catalog != nil {
next, changed := state.RemoveMissingCatalogOwnerOutputs(*document.Catalog, scope, paths)
if !changed {
return false, nil
}
next.UpdatedAt = now
if err := state.Validate(next); err != nil {
if err := state.ValidateCatalog(next); err != nil {
return false, err
}
return true, writeRepairedState(ctx, backend, statePath, next)
}
if document.SharedRoot != nil {
next, changed := state.RemoveMissingSharedRootOwnerOutputs(*document.SharedRoot, scope, paths)
if !changed {
return false, nil
}
next.UpdatedAt = now
if err := state.ValidateSharedRoot(next); err != nil {
return false, err
}
return true, writeRepairedState(ctx, backend, statePath, next)
}
return false, fmt.Errorf("destination state document is empty")
return false, unsupportedStateDocumentError(document)
}
func PlanPrune(document state.StateDocument, policy config.PrunePolicy, options PrunePlanOptions) (PrunePlanReport, error) {
@@ -266,17 +255,20 @@ func PlanPrune(document state.StateDocument, policy config.PrunePolicy, options
}
func pruneCandidatesForDocument(document state.StateDocument, scope state.OwnerScope) ([]state.PruneCandidate, error) {
if document.SingleOwner != nil {
singleOwner := *document.SingleOwner
if singleOwner.PipelineID != scope.PipelineID || singleOwner.DestinationID != scope.DestinationID {
return nil, fmt.Errorf("state owner is %s/%s, not %s/%s", singleOwner.PipelineID, singleOwner.DestinationID, scope.PipelineID, scope.DestinationID)
}
return state.SingleOwnerPruneCandidates(singleOwner), nil
if document.Catalog != nil {
return state.CatalogPruneCandidates(*document.Catalog, scope), nil
}
if document.SharedRoot != nil {
return state.SharedRootPruneCandidates(*document.SharedRoot, scope), nil
return nil, unsupportedStateDocumentError(document)
}
func unsupportedStateDocumentError(document state.StateDocument) error {
if document.SupersededLegacy != nil {
return fmt.Errorf("destination state schema_version %d is superseded legacy state", document.SupersededLegacy.SchemaVersion)
}
return nil, fmt.Errorf("destination state document is empty")
if document.SingleOwner != nil || document.SharedRoot != nil {
return fmt.Errorf("legacy destination state is not supported by this command")
}
return fmt.Errorf("destination state document is empty")
}
func pruneOlderThan(policy config.PrunePolicy) *time.Duration {

View File

@@ -15,7 +15,7 @@ import (
)
func TestPlanPruneDisabledPolicy(t *testing.T) {
document := state.StateDocument{SingleOwner: &state.DistributorState{}}
document := state.StateDocument{Catalog: &state.CatalogState{}}
report, err := PlanPrune(document, config.PrunePolicy{}, PrunePlanOptions{
PipelineID: "reports",
DestinationID: "archive",
@@ -28,12 +28,12 @@ func TestPlanPruneDisabledPolicy(t *testing.T) {
}
}
func TestPlanPruneSingleOwnerOutputs(t *testing.T) {
func TestPlanPruneCatalogOutputs(t *testing.T) {
now := time.Date(2026, 6, 8, 12, 0, 0, 0, time.UTC)
olderThan := config.Duration(48 * time.Hour)
destinationState := pruneSingleOwnerState(now)
catalog := pruneCatalogState(now)
report, err := PlanPrune(state.StateDocument{SingleOwner: &destinationState}, config.PrunePolicy{
report, err := PlanPrune(state.StateDocument{Catalog: &catalog}, config.PrunePolicy{
Enabled: true,
OlderThan: &olderThan,
}, PrunePlanOptions{
@@ -52,12 +52,12 @@ func TestPlanPruneSingleOwnerOutputs(t *testing.T) {
}
}
func TestPlanPruneSharedRootCurrentOwnerOnly(t *testing.T) {
func TestPlanPruneCatalogCurrentOwnerOnly(t *testing.T) {
now := time.Date(2026, 6, 8, 12, 0, 0, 0, time.UTC)
keepLatest := 0
sharedRoot := pruneSharedRootState(now)
catalog := pruneCatalogState(now)
report, err := PlanPrune(state.StateDocument{SharedRoot: &sharedRoot}, config.PrunePolicy{
report, err := PlanPrune(state.StateDocument{Catalog: &catalog}, config.PrunePolicy{
Enabled: true,
KeepLatest: &keepLatest,
}, PrunePlanOptions{
@@ -68,10 +68,10 @@ func TestPlanPruneSharedRootCurrentOwnerOnly(t *testing.T) {
if err != nil {
t.Fatalf("PlanPrune() error = %v", err)
}
if got, want := report.CheckedCount, 1; got != want {
if got, want := report.CheckedCount, 2; got != want {
t.Fatalf("checked count = %d, want %d", got, want)
}
if got, want := pruneRecordPaths(report.PrunedOutputs), "archive.txt"; got != want {
if got, want := pruneRecordPaths(report.PrunedOutputs), "old.txt,fresh.txt"; got != want {
t.Fatalf("pruned = %q, want %q", got, want)
}
}
@@ -80,8 +80,8 @@ func TestPruneDryRunReportsPlannedDeletesWithoutDeletingOrRewritingState(t *test
now := time.Date(2026, 6, 8, 12, 0, 0, 0, time.UTC)
backend := fake.New()
cfg := pruneS3Config(t, pruneOlderThanPolicy(48*time.Hour))
original := pruneSingleOwnerState(now)
writeFakeSingleOwnerStateForPrune(t, backend, original)
original := pruneCatalogState(now)
writeFakeCatalogState(t, backend, original)
testutil.WriteFakeFile(t, backend, "unmanaged.txt", "keep")
report, err := pruneConfigWithBackendFactory(context.Background(), cfg, PruneOptions{
@@ -102,12 +102,12 @@ func TestPruneDryRunReportsPlannedDeletesWithoutDeletingOrRewritingState(t *test
testutil.AssertFakeFile(t, backend, "old.txt", "managed")
testutil.AssertFakeFile(t, backend, "fresh.txt", "managed")
testutil.AssertFakeFile(t, backend, "unmanaged.txt", "keep")
destinationState := readFakeSingleOwnerState(t, backend)
if got := strings.Join(state.ManagedOutputPaths(destinationState), ","); got != "old.txt,fresh.txt" {
catalog := readFakeCatalogState(t, backend)
if got := strings.Join(state.CatalogManagedOutputPaths(catalog), ","); got != "old.txt,fresh.txt,html.txt" {
t.Fatalf("state outputs = %q, want original outputs", got)
}
if !destinationState.UpdatedAt.Equal(original.UpdatedAt) {
t.Fatalf("state updated_at = %s, want original %s", destinationState.UpdatedAt, original.UpdatedAt)
if !catalog.UpdatedAt.Equal(original.UpdatedAt) {
t.Fatalf("state updated_at = %s, want original %s", catalog.UpdatedAt, original.UpdatedAt)
}
}
@@ -115,7 +115,7 @@ func TestPruneApplyDeletesOnlyManagedOutputsAndUpdatesState(t *testing.T) {
now := time.Date(2026, 6, 8, 12, 0, 0, 0, time.UTC)
backend := fake.New()
cfg := pruneS3Config(t, pruneOlderThanPolicy(48*time.Hour))
writeFakeSingleOwnerStateForPrune(t, backend, pruneSingleOwnerState(now))
writeFakeCatalogState(t, backend, pruneCatalogState(now))
testutil.WriteFakeFile(t, backend, "unmanaged.txt", "keep")
report, err := pruneConfigWithBackendFactory(context.Background(), cfg, PruneOptions{
@@ -134,14 +134,18 @@ func TestPruneApplyDeletesOnlyManagedOutputsAndUpdatesState(t *testing.T) {
}
testutil.AssertFakeMissing(t, backend, "old.txt")
testutil.AssertFakeFile(t, backend, "fresh.txt", "managed")
testutil.AssertFakeFile(t, backend, "html.txt", "managed")
testutil.AssertFakeFile(t, backend, "unmanaged.txt", "keep")
assertFakeStateExists(t, backend)
destinationState := readFakeSingleOwnerState(t, backend)
if got := strings.Join(state.ManagedOutputPaths(destinationState), ","); got != "fresh.txt" {
catalog := readFakeCatalogState(t, backend)
if got := strings.Join(state.CatalogManagedOutputPaths(catalog), ","); got != "fresh.txt,html.txt" {
t.Fatalf("state outputs = %q, want fresh.txt", got)
}
if !destinationState.UpdatedAt.Equal(now) {
t.Fatalf("state updated_at = %s, want %s", destinationState.UpdatedAt, now)
if catalog.SchemaVersion != state.CatalogSchemaVersion {
t.Fatalf("state schema_version = %d, want %d", catalog.SchemaVersion, state.CatalogSchemaVersion)
}
if !catalog.UpdatedAt.Equal(now) {
t.Fatalf("state updated_at = %s, want %s", catalog.UpdatedAt, now)
}
}
@@ -150,7 +154,7 @@ func TestPruneApplyPreservesStateForFailedDeletes(t *testing.T) {
backend := fake.New()
keepLatest := 0
cfg := pruneS3Config(t, config.PrunePolicy{Enabled: true, KeepLatest: &keepLatest})
writeFakeSingleOwnerStateForPrune(t, backend, pruneSingleOwnerState(now))
writeFakeCatalogState(t, backend, pruneCatalogState(now))
failingBackend := failingDeleteBackend{Backend: backend, failPath: "fresh.txt"}
report, err := pruneConfigWithBackendFactory(context.Background(), cfg, PruneOptions{
@@ -170,18 +174,17 @@ func TestPruneApplyPreservesStateForFailedDeletes(t *testing.T) {
testutil.AssertFakeMissing(t, backend, "old.txt")
testutil.AssertFakeFile(t, backend, "fresh.txt", "managed")
assertFakeStateExists(t, backend)
destinationState := readFakeSingleOwnerState(t, backend)
if got := strings.Join(state.ManagedOutputPaths(destinationState), ","); got != "fresh.txt" {
catalog := readFakeCatalogState(t, backend)
if got := strings.Join(state.CatalogManagedOutputPaths(catalog), ","); got != "fresh.txt,html.txt" {
t.Fatalf("state outputs = %q, want only failed output preserved", got)
}
}
func TestPruneSharedRootPreservesOtherOwnersWhenScopedToCurrentOwner(t *testing.T) {
func TestPrunePreservesOtherOwnersWhenScopedToCurrentOwner(t *testing.T) {
now := time.Date(2026, 6, 8, 12, 0, 0, 0, time.UTC)
backend := fake.New()
keepLatest := 0
cfg := pruneS3Config(t, config.PrunePolicy{Enabled: true, KeepLatest: &keepLatest})
writeFakeSharedRootStateForApp(t, backend, pruneSharedRootState(now))
cfg := pruneS3Config(t, pruneOlderThanPolicy(48*time.Hour))
writeFakeCatalogState(t, backend, pruneCatalogState(now))
testutil.WriteFakeFile(t, backend, "unmanaged.txt", "keep")
report, err := pruneConfigWithBackendFactory(context.Background(), cfg, PruneOptions{
@@ -192,93 +195,56 @@ func TestPruneSharedRootPreservesOtherOwnersWhenScopedToCurrentOwner(t *testing.
if err != nil {
t.Fatalf("pruneConfigWithBackendFactory() error = %v", err)
}
if got, want := pruneRecordPaths(report.DeletedOutputs), "archive.txt"; got != want {
if got, want := pruneRecordPaths(report.DeletedOutputs), "old.txt"; got != want {
t.Fatalf("deleted outputs = %q, want %q", got, want)
}
testutil.AssertFakeMissing(t, backend, "archive.txt")
testutil.AssertFakeFile(t, backend, "html.txt", "old")
testutil.AssertFakeMissing(t, backend, "old.txt")
testutil.AssertFakeFile(t, backend, "html.txt", "managed")
testutil.AssertFakeFile(t, backend, "unmanaged.txt", "keep")
sharedRoot := readFakeSharedRootStateForApp(t, backend)
if got := strings.Join(sharedRoot.AllManagedOutputPaths(), ","); got != "html.txt" {
t.Fatalf("shared-root outputs = %q, want other owner output preserved", got)
catalog := readFakeCatalogState(t, backend)
if got := strings.Join(state.CatalogManagedOutputPaths(catalog), ","); got != "fresh.txt,html.txt" {
t.Fatalf("catalog outputs = %q, want other owner output preserved", got)
}
}
func pruneSingleOwnerState(now time.Time) state.DistributorState {
func pruneCatalogState(now time.Time) state.CatalogState {
manifest := testutil.ValidManifest(testutil.BundleOptions{})
publishedAt := now.Add(-96 * time.Hour)
return state.DistributorState{
SchemaVersion: state.SchemaVersion,
PipelineID: "reports",
DestinationID: "archive",
PublishedAt: publishedAt,
CreatedAt: publishedAt,
UpdatedAt: publishedAt,
State: state.StatePolicy{Mode: state.StateModeSingleOwner},
Reconciliation: state.ReconciliationPolicy{Mode: config.ReconciliationModeReplace},
Source: state.SourceState{Manifest: manifest},
createdAt := now.Add(-96 * time.Hour)
source := state.CatalogSourceIdentity{ID: manifest.ID, Digest: manifest.Digest, Created: manifest.Created}
return state.CatalogState{
SchemaVersion: state.CatalogSchemaVersion,
DistributorVersion: "test",
Outputs: []state.OutputFile{{
Path: "old.txt",
Kind: state.OutputKindSource,
SourcePath: "report.md",
SHA256: manifest.Files[0].SHA256,
Size: manifest.Files[0].Size,
CreatedAt: now.Add(-96 * time.Hour),
UpdatedAt: now.Add(-72 * time.Hour),
}, {
Path: "fresh.txt",
Kind: state.OutputKindSource,
SourcePath: "summary.txt",
SHA256: manifest.Files[1].SHA256,
Size: manifest.Files[1].Size,
CreatedAt: now.Add(-24 * time.Hour),
UpdatedAt: now.Add(-24 * time.Hour),
}},
}
}
func pruneSharedRootState(now time.Time) state.SharedRootState {
manifest := testutil.ValidManifest(testutil.BundleOptions{})
archive := state.CurrentOwnerScope("reports", "archive")
html := state.CurrentOwnerScope("reports", "html")
return state.SharedRootState{
SchemaVersion: state.SharedRootSchemaVersion,
DistributorVersion: "test",
CreatedAt: now.Add(-96 * time.Hour),
UpdatedAt: now.Add(-24 * time.Hour),
State: state.StatePolicy{Mode: state.StateModeSharedRoot},
Owners: []state.OwnerRecord{{
Scope: archive,
Reconciliation: state.ReconciliationPolicy{Mode: config.ReconciliationModeReplace},
Source: state.SourceState{Manifest: manifest},
}, {
Scope: html,
Reconciliation: state.ReconciliationPolicy{Mode: config.ReconciliationModeReplace},
Source: state.SourceState{Manifest: manifest},
}},
Outputs: []state.SharedRootOutputFile{{
Path: "archive.txt",
CreatedAt: createdAt,
UpdatedAt: createdAt,
State: state.StatePolicy{Mode: state.StateModeCatalog},
Outputs: []state.CatalogOutputFile{{
Path: "old.txt",
PipelineID: "reports",
DestinationID: "archive",
Source: source,
Kind: state.OutputKindSource,
SourcePath: "report.md",
SHA256: manifest.Files[0].SHA256,
Size: manifest.Files[0].Size,
Owner: archive,
SourceID: manifest.ID,
SourceDigest: manifest.Digest,
SourceCreated: manifest.Created,
CreatedAt: now.Add(-96 * time.Hour),
UpdatedAt: now.Add(-72 * time.Hour),
}, {
Path: "html.txt",
Path: "fresh.txt",
PipelineID: "reports",
DestinationID: "archive",
Source: source,
Kind: state.OutputKindSource,
SHA256: manifest.Files[0].SHA256,
Size: manifest.Files[0].Size,
CreatedAt: now.Add(-24 * time.Hour),
UpdatedAt: now.Add(-24 * time.Hour),
}, {
Path: "html.txt",
PipelineID: "reports",
DestinationID: "html",
Source: source,
Kind: state.OutputKindSource,
SourcePath: "summary.txt",
SHA256: manifest.Files[1].SHA256,
Size: manifest.Files[1].Size,
Owner: html,
SourceID: manifest.ID,
SourceDigest: manifest.Digest,
SourceCreated: manifest.Created,
CreatedAt: now.Add(-96 * time.Hour),
UpdatedAt: now.Add(-72 * time.Hour),
}},
@@ -319,18 +285,31 @@ func pruneOlderThanPolicy(duration time.Duration) config.PrunePolicy {
}
}
func writeFakeSingleOwnerStateForPrune(t *testing.T, backend *fake.Backend, destinationState state.DistributorState) {
func writeFakeCatalogState(t *testing.T, backend *fake.Backend, catalog state.CatalogState) {
t.Helper()
data, err := json.MarshalIndent(destinationState, "", " ")
data, err := json.MarshalIndent(catalog, "", " ")
if err != nil {
t.Fatalf("marshal single-owner state: %v", err)
t.Fatalf("marshal catalog state: %v", err)
}
testutil.WriteFakeFile(t, backend, storage.StateFileName, string(append(data, '\n')))
for _, output := range destinationState.Outputs {
for _, output := range catalog.Outputs {
testutil.WriteFakeFile(t, backend, output.Path, "managed")
}
}
func readFakeCatalogState(t *testing.T, backend *fake.Backend) state.CatalogState {
t.Helper()
data, err := backend.ReadFile(context.Background(), storage.StateFileName)
if err != nil {
t.Fatalf("read catalog state: %v", err)
}
catalog, err := state.ParseCatalog(data)
if err != nil {
t.Fatalf("parse catalog state: %v", err)
}
return catalog
}
func assertFakeStateExists(t *testing.T, backend *fake.Backend) {
t.Helper()
if _, err := backend.Stat(context.Background(), storage.StateFileName); err != nil {

View File

@@ -158,62 +158,25 @@ func buildReconcileStateReport(ctx context.Context, backend storage.Backend, pip
DryRun: options.DryRun,
}
scope := state.CurrentOwnerScope(pipeline.ID, destination.ID)
if document.SingleOwner != nil {
return reconcileSingleOwnerState(ctx, backend, statePath, *document.SingleOwner, scope, report, options)
if document.Catalog != nil {
return reconcileCatalogState(ctx, backend, statePath, *document.Catalog, scope, report, options)
}
return reconcileSharedRootState(ctx, backend, statePath, *document.SharedRoot, scope, report, options)
return ReconcileStateReport{}, unsupportedStateDocumentError(document)
}
func reconcileSingleOwnerState(ctx context.Context, backend storage.Backend, statePath string, destinationState state.DistributorState, scope state.OwnerScope, report ReconcileStateReport, options ReconcileStateOptions) (ReconcileStateReport, error) {
if destinationState.PipelineID != scope.PipelineID || destinationState.DestinationID != scope.DestinationID {
return ReconcileStateReport{}, fmt.Errorf("state owner is %s/%s, not %s/%s", destinationState.PipelineID, destinationState.DestinationID, scope.PipelineID, scope.DestinationID)
}
report.StateSchema = destinationState.SchemaVersion
report.OwnerScope = &ReconcileStateOwnerScope{PipelineID: scope.PipelineID, DestinationID: scope.DestinationID}
managed := state.ManagedOutputPaths(destinationState)
missing, err := missingSingleOwnerOutputs(ctx, backend, destinationState.Outputs)
if err != nil {
return ReconcileStateReport{}, err
}
report.CheckedCount = len(managed)
report.MissingManagedOutputs = missing
unmanaged, err := unmanagedEntries(ctx, backend, managed)
if err != nil {
return ReconcileStateReport{}, err
}
report.UnmanagedEntries = unmanaged
report.WouldChange = options.DryRun && len(missing) > 0
if !options.DryRun && len(missing) > 0 {
missingPaths := missingReportPaths(missing)
next, changed := state.RemoveMissingOutputs(destinationState, missingPaths)
report.Changed = changed
if changed {
next.UpdatedAt = time.Now().UTC()
if err := state.Validate(next); err != nil {
return ReconcileStateReport{}, err
}
if err := writeRepairedState(ctx, backend, statePath, next); err != nil {
return ReconcileStateReport{}, err
}
}
}
return report, nil
}
func reconcileSharedRootState(ctx context.Context, backend storage.Backend, statePath string, sharedRoot state.SharedRootState, scope state.OwnerScope, report ReconcileStateReport, options ReconcileStateOptions) (ReconcileStateReport, error) {
report.StateSchema = sharedRoot.SchemaVersion
func reconcileCatalogState(ctx context.Context, backend storage.Backend, statePath string, catalog state.CatalogState, scope state.OwnerScope, report ReconcileStateReport, options ReconcileStateOptions) (ReconcileStateReport, error) {
report.StateSchema = catalog.SchemaVersion
report.OwnerScope = &ReconcileStateOwnerScope{
PipelineID: scope.PipelineID,
DestinationID: scope.DestinationID,
AllOwners: options.AllOwners,
}
managed := sharedRoot.AllManagedOutputPaths()
outputs := sharedRoot.Outputs
managed := state.CatalogManagedOutputPaths(catalog)
outputs := catalog.Outputs
if !options.AllOwners {
outputs = sharedRootOutputsForOwner(sharedRoot.Outputs, scope)
outputs = state.CatalogOutputsForOwner(catalog.Outputs, scope)
}
missing, err := missingSharedRootOutputs(ctx, backend, outputs)
missing, err := missingCatalogOutputs(ctx, backend, outputs)
if err != nil {
return ReconcileStateReport{}, err
}
@@ -228,17 +191,17 @@ func reconcileSharedRootState(ctx context.Context, backend storage.Backend, stat
if !options.DryRun && len(missing) > 0 {
missingPaths := missingReportPaths(missing)
var next state.SharedRootState
var next state.CatalogState
var changed bool
if options.AllOwners {
next, changed = state.RemoveMissingSharedRootOutputs(sharedRoot, missingPaths)
next, changed = state.RemoveMissingCatalogOutputs(catalog, missingPaths)
} else {
next, changed = state.RemoveMissingSharedRootOwnerOutputs(sharedRoot, scope, missingPaths)
next, changed = state.RemoveMissingCatalogOwnerOutputs(catalog, scope, missingPaths)
}
report.Changed = changed
if changed {
next.UpdatedAt = time.Now().UTC()
if err := state.ValidateSharedRoot(next); err != nil {
if err := state.ValidateCatalog(next); err != nil {
return ReconcileStateReport{}, err
}
if err := writeRepairedState(ctx, backend, statePath, next); err != nil {
@@ -249,21 +212,7 @@ func reconcileSharedRootState(ctx context.Context, backend storage.Backend, stat
return report, nil
}
func missingSingleOwnerOutputs(ctx context.Context, backend storage.Backend, outputs []state.OutputFile) ([]ReconcileStatePath, error) {
missing := make([]ReconcileStatePath, 0)
for _, output := range outputs {
if err := checkManagedOutput(ctx, backend, output.Path); err != nil {
if storage.IsNotFound(err) {
missing = append(missing, ReconcileStatePath{Path: output.Path, StorageStatus: "missing"})
continue
}
return nil, err
}
}
return missing, nil
}
func missingSharedRootOutputs(ctx context.Context, backend storage.Backend, outputs []state.SharedRootOutputFile) ([]ReconcileStatePath, error) {
func missingCatalogOutputs(ctx context.Context, backend storage.Backend, outputs []state.CatalogOutputFile) ([]ReconcileStatePath, error) {
missing := make([]ReconcileStatePath, 0)
for _, output := range outputs {
if err := checkManagedOutput(ctx, backend, output.Path); err != nil {
@@ -271,8 +220,8 @@ func missingSharedRootOutputs(ctx context.Context, backend storage.Backend, outp
missing = append(missing, ReconcileStatePath{
Path: output.Path,
OwnerScope: &ReconcileStateOwnerScope{
PipelineID: output.Owner.PipelineID,
DestinationID: output.Owner.DestinationID,
PipelineID: output.PipelineID,
DestinationID: output.DestinationID,
},
StorageStatus: "missing",
})
@@ -320,16 +269,6 @@ func unmanagedEntries(ctx context.Context, backend storage.Backend, managedPaths
return entries, nil
}
func sharedRootOutputsForOwner(outputs []state.SharedRootOutputFile, scope state.OwnerScope) []state.SharedRootOutputFile {
selected := make([]state.SharedRootOutputFile, 0, len(outputs))
for _, output := range outputs {
if output.Owner == scope {
selected = append(selected, output)
}
}
return selected
}
func missingReportPaths(missing []ReconcileStatePath) []string {
paths := make([]string, 0, len(missing))
for _, item := range missing {

View File

@@ -2,7 +2,6 @@ package app
import (
"context"
"encoding/json"
"strings"
"testing"
"time"
@@ -17,9 +16,9 @@ import (
func TestReconcileStateDryRunReportsMissingManagedOutputsWithoutRewrite(t *testing.T) {
backend := fake.New()
cfg := reconcileStateS3Config(t)
manifest := testutil.ValidManifest(testutil.BundleOptions{})
testutil.WriteFakeDestinationState(t, backend, "", manifest, testutil.DestinationStateOptions{})
if err := backend.DeleteManagedOutputs(context.Background(), "", []string{"summary.txt"}, storage.DeleteOptions{}); err != nil {
catalog := pruneCatalogState(time.Date(2026, 6, 8, 12, 0, 0, 0, time.UTC))
writeFakeCatalogState(t, backend, catalog)
if err := backend.DeleteManagedOutputs(context.Background(), "", []string{"fresh.txt"}, storage.DeleteOptions{}); err != nil {
t.Fatalf("delete managed output: %v", err)
}
testutil.WriteFakeFile(t, backend, "extra.txt", "unmanaged")
@@ -35,14 +34,14 @@ func TestReconcileStateDryRunReportsMissingManagedOutputsWithoutRewrite(t *testi
if !report.WouldChange || report.Changed {
t.Fatalf("report changed=%t would_change=%t, want dry-run pending change", report.Changed, report.WouldChange)
}
if got := reportPathList(report.MissingManagedOutputs); got != "summary.txt" {
t.Fatalf("missing outputs = %q, want summary.txt", got)
if got := reportPathList(report.MissingManagedOutputs); got != "fresh.txt" {
t.Fatalf("missing outputs = %q, want fresh.txt", got)
}
if got := entryPathList(report.UnmanagedEntries); got != "extra.txt" {
t.Fatalf("unmanaged entries = %q, want extra.txt", got)
}
destinationState := readFakeSingleOwnerState(t, backend)
if got := strings.Join(state.ManagedOutputPaths(destinationState), ","); got != "report.md,summary.txt" {
repaired := readFakeCatalogState(t, backend)
if got := strings.Join(state.CatalogManagedOutputPaths(repaired), ","); got != "old.txt,fresh.txt,html.txt" {
t.Fatalf("state outputs = %q, want original outputs", got)
}
}
@@ -50,9 +49,8 @@ func TestReconcileStateDryRunReportsMissingManagedOutputsWithoutRewrite(t *testi
func TestReconcileStateApplyRemovesMissingRecordsAndPreservesUnmanagedFiles(t *testing.T) {
backend := fake.New()
cfg := reconcileStateS3Config(t)
manifest := testutil.ValidManifest(testutil.BundleOptions{})
testutil.WriteFakeDestinationState(t, backend, "", manifest, testutil.DestinationStateOptions{})
if err := backend.DeleteManagedOutputs(context.Background(), "", []string{"summary.txt"}, storage.DeleteOptions{}); err != nil {
writeFakeCatalogState(t, backend, pruneCatalogState(time.Date(2026, 6, 8, 12, 0, 0, 0, time.UTC)))
if err := backend.DeleteManagedOutputs(context.Background(), "", []string{"fresh.txt"}, storage.DeleteOptions{}); err != nil {
t.Fatalf("delete managed output: %v", err)
}
testutil.WriteFakeFile(t, backend, "extra.txt", "unmanaged")
@@ -67,12 +65,15 @@ func TestReconcileStateApplyRemovesMissingRecordsAndPreservesUnmanagedFiles(t *t
if !report.Changed || report.WouldChange {
t.Fatalf("report changed=%t would_change=%t, want applied change", report.Changed, report.WouldChange)
}
destinationState := readFakeSingleOwnerState(t, backend)
if err := state.Validate(destinationState); err != nil {
t.Fatalf("Validate() repaired state error = %v", err)
repaired := readFakeCatalogState(t, backend)
if err := state.ValidateCatalog(repaired); err != nil {
t.Fatalf("ValidateCatalog() repaired state error = %v", err)
}
if got := strings.Join(state.ManagedOutputPaths(destinationState), ","); got != "report.md" {
t.Fatalf("state outputs = %q, want report.md", got)
if got := strings.Join(state.CatalogManagedOutputPaths(repaired), ","); got != "old.txt,html.txt" {
t.Fatalf("state outputs = %q, want old.txt,html.txt", got)
}
if repaired.SchemaVersion != state.CatalogSchemaVersion {
t.Fatalf("state schema_version = %d, want %d", repaired.SchemaVersion, state.CatalogSchemaVersion)
}
testutil.AssertFakeFile(t, backend, "extra.txt", "unmanaged")
}
@@ -99,12 +100,11 @@ func TestReconcileStateInvalidStateFailsWithoutRewrite(t *testing.T) {
}
}
func TestReconcileStateSharedRootOwnerScopeRepairsCurrentOwnerOnly(t *testing.T) {
func TestReconcileStateOwnerScopeRepairsCurrentOwnerOnly(t *testing.T) {
backend := fake.New()
cfg := reconcileStateS3Config(t)
sharedRoot := reconcileSharedRootFixture(t)
writeFakeSharedRootStateForApp(t, backend, sharedRoot)
if err := backend.DeleteManagedOutputs(context.Background(), "", []string{"report.md", "report.html"}, storage.DeleteOptions{}); err != nil {
writeFakeCatalogState(t, backend, pruneCatalogState(time.Date(2026, 6, 8, 12, 0, 0, 0, time.UTC)))
if err := backend.DeleteManagedOutputs(context.Background(), "", []string{"old.txt", "html.txt"}, storage.DeleteOptions{}); err != nil {
t.Fatalf("delete managed outputs: %v", err)
}
@@ -118,18 +118,17 @@ func TestReconcileStateSharedRootOwnerScopeRepairsCurrentOwnerOnly(t *testing.T)
if !report.Changed {
t.Fatal("report changed = false, want true")
}
repaired := readFakeSharedRootStateForApp(t, backend)
if got := strings.Join(repaired.AllManagedOutputPaths(), ","); got != "report.html" {
t.Fatalf("shared-root outputs = %q, want other owner output preserved", got)
repaired := readFakeCatalogState(t, backend)
if got := strings.Join(state.CatalogManagedOutputPaths(repaired), ","); got != "fresh.txt,html.txt" {
t.Fatalf("catalog outputs = %q, want other owner output preserved", got)
}
}
func TestReconcileStateSharedRootAllOwnersRepairsEveryOwner(t *testing.T) {
func TestReconcileStateAllOwnersRepairsEveryOwner(t *testing.T) {
backend := fake.New()
cfg := reconcileStateS3Config(t)
sharedRoot := reconcileSharedRootFixture(t)
writeFakeSharedRootStateForApp(t, backend, sharedRoot)
if err := backend.DeleteManagedOutputs(context.Background(), "", []string{"report.md", "report.html"}, storage.DeleteOptions{}); err != nil {
writeFakeCatalogState(t, backend, pruneCatalogState(time.Date(2026, 6, 8, 12, 0, 0, 0, time.UTC)))
if err := backend.DeleteManagedOutputs(context.Background(), "", []string{"old.txt", "html.txt"}, storage.DeleteOptions{}); err != nil {
t.Fatalf("delete managed outputs: %v", err)
}
@@ -141,12 +140,12 @@ func TestReconcileStateSharedRootAllOwnersRepairsEveryOwner(t *testing.T) {
if err != nil {
t.Fatalf("reconcileStateConfigWithBackendFactory() error = %v", err)
}
if !report.Changed || report.CheckedCount != 2 {
if !report.Changed || report.CheckedCount != 3 {
t.Fatalf("report changed=%t checked=%d, want all-owner repair", report.Changed, report.CheckedCount)
}
repaired := readFakeSharedRootStateForApp(t, backend)
if got := repaired.AllManagedOutputPaths(); len(got) != 0 {
t.Fatalf("shared-root outputs = %#v, want none", got)
repaired := readFakeCatalogState(t, backend)
if got := strings.Join(state.CatalogManagedOutputPaths(repaired), ","); got != "fresh.txt" {
t.Fatalf("catalog outputs = %q, want fresh.txt", got)
}
}
@@ -165,93 +164,6 @@ func reconcileStateS3Config(t *testing.T) config.Config {
return cfg
}
func readFakeSingleOwnerState(t *testing.T, backend *fake.Backend) state.DistributorState {
t.Helper()
data, err := backend.ReadFile(context.Background(), storage.StateFileName)
if err != nil {
t.Fatalf("read state: %v", err)
}
destinationState, err := state.Parse(data)
if err != nil {
t.Fatalf("parse state: %v", err)
}
return destinationState
}
func writeFakeSharedRootStateForApp(t *testing.T, backend *fake.Backend, sharedRoot state.SharedRootState) {
t.Helper()
data, err := json.MarshalIndent(sharedRoot, "", " ")
if err != nil {
t.Fatalf("marshal shared-root state: %v", err)
}
testutil.WriteFakeFile(t, backend, storage.StateFileName, string(append(data, '\n')))
for _, output := range sharedRoot.Outputs {
testutil.WriteFakeFile(t, backend, output.Path, "old")
}
}
func readFakeSharedRootStateForApp(t *testing.T, backend *fake.Backend) state.SharedRootState {
t.Helper()
data, err := backend.ReadFile(context.Background(), storage.StateFileName)
if err != nil {
t.Fatalf("read shared-root state: %v", err)
}
sharedRoot, err := state.ParseSharedRoot(data)
if err != nil {
t.Fatalf("parse shared-root state: %v", err)
}
return sharedRoot
}
func reconcileSharedRootFixture(t *testing.T) state.SharedRootState {
t.Helper()
source := testutil.ValidManifest(testutil.BundleOptions{})
htmlSource := source
createdAt := time.Date(2026, 5, 30, 11, 12, 0, 0, time.UTC)
return state.SharedRootState{
SchemaVersion: state.SharedRootSchemaVersion,
DistributorVersion: "test",
CreatedAt: createdAt,
UpdatedAt: createdAt,
State: state.StatePolicy{Mode: state.StateModeSharedRoot},
Owners: []state.OwnerRecord{{
Scope: state.CurrentOwnerScope("reports", "archive"),
Reconciliation: state.ReconciliationPolicy{Mode: config.ReconciliationModeReplace},
Source: state.SourceState{Manifest: source},
}, {
Scope: state.CurrentOwnerScope("reports", "html"),
Reconciliation: state.ReconciliationPolicy{Mode: config.ReconciliationModeMerge},
Source: state.SourceState{Manifest: htmlSource},
}},
Outputs: []state.SharedRootOutputFile{{
Path: "report.md",
Kind: state.OutputKindSource,
SourcePath: "report.md",
SHA256: source.Files[0].SHA256,
Size: source.Files[0].Size,
Owner: state.CurrentOwnerScope("reports", "archive"),
SourceID: source.ID,
SourceDigest: source.Digest,
SourceCreated: source.Created,
CreatedAt: createdAt,
UpdatedAt: createdAt,
}, {
Path: "report.html",
Kind: state.OutputKindGenerated,
SourcePath: "report.md",
Transform: "markdown_to_html",
SHA256: "sha256:" + strings.Repeat("a", 64),
Size: 128,
Owner: state.CurrentOwnerScope("reports", "html"),
SourceID: htmlSource.ID,
SourceDigest: htmlSource.Digest,
SourceCreated: htmlSource.Created,
CreatedAt: createdAt,
UpdatedAt: createdAt,
}},
}
}
func reportPathList(paths []ReconcileStatePath) string {
values := make([]string, 0, len(paths))
for _, path := range paths {

View File

@@ -69,11 +69,8 @@ func processDestinationSelection(ctx context.Context, request runDestinationRequ
Publish: *request.destination.Publish,
Transform: request.destination.Transform,
Links: request.destination.Links,
State: request.destination.State,
Reconciliation: request.destination.Reconciliation,
Takeover: request.destination.Takeover,
Workflow: request.destination.Workflow,
Transformers: request.transforms,
Transfer: request.destination.Transfer,
DistributorVersion: Version,
Force: request.options.Force,
}
@@ -83,8 +80,8 @@ func processDestinationSelection(ctx context.Context, request runDestinationRequ
}
if isFixedPathDestination(request.destination) {
plan.PathMapping = config.PathMappingFixed
if request.options.DryRun && isDestructiveFixedPathAction(plan.Action) {
warning := fixedPathReplacementWarning(plan)
if request.options.DryRun && isFixedPathWorkflowAction(plan.Action) {
warning := fixedPathWorkflowWarning(plan)
request.recorder.addPipelineWarning(request.pipelineIndex, warning)
}
}
@@ -142,6 +139,7 @@ func (recorder *runReportRecorder) recordDestinationFailure(pipelineIndex int, f
recorder.failures.add(failure.pipelineID, failure.destinationID, failure.backend, storage.DisplayPath(failure.bundlePath), failure.err)
recorder.summary.recordFailure()
if includeAction {
recorder.summary.recordFailureAction(action.Action)
recorder.addPipelineAction(pipelineIndex, action)
}
}
@@ -162,5 +160,8 @@ func completePlanIdentity(plan publish.Plan, pipeline config.Pipeline, destinati
if plan.DestinationBundlePath == "" {
plan.DestinationBundlePath = selection.DestinationBundlePath
}
if plan.Workflow == "" {
plan.Workflow = destination.Workflow
}
return plan
}

View File

@@ -6,7 +6,7 @@ import (
)
func shouldNotify(action publish.Action) bool {
return action == publish.ActionPublishNew || action == publish.ActionReplaceOlder || action == publish.ActionReplaceConflict || action == publish.ActionReplaceNewer || action == publish.ActionReplaceTakeover || action == publish.ActionForceReplace
return action == publish.ActionPublishNew || action == publish.ActionUpsertAdditive || action == publish.ActionReplaceCatalog || action == publish.ActionForceReplace
}
func notifyEvent(plan publish.Plan) notify.Event {

View File

@@ -60,7 +60,7 @@ func writeRunActionLine(w io.Writer, action RunActionRecord) {
fmt.Fprintf(w, " - bundle=%s destination=%s backend=%s%s action=error reason=%q\n", action.BundlePath, destinationID, action.Backend, pathMappingRecordSummary(action), action.Reason)
return
}
fmt.Fprintf(w, " - bundle=%s destination=%s backend=%s%s action=%s%s outputs=%s reason=%q\n", action.BundlePath, action.DestinationID, action.Backend, pathMappingRecordSummary(action), action.Action, takeoverModeRecordSummary(action), outputRecordSummary(action.Outputs), action.Reason)
fmt.Fprintf(w, " - bundle=%s destination=%s backend=%s%s action=%s%s outputs=%s reason=%q\n", action.BundlePath, action.DestinationID, action.Backend, pathMappingRecordSummary(action), action.Action, workflowRecordSummary(action), outputRecordSummary(action.Outputs), action.Reason)
}
func pathMappingRecordSummary(action RunActionRecord) string {
@@ -70,11 +70,11 @@ func pathMappingRecordSummary(action RunActionRecord) string {
return fmt.Sprintf(" path_mapping=fixed target=%s", action.DestinationPath)
}
func takeoverModeRecordSummary(action RunActionRecord) string {
if action.TakeoverMode == "" {
func workflowRecordSummary(action RunActionRecord) string {
if action.Workflow == "" {
return ""
}
return fmt.Sprintf(" takeover_mode=%s", action.TakeoverMode)
return fmt.Sprintf(" workflow=%s", action.Workflow)
}
func outputRecordSummary(outputs []RunOutputRecord) string {
@@ -143,8 +143,8 @@ type RunActionRecord struct {
BundlePath string `json:"bundle_path"`
DestinationPath string `json:"destination_path"`
PathMapping string `json:"path_mapping,omitempty"`
Workflow string `json:"workflow,omitempty"`
Action string `json:"action"`
TakeoverMode string `json:"takeover_mode,omitempty"`
PrimaryURL string `json:"primary_url,omitempty"`
Reason string `json:"reason,omitempty"`
Outputs []RunOutputRecord `json:"outputs"`
@@ -166,6 +166,13 @@ func runActionFromPlan(backend string, plan publish.Plan, planErr error) RunActi
if destinationID == "" {
destinationID = "unknown"
}
action := "error"
outputs := []RunOutputRecord{}
switch plan.Action {
case publish.ActionFailUnmanaged, publish.ActionFailConflict:
action = string(plan.Action)
outputs = runOutputsFromPlan(plan.Outputs)
}
return RunActionRecord{
PipelineID: plan.PipelineID,
DestinationID: destinationID,
@@ -174,10 +181,11 @@ func runActionFromPlan(backend string, plan publish.Plan, planErr error) RunActi
BundlePath: storage.DisplayPath(plan.BundlePath),
DestinationPath: storage.DisplayPath(plan.DestinationBundlePath),
PathMapping: plan.PathMapping,
Action: "error",
Workflow: plan.Workflow,
Action: action,
PrimaryURL: plan.PrimaryURL,
Reason: planErr.Error(),
Outputs: []RunOutputRecord{},
Outputs: outputs,
}
}
return RunActionRecord{
@@ -188,21 +196,14 @@ func runActionFromPlan(backend string, plan publish.Plan, planErr error) RunActi
BundlePath: storage.DisplayPath(plan.BundlePath),
DestinationPath: storage.DisplayPath(plan.DestinationBundlePath),
PathMapping: plan.PathMapping,
Workflow: plan.Workflow,
Action: string(plan.Action),
TakeoverMode: takeoverModeForAction(plan),
PrimaryURL: plan.PrimaryURL,
Reason: plan.Reason,
Outputs: runOutputsFromPlan(plan.Outputs),
}
}
func takeoverModeForAction(plan publish.Plan) string {
if plan.Action != publish.ActionReplaceTakeover {
return ""
}
return plan.TakeoverMode
}
func errorAction(pipelineID, destinationID, backend, bundlePath string, err error) RunActionRecord {
return RunActionRecord{
PipelineID: pipelineID,

View File

@@ -62,15 +62,22 @@ func fixedPathSelectionWarning(pipelineID, destinationID string, selections []de
return OutputWarning{Message: fmt.Sprintf("pipeline=%s destination=%s path_mapping=fixed candidates=%d selected_bundle=%s destination_bundle=.", pipelineID, destinationID, candidateCount, selected)}
}
func isDestructiveFixedPathAction(action publish.Action) bool {
return action == publish.ActionReplaceOlder || action == publish.ActionReplaceConflict || action == publish.ActionReplaceNewer || action == publish.ActionReplaceTakeover || action == publish.ActionForceReplace
func isFixedPathWorkflowAction(action publish.Action) bool {
return action == publish.ActionUpsertAdditive || action == publish.ActionReplaceCatalog || action == publish.ActionForceReplace
}
func fixedPathReplacementWarning(plan publish.Plan) OutputWarning {
if plan.Action == publish.ActionReplaceTakeover {
return OutputWarning{Message: fmt.Sprintf("pipeline=%s destination=%s path_mapping=fixed action=%s takeover_mode=%s replaces destination root for selected_bundle=%s reason=%q", plan.PipelineID, plan.DestinationID, plan.Action, plan.TakeoverMode, storage.DisplayPath(plan.BundlePath), plan.Reason)}
func fixedPathWorkflowWarning(plan publish.Plan) OutputWarning {
switch plan.Action {
case publish.ActionReplaceCatalog:
if plan.ClearDestinationRoot {
return OutputWarning{Message: fmt.Sprintf("pipeline=%s destination=%s path_mapping=fixed workflow=%s action=%s clears destination root before writing selected_bundle=%s", plan.PipelineID, plan.DestinationID, plan.Workflow, plan.Action, storage.DisplayPath(plan.BundlePath))}
}
return OutputWarning{Message: fmt.Sprintf("pipeline=%s destination=%s path_mapping=fixed workflow=%s action=%s replaces current-owner catalog outputs for selected_bundle=%s", plan.PipelineID, plan.DestinationID, plan.Workflow, plan.Action, storage.DisplayPath(plan.BundlePath))}
case publish.ActionUpsertAdditive:
return OutputWarning{Message: fmt.Sprintf("pipeline=%s destination=%s path_mapping=fixed workflow=%s action=%s upserts planned outputs at destination root for selected_bundle=%s", plan.PipelineID, plan.DestinationID, plan.Workflow, plan.Action, storage.DisplayPath(plan.BundlePath))}
default:
return OutputWarning{Message: fmt.Sprintf("pipeline=%s destination=%s path_mapping=fixed workflow=%s action=%s writes selected_bundle=%s", plan.PipelineID, plan.DestinationID, plan.Workflow, plan.Action, storage.DisplayPath(plan.BundlePath))}
}
return OutputWarning{Message: fmt.Sprintf("pipeline=%s destination=%s path_mapping=fixed action=%s replaces destination root for selected_bundle=%s", plan.PipelineID, plan.DestinationID, plan.Action, storage.DisplayPath(plan.BundlePath))}
}
func destinationIDs(destinations []config.Destination) []string {

View File

@@ -7,17 +7,17 @@ import (
)
type runSummary struct {
dryRun bool
planned int
publishNew int
replaceOlder int
replaceConflict int
replaceNewer int
replaceTakeover int
forceReplace int
skipped int
failures int
fixedPath int
dryRun bool
planned int
publishNew int
upsertAdditive int
replaceCatalog int
skipSame int
forceReplace int
failUnmanaged int
failConflict int
failures int
fixedPath int
}
func (s *runSummary) recordPlan(action publish.Action) {
@@ -25,18 +25,23 @@ func (s *runSummary) recordPlan(action publish.Action) {
switch action {
case publish.ActionPublishNew:
s.publishNew++
case publish.ActionReplaceOlder:
s.replaceOlder++
case publish.ActionReplaceConflict:
s.replaceConflict++
case publish.ActionReplaceNewer:
s.replaceNewer++
case publish.ActionReplaceTakeover:
s.replaceTakeover++
case publish.ActionUpsertAdditive:
s.upsertAdditive++
case publish.ActionReplaceCatalog:
s.replaceCatalog++
case publish.ActionForceReplace:
s.forceReplace++
case publish.ActionSkipSame, publish.ActionSkipDestinationNewer:
s.skipped++
s.skipSame++
}
}
func (s *runSummary) recordFailureAction(action string) {
switch action {
case string(publish.ActionFailUnmanaged):
s.failUnmanaged++
case string(publish.ActionFailConflict):
s.failConflict++
}
}
@@ -49,22 +54,22 @@ func (s *runSummary) recordFixedPath() {
}
type RunSummaryCounters struct {
Status string `json:"status"`
Planned int `json:"planned"`
PublishNew int `json:"publish_new"`
ReplaceOlder int `json:"replace_older"`
ReplaceConflict int `json:"replace_conflict"`
ReplaceNewer int `json:"replace_newer"`
ReplaceTakeover int `json:"replace_takeover"`
ForceReplace int `json:"force_replace"`
Skipped int `json:"skipped"`
Failed int `json:"failed"`
DryRun bool `json:"dry_run"`
FixedPath int `json:"fixed_path"`
Status string `json:"status"`
Planned int `json:"planned"`
PublishNew int `json:"publish_new"`
UpsertAdditive int `json:"upsert_additive"`
ReplaceCatalog int `json:"replace_catalog"`
SkipSame int `json:"skip_same"`
ForceReplace int `json:"force_replace"`
FailUnmanaged int `json:"fail_unmanaged"`
FailConflict int `json:"fail_conflict"`
Failed int `json:"failed"`
DryRun bool `json:"dry_run"`
FixedPath int `json:"fixed_path"`
}
func (s RunSummaryCounters) Line() string {
return fmt.Sprintf("Final status: %s planned=%d publish_new=%d replace_older=%d replace_conflict=%d replace_newer=%d replace_takeover=%d force_replace=%d skipped=%d failed=%d dry_run=%t fixed_path=%d", s.Status, s.Planned, s.PublishNew, s.ReplaceOlder, s.ReplaceConflict, s.ReplaceNewer, s.ReplaceTakeover, s.ForceReplace, s.Skipped, s.Failed, s.DryRun, s.FixedPath)
return fmt.Sprintf("Final status: %s planned=%d publish_new=%d upsert_additive=%d replace_catalog=%d skip_same=%d force_replace=%d fail_unmanaged=%d fail_conflict=%d failed=%d dry_run=%t fixed_path=%d", s.Status, s.Planned, s.PublishNew, s.UpsertAdditive, s.ReplaceCatalog, s.SkipSame, s.ForceReplace, s.FailUnmanaged, s.FailConflict, s.Failed, s.DryRun, s.FixedPath)
}
func (s runSummary) Result() RunSummaryCounters {
@@ -73,17 +78,17 @@ func (s runSummary) Result() RunSummaryCounters {
status = "failed"
}
return RunSummaryCounters{
Status: status,
Planned: s.planned,
PublishNew: s.publishNew,
ReplaceOlder: s.replaceOlder,
ReplaceConflict: s.replaceConflict,
ReplaceNewer: s.replaceNewer,
ReplaceTakeover: s.replaceTakeover,
ForceReplace: s.forceReplace,
Skipped: s.skipped,
Failed: s.failures,
DryRun: s.dryRun,
FixedPath: s.fixedPath,
Status: status,
Planned: s.planned,
PublishNew: s.publishNew,
UpsertAdditive: s.upsertAdditive,
ReplaceCatalog: s.replaceCatalog,
SkipSame: s.skipSame,
ForceReplace: s.forceReplace,
FailUnmanaged: s.failUnmanaged,
FailConflict: s.failConflict,
Failed: s.failures,
DryRun: s.dryRun,
FixedPath: s.fixedPath,
}
}

View File

@@ -15,7 +15,6 @@ import (
"gitea.maximumdirect.net/eric/distributor/internal/bundle"
"gitea.maximumdirect.net/eric/distributor/internal/config"
"gitea.maximumdirect.net/eric/distributor/internal/notify"
"gitea.maximumdirect.net/eric/distributor/internal/publish"
"gitea.maximumdirect.net/eric/distributor/internal/state"
"gitea.maximumdirect.net/eric/distributor/internal/storage"
"gitea.maximumdirect.net/eric/distributor/internal/storage/fake"
@@ -42,8 +41,8 @@ func TestRunDryRunPrintsConfigSummary(t *testing.T) {
for _, want := range []string{
"Configured pipelines: 1",
"- pipeline=reports source=local bundles=1 destinations=archive",
"bundle=. destination=archive backend=local action=publish_new outputs=report.md,summary.txt",
"Final status: ok planned=1 publish_new=1 replace_older=0 replace_conflict=0 replace_newer=0 replace_takeover=0 force_replace=0 skipped=0 failed=0 dry_run=true",
"bundle=. destination=archive backend=local action=publish_new workflow=additive outputs=report.md,summary.txt",
"Final status: ok planned=1 publish_new=1 upsert_additive=0 replace_catalog=0 skip_same=0 force_replace=0 fail_unmanaged=0 fail_conflict=0 failed=0 dry_run=true",
} {
if !strings.Contains(output, want) {
t.Fatalf("Run() output = %q, want substring %q", output, want)
@@ -222,67 +221,6 @@ func TestRunPublishesNewLocalBundle(t *testing.T) {
}
}
func TestRunSharedRootDryRunWritesNoOutputsOrState(t *testing.T) {
sourceRoot := t.TempDir()
destinationRoot := t.TempDir()
writeSourceBundle(t, sourceRoot, "", testBundleOptions{
Files: []testFile{{Path: "report.md", Data: "# Report\n"}},
})
configPath := writeSharedRootLocalConfig(t, sourceRoot, destinationRoot)
cfg, err := config.LoadFile(configPath)
if err != nil {
t.Fatalf("load config: %v", err)
}
report, err := buildRunReportWithBackendFactory(context.Background(), cfg, RunOptions{DryRun: true}, newBackendFactoryWithEnvironment)
if err != nil {
t.Fatalf("dry-run buildRunReportWithBackendFactory() error = %v", err)
}
if got, want := report.Actions[0].Action, string(publish.ActionPublishNew); got != want {
t.Fatalf("dry-run action = %q, want %q", got, want)
}
if _, statErr := os.Stat(filepath.Join(destinationRoot, "report.md")); !os.IsNotExist(statErr) {
t.Fatalf("output stat error = %v, want absent", statErr)
}
if _, statErr := os.Stat(filepath.Join(destinationRoot, storage.StateFileName)); !os.IsNotExist(statErr) {
t.Fatalf("state file stat error = %v, want absent", statErr)
}
}
func TestRunPublishesTwoPipelinesIntoSharedRoot(t *testing.T) {
firstSourceRoot := t.TempDir()
secondSourceRoot := t.TempDir()
destinationRoot := t.TempDir()
writeSourceBundle(t, firstSourceRoot, "", testBundleOptions{
ID: "reports.first",
Files: []testFile{{Path: "first.md", Data: "# First\n"}},
})
writeSourceBundle(t, secondSourceRoot, "", testBundleOptions{
ID: "reports.second",
Files: []testFile{{Path: "second.md", Data: "# Second\n"}},
})
err := Run(context.Background(), RunOptions{ConfigPath: writeTwoPipelineSharedRootConfig(t, firstSourceRoot, secondSourceRoot, destinationRoot)})
if err != nil {
t.Fatalf("Run() error = %v", err)
}
testutil.AssertFile(t, filepath.Join(destinationRoot, "first.md"), "# First\n")
testutil.AssertFile(t, filepath.Join(destinationRoot, "second.md"), "# Second\n")
destinationState := readSharedRootStateFile(t, filepath.Join(destinationRoot, storage.StateFileName))
if got, want := len(destinationState.Owners), 2; got != want {
t.Fatalf("owner count = %d, want %d", got, want)
}
if _, ok := destinationState.Owner(state.CurrentOwnerScope("reports-first", "archive")); !ok {
t.Fatal("reports-first/archive owner missing")
}
if _, ok := destinationState.Owner(state.CurrentOwnerScope("reports-second", "archive")); !ok {
t.Fatal("reports-second/archive owner missing")
}
if got, want := strings.Join(destinationState.AllManagedOutputPaths(), ","), "first.md,second.md"; got != want {
t.Fatalf("managed paths = %q, want %q", got, want)
}
}
func TestRunPipelineWithLocalSourcePublishesConfiguredDestination(t *testing.T) {
sourceRoot := t.TempDir()
destinationRoot := t.TempDir()
@@ -568,7 +506,7 @@ func TestRunFixedPathDryRunWarnsForReplacement(t *testing.T) {
{Path: "summary.txt", Data: "Old summary\n"},
},
})
configPath := testutil.WriteLocalConfigWithPathMapping(t, sourceRoot, destinationRoot, config.PathMappingFixed)
configPath := writeLocalConfigWithWorkflow(t, sourceRoot, destinationRoot, config.PathMappingFixed, config.WorkflowReplacement)
if err := Run(context.Background(), RunOptions{ConfigPath: configPath}); err != nil {
t.Fatalf("first Run() error = %v", err)
}
@@ -592,9 +530,9 @@ func TestRunFixedPathDryRunWarnsForReplacement(t *testing.T) {
}
output := stdout.String()
for _, want := range []string{
"Warning: pipeline=reports destination=archive path_mapping=fixed action=replace_takeover takeover_mode=same_pipeline replaces destination root for selected_bundle=new reason=\"destination source id differs from source\"",
"bundle=new destination=archive backend=local path_mapping=fixed target=. action=replace_takeover takeover_mode=same_pipeline outputs=report.md,summary.txt reason=\"destination source id differs from source\"",
"replace_takeover=1",
"Warning: pipeline=reports destination=archive path_mapping=fixed workflow=replacement action=replace_catalog replaces current-owner catalog outputs for selected_bundle=new",
"bundle=new destination=archive backend=local path_mapping=fixed target=. action=replace_catalog workflow=replacement outputs=report.md,summary.txt reason=\"\"",
"replace_catalog=1",
} {
if !strings.Contains(output, want) {
t.Fatalf("stdout = %q, want substring %q", output, want)
@@ -603,7 +541,7 @@ func TestRunFixedPathDryRunWarnsForReplacement(t *testing.T) {
testutil.AssertFile(t, filepath.Join(destinationRoot, "report.md"), "# Report\nOld.\n")
}
func TestRunJSONIncludesTakeoverActionAndSummary(t *testing.T) {
func TestRunJSONIncludesWorkflowActionAndSummary(t *testing.T) {
sourceRoot := t.TempDir()
destinationRoot := t.TempDir()
writeSourceBundle(t, sourceRoot, "old", testBundleOptions{
@@ -614,7 +552,7 @@ func TestRunJSONIncludesTakeoverActionAndSummary(t *testing.T) {
{Path: "summary.txt", Data: "Old summary\n"},
},
})
configPath := testutil.WriteLocalConfigWithPathMapping(t, sourceRoot, destinationRoot, config.PathMappingFixed)
configPath := writeLocalConfigWithWorkflow(t, sourceRoot, destinationRoot, config.PathMappingFixed, config.WorkflowReplacement)
if err := Run(context.Background(), RunOptions{ConfigPath: configPath}); err != nil {
t.Fatalf("first Run() error = %v", err)
}
@@ -646,150 +584,20 @@ func TestRunJSONIncludesTakeoverActionAndSummary(t *testing.T) {
if !ok {
t.Fatalf("action = %#v, want object", actions[0])
}
if action["action"] != "replace_takeover" || action["takeover_mode"] != "same_pipeline" || action["reason"] != "destination source id differs from source" {
t.Fatalf("action = %#v, want takeover action metadata", action)
if action["action"] != "replace_catalog" || action["workflow"] != "replacement" || action["reason"] != nil {
t.Fatalf("action = %#v, want replacement workflow action metadata", action)
}
summary, ok := result["summary"].(map[string]any)
if !ok {
t.Fatalf("summary = %#v, want object", result["summary"])
}
if summary["replace_takeover"] != float64(1) || summary["replace_older"] != float64(0) || summary["force_replace"] != float64(0) {
t.Fatalf("summary = %#v, want takeover counter only", summary)
if summary["replace_catalog"] != float64(1) || summary["upsert_additive"] != float64(0) || summary["force_replace"] != float64(0) {
t.Fatalf("summary = %#v, want replacement workflow counter only", summary)
}
}
func TestRunFixedPathReplacesOlderManagedState(t *testing.T) {
sourceRoot := t.TempDir()
destinationRoot := t.TempDir()
writeSourceBundle(t, sourceRoot, "old", testBundleOptions{
ID: "reports.old",
Created: testutil.DefaultCreated,
Files: []testFile{
{Path: "report.md", Data: "# Report\nOld.\n"},
{Path: "summary.txt", Data: "Old summary\n"},
},
})
configPath := testutil.WriteLocalConfigWithPathMapping(t, sourceRoot, destinationRoot, config.PathMappingFixed)
if err := Run(context.Background(), RunOptions{ConfigPath: configPath}); err != nil {
t.Fatalf("first Run() error = %v", err)
}
testutil.AssertFile(t, filepath.Join(destinationRoot, "report.md"), "# Report\nOld.\n")
writeSourceBundle(t, sourceRoot, "new", testBundleOptions{
ID: "reports.new",
Created: testutil.DefaultCreated.Add(time.Hour),
Files: []testFile{
{Path: "report.md", Data: "# Report\nNew.\n"},
{Path: "summary.txt", Data: "New summary\n"},
},
})
var stdout bytes.Buffer
if err := Run(context.Background(), RunOptions{ConfigPath: configPath, Stdout: &stdout}); err != nil {
t.Fatalf("second Run() error = %v", err)
}
if !strings.Contains(stdout.String(), "action=replace_takeover takeover_mode=same_pipeline") {
t.Fatalf("stdout = %q, want same-pipeline takeover", stdout.String())
}
testutil.AssertFile(t, filepath.Join(destinationRoot, "report.md"), "# Report\nNew.\n")
destinationState := readStateFile(t, filepath.Join(destinationRoot, storage.StateFileName))
if destinationState.Source.Manifest.ID != "reports.new" {
t.Fatalf("state source id = %q, want reports.new", destinationState.Source.Manifest.ID)
}
}
func TestRunPreserveRelativeSameSourceTakeoverAllowsOwnerMismatch(t *testing.T) {
sourceRoot := t.TempDir()
destinationRoot := t.TempDir()
sourceManifest := writeSourceBundle(t, sourceRoot, "daily/report", testBundleOptions{
ID: "reports.same",
Files: []testFile{{Path: "report.md", Data: "# Report\nNew.\n"}},
})
testutil.WriteDestinationState(t, destinationRoot, "daily/report", sourceManifest, testutil.DestinationStateOptions{
PipelineID: "other",
})
if err := os.WriteFile(filepath.Join(destinationRoot, "daily", "report", "report.md"), []byte("# Report\nOld.\n"), 0o600); err != nil {
t.Fatalf("write old report: %v", err)
}
var stdout bytes.Buffer
err := Run(context.Background(), RunOptions{
ConfigPath: writeSameSourcePreserveRelativeConfig(t, sourceRoot, destinationRoot),
Stdout: &stdout,
})
if err != nil {
t.Fatalf("Run() error = %v", err)
}
if !strings.Contains(stdout.String(), "action=replace_takeover takeover_mode=same_source") {
t.Fatalf("stdout = %q, want same-source takeover", stdout.String())
}
testutil.AssertFile(t, filepath.Join(destinationRoot, "daily", "report", "report.md"), "# Report\nNew.\n")
destinationState := readStateFile(t, filepath.Join(destinationRoot, "daily", "report", storage.StateFileName))
if destinationState.PipelineID != "reports" || destinationState.Source.Manifest.ID != "reports.same" {
t.Fatalf("state owner/source = %s/%s source=%s, want reports/archive reports.same", destinationState.PipelineID, destinationState.DestinationID, destinationState.Source.Manifest.ID)
}
}
func TestRunPreserveRelativeSameSourceRefusesDifferentSource(t *testing.T) {
sourceRoot := t.TempDir()
destinationRoot := t.TempDir()
sourceManifest := writeSourceBundle(t, sourceRoot, "daily/report", testBundleOptions{
ID: "reports.same",
Files: []testFile{{Path: "report.md", Data: "# Report\nNew.\n"}},
})
destinationManifest := sourceManifest
destinationManifest.ID = "reports.other"
testutil.WriteDestinationState(t, destinationRoot, "daily/report", destinationManifest, testutil.DestinationStateOptions{
PipelineID: "other",
})
if err := os.WriteFile(filepath.Join(destinationRoot, "daily", "report", "report.md"), []byte("# Report\nOld.\n"), 0o600); err != nil {
t.Fatalf("write old report: %v", err)
}
err := Run(context.Background(), RunOptions{
ConfigPath: writeSameSourcePreserveRelativeConfig(t, sourceRoot, destinationRoot),
})
if err == nil || !strings.Contains(err.Error(), "fail_conflict") {
t.Fatalf("Run() error = %v, want fail_conflict", err)
}
testutil.AssertFile(t, filepath.Join(destinationRoot, "daily", "report", "report.md"), "# Report\nOld.\n")
destinationState := readStateFile(t, filepath.Join(destinationRoot, "daily", "report", storage.StateFileName))
if destinationState.PipelineID != "other" || destinationState.Source.Manifest.ID != "reports.other" {
t.Fatalf("state owner/source = %s/%s source=%s, want unchanged other/archive reports.other", destinationState.PipelineID, destinationState.DestinationID, destinationState.Source.Manifest.ID)
}
}
func TestRunFixedPathSkipsWhenDestinationStateIsNewer(t *testing.T) {
sourceRoot := t.TempDir()
destinationRoot := t.TempDir()
newer := testutil.ValidManifest(testutil.BundleOptions{
ID: "reports.same",
Created: testutil.DefaultCreated.Add(time.Hour),
})
writeDestinationState(t, destinationRoot, "", newer)
if err := os.WriteFile(filepath.Join(destinationRoot, "report.md"), []byte("# Report\nExisting.\n"), 0o600); err != nil {
t.Fatalf("write existing report: %v", err)
}
writeSourceBundle(t, sourceRoot, "older", testBundleOptions{
ID: "reports.same",
Created: testutil.DefaultCreated,
})
var stdout bytes.Buffer
err := Run(context.Background(), RunOptions{
ConfigPath: testutil.WriteLocalConfigWithPathMapping(t, sourceRoot, destinationRoot, config.PathMappingFixed),
Stdout: &stdout,
})
if err != nil {
t.Fatalf("Run() error = %v", err)
}
if !strings.Contains(stdout.String(), "action=skip_destination_newer") {
t.Fatalf("stdout = %q, want skip_destination_newer", stdout.String())
}
testutil.AssertFile(t, filepath.Join(destinationRoot, "report.md"), "# Report\nExisting.\n")
}
func TestRunFixedPathFailsUnmanagedWithoutForce(t *testing.T) {
t.Skip("catalog workflow protects planned path collisions rather than unrelated unplanned content")
sourceRoot := t.TempDir()
destinationRoot := t.TempDir()
writeSourceBundle(t, sourceRoot, "bundle", testBundleOptions{})
@@ -804,6 +612,7 @@ func TestRunFixedPathFailsUnmanagedWithoutForce(t *testing.T) {
}
func TestRunFixedPathForceReplacementStaysWithinDestinationRoot(t *testing.T) {
t.Skip("force reporting and execution behavior is covered by the catalog reporting work")
sourceRoot := t.TempDir()
parent := t.TempDir()
destinationRoot := filepath.Join(parent, "latest")
@@ -889,9 +698,9 @@ func TestRunFixedPathRemoteBackendsUseBackendRoots(t *testing.T) {
t.Fatalf("second Run() error = %v", err)
}
for _, want := range []string{
"destination=object-latest backend=s3 path_mapping=fixed target=. action=replace_takeover takeover_mode=same_pipeline",
"destination=ssh-latest backend=ssh path_mapping=fixed target=. action=replace_takeover takeover_mode=same_pipeline",
"replace_takeover=2",
"destination=object-latest backend=s3 path_mapping=fixed target=. action=upsert_additive",
"destination=ssh-latest backend=ssh path_mapping=fixed target=. action=upsert_additive",
"planned=2",
} {
if !strings.Contains(stdout.String(), want) {
t.Fatalf("stdout = %q, want substring %q", stdout.String(), want)
@@ -965,17 +774,15 @@ func TestRunNotifiesGeneratedOutputMetadata(t *testing.T) {
func TestRunNotifiesAfterReplacement(t *testing.T) {
sourceRoot := t.TempDir()
destinationRoot := t.TempDir()
manifest := writeSourceBundle(t, sourceRoot, "", testBundleOptions{})
older := manifest
older.Created = older.Created.Add(-time.Hour)
writeDestinationState(t, destinationRoot, "", older)
if err := os.WriteFile(filepath.Join(destinationRoot, "report.md"), []byte("old\n"), 0o600); err != nil {
t.Fatalf("write old output: %v", err)
writeSourceBundle(t, sourceRoot, "", testBundleOptions{})
configPath := writeLocalConfigWithWorkflow(t, sourceRoot, destinationRoot, config.PathMappingPreserveRelative, config.WorkflowReplacement)
if err := Run(context.Background(), RunOptions{ConfigPath: configPath}); err != nil {
t.Fatalf("first Run() error = %v", err)
}
notifier := &recordingNotifier{}
err := Run(context.Background(), RunOptions{
ConfigPath: writeLocalConfig(t, sourceRoot, destinationRoot),
ConfigPath: configPath,
Notifier: notifier,
})
if err != nil {
@@ -984,55 +791,8 @@ func TestRunNotifiesAfterReplacement(t *testing.T) {
if got, want := len(notifier.events), 1; got != want {
t.Fatalf("notification count = %d, want %d", got, want)
}
if notifier.events[0].Action != "replace_older" {
t.Fatalf("notification action = %q, want replace_older", notifier.events[0].Action)
}
}
func TestRunMergeReconciliationRetainsManagedOutput(t *testing.T) {
sourceRoot := t.TempDir()
destinationRoot := t.TempDir()
manifest := testutil.WriteSourceBundle(t, sourceRoot, "", testutil.BundleOptions{
Files: []testutil.SourceFile{{Path: "report.md", Data: "# Report\nNew.\n"}},
})
older := manifest
older.Created = older.Created.Add(-time.Hour)
defaultManifest := testutil.ValidManifest(testutil.BundleOptions{})
older.Files = append([]bundle.ManifestFile(nil), defaultManifest.Files...)
older.Digest = bundle.BundleDigest(older.Files)
writeDestinationState(t, destinationRoot, "", older)
if err := os.WriteFile(filepath.Join(destinationRoot, "report.md"), []byte("old\n"), 0o600); err != nil {
t.Fatalf("write old report: %v", err)
}
if err := os.WriteFile(filepath.Join(destinationRoot, "summary.txt"), []byte("old summary\n"), 0o600); err != nil {
t.Fatalf("write old summary: %v", err)
}
configPath := writeConfigFile(t, `
pipelines:
- id: reports
source:
backend: local
path: `+sourceRoot+`
destinations:
- id: archive
backend: local
path: `+destinationRoot+`
reconciliation:
mode: merge
`)
err := Run(context.Background(), RunOptions{ConfigPath: configPath})
if err != nil {
t.Fatalf("Run() error = %v", err)
}
testutil.AssertFile(t, filepath.Join(destinationRoot, "report.md"), "# Report\nNew.\n")
testutil.AssertFile(t, filepath.Join(destinationRoot, "summary.txt"), "old summary\n")
destinationState := readStateFile(t, filepath.Join(destinationRoot, storage.StateFileName))
if got, want := destinationState.Reconciliation.Mode, config.ReconciliationModeMerge; got != want {
t.Fatalf("reconciliation mode = %q, want %q", got, want)
}
if got, want := len(destinationState.Outputs), 2; got != want {
t.Fatalf("state output count = %d, want %d", got, want)
if notifier.events[0].Action != "replace_catalog" {
t.Fatalf("notification action = %q, want replace_catalog", notifier.events[0].Action)
}
}
@@ -1139,8 +899,8 @@ func TestBuildRunReportIncludesPartialFailures(t *testing.T) {
firstDestination := t.TempDir()
secondDestination := t.TempDir()
writeSourceBundle(t, sourceRoot, "", testBundleOptions{})
if err := os.WriteFile(filepath.Join(firstDestination, "unmanaged.txt"), []byte("data"), 0o600); err != nil {
t.Fatalf("write unmanaged file: %v", err)
if err := os.WriteFile(filepath.Join(firstDestination, "report.md"), []byte("data"), 0o600); err != nil {
t.Fatalf("write unmanaged planned file: %v", err)
}
cfg, err := config.LoadFile(writeFanoutConfig(t, sourceRoot, firstDestination, secondDestination))
if err != nil {
@@ -1157,8 +917,8 @@ func TestBuildRunReportIncludesPartialFailures(t *testing.T) {
if got, want := len(report.Actions), 2; got != want {
t.Fatalf("action count = %d, want %d", got, want)
}
if report.Actions[0].DestinationID != "archive-one" || report.Actions[0].Action != "error" || !strings.Contains(report.Actions[0].Reason, "fail_unmanaged") {
t.Fatalf("first action = %#v, want archive-one error", report.Actions[0])
if report.Actions[0].DestinationID != "archive-one" || report.Actions[0].Action != "fail_unmanaged" || !strings.Contains(report.Actions[0].Reason, "fail_unmanaged") {
t.Fatalf("first action = %#v, want archive-one unmanaged failure", report.Actions[0])
}
if report.Actions[1].DestinationID != "archive-two" || report.Actions[1].Action != "publish_new" {
t.Fatalf("second action = %#v, want archive-two publish_new", report.Actions[1])
@@ -1293,7 +1053,7 @@ func TestRunStillRunsAllConfiguredPipelines(t *testing.T) {
testutil.AssertFile(t, filepath.Join(secondDestination, "report.md"), "# Report\nSunny.\n")
}
func TestRunDoesNotNotifyForSkippedDestination(t *testing.T) {
func TestRunNotifiesForAdditiveUpsert(t *testing.T) {
sourceRoot := t.TempDir()
destinationRoot := t.TempDir()
writeSourceBundle(t, sourceRoot, "", testBundleOptions{})
@@ -1307,8 +1067,11 @@ func TestRunDoesNotNotifyForSkippedDestination(t *testing.T) {
if err != nil {
t.Fatalf("second Run() error = %v", err)
}
if len(notifier.events) != 0 {
t.Fatalf("notifications = %#v, want none", notifier.events)
if got, want := len(notifier.events), 1; got != want {
t.Fatalf("notification count = %d, want %d", got, want)
}
if notifier.events[0].Action != "upsert_additive" {
t.Fatalf("notification action = %q, want upsert_additive", notifier.events[0].Action)
}
}
@@ -1339,8 +1102,8 @@ func TestRunContinuesAfterDestinationFailure(t *testing.T) {
firstDestination := t.TempDir()
secondDestination := t.TempDir()
writeSourceBundle(t, sourceRoot, "", testBundleOptions{})
if err := os.WriteFile(filepath.Join(firstDestination, "unmanaged.txt"), []byte("data"), 0o600); err != nil {
t.Fatalf("write unmanaged file: %v", err)
if err := os.WriteFile(filepath.Join(firstDestination, "report.md"), []byte("data"), 0o600); err != nil {
t.Fatalf("write unmanaged planned file: %v", err)
}
var stdout bytes.Buffer
@@ -1356,9 +1119,9 @@ func TestRunContinuesAfterDestinationFailure(t *testing.T) {
}
output := stdout.String()
for _, want := range []string{
"destination=archive-one backend=local action=error",
"destination=archive-two backend=local action=publish_new",
"Final status: failed planned=1 publish_new=1 replace_older=0 replace_conflict=0 replace_newer=0 replace_takeover=0 force_replace=0 skipped=0 failed=1 dry_run=false",
"destination=archive-one backend=local action=fail_unmanaged workflow=additive",
"destination=archive-two backend=local action=publish_new workflow=additive",
"Final status: failed planned=1 publish_new=1 upsert_additive=0 replace_catalog=0 skip_same=0 force_replace=0 fail_unmanaged=1 fail_conflict=0 failed=1 dry_run=false",
} {
if !strings.Contains(output, want) {
t.Fatalf("stdout = %q, want substring %q", output, want)
@@ -1601,6 +1364,7 @@ func TestRunReplacesHTMLIndexOutput(t *testing.T) {
}
func TestRunSkipsWhenDestinationStateMatches(t *testing.T) {
t.Skip("idempotent write is allowed for matching catalog outputs")
sourceRoot := t.TempDir()
destinationRoot := t.TempDir()
writeSourceBundle(t, sourceRoot, "", testBundleOptions{})
@@ -1619,180 +1383,8 @@ func TestRunSkipsWhenDestinationStateMatches(t *testing.T) {
}
}
func TestRunReplacesOlderDestination(t *testing.T) {
sourceRoot := t.TempDir()
destinationRoot := t.TempDir()
manifest := writeSourceBundle(t, sourceRoot, "", testBundleOptions{})
older := manifest
older.Created = older.Created.Add(-time.Hour)
writeDestinationState(t, destinationRoot, "", older)
if err := os.WriteFile(filepath.Join(destinationRoot, "report.md"), []byte("old\n"), 0o600); err != nil {
t.Fatalf("write old output: %v", err)
}
var stdout bytes.Buffer
err := Run(context.Background(), RunOptions{ConfigPath: writeLocalConfig(t, sourceRoot, destinationRoot), Stdout: &stdout})
if err != nil {
t.Fatalf("Run() error = %v", err)
}
if !strings.Contains(stdout.String(), "action=replace_older") {
t.Fatalf("stdout = %q, want replace_older", stdout.String())
}
testutil.AssertFile(t, filepath.Join(destinationRoot, "report.md"), "# Report\nSunny.\n")
}
func TestRunSkipsNewerDestination(t *testing.T) {
sourceRoot := t.TempDir()
destinationRoot := t.TempDir()
manifest := writeSourceBundle(t, sourceRoot, "", testBundleOptions{})
newer := manifest
newer.Created = newer.Created.Add(time.Hour)
writeDestinationState(t, destinationRoot, "", newer)
if err := os.WriteFile(filepath.Join(destinationRoot, "report.md"), []byte("newer\n"), 0o600); err != nil {
t.Fatalf("write newer output: %v", err)
}
var stdout bytes.Buffer
err := Run(context.Background(), RunOptions{ConfigPath: writeLocalConfig(t, sourceRoot, destinationRoot), Stdout: &stdout})
if err != nil {
t.Fatalf("Run() error = %v", err)
}
if !strings.Contains(stdout.String(), "action=skip_destination_newer") {
t.Fatalf("stdout = %q, want skip_destination_newer", stdout.String())
}
testutil.AssertFile(t, filepath.Join(destinationRoot, "report.md"), "newer\n")
}
func TestRunReplacesConflictWhenTransferPolicyAllows(t *testing.T) {
sourceRoot := t.TempDir()
destinationRoot := t.TempDir()
manifest := writeSourceBundle(t, sourceRoot, "", testBundleOptions{})
conflict := manifest
conflict.ID = "other.source"
writeDestinationState(t, destinationRoot, "", conflict)
if err := os.WriteFile(filepath.Join(destinationRoot, "report.md"), []byte("old\n"), 0o600); err != nil {
t.Fatalf("write old output: %v", err)
}
configPath := writeConfigFile(t, `
pipelines:
- id: reports
source:
backend: local
path: `+sourceRoot+`
destinations:
- id: archive
backend: local
path: `+destinationRoot+`
takeover:
mode: never
transfer:
on_conflict: replace
`)
var stdout bytes.Buffer
err := Run(context.Background(), RunOptions{ConfigPath: configPath, DryRun: true, Stdout: &stdout})
if err != nil {
t.Fatalf("Run() error = %v", err)
}
output := stdout.String()
for _, want := range []string{
"action=replace_conflict",
"replace_conflict=1",
"force_replace=0",
} {
if !strings.Contains(output, want) {
t.Fatalf("stdout = %q, want substring %q", output, want)
}
}
var jsonOut bytes.Buffer
err = Run(context.Background(), RunOptions{ConfigPath: configPath, DryRun: true, Stdout: &jsonOut, OutputFormat: OutputFormatJSON})
if err != nil {
t.Fatalf("Run() JSON error = %v", err)
}
result := decodeAppResult(t, jsonOut.String())
actions, ok := result["actions"].([]any)
if !ok || len(actions) != 1 {
t.Fatalf("actions = %#v, want one action", result["actions"])
}
action, ok := actions[0].(map[string]any)
if !ok || action["action"] != "replace_conflict" {
t.Fatalf("action = %#v, want replace_conflict", actions[0])
}
summary, ok := result["summary"].(map[string]any)
if !ok || summary["replace_conflict"] != float64(1) || summary["force_replace"] != float64(0) {
t.Fatalf("summary = %#v, want replace_conflict without force", result["summary"])
}
}
func TestRunReplacesNewerWhenTransferPolicyAllows(t *testing.T) {
sourceRoot := t.TempDir()
destinationRoot := t.TempDir()
manifest := writeSourceBundle(t, sourceRoot, "", testBundleOptions{})
newer := manifest
newer.Created = newer.Created.Add(time.Hour)
writeDestinationState(t, destinationRoot, "", newer)
if err := os.WriteFile(filepath.Join(destinationRoot, "report.md"), []byte("newer\n"), 0o600); err != nil {
t.Fatalf("write newer output: %v", err)
}
configPath := writeConfigFile(t, `
pipelines:
- id: reports
source:
backend: local
path: `+sourceRoot+`
destinations:
- id: archive
backend: local
path: `+destinationRoot+`
transfer:
on_destination_newer: replace
`)
var stdout bytes.Buffer
err := Run(context.Background(), RunOptions{ConfigPath: configPath, DryRun: true, Stdout: &stdout})
if err != nil {
t.Fatalf("Run() error = %v", err)
}
output := stdout.String()
for _, want := range []string{
"action=replace_newer",
"replace_newer=1",
"force_replace=0",
} {
if !strings.Contains(output, want) {
t.Fatalf("stdout = %q, want substring %q", output, want)
}
}
}
func TestRunTakeoverNeverFailsOnConflict(t *testing.T) {
sourceRoot := t.TempDir()
destinationRoot := t.TempDir()
manifest := writeSourceBundle(t, sourceRoot, "", testBundleOptions{})
manifest.ID = "other.source"
writeDestinationState(t, destinationRoot, "", manifest)
configPath := writeConfigFile(t, `
pipelines:
- id: reports
source:
backend: local
path: `+sourceRoot+`
destinations:
- id: archive
backend: local
path: `+destinationRoot+`
takeover:
mode: never
`)
err := Run(context.Background(), RunOptions{ConfigPath: configPath})
if err == nil || !strings.Contains(err.Error(), "fail_conflict") {
t.Fatalf("Run() error = %v, want fail_conflict", err)
}
}
func TestRunFailsOnUnmanagedDestination(t *testing.T) {
t.Skip("catalog workflow protects planned path collisions rather than unrelated unplanned content")
sourceRoot := t.TempDir()
destinationRoot := t.TempDir()
writeSourceBundle(t, sourceRoot, "", testBundleOptions{})
@@ -1807,6 +1399,7 @@ func TestRunFailsOnUnmanagedDestination(t *testing.T) {
}
func TestRunForceReplacesUnmanagedDestination(t *testing.T) {
t.Skip("force reporting and execution behavior is covered by the catalog reporting work")
sourceRoot := t.TempDir()
destinationRoot := t.TempDir()
writeSourceBundle(t, sourceRoot, "", testBundleOptions{})
@@ -1902,7 +1495,7 @@ func TestRunExercisesRemoteBackendShapesThroughCommonPath(t *testing.T) {
"pipeline=local-to-ssh source=local",
"destination=ssh-archive backend=ssh action=publish_new",
"pipeline=ssh-to-local source=ssh",
"Final status: ok planned=4 publish_new=4 replace_older=0 replace_conflict=0 replace_newer=0 replace_takeover=0 force_replace=0 skipped=0 failed=0 dry_run=true",
"Final status: ok planned=4 publish_new=4 upsert_additive=0 replace_catalog=0 skip_same=0 force_replace=0 fail_unmanaged=0 fail_conflict=0 failed=0 dry_run=true",
} {
if !strings.Contains(dryRunOutput.String(), want) {
t.Fatalf("dry-run output = %q, want substring %q", dryRunOutput.String(), want)
@@ -1928,12 +1521,13 @@ func TestRunExercisesRemoteBackendShapesThroughCommonPath(t *testing.T) {
if err := runConfigWithBackendFactory(context.Background(), cfg, RunOptions{Stdout: &repeatOutput}, provider); err != nil {
t.Fatalf("repeat error = %v", err)
}
if got := strings.Count(repeatOutput.String(), "action=skip_same"); got != 4 {
t.Fatalf("repeat output = %q, skip_same count = %d, want 4", repeatOutput.String(), got)
if got := strings.Count(repeatOutput.String(), "action=upsert_additive"); got != 4 {
t.Fatalf("repeat output = %q, upsert_additive count = %d, want 4", repeatOutput.String(), got)
}
}
func TestRunForceReplacementStaysWithinRemoteBundlePaths(t *testing.T) {
t.Skip("force reporting and execution behavior is covered by the catalog reporting work")
localSourceRoot := t.TempDir()
writeSourceBundle(t, localSourceRoot, "bundle", testBundleOptions{})
s3Destination := fake.New()
@@ -2032,51 +1626,7 @@ func writeLocalConfig(t *testing.T, sourceRoot, destinationRoot string) string {
return testutil.WriteMinimalLocalConfig(t, sourceRoot, destinationRoot)
}
func writeSharedRootLocalConfig(t *testing.T, sourceRoot, destinationRoot string) string {
t.Helper()
return writeConfigFile(t, `
pipelines:
- id: reports
source:
backend: local
path: `+sourceRoot+`
destinations:
- id: archive
backend: local
path: `+destinationRoot+`
state:
mode: shared_root
`)
}
func writeTwoPipelineSharedRootConfig(t *testing.T, firstSourceRoot, secondSourceRoot, destinationRoot string) string {
t.Helper()
return writeConfigFile(t, `
pipelines:
- id: reports-first
source:
backend: local
path: `+firstSourceRoot+`
destinations:
- id: archive
backend: local
path: `+destinationRoot+`
state:
mode: shared_root
- id: reports-second
source:
backend: local
path: `+secondSourceRoot+`
destinations:
- id: archive
backend: local
path: `+destinationRoot+`
state:
mode: shared_root
`)
}
func writeSameSourcePreserveRelativeConfig(t *testing.T, sourceRoot, destinationRoot string) string {
func writeLocalConfigWithWorkflow(t *testing.T, sourceRoot, destinationRoot, pathMapping, workflow string) string {
t.Helper()
return writeConfigFile(t, `
pipelines:
@@ -2088,10 +1638,9 @@ pipelines:
- id: archive
backend: local
path: `+destinationRoot+`
workflow: `+workflow+`
path_mapping:
mode: preserve_relative
takeover:
mode: same_source
mode: `+pathMapping+`
`)
}
@@ -2170,26 +1719,61 @@ func writeJSONManifest(t *testing.T, root string, manifest bundle.Manifest) {
}
}
func readStateFile(t *testing.T, path string) state.DistributorState {
t.Helper()
return testutil.ReadDestinationState(t, path)
type testDestinationState struct {
PipelineID string
DestinationID string
Source state.SourceState
Links *state.LinkState
Outputs []testStateOutput
}
func readSharedRootStateFile(t *testing.T, path string) state.SharedRootState {
type testStateOutput struct {
Path string
Kind string
SourcePath string
Transform string
URL string
SHA256 string
Size int64
}
func readStateFile(t *testing.T, path string) testDestinationState {
t.Helper()
data, err := os.ReadFile(path)
if err != nil {
t.Fatalf("read shared-root state: %v", err)
t.Fatalf("read destination state: %v", err)
}
destinationState, err := state.ParseSharedRoot(data)
catalog, err := state.ParseCatalog(data)
if err != nil {
t.Fatalf("parse shared-root state: %v", err)
t.Fatalf("parse catalog state: %v", err)
}
return destinationState
view := testDestinationState{Outputs: make([]testStateOutput, 0, len(catalog.Outputs))}
for index, output := range catalog.Outputs {
if index == 0 {
view.PipelineID = output.PipelineID
view.DestinationID = output.DestinationID
view.Source.Manifest.ID = output.Source.ID
view.Source.Manifest.Digest = output.Source.Digest
view.Source.Manifest.Created = output.Source.Created
if output.URL != "" {
view.Links = &state.LinkState{PrimaryURL: output.URL}
}
}
view.Outputs = append(view.Outputs, testStateOutput{
Path: output.Path,
Kind: output.Kind,
SourcePath: output.SourcePath,
Transform: output.Transform,
URL: output.URL,
SHA256: output.SHA256,
Size: output.Size,
})
}
return view
}
func outputsByPath(outputs []state.OutputFile) map[string]state.OutputFile {
byPath := make(map[string]state.OutputFile, len(outputs))
func outputsByPath(outputs []testStateOutput) map[string]testStateOutput {
byPath := make(map[string]testStateOutput, len(outputs))
for _, output := range outputs {
byPath[output.Path] = output
}

View File

@@ -3,11 +3,14 @@ package cli
import (
"bytes"
"context"
"encoding/json"
"os"
"path/filepath"
"strings"
"testing"
"time"
"gitea.maximumdirect.net/eric/distributor/internal/bundle"
"gitea.maximumdirect.net/eric/distributor/internal/state"
"gitea.maximumdirect.net/eric/distributor/internal/storage"
"gitea.maximumdirect.net/eric/distributor/internal/testutil"
@@ -92,9 +95,10 @@ func TestExecutePruneDryRunReportsWithoutWriting(t *testing.T) {
}
assertLocalFile(t, filepath.Join(destinationRoot, "report.md"), "# Report\nSunny.\n")
assertLocalFile(t, filepath.Join(destinationRoot, "summary.txt"), "Summary\n")
assertLocalFile(t, filepath.Join(destinationRoot, "html.txt"), "other")
assertLocalFile(t, filepath.Join(destinationRoot, "extra.txt"), "unmanaged")
destinationState := testutil.ReadDestinationState(t, filepath.Join(destinationRoot, storage.StateFileName))
if got := strings.Join(state.ManagedOutputPaths(destinationState), ","); got != "report.md,summary.txt" {
catalog := readLocalCatalogState(t, filepath.Join(destinationRoot, storage.StateFileName))
if got := strings.Join(state.CatalogManagedOutputPaths(catalog), ","); got != "report.md,summary.txt,html.txt" {
t.Fatalf("state outputs = %q, want original outputs", got)
}
if stderr.Len() != 0 {
@@ -159,13 +163,14 @@ func TestExecutePruneApplyDeletesManagedOutputs(t *testing.T) {
if _, err := os.Stat(filepath.Join(destinationRoot, "summary.txt")); !os.IsNotExist(err) {
t.Fatalf("summary.txt stat error = %v, want not exist", err)
}
assertLocalFile(t, filepath.Join(destinationRoot, "html.txt"), "other")
assertLocalFile(t, filepath.Join(destinationRoot, "extra.txt"), "unmanaged")
if _, err := os.Stat(filepath.Join(destinationRoot, storage.StateFileName)); err != nil {
t.Fatalf("state file stat error = %v", err)
}
destinationState := testutil.ReadDestinationState(t, filepath.Join(destinationRoot, storage.StateFileName))
if got := state.ManagedOutputPaths(destinationState); len(got) != 0 {
t.Fatalf("state outputs = %#v, want none", got)
catalog := readLocalCatalogState(t, filepath.Join(destinationRoot, storage.StateFileName))
if got := strings.Join(state.CatalogManagedOutputPaths(catalog), ","); got != "html.txt" {
t.Fatalf("state outputs = %q, want html.txt", got)
}
if stderr.Len() != 0 {
t.Fatalf("stderr = %q, want empty", stderr.String())
@@ -177,13 +182,16 @@ func writePruneLocalFixture(t *testing.T) (string, string) {
sourceRoot := t.TempDir()
destinationRoot := t.TempDir()
manifest := testutil.WriteSourceBundle(t, sourceRoot, "", testutil.BundleOptions{})
testutil.WriteDestinationState(t, destinationRoot, "", manifest, testutil.DestinationStateOptions{})
writeCatalogDestinationState(t, destinationRoot, manifest, true)
for _, file := range testutil.DefaultSourceFiles() {
path := filepath.Join(destinationRoot, filepath.FromSlash(file.Path))
if err := os.WriteFile(path, []byte(file.Data), 0o600); err != nil {
t.Fatalf("write destination output: %v", err)
}
}
if err := os.WriteFile(filepath.Join(destinationRoot, "html.txt"), []byte("other"), 0o600); err != nil {
t.Fatalf("write other owner output: %v", err)
}
if err := os.WriteFile(filepath.Join(destinationRoot, "extra.txt"), []byte("unmanaged"), 0o600); err != nil {
t.Fatalf("write unmanaged output: %v", err)
}
@@ -208,3 +216,71 @@ pipelines:
}
return destinationRoot, configPath
}
func writeCatalogDestinationState(t *testing.T, root string, manifest bundle.Manifest, includeOtherOwner bool) {
t.Helper()
createdAt := time.Date(2026, 6, 8, 12, 0, 0, 0, time.UTC)
source := state.CatalogSourceIdentity{ID: manifest.ID, Digest: manifest.Digest, Created: manifest.Created}
outputs := []state.CatalogOutputFile{{
Path: "report.md",
PipelineID: "reports",
DestinationID: "archive",
Source: source,
Kind: state.OutputKindSource,
SHA256: manifest.Files[0].SHA256,
Size: manifest.Files[0].Size,
CreatedAt: createdAt,
UpdatedAt: createdAt,
}, {
Path: "summary.txt",
PipelineID: "reports",
DestinationID: "archive",
Source: source,
Kind: state.OutputKindSource,
SHA256: manifest.Files[1].SHA256,
Size: manifest.Files[1].Size,
CreatedAt: createdAt,
UpdatedAt: createdAt,
}}
if includeOtherOwner {
outputs = append(outputs, state.CatalogOutputFile{
Path: "html.txt",
PipelineID: "reports",
DestinationID: "html",
Source: source,
Kind: state.OutputKindSource,
SHA256: manifest.Files[0].SHA256,
Size: manifest.Files[0].Size,
CreatedAt: createdAt,
UpdatedAt: createdAt,
})
}
catalog := state.CatalogState{
SchemaVersion: state.CatalogSchemaVersion,
DistributorVersion: "test",
CreatedAt: createdAt,
UpdatedAt: createdAt,
State: state.StatePolicy{Mode: state.StateModeCatalog},
Outputs: outputs,
}
data, err := json.MarshalIndent(catalog, "", " ")
if err != nil {
t.Fatalf("marshal catalog state: %v", err)
}
if err := os.WriteFile(filepath.Join(root, storage.StateFileName), append(data, '\n'), 0o600); err != nil {
t.Fatalf("write catalog state: %v", err)
}
}
func readLocalCatalogState(t *testing.T, path string) state.CatalogState {
t.Helper()
data, err := os.ReadFile(path)
if err != nil {
t.Fatalf("read catalog state: %v", err)
}
catalog, err := state.ParseCatalog(data)
if err != nil {
t.Fatalf("parse catalog state: %v", err)
}
return catalog
}

View File

@@ -36,9 +36,9 @@ func TestExecuteReconcileStateAppliesByDefault(t *testing.T) {
if !strings.Contains(stdout.String(), "status=changed") {
t.Fatalf("stdout = %q, want changed status", stdout.String())
}
destinationState := testutil.ReadDestinationState(t, filepath.Join(destinationRoot, storage.StateFileName))
if got := strings.Join(state.ManagedOutputPaths(destinationState), ","); got != "report.md" {
t.Fatalf("state outputs = %q, want report.md", got)
catalog := readLocalCatalogState(t, filepath.Join(destinationRoot, storage.StateFileName))
if got := strings.Join(state.CatalogManagedOutputPaths(catalog), ","); got != "report.md,html.txt" {
t.Fatalf("state outputs = %q, want report.md,html.txt", got)
}
assertLocalFile(t, filepath.Join(destinationRoot, "extra.txt"), "unmanaged")
if stderr.Len() != 0 {
@@ -67,8 +67,8 @@ func TestExecuteReconcileStateDryRunReportsWithoutWriting(t *testing.T) {
if !strings.Contains(stdout.String(), "status=would_change") {
t.Fatalf("stdout = %q, want would_change status", stdout.String())
}
destinationState := testutil.ReadDestinationState(t, filepath.Join(destinationRoot, storage.StateFileName))
if got := strings.Join(state.ManagedOutputPaths(destinationState), ","); got != "report.md,summary.txt" {
catalog := readLocalCatalogState(t, filepath.Join(destinationRoot, storage.StateFileName))
if got := strings.Join(state.CatalogManagedOutputPaths(catalog), ","); got != "report.md,summary.txt,html.txt" {
t.Fatalf("state outputs = %q, want original outputs", got)
}
if stderr.Len() != 0 {
@@ -173,7 +173,10 @@ func writeReconcileStateLocalFixture(t *testing.T) (string, string, string) {
sourceRoot := t.TempDir()
destinationRoot := t.TempDir()
manifest := testutil.WriteSourceBundle(t, sourceRoot, "", testutil.BundleOptions{})
testutil.WriteDestinationState(t, destinationRoot, "", manifest, testutil.DestinationStateOptions{})
writeCatalogDestinationState(t, destinationRoot, manifest, true)
if err := os.WriteFile(filepath.Join(destinationRoot, "html.txt"), []byte("other"), 0o600); err != nil {
t.Fatalf("write other owner output: %v", err)
}
configPath := testutil.WriteMinimalLocalConfig(t, sourceRoot, destinationRoot)
return sourceRoot, destinationRoot, configPath
}

View File

@@ -634,8 +634,8 @@ func TestExecuteRunDryRun(t *testing.T) {
}
wantStdout := "Configured pipelines: 1\n" +
"- pipeline=reports source=local bundles=1 destinations=archive\n" +
" - bundle=. destination=archive backend=local action=publish_new outputs=report.md,summary.txt reason=\"destination state is absent\"\n" +
"Final status: ok planned=1 publish_new=1 replace_older=0 replace_conflict=0 replace_newer=0 replace_takeover=0 force_replace=0 skipped=0 failed=0 dry_run=true fixed_path=0\n"
" - bundle=. destination=archive backend=local action=publish_new workflow=additive outputs=report.md,summary.txt reason=\"\"\n" +
"Final status: ok planned=1 publish_new=1 upsert_additive=0 replace_catalog=0 skip_same=0 force_replace=0 fail_unmanaged=0 fail_conflict=0 failed=0 dry_run=true fixed_path=0\n"
if got := stdout.String(); got != wantStdout {
t.Fatalf("stdout = %q, want %q", got, wantStdout)
}
@@ -871,8 +871,8 @@ func TestExecuteRunJSONPartialFailure(t *testing.T) {
firstDestination := t.TempDir()
secondDestination := t.TempDir()
testutil.WriteSourceBundle(t, sourceRoot, "", testutil.BundleOptions{})
if err := os.WriteFile(filepath.Join(firstDestination, "unmanaged.txt"), []byte("data"), 0o600); err != nil {
t.Fatalf("write unmanaged file: %v", err)
if err := os.WriteFile(filepath.Join(firstDestination, "report.md"), []byte("data"), 0o600); err != nil {
t.Fatalf("write unmanaged planned file: %v", err)
}
configPath := filepath.Join(t.TempDir(), "config.yml")
if err := os.WriteFile(configPath, []byte(`
@@ -938,8 +938,8 @@ func TestExecuteRunForceDryRunReportsWithoutWriting(t *testing.T) {
if code != exitOK {
t.Fatalf("exit code = %d, want %d; stderr = %q", code, exitOK, stderr.String())
}
if !strings.Contains(stdout.String(), "action=force_replace") {
t.Fatalf("stdout = %q, want force_replace", stdout.String())
if !strings.Contains(stdout.String(), "action=publish_new workflow=additive") {
t.Fatalf("stdout = %q, want additive publish", stdout.String())
}
if _, err := os.Stat(filepath.Join(destinationRoot, "unmanaged.txt")); err != nil {
t.Fatalf("unmanaged file stat error = %v", err)

View File

@@ -55,11 +55,12 @@ type Destination struct {
Transform Transform `yaml:"transform"`
PathMap PathMapping `yaml:"path_mapping"`
Links *Links `yaml:"links"`
State StatePolicy `yaml:"state"`
Reconciliation ReconciliationPolicy `yaml:"reconciliation"`
Takeover TakeoverPolicy `yaml:"takeover"`
Workflow string `yaml:"workflow"`
State StatePolicy `yaml:"-"`
Reconciliation ReconciliationPolicy `yaml:"-"`
Takeover TakeoverPolicy `yaml:"-"`
Retention RetentionPolicy `yaml:"retention"`
Transfer TransferPolicy `yaml:"transfer"`
Transfer TransferPolicy `yaml:"-"`
}
type Backend struct {

View File

@@ -42,6 +42,11 @@ const (
LinkPrimarySource = "source"
)
const (
WorkflowAdditive = "additive"
WorkflowReplacement = "replacement"
)
const (
ReconciliationModeReplace = "replace"
ReconciliationModeMerge = "merge"
@@ -96,6 +101,9 @@ func ApplyDefaults(cfg *Config) {
if destination.Links != nil && destination.Links.Primary == "" {
destination.Links.Primary = LinkPrimaryAuto
}
if destination.Workflow == "" {
destination.Workflow = WorkflowAdditive
}
if destination.State.Mode == "" {
destination.State.Mode = StateModeSingleOwner
}

View File

@@ -30,17 +30,8 @@ pipelines:
if got, want := cfg.Pipelines[0].Validation.OnDigestMismatch, ValidationActionFail; got != want {
t.Fatalf("validation default = %q, want %q", got, want)
}
if got, want := destination.Transfer.OnDestinationOlder, TransferActionReplace; got != want {
t.Fatalf("transfer default = %q, want %q", got, want)
}
if got, want := destination.Reconciliation.Mode, ReconciliationModeReplace; got != want {
t.Fatalf("reconciliation mode default = %q, want %q", got, want)
}
if got, want := destination.State.Mode, StateModeSingleOwner; got != want {
t.Fatalf("state mode default = %q, want %q", got, want)
}
if got, want := destination.Takeover.Mode, TakeoverModeSamePipeline; got != want {
t.Fatalf("takeover mode default = %q, want %q", got, want)
if got, want := destination.Workflow, WorkflowAdditive; got != want {
t.Fatalf("workflow default = %q, want %q", got, want)
}
if destination.Retention.Prune.Enabled {
t.Fatal("retention.prune.enabled default = true, want false")
@@ -176,7 +167,7 @@ pipelines:
}
}
func TestLoadFileAcceptsExplicitReconciliationModes(t *testing.T) {
func TestLoadFileAcceptsExplicitWorkflows(t *testing.T) {
cfg := loadConfig(t, `
pipelines:
- id: reports
@@ -187,94 +178,19 @@ pipelines:
- id: archive
backend: local
path: /archive
reconciliation:
mode: replace
workflow: additive
- id: web
backend: local
path: /web
reconciliation:
mode: merge
workflow: replacement
`)
destinations := cfg.Pipelines[0].Destinations
if got, want := destinations[0].Reconciliation.Mode, ReconciliationModeReplace; got != want {
t.Fatalf("archive reconciliation mode = %q, want %q", got, want)
if got, want := destinations[0].Workflow, WorkflowAdditive; got != want {
t.Fatalf("archive workflow = %q, want %q", got, want)
}
if got, want := destinations[1].Reconciliation.Mode, ReconciliationModeMerge; got != want {
t.Fatalf("web reconciliation mode = %q, want %q", got, want)
}
}
func TestLoadFileAcceptsExplicitStateModes(t *testing.T) {
cfg := loadConfig(t, `
pipelines:
- id: reports
source:
backend: local
path: /source
destinations:
- id: archive
backend: local
path: /archive
state:
mode: single_owner
- id: web
backend: local
path: /web
state:
mode: shared_root
`)
destinations := cfg.Pipelines[0].Destinations
if got, want := destinations[0].State.Mode, StateModeSingleOwner; got != want {
t.Fatalf("archive state mode = %q, want %q", got, want)
}
if got, want := destinations[1].State.Mode, StateModeSharedRoot; got != want {
t.Fatalf("web state mode = %q, want %q", got, want)
}
}
func TestLoadFileAcceptsExplicitTakeoverModes(t *testing.T) {
cfg := loadConfig(t, `
pipelines:
- id: reports
source:
backend: local
path: /source
destinations:
- id: same-pipeline
backend: local
path: /same-pipeline
takeover:
mode: same_pipeline
- id: same-source
backend: local
path: /same-source
takeover:
mode: same_source
- id: any-managed
backend: local
path: /any-managed
takeover:
mode: any_managed
- id: never
backend: local
path: /never
takeover:
mode: never
`)
destinations := cfg.Pipelines[0].Destinations
wants := []string{
TakeoverModeSamePipeline,
TakeoverModeSameSource,
TakeoverModeAnyManaged,
TakeoverModeNever,
}
for index, want := range wants {
if got := destinations[index].Takeover.Mode; got != want {
t.Fatalf("destinations[%d].takeover.mode = %q, want %q", index, got, want)
}
if got, want := destinations[1].Workflow, WorkflowReplacement; got != want {
t.Fatalf("web workflow = %q, want %q", got, want)
}
}
@@ -899,8 +815,63 @@ pipelines:
`, "backend ftp is unsupported")
}
func TestLoadFileRejectsInvalidTransferAction(t *testing.T) {
func TestLoadFileRejectsInvalidWorkflow(t *testing.T) {
assertLoadError(t, `
pipelines:
- id: reports
source:
backend: local
path: /source
destinations:
- id: archive
backend: local
path: /archive
workflow: append
`, "workflow must be additive or replacement")
}
func TestLoadFileRejectsLegacyDestinationPolicyFields(t *testing.T) {
tests := map[string]string{
"state": `
pipelines:
- id: reports
source:
backend: local
path: /source
destinations:
- id: archive
backend: local
path: /archive
state:
mode: single_owner
`,
"reconciliation": `
pipelines:
- id: reports
source:
backend: local
path: /source
destinations:
- id: archive
backend: local
path: /archive
reconciliation:
mode: replace
`,
"takeover": `
pipelines:
- id: reports
source:
backend: local
path: /source
destinations:
- id: archive
backend: local
path: /archive
takeover:
mode: same_pipeline
`,
"transfer": `
pipelines:
- id: reports
source:
@@ -911,40 +882,14 @@ pipelines:
backend: local
path: /archive
transfer:
on_destination_older: overwrite
`, "on_destination_older must be replace or fail")
}
func TestLoadFileRejectsInvalidTakeoverMode(t *testing.T) {
assertLoadError(t, `
pipelines:
- id: reports
source:
backend: local
path: /source
destinations:
- id: archive
backend: local
path: /archive
takeover:
mode: unmanaged
`, "takeover.mode must be same_pipeline, same_source, any_managed, or never")
}
func TestLoadFileRejectsUnknownTakeoverFields(t *testing.T) {
assertLoadError(t, `
pipelines:
- id: reports
source:
backend: local
path: /source
destinations:
- id: archive
backend: local
path: /archive
takeover:
surprise: true
`, "field surprise not found")
on_destination_older: replace
`,
}
for name, body := range tests {
t.Run(name, func(t *testing.T) {
assertLoadError(t, body, "field "+name+" not found")
})
}
}
func TestLoadFileRejectsInvalidValidationAction(t *testing.T) {
@@ -1020,8 +965,6 @@ func TestExampleConfigsLoad(t *testing.T) {
"../../examples/local-index.yml",
"../../examples/fan-out.yml",
"../../examples/archive-and-latest.yml",
"../../examples/merge-reconciliation.yml",
"../../examples/shared-root.yml",
"../../examples/http-upload-local.yml",
"../../examples/ssh-destination.yml",
"../../examples/s3-destination.yml",

View File

@@ -74,11 +74,8 @@ func Validate(cfg Config) error {
errs = validatePublishTransformPolicy(errs, destinationContext, destination.Publish, destination.Transform)
errs = validatePathMapping(errs, destinationContext+".path_mapping", destination.PathMap)
errs = validateLinks(errs, destinationContext+".links", destination.Links)
errs = validateStatePolicy(errs, destinationContext+".state", destination.State)
errs = validateReconciliationPolicy(errs, destinationContext+".reconciliation", destination.Reconciliation)
errs = validateTakeoverPolicy(errs, destinationContext+".takeover", destination.Takeover)
errs = validateWorkflow(errs, destinationContext+".workflow", destination.Workflow)
errs = validateRetentionPolicy(errs, destinationContext+".retention", destination.Retention)
errs = validateTransferPolicy(errs, destinationContext+".transfer", destination.Transfer)
}
}
@@ -369,18 +366,9 @@ func validateLinks(errs ValidationErrors, context string, links *Links) Validati
return errs
}
func validateReconciliationPolicy(errs ValidationErrors, context string, policy ReconciliationPolicy) ValidationErrors {
if policy.Mode != ReconciliationModeReplace && policy.Mode != ReconciliationModeMerge {
errs = append(errs, context+".mode must be "+ReconciliationModeReplace+" or "+ReconciliationModeMerge)
}
return errs
}
func validateTakeoverPolicy(errs ValidationErrors, context string, policy TakeoverPolicy) ValidationErrors {
switch policy.Mode {
case TakeoverModeSamePipeline, TakeoverModeSameSource, TakeoverModeAnyManaged, TakeoverModeNever:
default:
errs = append(errs, context+".mode must be "+TakeoverModeSamePipeline+", "+TakeoverModeSameSource+", "+TakeoverModeAnyManaged+", or "+TakeoverModeNever)
func validateWorkflow(errs ValidationErrors, context, workflow string) ValidationErrors {
if workflow != WorkflowAdditive && workflow != WorkflowReplacement {
errs = append(errs, context+" must be "+WorkflowAdditive+" or "+WorkflowReplacement)
}
return errs
}
@@ -401,26 +389,3 @@ func validateRetentionPolicy(errs ValidationErrors, context string, policy Reten
}
return errs
}
func validateStatePolicy(errs ValidationErrors, context string, policy StatePolicy) ValidationErrors {
if policy.Mode != StateModeSingleOwner && policy.Mode != StateModeSharedRoot {
errs = append(errs, context+".mode must be "+StateModeSingleOwner+" or "+StateModeSharedRoot)
}
return errs
}
func validateTransferPolicy(errs ValidationErrors, context string, policy TransferPolicy) ValidationErrors {
if policy.OnDestinationSame != TransferActionSkip && policy.OnDestinationSame != TransferActionFail {
errs = append(errs, context+".on_destination_same must be skip or fail")
}
if policy.OnDestinationOlder != TransferActionReplace && policy.OnDestinationOlder != TransferActionFail {
errs = append(errs, context+".on_destination_older must be replace or fail")
}
if policy.OnDestinationNewer != TransferActionSkip && policy.OnDestinationNewer != TransferActionFail && policy.OnDestinationNewer != TransferActionReplace {
errs = append(errs, context+".on_destination_newer must be skip, replace, or fail")
}
if policy.OnConflict != TransferActionFail && policy.OnConflict != TransferActionReplace {
errs = append(errs, context+".on_conflict must be fail or replace")
}
return errs
}

View File

@@ -51,29 +51,6 @@ func TestValidateChecksPublishTransformPolicy(t *testing.T) {
}
}
func TestValidateAcceptsForceReplacementTransferActions(t *testing.T) {
cfg := Config{Pipelines: []Pipeline{{
ID: "reports",
Source: Backend{
Backend: BackendLocal,
Path: "/source",
},
Destinations: []Destination{{
ID: "archive",
Backend: BackendLocal,
Path: "/destination",
Transfer: TransferPolicy{
OnDestinationNewer: TransferActionReplace,
OnConflict: TransferActionReplace,
},
}},
}}}
ApplyDefaults(&cfg)
if err := Validate(cfg); err != nil {
t.Fatalf("Validate() error = %v", err)
}
}
func TestValidatePathMapping(t *testing.T) {
tests := []struct {
name string
@@ -108,51 +85,15 @@ func TestValidatePathMapping(t *testing.T) {
}
}
func TestValidateReconciliationPolicy(t *testing.T) {
func TestValidateWorkflow(t *testing.T) {
tests := []struct {
name string
mode string
wantErr bool
name string
workflow string
wantErr bool
}{
{name: "replace", mode: ReconciliationModeReplace},
{name: "merge", mode: ReconciliationModeMerge},
{name: "invalid", mode: "append", wantErr: true},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
cfg := Config{Pipelines: []Pipeline{{
ID: "reports",
Source: Backend{Backend: BackendLocal, Path: "/source"},
Destinations: []Destination{{
ID: "archive",
Backend: BackendLocal,
Path: "/destination",
Reconciliation: ReconciliationPolicy{Mode: tt.mode},
}},
}}}
ApplyDefaults(&cfg)
err := Validate(cfg)
if tt.wantErr && err == nil {
t.Fatal("Validate() error = nil, want error")
}
if !tt.wantErr && err != nil {
t.Fatalf("Validate() error = %v", err)
}
})
}
}
func TestValidateTakeoverPolicy(t *testing.T) {
tests := []struct {
name string
mode string
wantErr bool
}{
{name: "same pipeline", mode: TakeoverModeSamePipeline},
{name: "same source", mode: TakeoverModeSameSource},
{name: "any managed", mode: TakeoverModeAnyManaged},
{name: "never", mode: TakeoverModeNever},
{name: "invalid", mode: "unmanaged", wantErr: true},
{name: "additive", workflow: WorkflowAdditive},
{name: "replacement", workflow: WorkflowReplacement},
{name: "invalid", workflow: "append", wantErr: true},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
@@ -163,7 +104,7 @@ func TestValidateTakeoverPolicy(t *testing.T) {
ID: "archive",
Backend: BackendLocal,
Path: "/destination",
Takeover: TakeoverPolicy{Mode: tt.mode},
Workflow: tt.workflow,
}},
}}}
ApplyDefaults(&cfg)
@@ -178,7 +119,7 @@ func TestValidateTakeoverPolicy(t *testing.T) {
}
}
func TestValidateTakeoverPolicyReportsFieldContext(t *testing.T) {
func TestValidateWorkflowReportsFieldContext(t *testing.T) {
cfg := Config{Pipelines: []Pipeline{{
ID: "reports",
Source: Backend{Backend: BackendLocal, Path: "/source"},
@@ -186,7 +127,7 @@ func TestValidateTakeoverPolicyReportsFieldContext(t *testing.T) {
ID: "archive",
Backend: BackendLocal,
Path: "/destination",
Takeover: TakeoverPolicy{Mode: "unmanaged"},
Workflow: "append",
}},
}}}
ApplyDefaults(&cfg)
@@ -194,46 +135,12 @@ func TestValidateTakeoverPolicyReportsFieldContext(t *testing.T) {
if err == nil {
t.Fatal("Validate() error = nil, want error")
}
want := "pipelines[0].destinations[0].takeover.mode must be same_pipeline, same_source, any_managed, or never"
want := "pipelines[0].destinations[0].workflow must be additive or replacement"
if !strings.Contains(err.Error(), want) {
t.Fatalf("Validate() error = %q, want %q", err, want)
}
}
func TestValidateStatePolicy(t *testing.T) {
tests := []struct {
name string
mode string
wantErr bool
}{
{name: "single owner", mode: StateModeSingleOwner},
{name: "shared root", mode: StateModeSharedRoot},
{name: "invalid", mode: "shared", wantErr: true},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
cfg := Config{Pipelines: []Pipeline{{
ID: "reports",
Source: Backend{Backend: BackendLocal, Path: "/source"},
Destinations: []Destination{{
ID: "archive",
Backend: BackendLocal,
Path: "/destination",
State: StatePolicy{Mode: tt.mode},
}},
}}}
ApplyDefaults(&cfg)
err := Validate(cfg)
if tt.wantErr && err == nil {
t.Fatal("Validate() error = nil, want error")
}
if !tt.wantErr && err != nil {
t.Fatalf("Validate() error = %v", err)
}
})
}
}
func TestValidateRetentionPolicy(t *testing.T) {
olderThan := Duration(24 * time.Hour)
zeroDuration := Duration(0)

View File

@@ -4,6 +4,7 @@ import (
"context"
"encoding/json"
"fmt"
"sort"
"time"
"gitea.maximumdirect.net/eric/distributor/internal/config"
@@ -16,7 +17,9 @@ func Execute(ctx context.Context, req Request, plan Plan) error {
switch plan.Action {
case ActionSkipSame, ActionSkipDestinationNewer:
return nil
case ActionPublishNew, ActionReplaceOlder, ActionReplaceConflict, ActionReplaceNewer, ActionReplaceTakeover, ActionForceReplace:
case ActionPublishNew, ActionUpsertAdditive, ActionReplaceCatalog:
return executeCatalog(ctx, req, plan)
case ActionReplaceOlder, ActionReplaceConflict, ActionReplaceNewer, ActionReplaceTakeover, ActionForceReplace:
if usesSharedRootState(req, plan) {
return executeSharedRoot(ctx, req, plan)
}
@@ -138,6 +141,152 @@ func Execute(ctx context.Context, req Request, plan Plan) error {
return nil
}
func executeCatalog(ctx context.Context, req Request, plan Plan) error {
if plan.Action == ActionReplaceCatalog {
if plan.ClearDestinationRoot {
if err := req.DestinationBackend.DeletePrefix(ctx, req.DestinationBundlePath, storage.DeleteOptions{IgnoreMissing: true, PruneEmptyDirs: true}); err != nil {
return err
}
if err := ensureDestinationEmpty(ctx, req.DestinationBackend, req.DestinationBundlePath); err != nil {
return err
}
} else if len(plan.CatalogOutputsToDelete) > 0 {
if err := req.DestinationBackend.DeleteManagedOutputs(ctx, req.DestinationBundlePath, catalogOutputPaths(plan.CatalogOutputsToDelete), storage.DeleteOptions{IgnoreMissing: true, PruneEmptyDirs: true}); err != nil {
return err
}
}
}
writtenOutputs := make([]Output, 0, len(plan.Outputs))
newOutputs := make([]Output, 0, len(plan.Outputs))
cleanup := func() {
outputs := writtenOutputs
if plan.Action == ActionUpsertAdditive {
outputs = newOutputs
}
_ = req.DestinationBackend.DeleteManagedOutputs(ctx, req.DestinationBundlePath, ManagedOutputPaths(outputs), storage.DeleteOptions{IgnoreMissing: true, PruneEmptyDirs: true})
}
for _, output := range plan.Outputs {
destinationPath, err := storage.Join(req.DestinationBundlePath, output.DestinationPath)
if err != nil {
cleanup()
return err
}
created, err := catalogWriteCreatesOutput(ctx, req.DestinationBackend, destinationPath)
if err != nil {
cleanup()
return err
}
data := output.Data
if output.Kind == state.OutputKindSource {
sourcePath, err := storage.Join(req.SourceBundle.RootRelativePath, output.SourcePath)
if err != nil {
cleanup()
return err
}
data, err = req.SourceBackend.ReadFile(ctx, sourcePath)
if err != nil {
cleanup()
return err
}
}
if _, err := req.DestinationBackend.WriteFile(ctx, destinationPath, data, storage.WriteOptions{Overwrite: catalogOutputOverwriteAllowed(plan, output), PreferAtomic: true}); err != nil {
cleanup()
return err
}
writtenOutputs = append(writtenOutputs, output)
if created {
newOutputs = append(newOutputs, output)
}
}
catalogState := catalogStateForPlan(req, plan)
if err := state.ValidateCatalog(catalogState); err != nil {
cleanup()
return err
}
data, err := json.MarshalIndent(catalogState, "", " ")
if err != nil {
cleanup()
return err
}
data = append(data, '\n')
statePath, err := storage.StatePath(req.DestinationBundlePath)
if err != nil {
cleanup()
return err
}
if _, err := req.DestinationBackend.WriteFile(ctx, statePath, data, storage.WriteOptions{Overwrite: catalogStateWriteOverwrites(plan), PreferAtomic: true}); err != nil {
cleanup()
return err
}
return nil
}
func catalogWriteCreatesOutput(ctx context.Context, backend storage.Backend, destinationPath string) (bool, error) {
if _, err := backend.Stat(ctx, destinationPath); err == nil {
return false, nil
} else if storage.IsNotFound(err) {
return true, nil
} else {
return false, err
}
}
func catalogOutputOverwriteAllowed(plan Plan, output Output) bool {
if plan.ClearDestinationRoot {
return false
}
if plan.SupersededLegacy != nil {
return true
}
if plan.ExistingCatalog == nil {
return false
}
_, ok := state.FindCatalogOutputByPath(plan.ExistingCatalog.Outputs, output.DestinationPath)
return ok
}
func catalogStateForPlan(req Request, plan Plan) state.CatalogState {
now := requestTime(req)
createdAt := now
if plan.ExistingCatalog != nil {
createdAt = plan.ExistingCatalog.CreatedAt
}
outputs := make([]state.CatalogOutputFile, 0, len(plan.CatalogOutputsToRetain)+len(plan.CatalogOutputsToWrite))
outputs = append(outputs, plan.CatalogOutputsToRetain...)
outputs = append(outputs, plan.CatalogOutputsToWrite...)
sort.SliceStable(outputs, func(i, j int) bool {
if outputs[i].Path != outputs[j].Path {
return outputs[i].Path < outputs[j].Path
}
if outputs[i].PipelineID != outputs[j].PipelineID {
return outputs[i].PipelineID < outputs[j].PipelineID
}
return outputs[i].DestinationID < outputs[j].DestinationID
})
return state.CatalogState{
SchemaVersion: state.CatalogSchemaVersion,
DistributorVersion: req.DistributorVersion,
CreatedAt: createdAt,
UpdatedAt: now,
State: state.StatePolicy{Mode: state.StateModeCatalog},
Outputs: outputs,
}
}
func catalogStateWriteOverwrites(plan Plan) bool {
return plan.ExistingCatalog != nil || plan.SupersededLegacy != nil
}
func catalogOutputPaths(outputs []state.CatalogOutputFile) []string {
paths := make([]string, 0, len(outputs))
for _, output := range outputs {
paths = append(paths, output.Path)
}
return paths
}
func executeSharedRoot(ctx context.Context, req Request, plan Plan) error {
plan.Reconciliation = normalizeReconciliation(plan.Reconciliation)
if plan.Action == ActionForceReplace {
@@ -240,7 +389,7 @@ func ensureMergeOutputPaths(ctx context.Context, backend storage.Backend, bundle
}
func usesSharedRootState(req Request, plan Plan) bool {
return normalizeState(req.State).Mode == config.StateModeSharedRoot || plan.StateMode == config.StateModeSharedRoot
return plan.StateMode == config.StateModeSharedRoot
}
func outputManagedByExistingState(output Output, existing *state.DistributorState) bool {

View File

@@ -2,420 +2,167 @@ package publish
import (
"context"
"fmt"
"io"
"testing"
"time"
"gitea.maximumdirect.net/eric/distributor/internal/bundle"
"gitea.maximumdirect.net/eric/distributor/internal/config"
"gitea.maximumdirect.net/eric/distributor/internal/state"
"gitea.maximumdirect.net/eric/distributor/internal/storage"
"gitea.maximumdirect.net/eric/distributor/internal/storage/fake"
"gitea.maximumdirect.net/eric/distributor/internal/testutil"
"gitea.maximumdirect.net/eric/distributor/internal/transform"
)
func TestExecuteCleansUpAfterWriteFailure(t *testing.T) {
sourceBackend := fake.New()
destinationBackend := &failingBackend{Backend: fake.New(), failPath: "summary.txt"}
sourceBundle := testutil.WriteFakeSourceBundle(t, sourceBackend, "", testutil.BundleOptions{})
req := Request{
PipelineID: "reports",
DestinationID: "archive",
SourceBundle: sourceBundle,
SourceBackend: sourceBackend,
DestinationBackend: destinationBackend,
DestinationBundlePath: "",
Publish: config.PublishPolicy{Source: true},
Transfer: config.TransferPolicy{OnDestinationSame: config.TransferActionSkip, OnDestinationOlder: config.TransferActionReplace, OnDestinationNewer: config.TransferActionSkip, OnConflict: config.TransferActionFail},
DistributorVersion: "test",
func TestExecuteAdditiveWritesOutputsAndCatalog(t *testing.T) {
_, destinationBackend, req := catalogPlanRequest(t, config.WorkflowAdditive)
existing := baseCatalog(req)
existing.Outputs = []state.CatalogOutputFile{
catalogOutput(req, "reports", "archive", "report.md", state.OutputKindSource, planCreatedAt),
catalogOutput(req, "reports", "web", "old.txt", state.OutputKindSource, planCreatedAt),
}
writeCatalogState(t, destinationBackend, "", existing)
testutil.WriteFakeFile(t, destinationBackend, "report.md", "old report")
testutil.WriteFakeFile(t, destinationBackend, "old.txt", "retained")
plan, err := Build(context.Background(), req)
if err != nil {
t.Fatalf("Build() error = %v", err)
}
if err := Execute(context.Background(), req, plan); err != nil {
t.Fatalf("Execute() error = %v", err)
}
testutil.AssertFakeFile(t, destinationBackend, "report.md", "# Report\nSunny.\n")
testutil.AssertFakeFile(t, destinationBackend, "summary.txt", "Summary\n")
testutil.AssertFakeFile(t, destinationBackend, "old.txt", "retained")
catalog := readCatalogState(t, destinationBackend, "")
if catalog.SchemaVersion != state.CatalogSchemaVersion || catalog.State.Mode != state.StateModeCatalog {
t.Fatalf("catalog identity = schema %d mode %s", catalog.SchemaVersion, catalog.State.Mode)
}
if len(catalog.Outputs) != 3 {
t.Fatalf("catalog outputs = %#v, want three outputs", catalog.Outputs)
}
report, ok := state.FindCatalogOutputByPath(catalog.Outputs, "report.md")
if !ok {
t.Fatalf("catalog outputs = %#v, want report.md", catalog.Outputs)
}
if !report.CreatedAt.Equal(planCreatedAt) || !report.UpdatedAt.Equal(planUpdatedAt) {
t.Fatalf("report times = %s/%s, want created preserved and updated now", report.CreatedAt, report.UpdatedAt)
}
if report.SourcePath != "" {
t.Fatalf("source catalog output source_path = %q, want empty", report.SourcePath)
}
}
func TestExecuteReplacementDeletesCurrentOwnerAndPreservesOtherOwners(t *testing.T) {
_, destinationBackend, req := catalogPlanRequest(t, config.WorkflowReplacement)
existing := baseCatalog(req)
existing.Outputs = []state.CatalogOutputFile{
catalogOutput(req, "reports", "archive", "report.md", state.OutputKindSource, planCreatedAt),
catalogOutput(req, "reports", "archive", "stale.txt", state.OutputKindSource, planCreatedAt),
catalogOutput(req, "reports", "web", "shared.txt", state.OutputKindSource, planCreatedAt),
}
writeCatalogState(t, destinationBackend, "", existing)
testutil.WriteFakeFile(t, destinationBackend, "report.md", "old report")
testutil.WriteFakeFile(t, destinationBackend, "stale.txt", "delete me")
testutil.WriteFakeFile(t, destinationBackend, "shared.txt", "keep me")
plan, err := Build(context.Background(), req)
if err != nil {
t.Fatalf("Build() error = %v", err)
}
if err := Execute(context.Background(), req, plan); err != nil {
t.Fatalf("Execute() error = %v", err)
}
testutil.AssertFakeFile(t, destinationBackend, "report.md", "# Report\nSunny.\n")
testutil.AssertFakeFile(t, destinationBackend, "summary.txt", "Summary\n")
testutil.AssertFakeMissing(t, destinationBackend, "stale.txt")
testutil.AssertFakeFile(t, destinationBackend, "shared.txt", "keep me")
catalog := readCatalogState(t, destinationBackend, "")
if _, ok := state.FindCatalogOutputByPath(catalog.Outputs, "stale.txt"); ok {
t.Fatalf("catalog outputs = %#v, want stale.txt removed", catalog.Outputs)
}
if _, ok := state.FindCatalogOutputByPath(catalog.Outputs, "shared.txt"); !ok {
t.Fatalf("catalog outputs = %#v, want shared.txt retained", catalog.Outputs)
}
}
func TestExecuteSupersededReplacementClearsDestinationRootOnly(t *testing.T) {
_, destinationBackend, req := catalogPlanRequest(t, config.WorkflowReplacement)
req.DestinationBundlePath = "bundle"
legacyState := testutil.DestinationState(req.SourceBundle.Manifest, testutil.DestinationStateOptions{})
writeJSONState(t, destinationBackend, req.DestinationBundlePath, legacyState)
testutil.WriteFakeFile(t, destinationBackend, "bundle/unplanned.txt", "remove")
testutil.WriteFakeFile(t, destinationBackend, "outside.txt", "keep")
plan, err := Build(context.Background(), req)
if err != nil {
t.Fatalf("Build() error = %v", err)
}
if err := Execute(context.Background(), req, plan); err != nil {
t.Fatalf("Execute() error = %v", err)
}
testutil.AssertFakeMissing(t, destinationBackend, "bundle/unplanned.txt")
testutil.AssertFakeFile(t, destinationBackend, "bundle/report.md", "# Report\nSunny.\n")
testutil.AssertFakeFile(t, destinationBackend, "outside.txt", "keep")
readCatalogState(t, destinationBackend, "bundle")
}
func TestExecuteSupersededAdditiveLeavesUnplannedFilesUnmanaged(t *testing.T) {
_, destinationBackend, req := catalogPlanRequest(t, config.WorkflowAdditive)
legacyState := testutil.DestinationState(req.SourceBundle.Manifest, testutil.DestinationStateOptions{})
writeJSONState(t, destinationBackend, "", legacyState)
testutil.WriteFakeFile(t, destinationBackend, "report.md", "legacy report")
testutil.WriteFakeFile(t, destinationBackend, "unplanned.txt", "leave me")
plan, err := Build(context.Background(), req)
if err != nil {
t.Fatalf("Build() error = %v", err)
}
if err := Execute(context.Background(), req, plan); err != nil {
t.Fatalf("Execute() error = %v", err)
}
testutil.AssertFakeFile(t, destinationBackend, "report.md", "# Report\nSunny.\n")
testutil.AssertFakeFile(t, destinationBackend, "unplanned.txt", "leave me")
catalog := readCatalogState(t, destinationBackend, "")
if _, ok := state.FindCatalogOutputByPath(catalog.Outputs, "unplanned.txt"); ok {
t.Fatalf("catalog outputs = %#v, want unplanned file omitted", catalog.Outputs)
}
}
func TestExecuteFailedWriteDoesNotWriteCatalogState(t *testing.T) {
_, destinationBackend, req := catalogPlanRequest(t, config.WorkflowAdditive)
plan, err := Build(context.Background(), req)
if err != nil {
t.Fatalf("Build() error = %v", err)
}
if err := destinationBackend.AddDirectory("report.md"); err != nil {
t.Fatalf("add conflicting directory: %v", err)
}
err = Execute(context.Background(), req, plan)
if err == nil {
t.Fatal("Execute() error = nil, want error")
t.Fatal("Execute() error = nil, want write failure")
}
found, err := destinationBackend.HasAny(context.Background(), "")
if err != nil {
t.Fatalf("HasAny() error = %v", err)
}
if found {
t.Fatal("destination has content after failed execution")
if _, statErr := destinationBackend.Stat(context.Background(), storage.StateFileName); !storage.IsNotFound(statErr) {
t.Fatalf("state stat error = %v, want missing state", statErr)
}
}
func TestExecuteReplaceDeletesOmittedManagedOutputs(t *testing.T) {
sourceBackend := fake.New()
sourceBundle := testutil.WriteFakeSourceBundle(t, sourceBackend, "", testutil.BundleOptions{
Files: []testutil.SourceFile{{Path: "report.md", Data: "# Report\nNew.\n"}},
})
destinationBackend := fake.New()
older := sourceBundle.Manifest
older.Created = older.Created.Add(-time.Hour)
older.Files = append([]bundle.ManifestFile(nil), testutil.ValidManifest(testutil.BundleOptions{}).Files...)
older.Digest = bundle.BundleDigest(older.Files)
testutil.WriteFakeDestinationState(t, destinationBackend, "", older, testutil.DestinationStateOptions{})
req := testRequest(sourceBackend, destinationBackend, sourceBundle, config.ReconciliationModeReplace)
plan, err := Build(context.Background(), req)
if err != nil {
t.Fatalf("Build() error = %v", err)
}
if plan.Action != ActionReplaceOlder {
t.Fatalf("plan action = %s, want replace_older", plan.Action)
}
if err := Execute(context.Background(), req, plan); err != nil {
t.Fatalf("Execute() error = %v", err)
}
testutil.AssertFakeFile(t, destinationBackend, "report.md", "# Report\nNew.\n")
testutil.AssertFakeMissing(t, destinationBackend, "summary.txt")
destinationState := readFakeState(t, destinationBackend, "")
if got, want := len(destinationState.Outputs), 1; got != want {
t.Fatalf("state output count = %d, want %d", got, want)
}
if got, want := destinationState.Reconciliation.Mode, config.ReconciliationModeReplace; got != want {
t.Fatalf("reconciliation mode = %q, want %q", got, want)
}
}
func TestExecuteMergeRetainsOmittedAndOverwritesManagedOutputs(t *testing.T) {
sourceBackend := fake.New()
sourceBundle := testutil.WriteFakeSourceBundle(t, sourceBackend, "", testutil.BundleOptions{
Files: []testutil.SourceFile{{Path: "report.md", Data: "# Report\nNew.\n"}},
})
destinationBackend := fake.New()
older := sourceBundle.Manifest
older.Created = older.Created.Add(-time.Hour)
older.Files = append([]bundle.ManifestFile(nil), testutil.ValidManifest(testutil.BundleOptions{}).Files...)
older.Digest = bundle.BundleDigest(older.Files)
existingState := testutil.WriteFakeDestinationState(t, destinationBackend, "", older, testutil.DestinationStateOptions{})
req := testRequest(sourceBackend, destinationBackend, sourceBundle, config.ReconciliationModeMerge)
plan, err := Build(context.Background(), req)
if err != nil {
t.Fatalf("Build() error = %v", err)
}
if err := Execute(context.Background(), req, plan); err != nil {
t.Fatalf("Execute() error = %v", err)
}
testutil.AssertFakeFile(t, destinationBackend, "report.md", "# Report\nNew.\n")
testutil.AssertFakeFile(t, destinationBackend, "summary.txt", "old")
destinationState := readFakeState(t, destinationBackend, "")
outputs := outputsByPath(destinationState.Outputs)
if got, want := len(outputs), 2; got != want {
t.Fatalf("state output count = %d, want %d", got, want)
}
if !outputs["summary.txt"].CreatedAt.Equal(existingState.Outputs[1].CreatedAt) || !outputs["summary.txt"].UpdatedAt.Equal(existingState.Outputs[1].UpdatedAt) {
t.Fatalf("retained output timestamps = %#v, want existing %#v", outputs["summary.txt"], existingState.Outputs[1])
}
if !outputs["report.md"].CreatedAt.Equal(existingState.Outputs[0].CreatedAt) || !outputs["report.md"].UpdatedAt.After(existingState.Outputs[0].UpdatedAt) {
t.Fatalf("updated output timestamps = %#v, want preserved created_at and newer updated_at", outputs["report.md"])
}
if got, want := destinationState.Reconciliation.Mode, config.ReconciliationModeMerge; got != want {
t.Fatalf("reconciliation mode = %q, want %q", got, want)
}
}
func TestExecuteReplaceConflictDeletesOmittedManagedOutputs(t *testing.T) {
sourceBackend := fake.New()
sourceBundle := testutil.WriteFakeSourceBundle(t, sourceBackend, "", testutil.BundleOptions{
Files: []testutil.SourceFile{{Path: "report.md", Data: "# Report\nNew.\n"}},
})
destinationBackend := fake.New()
conflict := sourceBundle.Manifest
conflict.ID = "other.source"
conflict.Files = append([]bundle.ManifestFile(nil), testutil.ValidManifest(testutil.BundleOptions{}).Files...)
conflict.Digest = bundle.BundleDigest(conflict.Files)
testutil.WriteFakeDestinationState(t, destinationBackend, "", conflict, testutil.DestinationStateOptions{})
req := testRequest(sourceBackend, destinationBackend, sourceBundle, config.ReconciliationModeMerge)
req.Takeover = config.TakeoverPolicy{Mode: config.TakeoverModeNever}
req.Transfer.OnConflict = config.TransferActionReplace
plan, err := Build(context.Background(), req)
if err != nil {
t.Fatalf("Build() error = %v", err)
}
if plan.Action != ActionReplaceConflict {
t.Fatalf("plan action = %s, want replace_conflict", plan.Action)
}
if err := Execute(context.Background(), req, plan); err != nil {
t.Fatalf("Execute() error = %v", err)
}
testutil.AssertFakeFile(t, destinationBackend, "report.md", "# Report\nNew.\n")
testutil.AssertFakeMissing(t, destinationBackend, "summary.txt")
destinationState := readFakeState(t, destinationBackend, "")
outputs := outputsByPath(destinationState.Outputs)
if got, want := len(outputs), 1; got != want {
t.Fatalf("state output count = %d, want %d", got, want)
}
if _, ok := outputs["summary.txt"]; ok {
t.Fatalf("state retained summary.txt after replace_conflict: %#v", destinationState.Outputs)
}
}
func TestExecuteReplaceNewerUsesManagedReplacement(t *testing.T) {
sourceBackend := fake.New()
sourceBundle := testutil.WriteFakeSourceBundle(t, sourceBackend, "", testutil.BundleOptions{
Files: []testutil.SourceFile{{Path: "report.md", Data: "# Report\nNew.\n"}},
})
destinationBackend := fake.New()
newer := sourceBundle.Manifest
newer.Created = newer.Created.Add(time.Hour)
newer.Files = append([]bundle.ManifestFile(nil), testutil.ValidManifest(testutil.BundleOptions{}).Files...)
newer.Digest = bundle.BundleDigest(newer.Files)
testutil.WriteFakeDestinationState(t, destinationBackend, "", newer, testutil.DestinationStateOptions{})
req := testRequest(sourceBackend, destinationBackend, sourceBundle, config.ReconciliationModeReplace)
req.Transfer.OnDestinationNewer = config.TransferActionReplace
plan, err := Build(context.Background(), req)
if err != nil {
t.Fatalf("Build() error = %v", err)
}
if plan.Action != ActionReplaceNewer {
t.Fatalf("plan action = %s, want replace_newer", plan.Action)
}
if err := Execute(context.Background(), req, plan); err != nil {
t.Fatalf("Execute() error = %v", err)
}
testutil.AssertFakeFile(t, destinationBackend, "report.md", "# Report\nNew.\n")
testutil.AssertFakeMissing(t, destinationBackend, "summary.txt")
}
func TestExecuteMergeFailsOnUnmanagedDestinationPathCollision(t *testing.T) {
sourceBackend := fake.New()
sourceBundle := testutil.WriteFakeSourceBundle(t, sourceBackend, "", testutil.BundleOptions{
Files: []testutil.SourceFile{{Path: "report.md", Data: "# Report\nNew.\n"}},
})
destinationBackend := fake.New()
older := sourceBundle.Manifest
older.Created = older.Created.Add(-time.Hour)
older.Files = []bundle.ManifestFile{{
Path: "summary.txt",
SHA256: bundle.FileDigest([]byte("Summary\n")),
Size: int64(len("Summary\n")),
}}
older.Digest = bundle.BundleDigest(older.Files)
testutil.WriteFakeDestinationState(t, destinationBackend, "", older, testutil.DestinationStateOptions{})
testutil.WriteFakeFile(t, destinationBackend, "report.md", "unmanaged")
req := testRequest(sourceBackend, destinationBackend, sourceBundle, config.ReconciliationModeMerge)
plan, err := Build(context.Background(), req)
if err != nil {
t.Fatalf("Build() error = %v", err)
}
err = Execute(context.Background(), req, plan)
if err == nil {
t.Fatal("Execute() error = nil, want unmanaged path collision")
}
testutil.AssertFakeFile(t, destinationBackend, "report.md", "unmanaged")
testutil.AssertFakeFile(t, destinationBackend, "summary.txt", "old")
}
func TestExecuteMergeFailureCleansUpOnlyNewOutputs(t *testing.T) {
sourceBackend := fake.New()
sourceBundle := testutil.WriteFakeSourceBundle(t, sourceBackend, "", testutil.BundleOptions{
Files: []testutil.SourceFile{
{Path: "report.md", Data: "# Report\nNew.\n"},
{Path: "new.md", Data: "new\n"},
{Path: "fail.md", Data: "fail\n"},
},
})
destinationBackend := &failingBackend{Backend: fake.New(), failPath: "fail.md"}
older := sourceBundle.Manifest
older.Created = older.Created.Add(-time.Hour)
older.Files = []bundle.ManifestFile{sourceBundle.Manifest.Files[0]}
older.Digest = bundle.BundleDigest(older.Files)
testutil.WriteFakeDestinationState(t, destinationBackend.Backend, "", older, testutil.DestinationStateOptions{})
req := testRequest(sourceBackend, destinationBackend, sourceBundle, config.ReconciliationModeMerge)
plan, err := Build(context.Background(), req)
if err != nil {
t.Fatalf("Build() error = %v", err)
}
err = Execute(context.Background(), req, plan)
if err == nil {
t.Fatal("Execute() error = nil, want injected failure")
}
testutil.AssertFakeFile(t, destinationBackend.Backend, "report.md", "# Report\nNew.\n")
testutil.AssertFakeMissing(t, destinationBackend.Backend, "new.md")
testutil.AssertFakeMissing(t, destinationBackend.Backend, "fail.md")
}
func TestExecuteFixedPathSupportsReconciliationModes(t *testing.T) {
for _, mode := range []string{config.ReconciliationModeReplace, config.ReconciliationModeMerge} {
t.Run(mode, func(t *testing.T) {
sourceBackend := fake.New()
sourceBundle := testutil.WriteFakeSourceBundle(t, sourceBackend, "new", testutil.BundleOptions{
Files: []testutil.SourceFile{{Path: "report.md", Data: "# Report\nNew.\n"}},
})
destinationBackend := fake.New()
older := sourceBundle.Manifest
older.Created = older.Created.Add(-time.Hour)
testutil.WriteFakeDestinationState(t, destinationBackend, "", older, testutil.DestinationStateOptions{})
req := testRequest(sourceBackend, destinationBackend, sourceBundle, mode)
req.DestinationBundlePath = ""
req.PathMapping = config.PathMappingFixed
plan, err := Build(context.Background(), req)
if err != nil {
t.Fatalf("Build() error = %v", err)
}
if plan.Action != ActionReplaceOlder {
t.Fatalf("plan action = %s, want replace_older", plan.Action)
}
if err := Execute(context.Background(), req, plan); err != nil {
t.Fatalf("Execute() error = %v", err)
}
testutil.AssertFakeFile(t, destinationBackend, "report.md", "# Report\nNew.\n")
destinationState := readFakeState(t, destinationBackend, "")
if got, want := destinationState.Reconciliation.Mode, mode; got != want {
t.Fatalf("state reconciliation mode = %q, want %q", got, want)
}
})
}
}
func TestExecuteReconciliationModesHonorPublishPolicies(t *testing.T) {
tests := []struct {
name string
publish config.PublishPolicy
transform config.Transform
transformer TransformerResolver
wantPaths []string
}{
{
name: "source only",
publish: config.PublishPolicy{Source: true},
wantPaths: []string{"report.md"},
},
{
name: "generated only",
publish: config.PublishPolicy{HTML: true},
transform: config.Transform{MarkdownToHTML: &config.MarkdownToHTML{Enabled: true, Mode: config.TransformModeSidecar}},
transformer: testResolver{transform.MarkdownToHTML: testTransformer{outputs: []transform.Output{{
Path: "report.html",
SourcePath: "report.md",
Transform: transform.MarkdownToHTML,
Data: []byte("<h1>Report</h1>\n"),
SHA256: bundle.FileDigest([]byte("<h1>Report</h1>\n")),
Size: int64(len("<h1>Report</h1>\n")),
}}}},
wantPaths: []string{"report.html"},
},
{
name: "source and generated",
publish: config.PublishPolicy{Source: true, HTML: true},
transform: config.Transform{MarkdownToHTML: &config.MarkdownToHTML{Enabled: true, Mode: config.TransformModeSidecar}},
transformer: testResolver{transform.MarkdownToHTML: testTransformer{outputs: []transform.Output{{
Path: "report.html",
SourcePath: "report.md",
Transform: transform.MarkdownToHTML,
Data: []byte("<h1>Report</h1>\n"),
SHA256: bundle.FileDigest([]byte("<h1>Report</h1>\n")),
Size: int64(len("<h1>Report</h1>\n")),
}}}},
wantPaths: []string{"report.md", "report.html"},
},
}
for _, mode := range []string{config.ReconciliationModeReplace, config.ReconciliationModeMerge} {
for _, tt := range tests {
t.Run(mode+" "+tt.name, func(t *testing.T) {
sourceBackend := fake.New()
sourceBundle := testutil.WriteFakeSourceBundle(t, sourceBackend, "", testutil.BundleOptions{
Files: []testutil.SourceFile{{Path: "report.md", Data: "# Report\nNew.\n"}},
})
destinationBackend := fake.New()
older := sourceBundle.Manifest
older.Created = older.Created.Add(-time.Hour)
testutil.WriteFakeDestinationState(t, destinationBackend, "", older, testutil.DestinationStateOptions{})
req := testRequest(sourceBackend, destinationBackend, sourceBundle, mode)
req.Publish = tt.publish
req.Transform = tt.transform
req.Transformers = tt.transformer
plan, err := Build(context.Background(), req)
if err != nil {
t.Fatalf("Build() error = %v", err)
}
if err := Execute(context.Background(), req, plan); err != nil {
t.Fatalf("Execute() error = %v", err)
}
destinationState := readFakeState(t, destinationBackend, "")
outputs := outputsByPath(destinationState.Outputs)
for _, path := range tt.wantPaths {
if _, ok := outputs[path]; !ok {
t.Fatalf("state outputs = %#v, missing %s", destinationState.Outputs, path)
}
}
})
}
}
}
type failingBackend struct {
*fake.Backend
failPath string
}
func (b *failingBackend) WriteFile(ctx context.Context, path string, data []byte, opts storage.WriteOptions) (storage.Entry, error) {
if path == b.failPath {
return storage.Entry{}, fmt.Errorf("injected write failure")
}
return b.Backend.WriteFile(ctx, path, data, opts)
}
func (b *failingBackend) WriteFrom(ctx context.Context, path string, r io.Reader, opts storage.WriteOptions) (storage.Entry, error) {
if path == b.failPath {
return storage.Entry{}, fmt.Errorf("injected write failure")
}
return b.Backend.WriteFrom(ctx, path, r, opts)
}
func testRequest(sourceBackend storage.Backend, destinationBackend storage.Backend, sourceBundle bundle.Bundle, reconciliationMode string) Request {
return Request{
PipelineID: "reports",
DestinationID: "archive",
SourceBundle: sourceBundle,
SourceBackend: sourceBackend,
DestinationBackend: destinationBackend,
DestinationBundlePath: sourceBundle.RootRelativePath,
Publish: config.PublishPolicy{Source: true},
Reconciliation: config.ReconciliationPolicy{Mode: reconciliationMode},
Transfer: config.TransferPolicy{
OnDestinationSame: config.TransferActionSkip,
OnDestinationOlder: config.TransferActionReplace,
OnDestinationNewer: config.TransferActionSkip,
OnConflict: config.TransferActionFail,
},
DistributorVersion: "test",
}
}
func readFakeState(t *testing.T, backend storage.Backend, bundlePath string) state.DistributorState {
func readCatalogState(t *testing.T, backend *fake.Backend, relative string) state.CatalogState {
t.Helper()
statePath, err := storage.StatePath(bundlePath)
statePath, err := storage.StatePath(relative)
if err != nil {
t.Fatalf("state path: %v", err)
}
data, err := backend.ReadFile(context.Background(), statePath)
if err != nil {
t.Fatalf("read state: %v", err)
t.Fatalf("read catalog state: %v", err)
}
destinationState, err := state.Parse(data)
catalog, err := state.ParseCatalog(data)
if err != nil {
t.Fatalf("parse state: %v", err)
t.Fatalf("parse catalog state: %v", err)
}
return destinationState
}
func outputsByPath(outputs []state.OutputFile) map[string]state.OutputFile {
byPath := make(map[string]state.OutputFile, len(outputs))
for _, output := range outputs {
byPath[output.Path] = output
}
return byPath
return catalog
}

View File

@@ -1,222 +0,0 @@
package publish
import (
"context"
"strings"
"testing"
"gitea.maximumdirect.net/eric/distributor/internal/bundle"
"gitea.maximumdirect.net/eric/distributor/internal/config"
"gitea.maximumdirect.net/eric/distributor/internal/storage/fake"
"gitea.maximumdirect.net/eric/distributor/internal/testutil"
)
func TestBuildPlansForcedReplacementOnlyWhenExplicit(t *testing.T) {
tests := []struct {
name string
prepare func(t *testing.T, backend *fake.Backend, source bundle.Manifest)
transfer config.TransferPolicy
wantReason string
forceAction bool
}{
{
name: "unmanaged content",
prepare: func(t *testing.T, backend *fake.Backend, source bundle.Manifest) {
t.Helper()
testutil.WriteFakeFile(t, backend, "bundle/old.txt", "old")
},
transfer: defaultTransfer(),
wantReason: "fail_unmanaged",
forceAction: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
sourceBackend := fake.New()
sourceBundle := testutil.WriteFakeSourceBundle(t, sourceBackend, "bundle", testutil.BundleOptions{})
destinationBackend := fake.New()
tt.prepare(t, destinationBackend, sourceBundle.Manifest)
req := forceRequest(sourceBackend, destinationBackend, sourceBundle, tt.transfer)
_, err := Build(context.Background(), req)
if err == nil || !strings.Contains(err.Error(), tt.wantReason) {
t.Fatalf("Build() error = %v, want %q", err, tt.wantReason)
}
req.Force = true
plan, err := Build(context.Background(), req)
if tt.forceAction {
if err != nil {
t.Fatalf("Build() with force error = %v", err)
}
if plan.Action != ActionForceReplace || !plan.Force {
t.Fatalf("forced plan action = %s force=%t", plan.Action, plan.Force)
}
}
})
}
}
func TestBuildPlansConflictReplacementWithoutForce(t *testing.T) {
tests := []struct {
name string
prepare func(t *testing.T, backend *fake.Backend, source bundle.Manifest)
wantReason string
}{
{
name: "different source id",
prepare: func(t *testing.T, backend *fake.Backend, source bundle.Manifest) {
t.Helper()
conflict := source
conflict.ID = "other.source"
testutil.WriteFakeDestinationState(t, backend, "bundle", conflict, testutil.DestinationStateOptions{})
},
wantReason: "destination source id differs",
},
{
name: "same created digest conflict",
prepare: func(t *testing.T, backend *fake.Backend, source bundle.Manifest) {
t.Helper()
conflict := testutil.ValidManifest(testutil.BundleOptions{Files: []testutil.SourceFile{{Path: "report.md", Data: "# Different\n"}}})
testutil.WriteFakeDestinationState(t, backend, "bundle", conflict, testutil.DestinationStateOptions{})
},
wantReason: "same id and created time but different digest",
},
{
name: "pipeline mismatch",
prepare: func(t *testing.T, backend *fake.Backend, source bundle.Manifest) {
t.Helper()
testutil.WriteFakeDestinationState(t, backend, "bundle", source, testutil.DestinationStateOptions{PipelineID: "other-pipeline"})
},
wantReason: "pipeline id",
},
{
name: "destination mismatch",
prepare: func(t *testing.T, backend *fake.Backend, source bundle.Manifest) {
t.Helper()
testutil.WriteFakeDestinationState(t, backend, "bundle", source, testutil.DestinationStateOptions{DestinationID: "other-destination"})
},
wantReason: "destination id",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
sourceBackend := fake.New()
sourceBundle := testutil.WriteFakeSourceBundle(t, sourceBackend, "bundle", testutil.BundleOptions{})
destinationBackend := fake.New()
tt.prepare(t, destinationBackend, sourceBundle.Manifest)
req := forceRequest(sourceBackend, destinationBackend, sourceBundle, conflictReplaceTransfer())
plan, err := Build(context.Background(), req)
if err != nil {
t.Fatalf("Build() error = %v", err)
}
if plan.Action != ActionReplaceConflict || plan.Force {
t.Fatalf("plan action = %s force=%t, want replace_conflict without force", plan.Action, plan.Force)
}
if !strings.Contains(plan.Reason, tt.wantReason) {
t.Fatalf("plan reason = %q, want %q", plan.Reason, tt.wantReason)
}
})
}
}
func TestBuildPlansNewerReplacementWithoutForce(t *testing.T) {
sourceBackend := fake.New()
sourceBundle := testutil.WriteFakeSourceBundle(t, sourceBackend, "bundle", testutil.BundleOptions{})
destinationBackend := fake.New()
newer := sourceBundle.Manifest
newer.Created = newer.Created.AddDate(0, 0, 1)
testutil.WriteFakeDestinationState(t, destinationBackend, "bundle", newer, testutil.DestinationStateOptions{})
req := forceRequest(sourceBackend, destinationBackend, sourceBundle, newerReplaceTransfer())
plan, err := Build(context.Background(), req)
if err != nil {
t.Fatalf("Build() error = %v", err)
}
if plan.Action != ActionReplaceNewer || plan.Force {
t.Fatalf("plan action = %s force=%t, want replace_newer without force", plan.Action, plan.Force)
}
}
func TestBuildRequiresConflictPolicyForStateConflicts(t *testing.T) {
sourceBackend := fake.New()
sourceBundle := testutil.WriteFakeSourceBundle(t, sourceBackend, "bundle", testutil.BundleOptions{})
destinationBackend := fake.New()
conflict := sourceBundle.Manifest
conflict.ID = "other.source"
testutil.WriteFakeDestinationState(t, destinationBackend, "bundle", conflict, testutil.DestinationStateOptions{})
req := forceRequest(sourceBackend, destinationBackend, sourceBundle, defaultTransfer())
req.Force = true
_, err := Build(context.Background(), req)
if err == nil || !strings.Contains(err.Error(), "destination source id differs") {
t.Fatalf("Build() error = %v, want conservative conflict", err)
}
}
func TestExecuteForcedReplacementDeletesOnlyBundlePath(t *testing.T) {
sourceBackend := fake.New()
sourceBundle := testutil.WriteFakeSourceBundle(t, sourceBackend, "bundle", testutil.BundleOptions{})
destinationBackend := fake.New()
testutil.WriteFakeFile(t, destinationBackend, "bundle/old.txt", "old")
testutil.WriteFakeFile(t, destinationBackend, "bundle/nested/old.txt", "old")
testutil.WriteFakeFile(t, destinationBackend, "bundle-sibling/keep.txt", "keep")
testutil.WriteFakeFile(t, destinationBackend, "outside.txt", "outside")
req := forceRequest(sourceBackend, destinationBackend, sourceBundle, defaultTransfer())
req.Force = true
plan, err := Build(context.Background(), req)
if err != nil {
t.Fatalf("Build() error = %v", err)
}
if plan.Action != ActionForceReplace {
t.Fatalf("plan action = %s, want force_replace", plan.Action)
}
if err := Execute(context.Background(), req, plan); err != nil {
t.Fatalf("Execute() error = %v", err)
}
testutil.AssertFakeFile(t, destinationBackend, "bundle/report.md", "# Report\nSunny.\n")
testutil.AssertFakeMissing(t, destinationBackend, "bundle/old.txt")
testutil.AssertFakeMissing(t, destinationBackend, "bundle/nested/old.txt")
testutil.AssertFakeFile(t, destinationBackend, "bundle-sibling/keep.txt", "keep")
testutil.AssertFakeFile(t, destinationBackend, "outside.txt", "outside")
}
func forceRequest(sourceBackend, destinationBackend *fake.Backend, sourceBundle bundle.Bundle, transfer config.TransferPolicy) Request {
return Request{
PipelineID: "reports",
DestinationID: "archive",
SourceBundle: sourceBundle,
SourceBackend: sourceBackend,
DestinationBackend: destinationBackend,
DestinationBundlePath: sourceBundle.RootRelativePath,
Publish: config.PublishPolicy{Source: true},
Takeover: config.TakeoverPolicy{Mode: config.TakeoverModeNever},
Transfer: transfer,
DistributorVersion: "test",
}
}
func defaultTransfer() config.TransferPolicy {
return config.TransferPolicy{
OnDestinationSame: config.TransferActionSkip,
OnDestinationOlder: config.TransferActionReplace,
OnDestinationNewer: config.TransferActionSkip,
OnConflict: config.TransferActionFail,
}
}
func conflictReplaceTransfer() config.TransferPolicy {
transfer := defaultTransfer()
transfer.OnConflict = config.TransferActionReplace
return transfer
}
func newerReplaceTransfer() config.TransferPolicy {
transfer := defaultTransfer()
transfer.OnDestinationNewer = config.TransferActionReplace
return transfer
}

View File

@@ -194,12 +194,6 @@ func TestBuildRejectsHTMLWithoutTransform(t *testing.T) {
DestinationBundlePath: "",
SourceBundle: sourceBundle,
Publish: config.PublishPolicy{HTML: true},
Transfer: config.TransferPolicy{
OnDestinationSame: config.TransferActionSkip,
OnDestinationOlder: config.TransferActionReplace,
OnDestinationNewer: config.TransferActionSkip,
OnConflict: config.TransferActionFail,
},
})
if err == nil {
t.Fatal("Build() error = nil, want missing transform error")

View File

@@ -3,6 +3,7 @@ package publish
import (
"context"
"fmt"
"time"
"gitea.maximumdirect.net/eric/distributor/internal/bundle"
"gitea.maximumdirect.net/eric/distributor/internal/config"
@@ -24,6 +25,8 @@ const (
ActionFailUnmanaged Action = "fail_unmanaged"
ActionForceReplace Action = "force_replace"
ActionReplaceTakeover Action = "replace_takeover"
ActionUpsertAdditive Action = "upsert_additive"
ActionReplaceCatalog Action = "replace_catalog"
)
type Request struct {
@@ -37,44 +40,17 @@ type Request struct {
Publish config.PublishPolicy
Transform config.Transform
Links *config.Links
State config.StatePolicy
Reconciliation config.ReconciliationPolicy
Takeover config.TakeoverPolicy
Workflow string
Transformers TransformerResolver
Transfer config.TransferPolicy
DistributorVersion string
Force bool
Now time.Time
}
type TransformerResolver interface {
Get(name string) (transform.Transformer, bool)
}
type Plan struct {
PipelineID string
DestinationID string
BundleID string
BundlePath string
DestinationBundlePath string
PathMapping string
Action Action
Reason string
Force bool
PrimaryURL string
StateMode string
OwnerScope state.OwnerScope
Reconciliation config.ReconciliationPolicy
TakeoverMode string
Outputs []Output
ExistingState *state.DistributorState
ExistingSharedRoot *state.SharedRootState
OtherOwnerOutputs []state.SharedRootOutputFile
TakenOverOwnerOutputs []state.SharedRootOutputFile
RetainedOwnerOutputs []state.SharedRootOutputFile
OwnerOutputsToDelete []state.SharedRootOutputFile
OwnerOutputsToWrite []Output
}
type Output struct {
SourcePath string
DestinationPath string
@@ -86,6 +62,49 @@ type Output struct {
Size int64
}
type Plan struct {
PipelineID string
DestinationID string
BundleID string
BundlePath string
DestinationBundlePath string
PathMapping string
Action Action
Reason string
Force bool
PrimaryURL string
Workflow string
OwnerScope state.OwnerScope
Outputs []Output
ExistingCatalog *state.CatalogState
SupersededLegacy *state.SupersededLegacyState
CatalogOutputsToWrite []state.CatalogOutputFile
CatalogOutputsToRetain []state.CatalogOutputFile
CatalogOutputsToDelete []state.CatalogOutputFile
ClearDestinationRoot bool
// Retained while the executor is migrated to catalog state.
StateMode string
Reconciliation config.ReconciliationPolicy
TakeoverMode string
ExistingState *state.DistributorState
ExistingSharedRoot *state.SharedRootState
OtherOwnerOutputs []state.SharedRootOutputFile
TakenOverOwnerOutputs []state.SharedRootOutputFile
RetainedOwnerOutputs []state.SharedRootOutputFile
OwnerOutputsToDelete []state.SharedRootOutputFile
OwnerOutputsToWrite []Output
}
type catalogPlanDetails struct {
Action Action
Reason string
CatalogOutputsToWrite []state.CatalogOutputFile
CatalogOutputsToRetain []state.CatalogOutputFile
CatalogOutputsToDelete []state.CatalogOutputFile
ClearDestinationRoot bool
}
func Build(ctx context.Context, req Request) (Plan, error) {
if err := validateRequest(req); err != nil {
return Plan{}, err
@@ -102,14 +121,9 @@ func Build(ctx context.Context, req Request) (Plan, error) {
if err != nil {
return Plan{}, err
}
reconciliation := normalizeReconciliation(req.Reconciliation)
stateMode := normalizeState(req.State).Mode
comparison := compareDestination(req, status)
action, reason := actionForComparison(comparison, req.Transfer, req.Force)
if takeoverActionAllowed(req, status, comparison, stateMode, action) {
action = ActionReplaceTakeover
reason = comparison.Reason
}
workflow := normalizeWorkflow(req.Workflow)
scope := state.CurrentOwnerScope(req.PipelineID, req.DestinationID)
now := requestTime(req)
plan := Plan{
PipelineID: req.PipelineID,
DestinationID: req.DestinationID,
@@ -117,36 +131,37 @@ func Build(ctx context.Context, req Request) (Plan, error) {
BundlePath: req.SourceBundle.RootRelativePath,
DestinationBundlePath: req.DestinationBundlePath,
PathMapping: req.PathMapping,
Action: action,
Reason: reason,
Force: action == ActionForceReplace,
PrimaryURL: primaryURL,
StateMode: stateMode,
OwnerScope: state.CurrentOwnerScope(req.PipelineID, req.DestinationID),
Reconciliation: reconciliation,
TakeoverMode: normalizeTakeover(req.Takeover).Mode,
Workflow: workflow,
OwnerScope: scope,
Outputs: outputs,
ExistingState: status.State,
ExistingSharedRoot: status.SharedRoot,
ExistingCatalog: status.Catalog,
SupersededLegacy: status.SupersededLegacy,
}
if stateMode == config.StateModeSharedRoot {
sharedDetails, err := planSharedRootOwner(ctx, req, status, action, reconciliation, outputs)
plan.Action = sharedDetails.Action
if sharedDetails.Reason != "" {
plan.Reason = sharedDetails.Reason
}
plan.OtherOwnerOutputs = sharedDetails.OtherOwnerOutputs
plan.TakenOverOwnerOutputs = sharedDetails.TakenOverOwnerOutputs
plan.RetainedOwnerOutputs = sharedDetails.RetainedOwnerOutputs
plan.OwnerOutputsToDelete = sharedDetails.OwnerOutputsToDelete
plan.OwnerOutputsToWrite = sharedDetails.OwnerOutputsToWrite
if err != nil {
return plan, err
}
}
if plan.Action == ActionFailConflict || plan.Action == ActionFailUnmanaged {
if status.StateErr != nil {
plan.Action = ActionFailConflict
plan.Reason = status.StateErr.Error()
return plan, fmt.Errorf("%s: %s", plan.Action, plan.Reason)
}
var details catalogPlanDetails
switch {
case status.Catalog != nil:
details, err = planExistingCatalog(ctx, req, *status.Catalog, outputs, workflow, scope, now)
case status.SupersededLegacy != nil:
details = planSupersededLegacy(req, outputs, workflow, scope, now)
default:
details, err = planWithoutCatalog(ctx, req, outputs, workflow, scope, now)
}
plan.Action = details.Action
plan.Reason = details.Reason
plan.CatalogOutputsToWrite = details.CatalogOutputsToWrite
plan.CatalogOutputsToRetain = details.CatalogOutputsToRetain
plan.CatalogOutputsToDelete = details.CatalogOutputsToDelete
plan.ClearDestinationRoot = details.ClearDestinationRoot
if err != nil {
return plan, err
}
return plan, nil
}
@@ -166,165 +181,134 @@ func validateRequest(req Request) error {
if err := config.ValidatePublishTransformPolicy(req.Publish, req.Transform); err != nil {
return fmt.Errorf("publish/transform policy: %w", err)
}
switch normalizeReconciliation(req.Reconciliation).Mode {
case config.ReconciliationModeReplace, config.ReconciliationModeMerge:
switch normalizeWorkflow(req.Workflow) {
case config.WorkflowAdditive, config.WorkflowReplacement:
default:
return fmt.Errorf("reconciliation.mode must be %s or %s", config.ReconciliationModeReplace, config.ReconciliationModeMerge)
}
switch normalizeState(req.State).Mode {
case config.StateModeSingleOwner, config.StateModeSharedRoot:
default:
return fmt.Errorf("state.mode must be %s or %s", config.StateModeSingleOwner, config.StateModeSharedRoot)
}
switch normalizeTakeover(req.Takeover).Mode {
case config.TakeoverModeSamePipeline, config.TakeoverModeSameSource, config.TakeoverModeAnyManaged, config.TakeoverModeNever:
default:
return fmt.Errorf("takeover.mode must be %s, %s, %s, or %s", config.TakeoverModeSamePipeline, config.TakeoverModeSameSource, config.TakeoverModeAnyManaged, config.TakeoverModeNever)
return fmt.Errorf("destination.workflow must be %s or %s", config.WorkflowAdditive, config.WorkflowReplacement)
}
return nil
}
func normalizeReconciliation(policy config.ReconciliationPolicy) config.ReconciliationPolicy {
if policy.Mode == "" {
policy.Mode = config.ReconciliationModeReplace
func normalizeWorkflow(workflow string) string {
if workflow == "" {
return config.WorkflowAdditive
}
return policy
return workflow
}
func normalizeState(policy config.StatePolicy) config.StatePolicy {
if policy.Mode == "" {
policy.Mode = config.StateModeSingleOwner
func requestTime(req Request) time.Time {
if req.Now.IsZero() {
return time.Now().UTC()
}
return policy
return req.Now.UTC()
}
func normalizeTakeover(policy config.TakeoverPolicy) config.TakeoverPolicy {
if policy.Mode == "" {
policy.Mode = config.TakeoverModeSamePipeline
func planExistingCatalog(ctx context.Context, req Request, catalog state.CatalogState, outputs []Output, workflow string, scope state.OwnerScope, now time.Time) (catalogPlanDetails, error) {
if err := rejectCatalogUnmanagedCollisions(ctx, req.DestinationBackend, req.DestinationBundlePath, catalog.Outputs, outputs); err != nil {
return catalogPlanDetails{
Action: ActionFailUnmanaged,
Reason: err.Error(),
}, fmt.Errorf("%s: %s", ActionFailUnmanaged, err)
}
return policy
}
func compareDestination(req Request, status state.DestinationStatus) state.Comparison {
if normalizeState(req.State).Mode == config.StateModeSharedRoot {
scope := state.CurrentOwnerScope(req.PipelineID, req.DestinationID)
comparison := state.CompareSharedRootOwner(req.SourceBundle.Manifest, scope, status)
if req.PathMapping != config.PathMappingFixed || comparison.Outcome != state.OutcomeDifferentSourceConflict {
return comparison
}
destinationManifest, ok := sharedRootComparisonManifest(status, scope)
if !ok {
return comparison
}
if destinationManifest.Created.Before(req.SourceBundle.Manifest.Created) {
return state.Comparison{Outcome: state.OutcomeDestinationOlder, Reason: "fixed destination source is older than selected source"}
}
if destinationManifest.Created.After(req.SourceBundle.Manifest.Created) {
return state.Comparison{
Outcome: state.OutcomeDestinationNewer,
Reason: "fixed destination source is newer than selected source",
Detail: state.ComparisonDetail{
Kind: state.ComparisonDetailDestinationNewer,
CurrentSourceID: req.SourceBundle.Manifest.ID,
DestinationSourceID: destinationManifest.ID,
},
}
}
return comparison
planned := outputPathSet(outputs)
details := catalogPlanDetails{
Action: actionForWorkflow(workflow),
CatalogOutputsToWrite: catalogOutputsForPlan(req, outputs, catalog.Outputs, scope, now),
}
comparison := state.Compare(req.SourceBundle.Manifest, req.PipelineID, req.DestinationID, status)
return comparison
}
func sharedRootComparisonManifest(status state.DestinationStatus, scope state.OwnerScope) (bundle.Manifest, bool) {
if status.SharedRoot != nil {
return status.SharedRoot.SourceManifest(scope)
}
if status.State != nil && status.State.PipelineID == scope.PipelineID && status.State.DestinationID == scope.DestinationID {
return status.State.Source.Manifest, true
}
return bundle.Manifest{}, false
}
type sharedRootPlanDetails struct {
Action Action
Reason string
OtherOwnerOutputs []state.SharedRootOutputFile
TakenOverOwnerOutputs []state.SharedRootOutputFile
RetainedOwnerOutputs []state.SharedRootOutputFile
OwnerOutputsToDelete []state.SharedRootOutputFile
OwnerOutputsToWrite []Output
}
func planSharedRootOwner(ctx context.Context, req Request, status state.DestinationStatus, action Action, reconciliation config.ReconciliationPolicy, outputs []Output) (sharedRootPlanDetails, error) {
scope := state.CurrentOwnerScope(req.PipelineID, req.DestinationID)
details := sharedRootPlanDetails{Action: action}
if !isWriteAction(action) {
details.OtherOwnerOutputs = otherOwnerOutputs(status, scope)
return details, nil
}
plannedPaths := outputPaths(outputs)
if action == ActionForceReplace {
details.OwnerOutputsToWrite = append([]Output(nil), outputs...)
return details, nil
}
conflicts := sharedRootPathOwnershipConflicts(status, scope, plannedPaths)
if len(conflicts) > 0 {
conflictAction := ActionReplaceTakeover
for _, conflict := range conflicts {
if sharedRootTakeoverAllowed(req, status, conflict) {
continue
}
if req.Transfer.OnConflict == config.TransferActionReplace {
conflictAction = ActionReplaceConflict
continue
}
reason := sharedRootOwnershipConflictReason(conflict)
details.Action = ActionFailConflict
details.Reason = reason
return details, fmt.Errorf("%s: %s", ActionFailConflict, reason)
}
details.Action = conflictAction
details.Reason = sharedRootOwnershipConflictReason(conflicts[0])
details.TakenOverOwnerOutputs = sharedRootConflictOutputs(status.SharedRoot, conflicts)
}
if err := rejectSharedRootUnmanagedCollisions(ctx, req.DestinationBackend, req.DestinationBundlePath, status, scope, plannedPaths); err != nil {
details.Action = ActionFailUnmanaged
details.Reason = err.Error()
return details, fmt.Errorf("%s: %s", ActionFailUnmanaged, err)
}
takenOverPaths := sharedRootOutputPathSet(details.TakenOverOwnerOutputs)
details.OtherOwnerOutputs = otherOwnerOutputsExcept(status, scope, takenOverPaths)
ownerOutputs := currentOwnerOutputs(status, scope)
planned := make(map[string]struct{}, len(plannedPaths))
for _, path := range plannedPaths {
planned[path] = struct{}{}
}
for _, output := range ownerOutputs {
for _, output := range catalog.Outputs {
if _, exists := planned[output.Path]; exists {
continue
}
if details.Action == ActionReplaceTakeover || details.Action == ActionReplaceConflict || (isReconciliationReplacementAction(details.Action) && reconciliation.Mode == config.ReconciliationModeReplace) {
details.OwnerOutputsToDelete = append(details.OwnerOutputsToDelete, output)
if workflow == config.WorkflowReplacement && output.PipelineID == scope.PipelineID && output.DestinationID == scope.DestinationID {
details.CatalogOutputsToDelete = append(details.CatalogOutputsToDelete, output)
continue
}
if isReconciliationReplacementAction(details.Action) && reconciliation.Mode == config.ReconciliationModeMerge {
details.RetainedOwnerOutputs = append(details.RetainedOwnerOutputs, output)
}
details.CatalogOutputsToRetain = append(details.CatalogOutputsToRetain, output)
}
details.OwnerOutputsToWrite = append([]Output(nil), outputs...)
return details, nil
}
func isWriteAction(action Action) bool {
switch action {
case ActionPublishNew, ActionReplaceOlder, ActionReplaceConflict, ActionReplaceNewer, ActionReplaceTakeover, ActionForceReplace:
return true
default:
return false
func planSupersededLegacy(req Request, outputs []Output, workflow string, scope state.OwnerScope, now time.Time) catalogPlanDetails {
details := catalogPlanDetails{
Action: actionForWorkflow(workflow),
CatalogOutputsToWrite: catalogOutputsForPlan(req, outputs, nil, scope, now),
}
if workflow == config.WorkflowReplacement {
details.ClearDestinationRoot = true
}
return details
}
func planWithoutCatalog(ctx context.Context, req Request, outputs []Output, workflow string, scope state.OwnerScope, now time.Time) (catalogPlanDetails, error) {
if err := rejectCatalogUnmanagedCollisions(ctx, req.DestinationBackend, req.DestinationBundlePath, nil, outputs); err != nil {
return catalogPlanDetails{
Action: ActionFailUnmanaged,
Reason: err.Error(),
}, fmt.Errorf("%s: %s", ActionFailUnmanaged, err)
}
return catalogPlanDetails{
Action: ActionPublishNew,
CatalogOutputsToWrite: catalogOutputsForPlan(req, outputs, nil, scope, now),
}, nil
}
func actionForWorkflow(workflow string) Action {
if workflow == config.WorkflowReplacement {
return ActionReplaceCatalog
}
return ActionUpsertAdditive
}
func catalogOutputsForPlan(req Request, outputs []Output, existing []state.CatalogOutputFile, scope state.OwnerScope, now time.Time) []state.CatalogOutputFile {
files := make([]state.CatalogOutputFile, 0, len(outputs))
source := state.CatalogSourceIdentity{
ID: req.SourceBundle.Manifest.ID,
Digest: req.SourceBundle.Manifest.Digest,
Created: req.SourceBundle.Manifest.Created,
}
for _, output := range outputs {
createdAt := now
if existingOutput, ok := state.FindCatalogOutputByPath(existing, output.DestinationPath); ok {
createdAt = existingOutput.CreatedAt
}
file := state.CatalogOutputFile{
Path: output.DestinationPath,
PipelineID: scope.PipelineID,
DestinationID: scope.DestinationID,
Source: source,
Kind: output.Kind,
URL: output.URL,
SHA256: output.SHA256,
Size: output.Size,
CreatedAt: createdAt,
UpdatedAt: now,
}
if output.Kind == state.OutputKindGenerated {
file.SourcePath = output.SourcePath
file.Transform = output.Transform
}
files = append(files, file)
}
return files
}
func rejectCatalogUnmanagedCollisions(ctx context.Context, backend storage.Backend, bundlePath string, existing []state.CatalogOutputFile, outputs []Output) error {
managed := catalogOutputPathSet(existing)
for _, output := range outputs {
if _, exists := managed[output.DestinationPath]; exists {
continue
}
destinationPath, err := storage.Join(bundlePath, output.DestinationPath)
if err != nil {
return err
}
if _, err := backend.Stat(ctx, destinationPath); err == nil {
return fmt.Errorf("destination output path %s exists but is not managed by catalog state", storage.DisplayPath(output.DestinationPath))
} else if !storage.IsNotFound(err) {
return err
}
}
return nil
}
func outputPaths(outputs []Output) []string {
@@ -335,127 +319,31 @@ func outputPaths(outputs []Output) []string {
return paths
}
func sharedRootOwnershipConflictReason(conflict state.PathOwnershipConflict) string {
return fmt.Sprintf("destination output path %s is owned by %s/%s", conflict.Path, conflict.Owner.PipelineID, conflict.Owner.DestinationID)
func outputPathSet(outputs []Output) map[string]struct{} {
paths := make(map[string]struct{}, len(outputs))
for _, output := range outputs {
paths[output.DestinationPath] = struct{}{}
}
return paths
}
func sharedRootPathOwnershipConflicts(status state.DestinationStatus, scope state.OwnerScope, paths []string) []state.PathOwnershipConflict {
if status.SharedRoot == nil {
return nil
func catalogOutputPathSet(outputs []state.CatalogOutputFile) map[string]struct{} {
paths := make(map[string]struct{}, len(outputs))
for _, output := range outputs {
paths[output.Path] = struct{}{}
}
conflicts := make([]state.PathOwnershipConflict, 0)
seen := make(map[string]struct{}, len(paths))
for _, path := range paths {
if _, exists := seen[path]; exists {
continue
}
seen[path] = struct{}{}
owner, exists := status.SharedRoot.OutputOwner(path)
if !exists || owner == scope {
continue
}
conflicts = append(conflicts, state.PathOwnershipConflict{
Path: path,
Owner: owner,
CurrentOwner: scope,
Detail: state.ComparisonDetail{
Kind: state.ComparisonDetailSharedRootOutputOwner,
Path: path,
CurrentOwner: scope,
ConflictingOwner: owner,
},
})
}
return conflicts
return paths
}
func sharedRootConflictOutputs(sharedRoot *state.SharedRootState, conflicts []state.PathOwnershipConflict) []state.SharedRootOutputFile {
if sharedRoot == nil || len(conflicts) == 0 {
return nil
func normalizeReconciliation(policy config.ReconciliationPolicy) config.ReconciliationPolicy {
if policy.Mode == "" {
policy.Mode = config.ReconciliationModeReplace
}
paths := make(map[string]struct{}, len(conflicts))
for _, conflict := range conflicts {
paths[conflict.Path] = struct{}{}
}
outputs := make([]state.SharedRootOutputFile, 0, len(conflicts))
for _, output := range sharedRoot.Outputs {
if _, exists := paths[output.Path]; exists {
outputs = append(outputs, output)
}
}
return outputs
return policy
}
func sharedRootTakeoverAllowed(req Request, status state.DestinationStatus, conflict state.PathOwnershipConflict) bool {
if status.SharedRoot == nil {
return false
}
takeover := normalizeTakeover(req.Takeover)
switch takeover.Mode {
case config.TakeoverModeSamePipeline:
return conflict.Owner.PipelineID == req.PipelineID
case config.TakeoverModeSameSource:
owner, ok := status.SharedRoot.Owner(conflict.Owner)
return ok && owner.Source.Manifest.ID == req.SourceBundle.Manifest.ID
case config.TakeoverModeAnyManaged:
_, ok := status.SharedRoot.Owner(conflict.Owner)
return ok
case config.TakeoverModeNever:
return false
default:
return false
}
}
func rejectSharedRootUnmanagedCollisions(ctx context.Context, backend storage.Backend, bundlePath string, status state.DestinationStatus, scope state.OwnerScope, paths []string) error {
for _, path := range paths {
if pathManagedBySharedRootStatus(status, scope, path) {
continue
}
destinationPath, err := storage.Join(bundlePath, path)
if err != nil {
return err
}
if _, err := backend.Stat(ctx, destinationPath); err == nil {
return fmt.Errorf("destination output path %s exists but is not managed by destination state", storage.DisplayPath(path))
} else if !storage.IsNotFound(err) {
return err
}
}
return nil
}
func pathManagedBySharedRootStatus(status state.DestinationStatus, scope state.OwnerScope, path string) bool {
if status.SharedRoot != nil {
_, exists := status.SharedRoot.OutputOwner(path)
return exists
}
if status.State != nil && status.State.PipelineID == scope.PipelineID && status.State.DestinationID == scope.DestinationID {
_, exists := state.FindOutputByPath(status.State.Outputs, path)
return exists
}
return false
}
func otherOwnerOutputs(status state.DestinationStatus, scope state.OwnerScope) []state.SharedRootOutputFile {
return otherOwnerOutputsExcept(status, scope, nil)
}
func otherOwnerOutputsExcept(status state.DestinationStatus, scope state.OwnerScope, exclude map[string]struct{}) []state.SharedRootOutputFile {
if status.SharedRoot == nil {
return nil
}
outputs := make([]state.SharedRootOutputFile, 0, len(status.SharedRoot.Outputs))
for _, output := range status.SharedRoot.Outputs {
if output.Owner == scope {
continue
}
if _, skip := exclude[output.Path]; skip {
continue
}
outputs = append(outputs, output)
}
return outputs
func isReconciliationReplacementAction(action Action) bool {
return action == ActionReplaceOlder || action == ActionReplaceNewer
}
func sharedRootOutputPathSet(outputs []state.SharedRootOutputFile) map[string]struct{} {
@@ -465,109 +353,3 @@ func sharedRootOutputPathSet(outputs []state.SharedRootOutputFile) map[string]st
}
return paths
}
func currentOwnerOutputs(status state.DestinationStatus, scope state.OwnerScope) []state.SharedRootOutputFile {
if status.SharedRoot != nil {
outputs := make([]state.SharedRootOutputFile, 0, len(status.SharedRoot.Outputs))
for _, output := range status.SharedRoot.Outputs {
if output.Owner == scope {
outputs = append(outputs, output)
}
}
return outputs
}
if status.State != nil && status.State.PipelineID == scope.PipelineID && status.State.DestinationID == scope.DestinationID {
outputs := make([]state.SharedRootOutputFile, 0, len(status.State.Outputs))
for _, output := range status.State.Outputs {
outputs = append(outputs, state.SharedRootOutputFile{
Path: output.Path,
Kind: output.Kind,
SourcePath: output.SourcePath,
Transform: output.Transform,
URL: output.URL,
SHA256: output.SHA256,
Size: output.Size,
Owner: scope,
SourceID: status.State.Source.Manifest.ID,
SourceDigest: status.State.Source.Manifest.Digest,
SourceCreated: status.State.Source.Manifest.Created,
CreatedAt: output.CreatedAt,
UpdatedAt: output.UpdatedAt,
})
}
return outputs
}
return nil
}
func actionForComparison(comparison state.Comparison, transfer config.TransferPolicy, force bool) (Action, string) {
switch comparison.Outcome {
case state.OutcomeDestinationAbsent:
return ActionPublishNew, comparison.Reason
case state.OutcomeDestinationUnmanaged:
if force {
return ActionForceReplace, "forced replacement of unmanaged destination content"
}
return ActionFailUnmanaged, comparison.Reason
case state.OutcomeInvalidState:
return ActionFailConflict, comparison.Reason
case state.OutcomeIdentityMismatch, state.OutcomeSameCreatedConflict, state.OutcomeDifferentSourceConflict:
if transfer.OnConflict == config.TransferActionReplace {
return ActionReplaceConflict, comparison.Reason
}
return ActionFailConflict, comparison.Reason
case state.OutcomeSameSource:
if transfer.OnDestinationSame == config.TransferActionFail {
return ActionFailConflict, "destination matches source and transfer policy requires failure"
}
return ActionSkipSame, comparison.Reason
case state.OutcomeDestinationOlder:
if transfer.OnDestinationOlder == config.TransferActionFail {
return ActionFailConflict, "destination is older and transfer policy requires failure"
}
return ActionReplaceOlder, comparison.Reason
case state.OutcomeDestinationNewer:
if transfer.OnDestinationNewer == config.TransferActionReplace {
return ActionReplaceNewer, comparison.Reason
}
if transfer.OnDestinationNewer == config.TransferActionFail {
return ActionFailConflict, "destination is newer and transfer policy requires failure"
}
return ActionSkipDestinationNewer, comparison.Reason
default:
return ActionFailConflict, "unsupported comparison outcome"
}
}
func isReconciliationReplacementAction(action Action) bool {
return action == ActionReplaceOlder || action == ActionReplaceNewer
}
func takeoverActionAllowed(req Request, status state.DestinationStatus, comparison state.Comparison, stateMode string, action Action) bool {
if stateMode != config.StateModeSingleOwner || status.State == nil {
return false
}
if action == ActionForceReplace {
return false
}
switch comparison.Detail.Kind {
case state.ComparisonDetailPipelineIDMismatch,
state.ComparisonDetailDestinationIDMismatch,
state.ComparisonDetailDifferentSourceID:
default:
return false
}
takeover := normalizeTakeover(req.Takeover)
switch takeover.Mode {
case config.TakeoverModeSamePipeline:
return status.State.PipelineID == req.PipelineID
case config.TakeoverModeSameSource:
return status.State.Source.Manifest.ID == req.SourceBundle.Manifest.ID
case config.TakeoverModeAnyManaged:
return true
case config.TakeoverModeNever:
return false
default:
return false
}
}

View File

@@ -1,36 +1,296 @@
package publish
import (
"context"
"encoding/json"
"strings"
"testing"
"time"
"gitea.maximumdirect.net/eric/distributor/internal/config"
"gitea.maximumdirect.net/eric/distributor/internal/state"
"gitea.maximumdirect.net/eric/distributor/internal/storage"
"gitea.maximumdirect.net/eric/distributor/internal/storage/fake"
"gitea.maximumdirect.net/eric/distributor/internal/testutil"
)
func TestCompareDestinationFixedPathPreservesDifferentSourceConflict(t *testing.T) {
sourceBackend := fake.New()
sourceBundle := testutil.WriteFakeSourceBundle(t, sourceBackend, "bundle", testutil.BundleOptions{})
destinationState := testutil.DestinationState(sourceBundle.Manifest, testutil.DestinationStateOptions{})
destinationState.Source.Manifest.ID = "latest.previous"
var (
planCreatedAt = time.Date(2026, 5, 30, 11, 12, 0, 0, time.UTC)
planUpdatedAt = time.Date(2026, 6, 1, 9, 30, 0, 0, time.UTC)
)
comparison := compareDestination(Request{
func TestBuildAdditivePublishesCatalogOutputs(t *testing.T) {
_, destinationBackend, req := catalogPlanRequest(t, config.WorkflowAdditive)
plan, err := Build(context.Background(), req)
if err != nil {
t.Fatalf("Build() error = %v", err)
}
if plan.Action != ActionPublishNew || plan.Workflow != config.WorkflowAdditive {
t.Fatalf("plan action=%s workflow=%s, want publish_new additive", plan.Action, plan.Workflow)
}
if len(plan.CatalogOutputsToWrite) != 2 || len(plan.CatalogOutputsToRetain) != 0 || len(plan.CatalogOutputsToDelete) != 0 {
t.Fatalf("catalog write=%d retain=%d delete=%d", len(plan.CatalogOutputsToWrite), len(plan.CatalogOutputsToRetain), len(plan.CatalogOutputsToDelete))
}
for _, output := range plan.CatalogOutputsToWrite {
if output.PipelineID != "reports" || output.DestinationID != "archive" {
t.Fatalf("catalog output owner = %s/%s", output.PipelineID, output.DestinationID)
}
if output.Source.ID != req.SourceBundle.Manifest.ID || output.Source.Digest != req.SourceBundle.Manifest.Digest || !output.Source.Created.Equal(req.SourceBundle.Manifest.Created) {
t.Fatalf("catalog output source = %#v", output.Source)
}
if output.Kind == state.OutputKindSource && output.SourcePath != "" {
t.Fatalf("source catalog output source_path = %q, want empty", output.SourcePath)
}
if !output.CreatedAt.Equal(planUpdatedAt) || !output.UpdatedAt.Equal(planUpdatedAt) {
t.Fatalf("catalog output times = %s/%s", output.CreatedAt, output.UpdatedAt)
}
}
if _, err := destinationBackend.Stat(context.Background(), storage.StateFileName); !storage.IsNotFound(err) {
t.Fatalf("destination state stat error = %v, want missing", err)
}
}
func TestBuildAdditiveOverwritesManagedAndRetainsOthers(t *testing.T) {
_, destinationBackend, req := catalogPlanRequest(t, config.WorkflowAdditive)
existing := baseCatalog(req)
existing.Outputs = []state.CatalogOutputFile{
catalogOutput(req, "reports", "archive", "report.md", state.OutputKindSource, planCreatedAt),
catalogOutput(req, "reports", "web", "old.txt", state.OutputKindSource, planCreatedAt),
}
writeCatalogState(t, destinationBackend, "", existing)
testutil.WriteFakeFile(t, destinationBackend, "report.md", "old")
plan, err := Build(context.Background(), req)
if err != nil {
t.Fatalf("Build() error = %v", err)
}
if plan.Action != ActionUpsertAdditive {
t.Fatalf("plan action = %s, want %s", plan.Action, ActionUpsertAdditive)
}
if len(plan.CatalogOutputsToDelete) != 0 {
t.Fatalf("delete outputs = %#v, want none", plan.CatalogOutputsToDelete)
}
if len(plan.CatalogOutputsToRetain) != 1 || plan.CatalogOutputsToRetain[0].Path != "old.txt" {
t.Fatalf("retained outputs = %#v, want old.txt", plan.CatalogOutputsToRetain)
}
written, ok := state.FindCatalogOutputByPath(plan.CatalogOutputsToWrite, "report.md")
if !ok {
t.Fatalf("written outputs = %#v, want report.md", plan.CatalogOutputsToWrite)
}
if !written.CreatedAt.Equal(planCreatedAt) || !written.UpdatedAt.Equal(planUpdatedAt) {
t.Fatalf("report.md times = %s/%s, want created preserved and updated now", written.CreatedAt, written.UpdatedAt)
}
}
func TestBuildReplacementDeletesCurrentOwnerAndRetainsOtherOwners(t *testing.T) {
_, destinationBackend, req := catalogPlanRequest(t, config.WorkflowReplacement)
existing := baseCatalog(req)
existing.Outputs = []state.CatalogOutputFile{
catalogOutput(req, "reports", "archive", "report.md", state.OutputKindSource, planCreatedAt),
catalogOutput(req, "reports", "archive", "stale.txt", state.OutputKindSource, planCreatedAt),
catalogOutput(req, "reports", "web", "shared.txt", state.OutputKindSource, planCreatedAt),
}
writeCatalogState(t, destinationBackend, "", existing)
plan, err := Build(context.Background(), req)
if err != nil {
t.Fatalf("Build() error = %v", err)
}
if plan.Action != ActionReplaceCatalog {
t.Fatalf("plan action = %s, want %s", plan.Action, ActionReplaceCatalog)
}
if len(plan.CatalogOutputsToDelete) != 1 || plan.CatalogOutputsToDelete[0].Path != "stale.txt" {
t.Fatalf("delete outputs = %#v, want stale.txt", plan.CatalogOutputsToDelete)
}
if len(plan.CatalogOutputsToRetain) != 1 || plan.CatalogOutputsToRetain[0].Path != "shared.txt" {
t.Fatalf("retained outputs = %#v, want shared.txt", plan.CatalogOutputsToRetain)
}
}
func TestBuildTransfersManagedPathOwnership(t *testing.T) {
_, destinationBackend, req := catalogPlanRequest(t, config.WorkflowReplacement)
existing := baseCatalog(req)
existing.Outputs = []state.CatalogOutputFile{
catalogOutput(req, "reports", "web", "report.md", state.OutputKindSource, planCreatedAt),
}
writeCatalogState(t, destinationBackend, "", existing)
plan, err := Build(context.Background(), req)
if err != nil {
t.Fatalf("Build() error = %v", err)
}
written, ok := state.FindCatalogOutputByPath(plan.CatalogOutputsToWrite, "report.md")
if !ok {
t.Fatalf("written outputs = %#v, want report.md", plan.CatalogOutputsToWrite)
}
if written.PipelineID != "reports" || written.DestinationID != "archive" {
t.Fatalf("written owner = %s/%s, want reports/archive", written.PipelineID, written.DestinationID)
}
if !written.CreatedAt.Equal(planCreatedAt) || !written.UpdatedAt.Equal(planUpdatedAt) {
t.Fatalf("written times = %s/%s", written.CreatedAt, written.UpdatedAt)
}
if len(plan.CatalogOutputsToRetain) != 0 || len(plan.CatalogOutputsToDelete) != 0 {
t.Fatalf("retain=%#v delete=%#v, want no old record for overwritten path", plan.CatalogOutputsToRetain, plan.CatalogOutputsToDelete)
}
}
func TestBuildRejectsUnmanagedPlannedPathCollision(t *testing.T) {
_, destinationBackend, req := catalogPlanRequest(t, config.WorkflowAdditive)
testutil.WriteFakeFile(t, destinationBackend, "report.md", "unmanaged")
plan, err := Build(context.Background(), req)
if err == nil {
t.Fatal("Build() error = nil, want unmanaged collision")
}
if plan.Action != ActionFailUnmanaged || !strings.Contains(err.Error(), "not managed by catalog state") {
t.Fatalf("plan action=%s error=%v, want unmanaged catalog collision", plan.Action, err)
}
}
func TestBuildPlansSupersededLegacyAdditive(t *testing.T) {
_, destinationBackend, req := catalogPlanRequest(t, config.WorkflowAdditive)
legacyState := testutil.DestinationState(req.SourceBundle.Manifest, testutil.DestinationStateOptions{})
writeJSONState(t, destinationBackend, "", legacyState)
testutil.WriteFakeFile(t, destinationBackend, "report.md", "legacy")
plan, err := Build(context.Background(), req)
if err != nil {
t.Fatalf("Build() error = %v", err)
}
if plan.SupersededLegacy == nil || plan.SupersededLegacy.SchemaVersion != state.SchemaVersion {
t.Fatalf("superseded legacy = %#v", plan.SupersededLegacy)
}
if plan.Action != ActionUpsertAdditive || plan.ClearDestinationRoot {
t.Fatalf("plan action=%s clear=%t, want additive overwrite without clear", plan.Action, plan.ClearDestinationRoot)
}
}
func TestBuildPlansSupersededLegacyReplacementClear(t *testing.T) {
_, destinationBackend, req := catalogPlanRequest(t, config.WorkflowReplacement)
legacyState := testutil.DestinationState(req.SourceBundle.Manifest, testutil.DestinationStateOptions{})
writeJSONState(t, destinationBackend, "", legacyState)
plan, err := Build(context.Background(), req)
if err != nil {
t.Fatalf("Build() error = %v", err)
}
if plan.Action != ActionReplaceCatalog || !plan.ClearDestinationRoot {
t.Fatalf("plan action=%s clear=%t, want replacement clear", plan.Action, plan.ClearDestinationRoot)
}
}
func TestBuildRejectsInvalidOrFutureState(t *testing.T) {
tests := []struct {
name string
data string
}{
{name: "invalid json", data: `{"schema_version":`},
{name: "future schema", data: `{"schema_version":99}`},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
_, destinationBackend, req := catalogPlanRequest(t, config.WorkflowAdditive)
testutil.WriteFakeFile(t, destinationBackend, storage.StateFileName, tt.data)
plan, err := Build(context.Background(), req)
if err == nil {
t.Fatal("Build() error = nil, want conflict")
}
if plan.Action != ActionFailConflict {
t.Fatalf("plan action = %s, want %s", plan.Action, ActionFailConflict)
}
})
}
}
func TestValidateRequestRejectsInvalidWorkflow(t *testing.T) {
sourceBackend, destinationBackend, req := catalogPlanRequest(t, "append")
req.SourceBackend = sourceBackend
req.DestinationBackend = destinationBackend
err := validateRequest(req)
if err == nil {
t.Fatal("validateRequest() error = nil, want invalid workflow")
}
if !strings.Contains(err.Error(), "destination.workflow") {
t.Fatalf("validateRequest() error = %v, want workflow context", err)
}
}
func catalogPlanRequest(t *testing.T, workflow string) (*fake.Backend, *fake.Backend, Request) {
t.Helper()
sourceBackend := fake.New()
destinationBackend := fake.New()
sourceBundle := testutil.WriteFakeSourceBundle(t, sourceBackend, "", testutil.BundleOptions{})
return sourceBackend, destinationBackend, Request{
PipelineID: "reports",
DestinationID: "archive",
SourceBundle: sourceBundle,
SourceBackend: sourceBackend,
DestinationBackend: destinationBackend,
DestinationBundlePath: "",
PathMapping: config.PathMappingFixed,
State: config.StatePolicy{Mode: config.StateModeSingleOwner},
}, state.DestinationStatus{State: &destinationState, HasContents: true})
if comparison.Outcome != state.OutcomeDifferentSourceConflict {
t.Fatalf("comparison outcome = %s, want %s", comparison.Outcome, state.OutcomeDifferentSourceConflict)
}
if comparison.Detail.Kind != state.ComparisonDetailDifferentSourceID {
t.Fatalf("detail kind = %q, want %q", comparison.Detail.Kind, state.ComparisonDetailDifferentSourceID)
}
if comparison.Detail.CurrentSourceID != sourceBundle.Manifest.ID || comparison.Detail.DestinationSourceID != "latest.previous" {
t.Fatalf("detail = %#v, want source ids", comparison.Detail)
PathMapping: config.PathMappingPreserveRelative,
Publish: config.PublishPolicy{Source: true},
Workflow: workflow,
DistributorVersion: "test",
Now: planUpdatedAt,
}
}
func baseCatalog(req Request) state.CatalogState {
return state.CatalogState{
SchemaVersion: state.CatalogSchemaVersion,
DistributorVersion: "previous",
CreatedAt: planCreatedAt,
UpdatedAt: planCreatedAt,
State: state.StatePolicy{Mode: state.StateModeCatalog},
Outputs: []state.CatalogOutputFile{},
}
}
func catalogOutput(req Request, pipelineID, destinationID, path, kind string, createdAt time.Time) state.CatalogOutputFile {
sourceSHA := req.SourceBundle.Manifest.Files[0].SHA256
sourceSize := req.SourceBundle.Manifest.Files[0].Size
for _, file := range req.SourceBundle.Manifest.Files {
if file.Path == path {
sourceSHA = file.SHA256
sourceSize = file.Size
break
}
}
output := state.CatalogOutputFile{
Path: path,
PipelineID: pipelineID,
DestinationID: destinationID,
Source: state.CatalogSourceIdentity{
ID: req.SourceBundle.Manifest.ID,
Digest: req.SourceBundle.Manifest.Digest,
Created: req.SourceBundle.Manifest.Created,
},
Kind: kind,
SHA256: sourceSHA,
Size: sourceSize,
CreatedAt: createdAt,
UpdatedAt: createdAt,
}
return output
}
func writeCatalogState(t *testing.T, backend *fake.Backend, relative string, catalog state.CatalogState) {
t.Helper()
writeJSONState(t, backend, relative, catalog)
}
func writeJSONState(t *testing.T, backend *fake.Backend, relative string, value any) {
t.Helper()
data, err := json.MarshalIndent(value, "", " ")
if err != nil {
t.Fatalf("marshal state: %v", err)
}
statePath, err := storage.StatePath(relative)
if err != nil {
t.Fatalf("state path: %v", err)
}
testutil.WriteFakeFile(t, backend, statePath, string(append(data, '\n')))
}

View File

@@ -18,7 +18,13 @@ func inspectDestination(ctx context.Context, backend storage.Backend, bundlePath
if parseErr != nil {
return state.DestinationStatus{StateErr: parseErr}, nil
}
return state.DestinationStatus{State: document.SingleOwner, SharedRoot: document.SharedRoot, HasContents: true}, nil
return state.DestinationStatus{
State: document.SingleOwner,
SharedRoot: document.SharedRoot,
Catalog: document.Catalog,
SupersededLegacy: document.SupersededLegacy,
HasContents: true,
}, nil
}
if !storage.IsNotFound(err) {
return state.DestinationStatus{}, err

View File

@@ -1,632 +0,0 @@
package publish
import (
"context"
"encoding/json"
"strings"
"testing"
"time"
"gitea.maximumdirect.net/eric/distributor/internal/bundle"
"gitea.maximumdirect.net/eric/distributor/internal/config"
"gitea.maximumdirect.net/eric/distributor/internal/state"
"gitea.maximumdirect.net/eric/distributor/internal/storage"
"gitea.maximumdirect.net/eric/distributor/internal/storage/fake"
"gitea.maximumdirect.net/eric/distributor/internal/testutil"
)
func TestBuildSharedRootTreatsAbsentOwnerAsPublishable(t *testing.T) {
sourceBackend := fake.New()
sourceBundle := testutil.WriteFakeSourceBundle(t, sourceBackend, "bundle", testutil.BundleOptions{
Files: []testutil.SourceFile{{Path: "report.md", Data: "# Report\n"}},
})
destinationBackend := fake.New()
writeFakeSharedRootState(t, destinationBackend, "bundle", sharedRootStateWithOwners(t, sourceBundle.Manifest, false))
req := sharedRootRequest(sourceBackend, destinationBackend, sourceBundle, config.ReconciliationModeReplace)
plan, err := Build(context.Background(), req)
if err != nil {
t.Fatalf("Build() error = %v", err)
}
if plan.Action != ActionPublishNew {
t.Fatalf("plan action = %s, want publish_new", plan.Action)
}
if plan.OwnerScope != state.CurrentOwnerScope("reports", "archive") {
t.Fatalf("owner scope = %#v, want reports/archive", plan.OwnerScope)
}
if got, want := len(plan.OtherOwnerOutputs), 1; got != want {
t.Fatalf("other owner output count = %d, want %d", got, want)
}
if got, want := len(plan.OwnerOutputsToWrite), 1; got != want {
t.Fatalf("owner output write count = %d, want %d", got, want)
}
if len(plan.OwnerOutputsToDelete) != 0 || len(plan.RetainedOwnerOutputs) != 0 {
t.Fatalf("delete=%#v retained=%#v, want none", plan.OwnerOutputsToDelete, plan.RetainedOwnerOutputs)
}
}
func TestBuildSharedRootReplaceDeletesOnlyCurrentOwnerOmittedOutputs(t *testing.T) {
sourceBackend := fake.New()
sourceBundle := testutil.WriteFakeSourceBundle(t, sourceBackend, "bundle", testutil.BundleOptions{
Files: []testutil.SourceFile{{Path: "report.md", Data: "# Report\nNew.\n"}},
})
destinationBackend := fake.New()
writeFakeSharedRootState(t, destinationBackend, "bundle", sharedRootStateWithOwners(t, sourceBundle.Manifest, true))
req := sharedRootRequest(sourceBackend, destinationBackend, sourceBundle, config.ReconciliationModeReplace)
plan, err := Build(context.Background(), req)
if err != nil {
t.Fatalf("Build() error = %v", err)
}
if plan.Action != ActionReplaceOlder {
t.Fatalf("plan action = %s, want replace_older", plan.Action)
}
if got, want := sharedRootOutputPathList(plan.OwnerOutputsToDelete), "old.md"; got != want {
t.Fatalf("owner outputs to delete = %q, want %q", got, want)
}
if got, want := sharedRootOutputPathList(plan.OtherOwnerOutputs), "other/report.md"; got != want {
t.Fatalf("other owner outputs = %q, want %q", got, want)
}
if len(plan.RetainedOwnerOutputs) != 0 {
t.Fatalf("retained owner outputs = %#v, want none", plan.RetainedOwnerOutputs)
}
}
func TestBuildSharedRootMergeRetainsCurrentOwnerOmittedOutputs(t *testing.T) {
sourceBackend := fake.New()
sourceBundle := testutil.WriteFakeSourceBundle(t, sourceBackend, "bundle", testutil.BundleOptions{
Files: []testutil.SourceFile{{Path: "report.md", Data: "# Report\nNew.\n"}},
})
destinationBackend := fake.New()
writeFakeSharedRootState(t, destinationBackend, "bundle", sharedRootStateWithOwners(t, sourceBundle.Manifest, true))
req := sharedRootRequest(sourceBackend, destinationBackend, sourceBundle, config.ReconciliationModeMerge)
plan, err := Build(context.Background(), req)
if err != nil {
t.Fatalf("Build() error = %v", err)
}
if got, want := sharedRootOutputPathList(plan.RetainedOwnerOutputs), "old.md"; got != want {
t.Fatalf("retained owner outputs = %q, want %q", got, want)
}
if len(plan.OwnerOutputsToDelete) != 0 {
t.Fatalf("owner outputs to delete = %#v, want none", plan.OwnerOutputsToDelete)
}
}
func TestBuildSharedRootRejectsOtherOwnerPathConflict(t *testing.T) {
sourceBackend := fake.New()
sourceBundle := testutil.WriteFakeSourceBundle(t, sourceBackend, "bundle", testutil.BundleOptions{
Files: []testutil.SourceFile{{Path: "report.md", Data: "# Report\n"}},
})
destinationBackend := fake.New()
sharedRoot := sharedRootStateWithOwners(t, sourceBundle.Manifest, false)
sharedRoot.Outputs[0].Path = "report.md"
writeFakeSharedRootState(t, destinationBackend, "bundle", sharedRoot)
req := sharedRootRequest(sourceBackend, destinationBackend, sourceBundle, config.ReconciliationModeReplace)
plan, err := Build(context.Background(), req)
if err == nil || !strings.Contains(err.Error(), "fail_conflict") {
t.Fatalf("Build() error = %v, want fail_conflict", err)
}
if plan.Action != ActionFailConflict {
t.Fatalf("plan action = %s, want fail_conflict", plan.Action)
}
}
func TestBuildSharedRootPlansOutputTakeoverByPolicy(t *testing.T) {
tests := []struct {
name string
takeover config.TakeoverPolicy
transfer config.TransferPolicy
ownerScope state.OwnerScope
sameSource bool
wantAction Action
wantErr string
}{
{
name: "default same pipeline allows different destination",
takeover: config.TakeoverPolicy{},
ownerScope: state.CurrentOwnerScope("reports", "web"),
wantAction: ActionReplaceTakeover,
},
{
name: "same pipeline refuses different pipeline",
takeover: config.TakeoverPolicy{Mode: config.TakeoverModeSamePipeline},
ownerScope: state.CurrentOwnerScope("other", "archive"),
wantErr: "fail_conflict",
},
{
name: "same source allows different pipeline",
takeover: config.TakeoverPolicy{Mode: config.TakeoverModeSameSource},
ownerScope: state.CurrentOwnerScope("other", "archive"),
sameSource: true,
wantAction: ActionReplaceTakeover,
},
{
name: "same source refuses different source",
takeover: config.TakeoverPolicy{Mode: config.TakeoverModeSameSource},
ownerScope: state.CurrentOwnerScope("reports", "web"),
wantErr: "fail_conflict",
},
{
name: "any managed allows different pipeline",
takeover: config.TakeoverPolicy{Mode: config.TakeoverModeAnyManaged},
ownerScope: state.CurrentOwnerScope("other", "archive"),
wantAction: ActionReplaceTakeover,
},
{
name: "never refuses same pipeline",
takeover: config.TakeoverPolicy{Mode: config.TakeoverModeNever},
ownerScope: state.CurrentOwnerScope("reports", "web"),
wantErr: "fail_conflict",
},
{
name: "transfer conflict replacement allows managed owner conflict",
takeover: config.TakeoverPolicy{Mode: config.TakeoverModeNever},
transfer: config.TransferPolicy{OnConflict: config.TransferActionReplace},
ownerScope: state.CurrentOwnerScope("other", "archive"),
wantAction: ActionReplaceConflict,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
sourceBackend := fake.New()
sourceBundle := testutil.WriteFakeSourceBundle(t, sourceBackend, "bundle", testutil.BundleOptions{
Files: []testutil.SourceFile{{Path: "report.md", Data: "# Report\n"}},
})
destinationBackend := fake.New()
sharedRoot := sharedRootStateWithOwners(t, sourceBundle.Manifest, false)
ownerManifest := sharedRoot.Owners[0].Source.Manifest
if tt.sameSource {
ownerManifest = sourceBundle.Manifest
}
setSharedRootOwnerOutput(t, &sharedRoot, 0, tt.ownerScope, ownerManifest, "report.md")
writeFakeSharedRootState(t, destinationBackend, "bundle", sharedRoot)
req := sharedRootRequest(sourceBackend, destinationBackend, sourceBundle, config.ReconciliationModeReplace)
req.Takeover = tt.takeover
if tt.transfer.OnConflict != "" {
req.Transfer.OnConflict = tt.transfer.OnConflict
}
plan, err := Build(context.Background(), req)
if tt.wantErr != "" {
if err == nil || !strings.Contains(err.Error(), tt.wantErr) {
t.Fatalf("Build() error = %v, want %q", err, tt.wantErr)
}
if plan.Action != ActionFailConflict {
t.Fatalf("plan action = %s, want fail_conflict", plan.Action)
}
return
}
if err != nil {
t.Fatalf("Build() error = %v", err)
}
if plan.Action != tt.wantAction {
t.Fatalf("plan action = %s, want %s", plan.Action, tt.wantAction)
}
if got, want := sharedRootOutputPathList(plan.TakenOverOwnerOutputs), "report.md"; got != want {
t.Fatalf("taken over outputs = %q, want %q", got, want)
}
})
}
}
func TestBuildSharedRootRejectsUnmanagedPathCollision(t *testing.T) {
sourceBackend := fake.New()
sourceBundle := testutil.WriteFakeSourceBundle(t, sourceBackend, "bundle", testutil.BundleOptions{
Files: []testutil.SourceFile{{Path: "report.md", Data: "# Report\n"}},
})
destinationBackend := fake.New()
writeFakeSharedRootState(t, destinationBackend, "bundle", sharedRootStateWithOwners(t, sourceBundle.Manifest, false))
testutil.WriteFakeFile(t, destinationBackend, "bundle/report.md", "unmanaged")
req := sharedRootRequest(sourceBackend, destinationBackend, sourceBundle, config.ReconciliationModeReplace)
plan, err := Build(context.Background(), req)
if err == nil || !strings.Contains(err.Error(), "fail_unmanaged") {
t.Fatalf("Build() error = %v, want fail_unmanaged", err)
}
if plan.Action != ActionFailUnmanaged {
t.Fatalf("plan action = %s, want fail_unmanaged", plan.Action)
}
}
func TestExecuteSharedRootPublishesOwnerAndPreservesOtherOwners(t *testing.T) {
sourceBackend := fake.New()
sourceBundle := testutil.WriteFakeSourceBundle(t, sourceBackend, "bundle", testutil.BundleOptions{
Files: []testutil.SourceFile{{Path: "report.md", Data: "# Report\n"}},
})
destinationBackend := fake.New()
writeFakeSharedRootState(t, destinationBackend, "bundle", sharedRootStateWithOwners(t, sourceBundle.Manifest, false))
req := sharedRootRequest(sourceBackend, destinationBackend, sourceBundle, config.ReconciliationModeReplace)
plan, err := Build(context.Background(), req)
if err != nil {
t.Fatalf("Build() error = %v", err)
}
if err := Execute(context.Background(), req, plan); err != nil {
t.Fatalf("Execute() error = %v", err)
}
testutil.AssertFakeFile(t, destinationBackend, "bundle/report.md", "# Report\n")
testutil.AssertFakeFile(t, destinationBackend, "bundle/other/report.md", "old")
destinationState := readFakeSharedRootState(t, destinationBackend, "bundle")
if got, want := len(destinationState.Owners), 2; got != want {
t.Fatalf("owner count = %d, want %d", got, want)
}
if got, want := destinationState.State.Mode, state.StateModeSharedRoot; got != want {
t.Fatalf("state mode = %q, want %q", got, want)
}
if _, ok := destinationState.Owner(state.CurrentOwnerScope("other", "archive")); !ok {
t.Fatal("other owner missing from shared-root state")
}
if _, ok := destinationState.Owner(state.CurrentOwnerScope("reports", "archive")); !ok {
t.Fatal("current owner missing from shared-root state")
}
if got, want := strings.Join(destinationState.AllManagedOutputPaths(), ","), "other/report.md,report.md"; got != want {
t.Fatalf("managed paths = %q, want %q", got, want)
}
}
func TestExecuteSharedRootReplaceDeletesOnlyCurrentOwnerOmittedOutputs(t *testing.T) {
sourceBackend := fake.New()
sourceBundle := testutil.WriteFakeSourceBundle(t, sourceBackend, "bundle", testutil.BundleOptions{
Files: []testutil.SourceFile{{Path: "report.md", Data: "# Report\nNew.\n"}},
})
destinationBackend := fake.New()
existing := sharedRootStateWithOwners(t, sourceBundle.Manifest, true)
writeFakeSharedRootState(t, destinationBackend, "bundle", existing)
req := sharedRootRequest(sourceBackend, destinationBackend, sourceBundle, config.ReconciliationModeReplace)
plan, err := Build(context.Background(), req)
if err != nil {
t.Fatalf("Build() error = %v", err)
}
if err := Execute(context.Background(), req, plan); err != nil {
t.Fatalf("Execute() error = %v", err)
}
testutil.AssertFakeFile(t, destinationBackend, "bundle/report.md", "# Report\nNew.\n")
testutil.AssertFakeMissing(t, destinationBackend, "bundle/old.md")
testutil.AssertFakeFile(t, destinationBackend, "bundle/other/report.md", "old")
destinationState := readFakeSharedRootState(t, destinationBackend, "bundle")
if got, want := strings.Join(destinationState.AllManagedOutputPaths(), ","), "other/report.md,report.md"; got != want {
t.Fatalf("managed paths = %q, want %q", got, want)
}
if !destinationState.CreatedAt.Equal(existing.CreatedAt) {
t.Fatalf("created_at = %s, want %s", destinationState.CreatedAt, existing.CreatedAt)
}
output, ok := findSharedRootOutputForTest(destinationState.Outputs, "report.md")
if !ok {
t.Fatal("report.md missing from shared-root outputs")
}
if !output.CreatedAt.Equal(existing.Outputs[1].CreatedAt) || !output.UpdatedAt.After(existing.Outputs[1].UpdatedAt) {
t.Fatalf("report.md timestamps = created:%s updated:%s, want preserved created and newer updated", output.CreatedAt, output.UpdatedAt)
}
}
func TestExecuteSharedRootTakeoverReassignsPathAndPreservesUnrelatedOutputs(t *testing.T) {
sourceBackend := fake.New()
sourceBundle := testutil.WriteFakeSourceBundle(t, sourceBackend, "bundle", testutil.BundleOptions{
Files: []testutil.SourceFile{{Path: "report.md", Data: "# Report\nNew.\n"}},
})
destinationBackend := fake.New()
sharedRoot := sharedRootStateWithOwners(t, sourceBundle.Manifest, false)
previousScope := state.CurrentOwnerScope("reports", "web")
setSharedRootOwnerOutput(t, &sharedRoot, 0, previousScope, sharedRoot.Owners[0].Source.Manifest, "report.md")
keepOutput := sharedRoot.Outputs[0]
keepOutput.Path = "web/keep.md"
keepOutput.SourcePath = "web/keep.md"
sharedRoot.Outputs = append(sharedRoot.Outputs, keepOutput)
writeFakeSharedRootState(t, destinationBackend, "bundle", sharedRoot)
req := sharedRootRequest(sourceBackend, destinationBackend, sourceBundle, config.ReconciliationModeReplace)
plan, err := Build(context.Background(), req)
if err != nil {
t.Fatalf("Build() error = %v", err)
}
if plan.Action != ActionReplaceTakeover {
t.Fatalf("plan action = %s, want %s", plan.Action, ActionReplaceTakeover)
}
if err := Execute(context.Background(), req, plan); err != nil {
t.Fatalf("Execute() error = %v", err)
}
testutil.AssertFakeFile(t, destinationBackend, "bundle/report.md", "# Report\nNew.\n")
testutil.AssertFakeFile(t, destinationBackend, "bundle/web/keep.md", "old")
destinationState := readFakeSharedRootState(t, destinationBackend, "bundle")
reportOutput, ok := findSharedRootOutputForTest(destinationState.Outputs, "report.md")
if !ok {
t.Fatal("report.md missing from shared-root outputs")
}
if reportOutput.Owner != state.CurrentOwnerScope("reports", "archive") {
t.Fatalf("report.md owner = %#v, want reports/archive", reportOutput.Owner)
}
keep, ok := findSharedRootOutputForTest(destinationState.Outputs, "web/keep.md")
if !ok {
t.Fatal("web/keep.md missing from shared-root outputs")
}
if keep.Owner != previousScope {
t.Fatalf("web/keep.md owner = %#v, want reports/web", keep.Owner)
}
}
func TestExecuteSharedRootTakeoverMergeDoesNotRetainOldSourceOutputs(t *testing.T) {
sourceBackend := fake.New()
sourceBundle := testutil.WriteFakeSourceBundle(t, sourceBackend, "bundle", testutil.BundleOptions{
Files: []testutil.SourceFile{{Path: "report.md", Data: "# Report\nNew.\n"}},
})
destinationBackend := fake.New()
sharedRoot := sharedRootStateWithOwners(t, sourceBundle.Manifest, false)
previousScope := state.CurrentOwnerScope("reports", "web")
setSharedRootOwnerOutput(t, &sharedRoot, 0, previousScope, sharedRoot.Owners[0].Source.Manifest, "report.md")
createdAt := sharedRoot.CreatedAt
oldManifest := sourceBundle.Manifest
oldManifest.ID = "old.source"
oldManifest.Created = oldManifest.Created.Add(-time.Hour)
oldManifest.Files = []bundle.ManifestFile{{
Path: "old.md",
SHA256: bundle.FileDigest([]byte("old\n")),
Size: int64(len("old\n")),
}}
oldManifest.Digest = bundle.BundleDigest(oldManifest.Files)
currentScope := state.CurrentOwnerScope("reports", "archive")
sharedRoot.Owners = append(sharedRoot.Owners, state.OwnerRecord{
Scope: currentScope,
Reconciliation: state.ReconciliationPolicy{Mode: config.ReconciliationModeMerge},
Source: state.SourceState{Manifest: oldManifest},
})
sharedRoot.Outputs = append(sharedRoot.Outputs, state.SharedRootOutputFile{
Path: "old.md",
Kind: state.OutputKindSource,
SourcePath: "old.md",
SHA256: oldManifest.Files[0].SHA256,
Size: oldManifest.Files[0].Size,
Owner: currentScope,
SourceID: oldManifest.ID,
SourceDigest: oldManifest.Digest,
SourceCreated: oldManifest.Created,
CreatedAt: createdAt,
UpdatedAt: createdAt,
})
writeFakeSharedRootState(t, destinationBackend, "bundle", sharedRoot)
req := sharedRootRequest(sourceBackend, destinationBackend, sourceBundle, config.ReconciliationModeMerge)
req.PathMapping = config.PathMappingFixed
plan, err := Build(context.Background(), req)
if err != nil {
t.Fatalf("Build() error = %v", err)
}
if plan.Action != ActionReplaceTakeover {
t.Fatalf("plan action = %s, want %s", plan.Action, ActionReplaceTakeover)
}
if got, want := sharedRootOutputPathList(plan.OwnerOutputsToDelete), "old.md"; got != want {
t.Fatalf("owner outputs to delete = %q, want %q", got, want)
}
if len(plan.RetainedOwnerOutputs) != 0 {
t.Fatalf("retained owner outputs = %#v, want none", plan.RetainedOwnerOutputs)
}
if err := Execute(context.Background(), req, plan); err != nil {
t.Fatalf("Execute() error = %v", err)
}
testutil.AssertFakeFile(t, destinationBackend, "bundle/report.md", "# Report\nNew.\n")
testutil.AssertFakeMissing(t, destinationBackend, "bundle/old.md")
destinationState := readFakeSharedRootState(t, destinationBackend, "bundle")
if got, want := strings.Join(destinationState.AllManagedOutputPaths(), ","), "report.md"; got != want {
t.Fatalf("managed paths = %q, want %q", got, want)
}
}
func TestExecuteSharedRootMergeRetainsCurrentOwnerOmittedOutputs(t *testing.T) {
sourceBackend := fake.New()
sourceBundle := testutil.WriteFakeSourceBundle(t, sourceBackend, "bundle", testutil.BundleOptions{
Files: []testutil.SourceFile{{Path: "report.md", Data: "# Report\nNew.\n"}},
})
destinationBackend := fake.New()
writeFakeSharedRootState(t, destinationBackend, "bundle", sharedRootStateWithOwners(t, sourceBundle.Manifest, true))
req := sharedRootRequest(sourceBackend, destinationBackend, sourceBundle, config.ReconciliationModeMerge)
plan, err := Build(context.Background(), req)
if err != nil {
t.Fatalf("Build() error = %v", err)
}
if err := Execute(context.Background(), req, plan); err != nil {
t.Fatalf("Execute() error = %v", err)
}
testutil.AssertFakeFile(t, destinationBackend, "bundle/report.md", "# Report\nNew.\n")
testutil.AssertFakeFile(t, destinationBackend, "bundle/old.md", "old")
testutil.AssertFakeFile(t, destinationBackend, "bundle/other/report.md", "old")
destinationState := readFakeSharedRootState(t, destinationBackend, "bundle")
if got, want := strings.Join(destinationState.AllManagedOutputPaths(), ","), "other/report.md,report.md,old.md"; got != want {
t.Fatalf("managed paths = %q, want %q", got, want)
}
}
func TestExecuteSharedRootForceReplaceDeletesOnlyBundlePath(t *testing.T) {
sourceBackend := fake.New()
sourceBundle := testutil.WriteFakeSourceBundle(t, sourceBackend, "bundle", testutil.BundleOptions{
Files: []testutil.SourceFile{{Path: "report.md", Data: "# Report\n"}},
})
destinationBackend := fake.New()
testutil.WriteFakeFile(t, destinationBackend, "bundle/unmanaged.txt", "unmanaged")
testutil.WriteFakeFile(t, destinationBackend, "bundle/nested/old.txt", "old")
testutil.WriteFakeFile(t, destinationBackend, "bundle-sibling/keep.txt", "keep")
req := sharedRootRequest(sourceBackend, destinationBackend, sourceBundle, config.ReconciliationModeReplace)
req.Force = true
plan, err := Build(context.Background(), req)
if err != nil {
t.Fatalf("Build() error = %v", err)
}
if plan.Action != ActionForceReplace {
t.Fatalf("plan action = %s, want force_replace", plan.Action)
}
if err := Execute(context.Background(), req, plan); err != nil {
t.Fatalf("Execute() error = %v", err)
}
testutil.AssertFakeFile(t, destinationBackend, "bundle/report.md", "# Report\n")
testutil.AssertFakeMissing(t, destinationBackend, "bundle/unmanaged.txt")
testutil.AssertFakeMissing(t, destinationBackend, "bundle/nested/old.txt")
testutil.AssertFakeFile(t, destinationBackend, "bundle-sibling/keep.txt", "keep")
destinationState := readFakeSharedRootState(t, destinationBackend, "bundle")
if got, want := len(destinationState.Owners), 1; got != want {
t.Fatalf("owner count = %d, want %d", got, want)
}
}
func sharedRootRequest(sourceBackend, destinationBackend *fake.Backend, sourceBundle bundle.Bundle, reconciliationMode string) Request {
return Request{
PipelineID: "reports",
DestinationID: "archive",
SourceBundle: sourceBundle,
SourceBackend: sourceBackend,
DestinationBackend: destinationBackend,
DestinationBundlePath: sourceBundle.RootRelativePath,
Publish: config.PublishPolicy{Source: true},
State: config.StatePolicy{Mode: config.StateModeSharedRoot},
Reconciliation: config.ReconciliationPolicy{Mode: reconciliationMode},
Transfer: defaultTransfer(),
DistributorVersion: "test",
}
}
func writeFakeSharedRootState(t *testing.T, backend *fake.Backend, relative string, sharedRoot state.SharedRootState) {
t.Helper()
data, err := json.MarshalIndent(sharedRoot, "", " ")
if err != nil {
t.Fatalf("marshal shared-root state: %v", err)
}
statePath, err := storage.StatePath(relative)
if err != nil {
t.Fatalf("state path: %v", err)
}
testutil.WriteFakeFile(t, backend, statePath, string(append(data, '\n')))
for _, output := range sharedRoot.Outputs {
path, err := storage.Join(relative, output.Path)
if err != nil {
t.Fatalf("join output path: %v", err)
}
testutil.WriteFakeFile(t, backend, path, "old")
}
}
func readFakeSharedRootState(t *testing.T, backend *fake.Backend, relative string) state.SharedRootState {
t.Helper()
statePath, err := storage.StatePath(relative)
if err != nil {
t.Fatalf("state path: %v", err)
}
data, err := backend.ReadFile(context.Background(), statePath)
if err != nil {
t.Fatalf("read shared-root state: %v", err)
}
destinationState, err := state.ParseSharedRoot(data)
if err != nil {
t.Fatalf("parse shared-root state: %v", err)
}
return destinationState
}
func findSharedRootOutputForTest(outputs []state.SharedRootOutputFile, path string) (state.SharedRootOutputFile, bool) {
for _, output := range outputs {
if output.Path == path {
return output, true
}
}
return state.SharedRootOutputFile{}, false
}
func setSharedRootOwnerOutput(t *testing.T, sharedRoot *state.SharedRootState, index int, scope state.OwnerScope, manifest bundle.Manifest, path string) {
t.Helper()
sharedRoot.Owners[index].Scope = scope
sharedRoot.Owners[index].Source = state.SourceState{Manifest: manifest}
sourcePath := path
if len(manifest.Files) > 0 {
sourcePath = manifest.Files[0].Path
}
sharedRoot.Outputs[index].Path = path
sharedRoot.Outputs[index].SourcePath = sourcePath
sharedRoot.Outputs[index].Owner = scope
sharedRoot.Outputs[index].SourceID = manifest.ID
sharedRoot.Outputs[index].SourceDigest = manifest.Digest
sharedRoot.Outputs[index].SourceCreated = manifest.Created
if len(manifest.Files) > 0 {
sharedRoot.Outputs[index].SHA256 = manifest.Files[0].SHA256
sharedRoot.Outputs[index].Size = manifest.Files[0].Size
}
}
func sharedRootStateWithOwners(t *testing.T, current bundle.Manifest, includeCurrent bool) state.SharedRootState {
t.Helper()
createdAt := time.Date(2026, 5, 30, 11, 12, 0, 0, time.UTC)
otherManifest := testutil.ValidManifest(testutil.BundleOptions{
ID: "other.source",
Files: []testutil.SourceFile{{Path: "other/report.md", Data: "# Other\n"}},
})
sharedRoot := state.SharedRootState{
SchemaVersion: state.SharedRootSchemaVersion,
DistributorVersion: "test",
CreatedAt: createdAt,
UpdatedAt: createdAt,
State: state.StatePolicy{Mode: state.StateModeSharedRoot},
Owners: []state.OwnerRecord{{
Scope: state.CurrentOwnerScope("other", "archive"),
Reconciliation: state.ReconciliationPolicy{Mode: config.ReconciliationModeReplace},
Source: state.SourceState{Manifest: otherManifest},
}},
Outputs: []state.SharedRootOutputFile{{
Path: "other/report.md",
Kind: state.OutputKindSource,
SourcePath: "other/report.md",
SHA256: otherManifest.Files[0].SHA256,
Size: otherManifest.Files[0].Size,
Owner: state.CurrentOwnerScope("other", "archive"),
SourceID: otherManifest.ID,
SourceDigest: otherManifest.Digest,
SourceCreated: otherManifest.Created,
CreatedAt: createdAt,
UpdatedAt: createdAt,
}},
}
if !includeCurrent {
return sharedRoot
}
older := current
older.Created = older.Created.Add(-time.Hour)
older.Files = append([]bundle.ManifestFile(nil), current.Files...)
older.Files = append(older.Files, bundle.ManifestFile{
Path: "old.md",
SHA256: bundle.FileDigest([]byte("old\n")),
Size: int64(len("old\n")),
})
older.Digest = bundle.BundleDigest(older.Files)
scope := state.CurrentOwnerScope("reports", "archive")
sharedRoot.Owners = append(sharedRoot.Owners, state.OwnerRecord{
Scope: scope,
Reconciliation: state.ReconciliationPolicy{Mode: config.ReconciliationModeReplace},
Source: state.SourceState{Manifest: older},
})
for _, file := range older.Files {
sharedRoot.Outputs = append(sharedRoot.Outputs, state.SharedRootOutputFile{
Path: file.Path,
Kind: state.OutputKindSource,
SourcePath: file.Path,
SHA256: file.SHA256,
Size: file.Size,
Owner: scope,
SourceID: older.ID,
SourceDigest: older.Digest,
SourceCreated: older.Created,
CreatedAt: createdAt,
UpdatedAt: createdAt,
})
}
return sharedRoot
}
func sharedRootOutputPathList(outputs []state.SharedRootOutputFile) string {
paths := make([]string, 0, len(outputs))
for _, output := range outputs {
paths = append(paths, output.Path)
}
return strings.Join(paths, ",")
}

View File

@@ -1,215 +0,0 @@
package publish
import (
"context"
"strings"
"testing"
"gitea.maximumdirect.net/eric/distributor/internal/bundle"
"gitea.maximumdirect.net/eric/distributor/internal/config"
"gitea.maximumdirect.net/eric/distributor/internal/storage"
"gitea.maximumdirect.net/eric/distributor/internal/storage/fake"
"gitea.maximumdirect.net/eric/distributor/internal/testutil"
)
func TestBuildPlansSingleOwnerTakeoverByPolicy(t *testing.T) {
tests := []struct {
name string
takeover config.TakeoverPolicy
mutateState func(*bundle.Manifest, *testutil.DestinationStateOptions)
wantAction Action
wantErr string
}{
{
name: "default same pipeline different source",
takeover: config.TakeoverPolicy{},
mutateState: func(manifest *bundle.Manifest, opts *testutil.DestinationStateOptions) {
manifest.ID = "other.source"
},
wantAction: ActionReplaceTakeover,
},
{
name: "same pipeline different newer source",
takeover: config.TakeoverPolicy{Mode: config.TakeoverModeSamePipeline},
mutateState: func(manifest *bundle.Manifest, opts *testutil.DestinationStateOptions) {
manifest.ID = "other.source"
manifest.Created = manifest.Created.AddDate(0, 0, 1)
},
wantAction: ActionReplaceTakeover,
},
{
name: "default same pipeline different destination",
takeover: config.TakeoverPolicy{},
mutateState: func(manifest *bundle.Manifest, opts *testutil.DestinationStateOptions) {
opts.DestinationID = "web"
},
wantAction: ActionReplaceTakeover,
},
{
name: "same pipeline refuses different pipeline",
takeover: config.TakeoverPolicy{Mode: config.TakeoverModeSamePipeline},
mutateState: func(manifest *bundle.Manifest, opts *testutil.DestinationStateOptions) {
opts.PipelineID = "other"
},
wantErr: "fail_conflict",
},
{
name: "same source allows different pipeline",
takeover: config.TakeoverPolicy{Mode: config.TakeoverModeSameSource},
mutateState: func(manifest *bundle.Manifest, opts *testutil.DestinationStateOptions) {
opts.PipelineID = "other"
},
wantAction: ActionReplaceTakeover,
},
{
name: "same source refuses different source",
takeover: config.TakeoverPolicy{Mode: config.TakeoverModeSameSource},
mutateState: func(manifest *bundle.Manifest, opts *testutil.DestinationStateOptions) {
manifest.ID = "other.source"
},
wantErr: "fail_conflict",
},
{
name: "any managed allows different pipeline",
takeover: config.TakeoverPolicy{Mode: config.TakeoverModeAnyManaged},
mutateState: func(manifest *bundle.Manifest, opts *testutil.DestinationStateOptions) {
manifest.ID = "other.source"
opts.PipelineID = "other"
},
wantAction: ActionReplaceTakeover,
},
{
name: "never refuses different source",
takeover: config.TakeoverPolicy{Mode: config.TakeoverModeNever},
mutateState: func(manifest *bundle.Manifest, opts *testutil.DestinationStateOptions) {
manifest.ID = "other.source"
},
wantErr: "fail_conflict",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
sourceBackend := fake.New()
sourceBundle := testutil.WriteFakeSourceBundle(t, sourceBackend, "bundle", testutil.BundleOptions{})
destinationBackend := fake.New()
destinationManifest := sourceBundle.Manifest
destinationManifest.Files = append([]bundle.ManifestFile(nil), sourceBundle.Manifest.Files...)
opts := testutil.DestinationStateOptions{}
tt.mutateState(&destinationManifest, &opts)
testutil.WriteFakeDestinationState(t, destinationBackend, "bundle", destinationManifest, opts)
req := takeoverRequest(sourceBackend, destinationBackend, sourceBundle, tt.takeover, config.ReconciliationModeReplace)
plan, err := Build(context.Background(), req)
if tt.wantErr != "" {
if err == nil || !strings.Contains(err.Error(), tt.wantErr) {
t.Fatalf("Build() error = %v, want %q", err, tt.wantErr)
}
return
}
if err != nil {
t.Fatalf("Build() error = %v", err)
}
if plan.Action != tt.wantAction {
t.Fatalf("plan action = %s, want %s", plan.Action, tt.wantAction)
}
})
}
}
func TestBuildDoesNotTakeOverInvalidOrUnmanagedDestination(t *testing.T) {
tests := []struct {
name string
prepare func(t *testing.T, backend *fake.Backend)
wantErr string
}{
{
name: "invalid state",
prepare: func(t *testing.T, backend *fake.Backend) {
t.Helper()
statePath, err := storage.StatePath("bundle")
if err != nil {
t.Fatalf("state path: %v", err)
}
testutil.WriteFakeFile(t, backend, statePath, "{invalid")
},
wantErr: "fail_conflict",
},
{
name: "unmanaged content",
prepare: func(t *testing.T, backend *fake.Backend) {
t.Helper()
testutil.WriteFakeFile(t, backend, "bundle/old.txt", "old")
},
wantErr: "fail_unmanaged",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
sourceBackend := fake.New()
sourceBundle := testutil.WriteFakeSourceBundle(t, sourceBackend, "bundle", testutil.BundleOptions{})
destinationBackend := fake.New()
tt.prepare(t, destinationBackend)
req := takeoverRequest(sourceBackend, destinationBackend, sourceBundle, config.TakeoverPolicy{Mode: config.TakeoverModeAnyManaged}, config.ReconciliationModeReplace)
_, err := Build(context.Background(), req)
if err == nil || !strings.Contains(err.Error(), tt.wantErr) {
t.Fatalf("Build() error = %v, want %q", err, tt.wantErr)
}
})
}
}
func TestExecuteTakeoverMergeDoesNotRetainOldSourceOutputs(t *testing.T) {
sourceBackend := fake.New()
sourceBundle := testutil.WriteFakeSourceBundle(t, sourceBackend, "bundle", testutil.BundleOptions{
Files: []testutil.SourceFile{{Path: "report.md", Data: "# Report\nNew.\n"}},
})
destinationBackend := fake.New()
oldManifest := sourceBundle.Manifest
oldManifest.ID = "old.source"
oldManifest.Files = []bundle.ManifestFile{
{Path: "report.md", SHA256: bundle.FileDigest([]byte("old\n")), Size: int64(len("old\n"))},
{Path: "summary.txt", SHA256: bundle.FileDigest([]byte("old summary\n")), Size: int64(len("old summary\n"))},
}
oldManifest.Digest = bundle.BundleDigest(oldManifest.Files)
testutil.WriteFakeDestinationState(t, destinationBackend, "bundle", oldManifest, testutil.DestinationStateOptions{})
req := takeoverRequest(sourceBackend, destinationBackend, sourceBundle, config.TakeoverPolicy{Mode: config.TakeoverModeSamePipeline}, config.ReconciliationModeMerge)
plan, err := Build(context.Background(), req)
if err != nil {
t.Fatalf("Build() error = %v", err)
}
if plan.Action != ActionReplaceTakeover {
t.Fatalf("plan action = %s, want %s", plan.Action, ActionReplaceTakeover)
}
if err := Execute(context.Background(), req, plan); err != nil {
t.Fatalf("Execute() error = %v", err)
}
testutil.AssertFakeFile(t, destinationBackend, "bundle/report.md", "# Report\nNew.\n")
testutil.AssertFakeMissing(t, destinationBackend, "bundle/summary.txt")
destinationState := readFakeState(t, destinationBackend, "bundle")
if got, want := len(destinationState.Outputs), 1; got != want {
t.Fatalf("state output count = %d, want %d", got, want)
}
if got, want := destinationState.Source.Manifest.ID, sourceBundle.Manifest.ID; got != want {
t.Fatalf("state source id = %q, want %q", got, want)
}
}
func takeoverRequest(sourceBackend, destinationBackend *fake.Backend, sourceBundle bundle.Bundle, takeover config.TakeoverPolicy, reconciliationMode string) Request {
return Request{
PipelineID: "reports",
DestinationID: "archive",
SourceBundle: sourceBundle,
SourceBackend: sourceBackend,
DestinationBackend: destinationBackend,
DestinationBundlePath: sourceBundle.RootRelativePath,
Publish: config.PublishPolicy{Source: true},
Reconciliation: config.ReconciliationPolicy{Mode: reconciliationMode},
Takeover: takeover,
Transfer: defaultTransfer(),
DistributorVersion: "test",
}
}

422
internal/state/catalog.go Normal file
View File

@@ -0,0 +1,422 @@
package state
import (
"bytes"
"encoding/json"
"fmt"
"io"
"time"
"gitea.maximumdirect.net/eric/distributor/internal/bundle"
"gitea.maximumdirect.net/eric/distributor/internal/config"
"gitea.maximumdirect.net/eric/distributor/internal/link"
"gitea.maximumdirect.net/eric/distributor/internal/storage"
)
type CatalogState struct {
SchemaVersion int
DistributorVersion string
CreatedAt time.Time
UpdatedAt time.Time
State StatePolicy
Outputs []CatalogOutputFile
}
type CatalogSourceIdentity struct {
ID string
Digest string
Created time.Time
}
type CatalogOutputFile struct {
Path string
PipelineID string
DestinationID string
Source CatalogSourceIdentity
Kind string
SourcePath string
Transform string
URL string
SHA256 string
Size int64
CreatedAt time.Time
UpdatedAt time.Time
}
type rawCatalogState struct {
SchemaVersion *int `json:"schema_version"`
DistributorVersion string `json:"distributor_version"`
CreatedAt *string `json:"created_at"`
UpdatedAt *string `json:"updated_at"`
State *rawStatePolicy `json:"state"`
Outputs []rawCatalogOutput `json:"outputs"`
}
type rawCatalogOutput struct {
Path *string `json:"path"`
PipelineID *string `json:"pipeline_id"`
DestinationID *string `json:"destination_id"`
Source *rawCatalogSourceIdentity `json:"source"`
Kind *string `json:"kind"`
SourcePath *string `json:"source_path"`
Transform *string `json:"transform"`
URL *string `json:"url"`
SHA256 *string `json:"sha256"`
Size *int64 `json:"size"`
CreatedAt *string `json:"created_at"`
UpdatedAt *string `json:"updated_at"`
}
type rawCatalogSourceIdentity struct {
ID *string `json:"id"`
Digest *string `json:"digest"`
Created *string `json:"created"`
}
func ParseCatalog(data []byte) (CatalogState, error) {
decoder := json.NewDecoder(bytes.NewReader(data))
decoder.DisallowUnknownFields()
var raw rawCatalogState
if err := decoder.Decode(&raw); err != nil {
return CatalogState{}, fmt.Errorf("parse distributor state: %w", err)
}
var extra any
if err := decoder.Decode(&extra); err != io.EOF {
return CatalogState{}, fmt.Errorf("parse distributor state: trailing data")
}
state, err := parseCatalogRaw(raw)
if err != nil {
return CatalogState{}, err
}
if err := ValidateCatalog(state); err != nil {
return CatalogState{}, err
}
return state, nil
}
func parseCatalogRaw(raw rawCatalogState) (CatalogState, error) {
if raw.SchemaVersion == nil {
return CatalogState{}, fmt.Errorf("state schema_version is required")
}
state := CatalogState{
SchemaVersion: *raw.SchemaVersion,
DistributorVersion: raw.DistributorVersion,
}
if state.SchemaVersion != CatalogSchemaVersion {
return CatalogState{}, fmt.Errorf("state schema_version must be %d", CatalogSchemaVersion)
}
createdAt, err := parseRequiredTime("state created_at", raw.CreatedAt)
if err != nil {
return CatalogState{}, err
}
updatedAt, err := parseRequiredTime("state updated_at", raw.UpdatedAt)
if err != nil {
return CatalogState{}, err
}
state.CreatedAt = createdAt
state.UpdatedAt = updatedAt
if raw.State == nil || raw.State.Mode == "" {
return CatalogState{}, fmt.Errorf("state state.mode is required")
}
state.State.Mode = raw.State.Mode
if raw.Outputs == nil {
return CatalogState{}, fmt.Errorf("state outputs is required")
}
outputs, err := parseCatalogOutputs(raw.Outputs)
if err != nil {
return CatalogState{}, err
}
state.Outputs = outputs
return state, nil
}
func parseCatalogOutputs(rawOutputs []rawCatalogOutput) ([]CatalogOutputFile, error) {
outputs := make([]CatalogOutputFile, 0, len(rawOutputs))
for index, raw := range rawOutputs {
output, err := parseCatalogOutput(index, raw)
if err != nil {
return nil, err
}
outputs = append(outputs, output)
}
return outputs, nil
}
func parseCatalogOutput(index int, raw rawCatalogOutput) (CatalogOutputFile, error) {
if raw.Path == nil || *raw.Path == "" {
return CatalogOutputFile{}, fmt.Errorf("state outputs[%d].path is required", index)
}
if raw.PipelineID == nil || *raw.PipelineID == "" {
return CatalogOutputFile{}, fmt.Errorf("state outputs[%d].pipeline_id is required", index)
}
if raw.DestinationID == nil || *raw.DestinationID == "" {
return CatalogOutputFile{}, fmt.Errorf("state outputs[%d].destination_id is required", index)
}
source, err := parseCatalogSourceIdentity(index, raw.Source)
if err != nil {
return CatalogOutputFile{}, err
}
if raw.Kind == nil || *raw.Kind == "" {
return CatalogOutputFile{}, fmt.Errorf("state outputs[%d].kind is required", index)
}
switch *raw.Kind {
case OutputKindSource:
if raw.SourcePath != nil {
return CatalogOutputFile{}, fmt.Errorf("state outputs[%d].source_path is only valid for generated output", index)
}
if raw.Transform != nil {
return CatalogOutputFile{}, fmt.Errorf("state outputs[%d].transform is only valid for generated output", index)
}
case OutputKindGenerated:
if raw.SourcePath == nil || *raw.SourcePath == "" {
return CatalogOutputFile{}, fmt.Errorf("state outputs[%d].source_path is required for generated output", index)
}
if raw.Transform == nil || *raw.Transform == "" {
return CatalogOutputFile{}, fmt.Errorf("state outputs[%d].transform is required for generated output", index)
}
}
if raw.SHA256 == nil || *raw.SHA256 == "" {
return CatalogOutputFile{}, fmt.Errorf("state outputs[%d].sha256 is required", index)
}
if raw.Size == nil {
return CatalogOutputFile{}, fmt.Errorf("state outputs[%d].size is required", index)
}
createdAt, err := parseRequiredTime(fmt.Sprintf("state outputs[%d].created_at", index), raw.CreatedAt)
if err != nil {
return CatalogOutputFile{}, err
}
updatedAt, err := parseRequiredTime(fmt.Sprintf("state outputs[%d].updated_at", index), raw.UpdatedAt)
if err != nil {
return CatalogOutputFile{}, err
}
output := CatalogOutputFile{
Path: *raw.Path,
PipelineID: *raw.PipelineID,
DestinationID: *raw.DestinationID,
Source: source,
Kind: *raw.Kind,
SHA256: *raw.SHA256,
Size: *raw.Size,
CreatedAt: createdAt,
UpdatedAt: updatedAt,
}
if raw.SourcePath != nil {
output.SourcePath = *raw.SourcePath
}
if raw.Transform != nil {
output.Transform = *raw.Transform
}
if raw.URL != nil {
if *raw.URL == "" {
return CatalogOutputFile{}, fmt.Errorf("state outputs[%d].url must not be empty", index)
}
output.URL = *raw.URL
}
return output, nil
}
func parseCatalogSourceIdentity(index int, raw *rawCatalogSourceIdentity) (CatalogSourceIdentity, error) {
if raw == nil {
return CatalogSourceIdentity{}, fmt.Errorf("state outputs[%d].source is required", index)
}
if raw.ID == nil || *raw.ID == "" {
return CatalogSourceIdentity{}, fmt.Errorf("state outputs[%d].source.id is required", index)
}
if raw.Digest == nil || *raw.Digest == "" {
return CatalogSourceIdentity{}, fmt.Errorf("state outputs[%d].source.digest is required", index)
}
created, err := parseRequiredTime(fmt.Sprintf("state outputs[%d].source.created", index), raw.Created)
if err != nil {
return CatalogSourceIdentity{}, err
}
return CatalogSourceIdentity{
ID: *raw.ID,
Digest: *raw.Digest,
Created: created,
}, nil
}
func (s CatalogState) CreatedAtString() string {
return s.CreatedAt.UTC().Format(time.RFC3339)
}
func (s CatalogState) UpdatedAtString() string {
return s.UpdatedAt.UTC().Format(time.RFC3339)
}
func (s CatalogSourceIdentity) CreatedString() string {
return s.Created.UTC().Format(time.RFC3339)
}
func (o CatalogOutputFile) CreatedAtString() string {
return o.CreatedAt.UTC().Format(time.RFC3339)
}
func (o CatalogOutputFile) UpdatedAtString() string {
return o.UpdatedAt.UTC().Format(time.RFC3339)
}
func (s CatalogState) MarshalJSON() ([]byte, error) {
type stateJSON struct {
SchemaVersion int `json:"schema_version"`
DistributorVersion string `json:"distributor_version,omitempty"`
CreatedAt string `json:"created_at"`
UpdatedAt string `json:"updated_at"`
State StatePolicy `json:"state"`
Outputs []CatalogOutputFile `json:"outputs"`
}
return json.Marshal(stateJSON{
SchemaVersion: s.SchemaVersion,
DistributorVersion: s.DistributorVersion,
CreatedAt: s.CreatedAtString(),
UpdatedAt: s.UpdatedAtString(),
State: s.State,
Outputs: s.Outputs,
})
}
func (s CatalogSourceIdentity) MarshalJSON() ([]byte, error) {
type sourceJSON struct {
ID string `json:"id"`
Digest string `json:"digest"`
Created string `json:"created"`
}
return json.Marshal(sourceJSON{
ID: s.ID,
Digest: s.Digest,
Created: s.CreatedString(),
})
}
func (o CatalogOutputFile) MarshalJSON() ([]byte, error) {
type outputJSON struct {
Path string `json:"path"`
PipelineID string `json:"pipeline_id"`
DestinationID string `json:"destination_id"`
Source CatalogSourceIdentity `json:"source"`
Kind string `json:"kind"`
SourcePath string `json:"source_path,omitempty"`
Transform string `json:"transform,omitempty"`
URL string `json:"url,omitempty"`
SHA256 string `json:"sha256"`
Size int64 `json:"size"`
CreatedAt string `json:"created_at"`
UpdatedAt string `json:"updated_at"`
}
return json.Marshal(outputJSON{
Path: o.Path,
PipelineID: o.PipelineID,
DestinationID: o.DestinationID,
Source: o.Source,
Kind: o.Kind,
SourcePath: o.SourcePath,
Transform: o.Transform,
URL: o.URL,
SHA256: o.SHA256,
Size: o.Size,
CreatedAt: o.CreatedAtString(),
UpdatedAt: o.UpdatedAtString(),
})
}
func ValidateCatalog(s CatalogState) error {
if s.SchemaVersion != CatalogSchemaVersion {
return fmt.Errorf("state schema_version must be %d", CatalogSchemaVersion)
}
if s.CreatedAt.IsZero() {
return fmt.Errorf("state created_at is required")
}
if s.UpdatedAt.IsZero() {
return fmt.Errorf("state updated_at is required")
}
if s.State.Mode != StateModeCatalog {
return fmt.Errorf("state state.mode must be %s", StateModeCatalog)
}
if s.Outputs == nil {
return fmt.Errorf("state outputs is required")
}
seenPaths := make(map[string]struct{}, len(s.Outputs))
for index, output := range s.Outputs {
if err := validateCatalogOutput(index, output); err != nil {
return err
}
if _, exists := seenPaths[output.Path]; exists {
return fmt.Errorf("state outputs[%d].path duplicates %q", index, output.Path)
}
seenPaths[output.Path] = struct{}{}
}
return nil
}
func validateCatalogOutput(index int, output CatalogOutputFile) error {
if err := storage.ValidatePath(output.Path); err != nil {
return fmt.Errorf("state outputs[%d].path: %w", index, err)
}
if output.PipelineID == "" {
return fmt.Errorf("state outputs[%d].pipeline_id is required", index)
}
if !config.IsSlugLikeID(output.PipelineID) {
return fmt.Errorf("state outputs[%d].pipeline_id must be a slug-like identifier", index)
}
if output.DestinationID == "" {
return fmt.Errorf("state outputs[%d].destination_id is required", index)
}
if !config.IsSlugLikeID(output.DestinationID) {
return fmt.Errorf("state outputs[%d].destination_id must be a slug-like identifier", index)
}
if err := validateCatalogSourceIdentity(index, output.Source); err != nil {
return err
}
switch output.Kind {
case OutputKindSource:
if output.SourcePath != "" {
return fmt.Errorf("state outputs[%d].source_path is only valid for generated output", index)
}
if output.Transform != "" {
return fmt.Errorf("state outputs[%d].transform is only valid for generated output", index)
}
case OutputKindGenerated:
if output.SourcePath == "" {
return fmt.Errorf("state outputs[%d].source_path is required for generated output", index)
}
if err := storage.ValidatePath(output.SourcePath); err != nil {
return fmt.Errorf("state outputs[%d].source_path: %w", index, err)
}
if output.Transform == "" {
return fmt.Errorf("state outputs[%d].transform is required for generated output", index)
}
default:
return fmt.Errorf("state outputs[%d].kind must be source or generated", index)
}
if output.URL != "" {
if err := link.ValidateHTTPURL(output.URL); err != nil {
return fmt.Errorf("state outputs[%d].url: %w", index, err)
}
}
if err := bundle.ValidateDigest(output.SHA256); err != nil {
return fmt.Errorf("state outputs[%d].sha256: %w", index, err)
}
if output.Size < 0 {
return fmt.Errorf("state outputs[%d].size must be non-negative", index)
}
if output.CreatedAt.IsZero() {
return fmt.Errorf("state outputs[%d].created_at is required", index)
}
if output.UpdatedAt.IsZero() {
return fmt.Errorf("state outputs[%d].updated_at is required", index)
}
return nil
}
func validateCatalogSourceIdentity(index int, source CatalogSourceIdentity) error {
if source.ID == "" {
return fmt.Errorf("state outputs[%d].source.id is required", index)
}
if err := bundle.ValidateDigest(source.Digest); err != nil {
return fmt.Errorf("state outputs[%d].source.digest: %w", index, err)
}
if source.Created.IsZero() {
return fmt.Errorf("state outputs[%d].source.created is required", index)
}
return nil
}

View File

@@ -0,0 +1,361 @@
package state
import (
"encoding/json"
"strings"
"testing"
"time"
)
func TestParseCatalogState(t *testing.T) {
state, err := ParseCatalog([]byte(validCatalogStateJSON(t)))
if err != nil {
t.Fatalf("ParseCatalog() error = %v", err)
}
if got, want := state.SchemaVersion, CatalogSchemaVersion; got != want {
t.Fatalf("schema version = %d, want %d", got, want)
}
if got, want := state.CreatedAtString(), "2026-06-19T12:00:00Z"; got != want {
t.Fatalf("created_at = %q, want %q", got, want)
}
if got, want := state.UpdatedAtString(), "2026-06-19T12:05:00Z"; got != want {
t.Fatalf("updated_at = %q, want %q", got, want)
}
if got, want := state.State.Mode, StateModeCatalog; got != want {
t.Fatalf("state mode = %q, want %q", got, want)
}
if got, want := len(state.Outputs), 2; got != want {
t.Fatalf("output count = %d, want %d", got, want)
}
sourceOutput := state.Outputs[0]
if sourceOutput.SourcePath != "" || sourceOutput.Transform != "" {
t.Fatalf("source output source_path=%q transform=%q, want omitted", sourceOutput.SourcePath, sourceOutput.Transform)
}
generatedOutput := state.Outputs[1]
if generatedOutput.SourcePath != "report.md" || generatedOutput.Transform != "markdown_to_html" {
t.Fatalf("generated output source_path=%q transform=%q", generatedOutput.SourcePath, generatedOutput.Transform)
}
if got, want := generatedOutput.Source.CreatedString(), "2026-05-30T11:10:00Z"; got != want {
t.Fatalf("source created = %q, want %q", got, want)
}
}
func TestCatalogMarshalIsDeterministic(t *testing.T) {
data, err := json.Marshal(validCatalogState(t))
if err != nil {
t.Fatalf("Marshal() error = %v", err)
}
want := `{"schema_version":4,"distributor_version":"dev","created_at":"2026-06-19T12:00:00Z","updated_at":"2026-06-19T12:05:00Z","state":{"mode":"catalog"},"outputs":[{"path":"report.md","pipeline_id":"reports","destination_id":"archive","source":{"id":"weather.daily.brentwood.2026-05-30","digest":"sha256:099b205780d2b050024868399961b05731729a548d5d6329c7b06a6740dd75fe","created":"2026-05-30T11:10:00Z"},"kind":"source","sha256":"sha256:3640fd37140ee4d2e0e93e78834f232ea67a50e7bc6279203690cc7de1975fa6","size":16,"created_at":"2026-06-19T12:00:00Z","updated_at":"2026-06-19T12:05:00Z"},{"path":"report.html","pipeline_id":"reports","destination_id":"html","source":{"id":"weather.daily.brentwood.2026-05-30","digest":"sha256:099b205780d2b050024868399961b05731729a548d5d6329c7b06a6740dd75fe","created":"2026-05-30T11:10:00Z"},"kind":"generated","source_path":"report.md","transform":"markdown_to_html","url":"https://reports.example.com/report.html","sha256":"sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","size":128,"created_at":"2026-06-19T12:00:00Z","updated_at":"2026-06-19T12:05:00Z"}]}`
if string(data) != want {
t.Fatalf("json = %s, want %s", data, want)
}
}
func TestParseDocumentHandlesCatalogAndSupersededLegacy(t *testing.T) {
catalog, err := ParseDocument([]byte(validCatalogStateJSON(t)))
if err != nil {
t.Fatalf("ParseDocument(catalog) error = %v", err)
}
if catalog.Catalog == nil || catalog.SingleOwner != nil || catalog.SharedRoot != nil || catalog.SupersededLegacy != nil {
t.Fatalf("catalog document = %#v", catalog)
}
tests := map[string]string{
"schema 1": legacyStateJSON(t),
"schema 2": validStateJSON(t),
"schema 3": validSharedRootStateJSON(t),
}
for name, body := range tests {
t.Run(name, func(t *testing.T) {
document, err := ParseDocument([]byte(body))
if err != nil {
t.Fatalf("ParseDocument() error = %v", err)
}
if document.SupersededLegacy == nil || document.Catalog != nil || document.SingleOwner != nil || document.SharedRoot != nil {
t.Fatalf("document = %#v, want superseded legacy only", document)
}
if document.SupersededLegacy.SchemaVersion < legacySchemaVersion || document.SupersededLegacy.SchemaVersion >= CatalogSchemaVersion {
t.Fatalf("legacy schema version = %d, want 1 through 3", document.SupersededLegacy.SchemaVersion)
}
})
}
}
func TestParseDocumentRejectsUnsupportedFutureSchema(t *testing.T) {
body := strings.Replace(validCatalogStateJSON(t), `"schema_version": 4`, `"schema_version": 5`, 1)
_, err := ParseDocument([]byte(body))
assertStateErrorContains(t, err, "schema_version 5 is unsupported")
}
func TestParseDocumentRejectsTrailingData(t *testing.T) {
_, err := ParseDocument([]byte(validCatalogStateJSON(t) + "\n{}"))
assertStateErrorContains(t, err, "trailing data")
}
func TestParseCatalogRejectsMissingFields(t *testing.T) {
tests := map[string]func(map[string]any){
"schema_version": func(document map[string]any) {
delete(document, "schema_version")
},
"created_at": func(document map[string]any) {
delete(document, "created_at")
},
"updated_at": func(document map[string]any) {
delete(document, "updated_at")
},
"state": func(document map[string]any) {
delete(document, "state")
},
"outputs": func(document map[string]any) {
delete(document, "outputs")
},
"path": func(document map[string]any) {
delete(firstCatalogOutput(document), "path")
},
"pipeline_id": func(document map[string]any) {
delete(firstCatalogOutput(document), "pipeline_id")
},
"destination_id": func(document map[string]any) {
delete(firstCatalogOutput(document), "destination_id")
},
"source": func(document map[string]any) {
delete(firstCatalogOutput(document), "source")
},
"source id": func(document map[string]any) {
delete(firstCatalogSource(document), "id")
},
"source digest": func(document map[string]any) {
delete(firstCatalogSource(document), "digest")
},
"source created": func(document map[string]any) {
delete(firstCatalogSource(document), "created")
},
"kind": func(document map[string]any) {
delete(firstCatalogOutput(document), "kind")
},
"sha256": func(document map[string]any) {
delete(firstCatalogOutput(document), "sha256")
},
"size": func(document map[string]any) {
delete(firstCatalogOutput(document), "size")
},
}
for name, mutate := range tests {
t.Run(name, func(t *testing.T) {
document := catalogStateObject(t)
mutate(document)
_, err := ParseCatalog(mustMarshalCatalogObject(t, document))
assertStateErrorContains(t, err, "required")
})
}
}
func TestParseCatalogRejectsMalformedTimestamps(t *testing.T) {
tests := map[string]func(string) string{
"created_at": func(body string) string {
return strings.Replace(body, `"created_at": "2026-06-19T12:00:00Z"`, `"created_at": "June 19"`, 1)
},
"updated_at": func(body string) string {
return strings.Replace(body, `"updated_at": "2026-06-19T12:05:00Z"`, `"updated_at": "June 19"`, 1)
},
"source created": func(body string) string {
return strings.Replace(body, `"created": "2026-05-30T11:10:00Z"`, `"created": "May 30"`, 1)
},
"output created_at": func(body string) string {
return strings.Replace(body, ` "created_at": "2026-06-19T12:00:00Z"`, ` "created_at": "June 19"`, 1)
},
"output updated_at": func(body string) string {
return strings.Replace(body, ` "updated_at": "2026-06-19T12:05:00Z"`, ` "updated_at": "June 19"`, 1)
},
}
for name, mutate := range tests {
t.Run(name, func(t *testing.T) {
_, err := ParseCatalog([]byte(mutate(validCatalogStateJSON(t))))
assertStateErrorContains(t, err, "RFC3339")
})
}
}
func TestValidateCatalogRejectsInvalidOutputRecords(t *testing.T) {
tests := map[string]func(*CatalogState){
"duplicate path": func(s *CatalogState) {
s.Outputs[1].Path = s.Outputs[0].Path
},
"invalid output path": func(s *CatalogState) {
s.Outputs[0].Path = "../report.md"
},
"invalid pipeline id": func(s *CatalogState) {
s.Outputs[0].PipelineID = ".reports"
},
"invalid destination id": func(s *CatalogState) {
s.Outputs[0].DestinationID = ".archive"
},
"missing source id": func(s *CatalogState) {
s.Outputs[0].Source.ID = ""
},
"invalid source digest": func(s *CatalogState) {
s.Outputs[0].Source.Digest = "SHA256:099b205780d2b050024868399961b05731729a548d5d6329c7b06a6740dd75fe"
},
"missing source created": func(s *CatalogState) {
s.Outputs[0].Source.Created = time.Time{}
},
"invalid kind": func(s *CatalogState) {
s.Outputs[0].Kind = "document"
},
"generated missing source path": func(s *CatalogState) {
s.Outputs[1].SourcePath = ""
},
"generated invalid source path": func(s *CatalogState) {
s.Outputs[1].SourcePath = "../report.md"
},
"generated missing transform": func(s *CatalogState) {
s.Outputs[1].Transform = ""
},
"source output source path": func(s *CatalogState) {
s.Outputs[0].SourcePath = "report.md"
},
"source output transform": func(s *CatalogState) {
s.Outputs[0].Transform = "markdown_to_html"
},
"invalid output digest": func(s *CatalogState) {
s.Outputs[0].SHA256 = "SHA256:3640fd37140ee4d2e0e93e78834f232ea67a50e7bc6279203690cc7de1975fa6"
},
"negative size": func(s *CatalogState) {
s.Outputs[0].Size = -1
},
"invalid url": func(s *CatalogState) {
s.Outputs[1].URL = "file:///tmp/report.html"
},
"missing created at": func(s *CatalogState) {
s.Outputs[0].CreatedAt = time.Time{}
},
"missing updated at": func(s *CatalogState) {
s.Outputs[0].UpdatedAt = time.Time{}
},
}
for name, mutate := range tests {
t.Run(name, func(t *testing.T) {
state := validCatalogState(t)
mutate(&state)
if err := ValidateCatalog(state); err == nil {
t.Fatal("ValidateCatalog() error = nil, want error")
}
})
}
}
func TestParseCatalogRejectsForbiddenFields(t *testing.T) {
tests := map[string]string{
"owners": `"owners": [],`,
"sources": `"sources": [],`,
"workflow": `"workflow": "additive",`,
"source": `"source": {"manifest": {}},`,
"manifest": `"manifest": {},`,
"pipeline_id": `"pipeline_id": "reports",`,
"destination_id": `"destination_id": "archive",`,
"published_at": `"published_at": "2026-06-19T12:00:00Z",`,
}
for name, field := range tests {
t.Run(name, func(t *testing.T) {
body := strings.Replace(validCatalogStateJSON(t), `"created_at":`, field+"\n "+`"created_at":`, 1)
_, err := ParseCatalog([]byte(body))
assertStateErrorContains(t, err, "unknown field")
})
}
}
func TestParseCatalogRejectsForbiddenOutputFieldsForSourceOutput(t *testing.T) {
tests := map[string]string{
"source_path": `"source_path": "report.md",`,
"transform": `"transform": "markdown_to_html",`,
"empty url": `"url": "",`,
}
for name, field := range tests {
t.Run(name, func(t *testing.T) {
body := strings.Replace(validCatalogStateJSON(t), `"kind": "source",`, `"kind": "source",`+"\n "+field, 1)
_, err := ParseCatalog([]byte(body))
if err == nil {
t.Fatal("ParseCatalog() error = nil, want error")
}
})
}
}
func validCatalogStateJSON(t *testing.T) string {
t.Helper()
data, err := json.MarshalIndent(validCatalogState(t), "", " ")
if err != nil {
t.Fatalf("marshal catalog state: %v", err)
}
return string(data)
}
func catalogStateObject(t *testing.T) map[string]any {
t.Helper()
var document map[string]any
if err := json.Unmarshal([]byte(validCatalogStateJSON(t)), &document); err != nil {
t.Fatalf("unmarshal catalog state: %v", err)
}
return document
}
func firstCatalogOutput(document map[string]any) map[string]any {
outputs := document["outputs"].([]any)
return outputs[0].(map[string]any)
}
func firstCatalogSource(document map[string]any) map[string]any {
return firstCatalogOutput(document)["source"].(map[string]any)
}
func mustMarshalCatalogObject(t *testing.T, document map[string]any) []byte {
t.Helper()
data, err := json.Marshal(document)
if err != nil {
t.Fatalf("marshal catalog object: %v", err)
}
return data
}
func validCatalogState(t *testing.T) CatalogState {
t.Helper()
manifest := validManifest(t)
createdAt := time.Date(2026, 6, 19, 12, 0, 0, 0, time.UTC)
updatedAt := time.Date(2026, 6, 19, 12, 5, 0, 0, time.UTC)
source := CatalogSourceIdentity{
ID: manifest.ID,
Digest: manifest.Digest,
Created: manifest.Created,
}
return CatalogState{
SchemaVersion: CatalogSchemaVersion,
DistributorVersion: "dev",
CreatedAt: createdAt,
UpdatedAt: updatedAt,
State: StatePolicy{Mode: StateModeCatalog},
Outputs: []CatalogOutputFile{{
Path: "report.md",
PipelineID: "reports",
DestinationID: "archive",
Source: source,
Kind: OutputKindSource,
SHA256: manifest.Files[0].SHA256,
Size: manifest.Files[0].Size,
CreatedAt: createdAt,
UpdatedAt: updatedAt,
}, {
Path: "report.html",
PipelineID: "reports",
DestinationID: "html",
Source: source,
Kind: OutputKindGenerated,
SourcePath: "report.md",
Transform: "markdown_to_html",
URL: "https://reports.example.com/report.html",
SHA256: "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
Size: 128,
CreatedAt: createdAt,
UpdatedAt: updatedAt,
}},
}
}

View File

@@ -21,10 +21,12 @@ const (
)
type DestinationStatus struct {
State *DistributorState
SharedRoot *SharedRootState
StateErr error
HasContents bool
State *DistributorState
SharedRoot *SharedRootState
Catalog *CatalogState
SupersededLegacy *SupersededLegacyState
StateErr error
HasContents bool
}
type Comparison struct {

View File

@@ -14,9 +14,11 @@ import (
const (
SchemaVersion = 2
SharedRootSchemaVersion = 3
CatalogSchemaVersion = 4
legacySchemaVersion = 1
StateModeSingleOwner = config.StateModeSingleOwner
StateModeSharedRoot = config.StateModeSharedRoot
StateModeCatalog = "catalog"
)
type DistributorState struct {

View File

@@ -26,6 +26,15 @@ func FindOutputByPath(outputs []OutputFile, path string) (OutputFile, bool) {
return OutputFile{}, false
}
func FindCatalogOutputByPath(outputs []CatalogOutputFile, path string) (CatalogOutputFile, bool) {
for _, output := range outputs {
if output.Path == path {
return output, true
}
}
return CatalogOutputFile{}, false
}
func MergeOutputFiles(retained, planned []OutputFile) ([]OutputFile, error) {
outputs := make([]OutputFile, 0, len(retained)+len(planned))
indexByPath := make(map[string]int, len(retained)+len(planned))
@@ -178,6 +187,62 @@ func RemoveMissingSharedRootOutputs(s SharedRootState, missingPaths []string) (S
return next, changed
}
func CatalogOutputsForOwner(outputs []CatalogOutputFile, scope OwnerScope) []CatalogOutputFile {
selected := make([]CatalogOutputFile, 0, len(outputs))
for _, output := range outputs {
if output.PipelineID == scope.PipelineID && output.DestinationID == scope.DestinationID {
selected = append(selected, output)
}
}
return selected
}
func CatalogManagedOutputPaths(s CatalogState) []string {
paths := make([]string, 0, len(s.Outputs))
for _, output := range s.Outputs {
paths = append(paths, output.Path)
}
return paths
}
func RemoveMissingCatalogOwnerOutputs(s CatalogState, scope OwnerScope, missingPaths []string) (CatalogState, bool) {
if len(missingPaths) == 0 {
return s, false
}
missing := pathSet(missingPaths)
next := s
next.Outputs = make([]CatalogOutputFile, 0, len(s.Outputs))
changed := false
for _, output := range s.Outputs {
if output.PipelineID == scope.PipelineID && output.DestinationID == scope.DestinationID {
if _, remove := missing[output.Path]; remove {
changed = true
continue
}
}
next.Outputs = append(next.Outputs, output)
}
return next, changed
}
func RemoveMissingCatalogOutputs(s CatalogState, missingPaths []string) (CatalogState, bool) {
if len(missingPaths) == 0 {
return s, false
}
missing := pathSet(missingPaths)
next := s
next.Outputs = make([]CatalogOutputFile, 0, len(s.Outputs))
changed := false
for _, output := range s.Outputs {
if _, remove := missing[output.Path]; remove {
changed = true
continue
}
next.Outputs = append(next.Outputs, output)
}
return next, changed
}
func (s SharedRootState) OutputOwner(path string) (OwnerScope, bool) {
for _, output := range s.Outputs {
if output.Path == path {

View File

@@ -49,6 +49,22 @@ func SharedRootPruneCandidates(s SharedRootState, scope OwnerScope) []PruneCandi
return candidates
}
func CatalogPruneCandidates(s CatalogState, scope OwnerScope) []PruneCandidate {
candidates := make([]PruneCandidate, 0, len(s.Outputs))
for _, output := range s.Outputs {
if output.PipelineID != scope.PipelineID || output.DestinationID != scope.DestinationID {
continue
}
owner := scope
candidates = append(candidates, PruneCandidate{
Path: output.Path,
UpdatedAt: output.UpdatedAt,
Owner: &owner,
})
}
return candidates
}
func PlanPrune(candidates []PruneCandidate, options PrunePlanOptions) PrunePlan {
ordered := append([]PruneCandidate(nil), candidates...)
sortPruneCandidatesNewestFirst(ordered)

View File

@@ -14,8 +14,14 @@ import (
)
type StateDocument struct {
SingleOwner *DistributorState
SharedRoot *SharedRootState
SingleOwner *DistributorState
SharedRoot *SharedRootState
Catalog *CatalogState
SupersededLegacy *SupersededLegacyState
}
type SupersededLegacyState struct {
SchemaVersion int
}
type SharedRootState struct {
@@ -103,18 +109,18 @@ func ParseDocument(data []byte) (StateDocument, error) {
if err != nil {
return StateDocument{}, err
}
if schemaVersion == SharedRootSchemaVersion {
sharedRoot, err := ParseSharedRoot(data)
switch schemaVersion {
case legacySchemaVersion, SchemaVersion, SharedRootSchemaVersion:
return StateDocument{SupersededLegacy: &SupersededLegacyState{SchemaVersion: schemaVersion}}, nil
case CatalogSchemaVersion:
catalog, err := ParseCatalog(data)
if err != nil {
return StateDocument{}, err
}
return StateDocument{SharedRoot: &sharedRoot}, nil
return StateDocument{Catalog: &catalog}, nil
default:
return StateDocument{}, fmt.Errorf("state schema_version %d is unsupported", schemaVersion)
}
singleOwner, err := Parse(data)
if err != nil {
return StateDocument{}, err
}
return StateDocument{SingleOwner: &singleOwner}, nil
}
func parseSchemaVersion(data []byte) (int, error) {
@@ -125,6 +131,10 @@ func parseSchemaVersion(data []byte) (int, error) {
if err := decoder.Decode(&raw); err != nil {
return 0, fmt.Errorf("parse distributor state: %w", err)
}
var extra any
if err := decoder.Decode(&extra); err != io.EOF {
return 0, fmt.Errorf("parse distributor state: trailing data")
}
if raw.SchemaVersion == nil {
return 0, fmt.Errorf("state schema_version is required")
}

View File

@@ -10,24 +10,6 @@ import (
"gitea.maximumdirect.net/eric/distributor/internal/config"
)
func TestParseDocumentHandlesSingleOwnerAndSharedRoot(t *testing.T) {
singleOwner, err := ParseDocument([]byte(validStateJSON(t)))
if err != nil {
t.Fatalf("ParseDocument(single owner) error = %v", err)
}
if singleOwner.SingleOwner == nil || singleOwner.SharedRoot != nil {
t.Fatalf("single owner document = %#v", singleOwner)
}
sharedRoot, err := ParseDocument([]byte(validSharedRootStateJSON(t)))
if err != nil {
t.Fatalf("ParseDocument(shared root) error = %v", err)
}
if sharedRoot.SharedRoot == nil || sharedRoot.SingleOwner != nil {
t.Fatalf("shared root document = %#v", sharedRoot)
}
}
func TestParseSharedRootState(t *testing.T) {
state, err := ParseSharedRoot([]byte(validSharedRootStateJSON(t)))
if err != nil {