Compare commits
6 Commits
d23c624179
...
cf7e62733e
| Author | SHA1 | Date | |
|---|---|---|---|
| cf7e62733e | |||
| 91b72478d5 | |||
| 52078e2195 | |||
| 77cde40296 | |||
| 20129bfff5 | |||
| a95662226f |
@@ -211,29 +211,18 @@ func removePrunedStateRecords(ctx context.Context, backend storage.Backend, stat
|
||||
if len(paths) == 0 {
|
||||
return false, nil
|
||||
}
|
||||
if document.SingleOwner != nil {
|
||||
next, changed := state.RemoveMissingOutputs(*document.SingleOwner, paths)
|
||||
if document.Catalog != nil {
|
||||
next, changed := state.RemoveMissingCatalogOwnerOutputs(*document.Catalog, scope, paths)
|
||||
if !changed {
|
||||
return false, nil
|
||||
}
|
||||
next.UpdatedAt = now
|
||||
if err := state.Validate(next); err != nil {
|
||||
if err := state.ValidateCatalog(next); err != nil {
|
||||
return false, err
|
||||
}
|
||||
return true, writeRepairedState(ctx, backend, statePath, next)
|
||||
}
|
||||
if document.SharedRoot != nil {
|
||||
next, changed := state.RemoveMissingSharedRootOwnerOutputs(*document.SharedRoot, scope, paths)
|
||||
if !changed {
|
||||
return false, nil
|
||||
}
|
||||
next.UpdatedAt = now
|
||||
if err := state.ValidateSharedRoot(next); err != nil {
|
||||
return false, err
|
||||
}
|
||||
return true, writeRepairedState(ctx, backend, statePath, next)
|
||||
}
|
||||
return false, fmt.Errorf("destination state document is empty")
|
||||
return false, unsupportedStateDocumentError(document)
|
||||
}
|
||||
|
||||
func PlanPrune(document state.StateDocument, policy config.PrunePolicy, options PrunePlanOptions) (PrunePlanReport, error) {
|
||||
@@ -266,17 +255,20 @@ func PlanPrune(document state.StateDocument, policy config.PrunePolicy, options
|
||||
}
|
||||
|
||||
func pruneCandidatesForDocument(document state.StateDocument, scope state.OwnerScope) ([]state.PruneCandidate, error) {
|
||||
if document.SingleOwner != nil {
|
||||
singleOwner := *document.SingleOwner
|
||||
if singleOwner.PipelineID != scope.PipelineID || singleOwner.DestinationID != scope.DestinationID {
|
||||
return nil, fmt.Errorf("state owner is %s/%s, not %s/%s", singleOwner.PipelineID, singleOwner.DestinationID, scope.PipelineID, scope.DestinationID)
|
||||
if document.Catalog != nil {
|
||||
return state.CatalogPruneCandidates(*document.Catalog, scope), nil
|
||||
}
|
||||
return state.SingleOwnerPruneCandidates(singleOwner), nil
|
||||
return nil, unsupportedStateDocumentError(document)
|
||||
}
|
||||
|
||||
func unsupportedStateDocumentError(document state.StateDocument) error {
|
||||
if document.SupersededLegacy != nil {
|
||||
return fmt.Errorf("destination state schema_version %d is superseded legacy state", document.SupersededLegacy.SchemaVersion)
|
||||
}
|
||||
if document.SharedRoot != nil {
|
||||
return state.SharedRootPruneCandidates(*document.SharedRoot, scope), nil
|
||||
if document.SingleOwner != nil || document.SharedRoot != nil {
|
||||
return fmt.Errorf("legacy destination state is not supported by this command")
|
||||
}
|
||||
return nil, fmt.Errorf("destination state document is empty")
|
||||
return fmt.Errorf("destination state document is empty")
|
||||
}
|
||||
|
||||
func pruneOlderThan(policy config.PrunePolicy) *time.Duration {
|
||||
|
||||
@@ -15,7 +15,7 @@ import (
|
||||
)
|
||||
|
||||
func TestPlanPruneDisabledPolicy(t *testing.T) {
|
||||
document := state.StateDocument{SingleOwner: &state.DistributorState{}}
|
||||
document := state.StateDocument{Catalog: &state.CatalogState{}}
|
||||
report, err := PlanPrune(document, config.PrunePolicy{}, PrunePlanOptions{
|
||||
PipelineID: "reports",
|
||||
DestinationID: "archive",
|
||||
@@ -28,12 +28,12 @@ func TestPlanPruneDisabledPolicy(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestPlanPruneSingleOwnerOutputs(t *testing.T) {
|
||||
func TestPlanPruneCatalogOutputs(t *testing.T) {
|
||||
now := time.Date(2026, 6, 8, 12, 0, 0, 0, time.UTC)
|
||||
olderThan := config.Duration(48 * time.Hour)
|
||||
destinationState := pruneSingleOwnerState(now)
|
||||
catalog := pruneCatalogState(now)
|
||||
|
||||
report, err := PlanPrune(state.StateDocument{SingleOwner: &destinationState}, config.PrunePolicy{
|
||||
report, err := PlanPrune(state.StateDocument{Catalog: &catalog}, config.PrunePolicy{
|
||||
Enabled: true,
|
||||
OlderThan: &olderThan,
|
||||
}, PrunePlanOptions{
|
||||
@@ -52,12 +52,12 @@ func TestPlanPruneSingleOwnerOutputs(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestPlanPruneSharedRootCurrentOwnerOnly(t *testing.T) {
|
||||
func TestPlanPruneCatalogCurrentOwnerOnly(t *testing.T) {
|
||||
now := time.Date(2026, 6, 8, 12, 0, 0, 0, time.UTC)
|
||||
keepLatest := 0
|
||||
sharedRoot := pruneSharedRootState(now)
|
||||
catalog := pruneCatalogState(now)
|
||||
|
||||
report, err := PlanPrune(state.StateDocument{SharedRoot: &sharedRoot}, config.PrunePolicy{
|
||||
report, err := PlanPrune(state.StateDocument{Catalog: &catalog}, config.PrunePolicy{
|
||||
Enabled: true,
|
||||
KeepLatest: &keepLatest,
|
||||
}, PrunePlanOptions{
|
||||
@@ -68,10 +68,10 @@ func TestPlanPruneSharedRootCurrentOwnerOnly(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("PlanPrune() error = %v", err)
|
||||
}
|
||||
if got, want := report.CheckedCount, 1; got != want {
|
||||
if got, want := report.CheckedCount, 2; got != want {
|
||||
t.Fatalf("checked count = %d, want %d", got, want)
|
||||
}
|
||||
if got, want := pruneRecordPaths(report.PrunedOutputs), "archive.txt"; got != want {
|
||||
if got, want := pruneRecordPaths(report.PrunedOutputs), "old.txt,fresh.txt"; got != want {
|
||||
t.Fatalf("pruned = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
@@ -80,8 +80,8 @@ func TestPruneDryRunReportsPlannedDeletesWithoutDeletingOrRewritingState(t *test
|
||||
now := time.Date(2026, 6, 8, 12, 0, 0, 0, time.UTC)
|
||||
backend := fake.New()
|
||||
cfg := pruneS3Config(t, pruneOlderThanPolicy(48*time.Hour))
|
||||
original := pruneSingleOwnerState(now)
|
||||
writeFakeSingleOwnerStateForPrune(t, backend, original)
|
||||
original := pruneCatalogState(now)
|
||||
writeFakeCatalogState(t, backend, original)
|
||||
testutil.WriteFakeFile(t, backend, "unmanaged.txt", "keep")
|
||||
|
||||
report, err := pruneConfigWithBackendFactory(context.Background(), cfg, PruneOptions{
|
||||
@@ -102,12 +102,12 @@ func TestPruneDryRunReportsPlannedDeletesWithoutDeletingOrRewritingState(t *test
|
||||
testutil.AssertFakeFile(t, backend, "old.txt", "managed")
|
||||
testutil.AssertFakeFile(t, backend, "fresh.txt", "managed")
|
||||
testutil.AssertFakeFile(t, backend, "unmanaged.txt", "keep")
|
||||
destinationState := readFakeSingleOwnerState(t, backend)
|
||||
if got := strings.Join(state.ManagedOutputPaths(destinationState), ","); got != "old.txt,fresh.txt" {
|
||||
catalog := readFakeCatalogState(t, backend)
|
||||
if got := strings.Join(state.CatalogManagedOutputPaths(catalog), ","); got != "old.txt,fresh.txt,html.txt" {
|
||||
t.Fatalf("state outputs = %q, want original outputs", got)
|
||||
}
|
||||
if !destinationState.UpdatedAt.Equal(original.UpdatedAt) {
|
||||
t.Fatalf("state updated_at = %s, want original %s", destinationState.UpdatedAt, original.UpdatedAt)
|
||||
if !catalog.UpdatedAt.Equal(original.UpdatedAt) {
|
||||
t.Fatalf("state updated_at = %s, want original %s", catalog.UpdatedAt, original.UpdatedAt)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -115,7 +115,7 @@ func TestPruneApplyDeletesOnlyManagedOutputsAndUpdatesState(t *testing.T) {
|
||||
now := time.Date(2026, 6, 8, 12, 0, 0, 0, time.UTC)
|
||||
backend := fake.New()
|
||||
cfg := pruneS3Config(t, pruneOlderThanPolicy(48*time.Hour))
|
||||
writeFakeSingleOwnerStateForPrune(t, backend, pruneSingleOwnerState(now))
|
||||
writeFakeCatalogState(t, backend, pruneCatalogState(now))
|
||||
testutil.WriteFakeFile(t, backend, "unmanaged.txt", "keep")
|
||||
|
||||
report, err := pruneConfigWithBackendFactory(context.Background(), cfg, PruneOptions{
|
||||
@@ -134,14 +134,18 @@ func TestPruneApplyDeletesOnlyManagedOutputsAndUpdatesState(t *testing.T) {
|
||||
}
|
||||
testutil.AssertFakeMissing(t, backend, "old.txt")
|
||||
testutil.AssertFakeFile(t, backend, "fresh.txt", "managed")
|
||||
testutil.AssertFakeFile(t, backend, "html.txt", "managed")
|
||||
testutil.AssertFakeFile(t, backend, "unmanaged.txt", "keep")
|
||||
assertFakeStateExists(t, backend)
|
||||
destinationState := readFakeSingleOwnerState(t, backend)
|
||||
if got := strings.Join(state.ManagedOutputPaths(destinationState), ","); got != "fresh.txt" {
|
||||
catalog := readFakeCatalogState(t, backend)
|
||||
if got := strings.Join(state.CatalogManagedOutputPaths(catalog), ","); got != "fresh.txt,html.txt" {
|
||||
t.Fatalf("state outputs = %q, want fresh.txt", got)
|
||||
}
|
||||
if !destinationState.UpdatedAt.Equal(now) {
|
||||
t.Fatalf("state updated_at = %s, want %s", destinationState.UpdatedAt, now)
|
||||
if catalog.SchemaVersion != state.CatalogSchemaVersion {
|
||||
t.Fatalf("state schema_version = %d, want %d", catalog.SchemaVersion, state.CatalogSchemaVersion)
|
||||
}
|
||||
if !catalog.UpdatedAt.Equal(now) {
|
||||
t.Fatalf("state updated_at = %s, want %s", catalog.UpdatedAt, now)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -150,7 +154,7 @@ func TestPruneApplyPreservesStateForFailedDeletes(t *testing.T) {
|
||||
backend := fake.New()
|
||||
keepLatest := 0
|
||||
cfg := pruneS3Config(t, config.PrunePolicy{Enabled: true, KeepLatest: &keepLatest})
|
||||
writeFakeSingleOwnerStateForPrune(t, backend, pruneSingleOwnerState(now))
|
||||
writeFakeCatalogState(t, backend, pruneCatalogState(now))
|
||||
failingBackend := failingDeleteBackend{Backend: backend, failPath: "fresh.txt"}
|
||||
|
||||
report, err := pruneConfigWithBackendFactory(context.Background(), cfg, PruneOptions{
|
||||
@@ -170,18 +174,17 @@ func TestPruneApplyPreservesStateForFailedDeletes(t *testing.T) {
|
||||
testutil.AssertFakeMissing(t, backend, "old.txt")
|
||||
testutil.AssertFakeFile(t, backend, "fresh.txt", "managed")
|
||||
assertFakeStateExists(t, backend)
|
||||
destinationState := readFakeSingleOwnerState(t, backend)
|
||||
if got := strings.Join(state.ManagedOutputPaths(destinationState), ","); got != "fresh.txt" {
|
||||
catalog := readFakeCatalogState(t, backend)
|
||||
if got := strings.Join(state.CatalogManagedOutputPaths(catalog), ","); got != "fresh.txt,html.txt" {
|
||||
t.Fatalf("state outputs = %q, want only failed output preserved", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPruneSharedRootPreservesOtherOwnersWhenScopedToCurrentOwner(t *testing.T) {
|
||||
func TestPrunePreservesOtherOwnersWhenScopedToCurrentOwner(t *testing.T) {
|
||||
now := time.Date(2026, 6, 8, 12, 0, 0, 0, time.UTC)
|
||||
backend := fake.New()
|
||||
keepLatest := 0
|
||||
cfg := pruneS3Config(t, config.PrunePolicy{Enabled: true, KeepLatest: &keepLatest})
|
||||
writeFakeSharedRootStateForApp(t, backend, pruneSharedRootState(now))
|
||||
cfg := pruneS3Config(t, pruneOlderThanPolicy(48*time.Hour))
|
||||
writeFakeCatalogState(t, backend, pruneCatalogState(now))
|
||||
testutil.WriteFakeFile(t, backend, "unmanaged.txt", "keep")
|
||||
|
||||
report, err := pruneConfigWithBackendFactory(context.Background(), cfg, PruneOptions{
|
||||
@@ -192,93 +195,56 @@ func TestPruneSharedRootPreservesOtherOwnersWhenScopedToCurrentOwner(t *testing.
|
||||
if err != nil {
|
||||
t.Fatalf("pruneConfigWithBackendFactory() error = %v", err)
|
||||
}
|
||||
if got, want := pruneRecordPaths(report.DeletedOutputs), "archive.txt"; got != want {
|
||||
if got, want := pruneRecordPaths(report.DeletedOutputs), "old.txt"; got != want {
|
||||
t.Fatalf("deleted outputs = %q, want %q", got, want)
|
||||
}
|
||||
testutil.AssertFakeMissing(t, backend, "archive.txt")
|
||||
testutil.AssertFakeFile(t, backend, "html.txt", "old")
|
||||
testutil.AssertFakeMissing(t, backend, "old.txt")
|
||||
testutil.AssertFakeFile(t, backend, "html.txt", "managed")
|
||||
testutil.AssertFakeFile(t, backend, "unmanaged.txt", "keep")
|
||||
sharedRoot := readFakeSharedRootStateForApp(t, backend)
|
||||
if got := strings.Join(sharedRoot.AllManagedOutputPaths(), ","); got != "html.txt" {
|
||||
t.Fatalf("shared-root outputs = %q, want other owner output preserved", got)
|
||||
catalog := readFakeCatalogState(t, backend)
|
||||
if got := strings.Join(state.CatalogManagedOutputPaths(catalog), ","); got != "fresh.txt,html.txt" {
|
||||
t.Fatalf("catalog outputs = %q, want other owner output preserved", got)
|
||||
}
|
||||
}
|
||||
|
||||
func pruneSingleOwnerState(now time.Time) state.DistributorState {
|
||||
func pruneCatalogState(now time.Time) state.CatalogState {
|
||||
manifest := testutil.ValidManifest(testutil.BundleOptions{})
|
||||
publishedAt := now.Add(-96 * time.Hour)
|
||||
return state.DistributorState{
|
||||
SchemaVersion: state.SchemaVersion,
|
||||
createdAt := now.Add(-96 * time.Hour)
|
||||
source := state.CatalogSourceIdentity{ID: manifest.ID, Digest: manifest.Digest, Created: manifest.Created}
|
||||
return state.CatalogState{
|
||||
SchemaVersion: state.CatalogSchemaVersion,
|
||||
DistributorVersion: "test",
|
||||
CreatedAt: createdAt,
|
||||
UpdatedAt: createdAt,
|
||||
State: state.StatePolicy{Mode: state.StateModeCatalog},
|
||||
Outputs: []state.CatalogOutputFile{{
|
||||
Path: "old.txt",
|
||||
PipelineID: "reports",
|
||||
DestinationID: "archive",
|
||||
PublishedAt: publishedAt,
|
||||
CreatedAt: publishedAt,
|
||||
UpdatedAt: publishedAt,
|
||||
State: state.StatePolicy{Mode: state.StateModeSingleOwner},
|
||||
Reconciliation: state.ReconciliationPolicy{Mode: config.ReconciliationModeReplace},
|
||||
Source: state.SourceState{Manifest: manifest},
|
||||
DistributorVersion: "test",
|
||||
Outputs: []state.OutputFile{{
|
||||
Path: "old.txt",
|
||||
Source: source,
|
||||
Kind: state.OutputKindSource,
|
||||
SourcePath: "report.md",
|
||||
SHA256: manifest.Files[0].SHA256,
|
||||
Size: manifest.Files[0].Size,
|
||||
CreatedAt: now.Add(-96 * time.Hour),
|
||||
UpdatedAt: now.Add(-72 * time.Hour),
|
||||
}, {
|
||||
Path: "fresh.txt",
|
||||
PipelineID: "reports",
|
||||
DestinationID: "archive",
|
||||
Source: source,
|
||||
Kind: state.OutputKindSource,
|
||||
SourcePath: "summary.txt",
|
||||
SHA256: manifest.Files[1].SHA256,
|
||||
Size: manifest.Files[1].Size,
|
||||
CreatedAt: now.Add(-24 * time.Hour),
|
||||
UpdatedAt: now.Add(-24 * time.Hour),
|
||||
}},
|
||||
}
|
||||
}
|
||||
|
||||
func pruneSharedRootState(now time.Time) state.SharedRootState {
|
||||
manifest := testutil.ValidManifest(testutil.BundleOptions{})
|
||||
archive := state.CurrentOwnerScope("reports", "archive")
|
||||
html := state.CurrentOwnerScope("reports", "html")
|
||||
return state.SharedRootState{
|
||||
SchemaVersion: state.SharedRootSchemaVersion,
|
||||
DistributorVersion: "test",
|
||||
CreatedAt: now.Add(-96 * time.Hour),
|
||||
UpdatedAt: now.Add(-24 * time.Hour),
|
||||
State: state.StatePolicy{Mode: state.StateModeSharedRoot},
|
||||
Owners: []state.OwnerRecord{{
|
||||
Scope: archive,
|
||||
Reconciliation: state.ReconciliationPolicy{Mode: config.ReconciliationModeReplace},
|
||||
Source: state.SourceState{Manifest: manifest},
|
||||
}, {
|
||||
Scope: html,
|
||||
Reconciliation: state.ReconciliationPolicy{Mode: config.ReconciliationModeReplace},
|
||||
Source: state.SourceState{Manifest: manifest},
|
||||
}},
|
||||
Outputs: []state.SharedRootOutputFile{{
|
||||
Path: "archive.txt",
|
||||
Kind: state.OutputKindSource,
|
||||
SourcePath: "report.md",
|
||||
SHA256: manifest.Files[0].SHA256,
|
||||
Size: manifest.Files[0].Size,
|
||||
Owner: archive,
|
||||
SourceID: manifest.ID,
|
||||
SourceDigest: manifest.Digest,
|
||||
SourceCreated: manifest.Created,
|
||||
CreatedAt: now.Add(-96 * time.Hour),
|
||||
UpdatedAt: now.Add(-72 * time.Hour),
|
||||
CreatedAt: now.Add(-24 * time.Hour),
|
||||
UpdatedAt: now.Add(-24 * time.Hour),
|
||||
}, {
|
||||
Path: "html.txt",
|
||||
PipelineID: "reports",
|
||||
DestinationID: "html",
|
||||
Source: source,
|
||||
Kind: state.OutputKindSource,
|
||||
SourcePath: "summary.txt",
|
||||
SHA256: manifest.Files[1].SHA256,
|
||||
Size: manifest.Files[1].Size,
|
||||
Owner: html,
|
||||
SourceID: manifest.ID,
|
||||
SourceDigest: manifest.Digest,
|
||||
SourceCreated: manifest.Created,
|
||||
CreatedAt: now.Add(-96 * time.Hour),
|
||||
UpdatedAt: now.Add(-72 * time.Hour),
|
||||
}},
|
||||
@@ -319,18 +285,31 @@ func pruneOlderThanPolicy(duration time.Duration) config.PrunePolicy {
|
||||
}
|
||||
}
|
||||
|
||||
func writeFakeSingleOwnerStateForPrune(t *testing.T, backend *fake.Backend, destinationState state.DistributorState) {
|
||||
func writeFakeCatalogState(t *testing.T, backend *fake.Backend, catalog state.CatalogState) {
|
||||
t.Helper()
|
||||
data, err := json.MarshalIndent(destinationState, "", " ")
|
||||
data, err := json.MarshalIndent(catalog, "", " ")
|
||||
if err != nil {
|
||||
t.Fatalf("marshal single-owner state: %v", err)
|
||||
t.Fatalf("marshal catalog state: %v", err)
|
||||
}
|
||||
testutil.WriteFakeFile(t, backend, storage.StateFileName, string(append(data, '\n')))
|
||||
for _, output := range destinationState.Outputs {
|
||||
for _, output := range catalog.Outputs {
|
||||
testutil.WriteFakeFile(t, backend, output.Path, "managed")
|
||||
}
|
||||
}
|
||||
|
||||
func readFakeCatalogState(t *testing.T, backend *fake.Backend) state.CatalogState {
|
||||
t.Helper()
|
||||
data, err := backend.ReadFile(context.Background(), storage.StateFileName)
|
||||
if err != nil {
|
||||
t.Fatalf("read catalog state: %v", err)
|
||||
}
|
||||
catalog, err := state.ParseCatalog(data)
|
||||
if err != nil {
|
||||
t.Fatalf("parse catalog state: %v", err)
|
||||
}
|
||||
return catalog
|
||||
}
|
||||
|
||||
func assertFakeStateExists(t *testing.T, backend *fake.Backend) {
|
||||
t.Helper()
|
||||
if _, err := backend.Stat(context.Background(), storage.StateFileName); err != nil {
|
||||
|
||||
@@ -158,62 +158,25 @@ func buildReconcileStateReport(ctx context.Context, backend storage.Backend, pip
|
||||
DryRun: options.DryRun,
|
||||
}
|
||||
scope := state.CurrentOwnerScope(pipeline.ID, destination.ID)
|
||||
if document.SingleOwner != nil {
|
||||
return reconcileSingleOwnerState(ctx, backend, statePath, *document.SingleOwner, scope, report, options)
|
||||
if document.Catalog != nil {
|
||||
return reconcileCatalogState(ctx, backend, statePath, *document.Catalog, scope, report, options)
|
||||
}
|
||||
return reconcileSharedRootState(ctx, backend, statePath, *document.SharedRoot, scope, report, options)
|
||||
return ReconcileStateReport{}, unsupportedStateDocumentError(document)
|
||||
}
|
||||
|
||||
func reconcileSingleOwnerState(ctx context.Context, backend storage.Backend, statePath string, destinationState state.DistributorState, scope state.OwnerScope, report ReconcileStateReport, options ReconcileStateOptions) (ReconcileStateReport, error) {
|
||||
if destinationState.PipelineID != scope.PipelineID || destinationState.DestinationID != scope.DestinationID {
|
||||
return ReconcileStateReport{}, fmt.Errorf("state owner is %s/%s, not %s/%s", destinationState.PipelineID, destinationState.DestinationID, scope.PipelineID, scope.DestinationID)
|
||||
}
|
||||
report.StateSchema = destinationState.SchemaVersion
|
||||
report.OwnerScope = &ReconcileStateOwnerScope{PipelineID: scope.PipelineID, DestinationID: scope.DestinationID}
|
||||
managed := state.ManagedOutputPaths(destinationState)
|
||||
missing, err := missingSingleOwnerOutputs(ctx, backend, destinationState.Outputs)
|
||||
if err != nil {
|
||||
return ReconcileStateReport{}, err
|
||||
}
|
||||
report.CheckedCount = len(managed)
|
||||
report.MissingManagedOutputs = missing
|
||||
unmanaged, err := unmanagedEntries(ctx, backend, managed)
|
||||
if err != nil {
|
||||
return ReconcileStateReport{}, err
|
||||
}
|
||||
report.UnmanagedEntries = unmanaged
|
||||
report.WouldChange = options.DryRun && len(missing) > 0
|
||||
|
||||
if !options.DryRun && len(missing) > 0 {
|
||||
missingPaths := missingReportPaths(missing)
|
||||
next, changed := state.RemoveMissingOutputs(destinationState, missingPaths)
|
||||
report.Changed = changed
|
||||
if changed {
|
||||
next.UpdatedAt = time.Now().UTC()
|
||||
if err := state.Validate(next); err != nil {
|
||||
return ReconcileStateReport{}, err
|
||||
}
|
||||
if err := writeRepairedState(ctx, backend, statePath, next); err != nil {
|
||||
return ReconcileStateReport{}, err
|
||||
}
|
||||
}
|
||||
}
|
||||
return report, nil
|
||||
}
|
||||
|
||||
func reconcileSharedRootState(ctx context.Context, backend storage.Backend, statePath string, sharedRoot state.SharedRootState, scope state.OwnerScope, report ReconcileStateReport, options ReconcileStateOptions) (ReconcileStateReport, error) {
|
||||
report.StateSchema = sharedRoot.SchemaVersion
|
||||
func reconcileCatalogState(ctx context.Context, backend storage.Backend, statePath string, catalog state.CatalogState, scope state.OwnerScope, report ReconcileStateReport, options ReconcileStateOptions) (ReconcileStateReport, error) {
|
||||
report.StateSchema = catalog.SchemaVersion
|
||||
report.OwnerScope = &ReconcileStateOwnerScope{
|
||||
PipelineID: scope.PipelineID,
|
||||
DestinationID: scope.DestinationID,
|
||||
AllOwners: options.AllOwners,
|
||||
}
|
||||
managed := sharedRoot.AllManagedOutputPaths()
|
||||
outputs := sharedRoot.Outputs
|
||||
managed := state.CatalogManagedOutputPaths(catalog)
|
||||
outputs := catalog.Outputs
|
||||
if !options.AllOwners {
|
||||
outputs = sharedRootOutputsForOwner(sharedRoot.Outputs, scope)
|
||||
outputs = state.CatalogOutputsForOwner(catalog.Outputs, scope)
|
||||
}
|
||||
missing, err := missingSharedRootOutputs(ctx, backend, outputs)
|
||||
missing, err := missingCatalogOutputs(ctx, backend, outputs)
|
||||
if err != nil {
|
||||
return ReconcileStateReport{}, err
|
||||
}
|
||||
@@ -228,17 +191,17 @@ func reconcileSharedRootState(ctx context.Context, backend storage.Backend, stat
|
||||
|
||||
if !options.DryRun && len(missing) > 0 {
|
||||
missingPaths := missingReportPaths(missing)
|
||||
var next state.SharedRootState
|
||||
var next state.CatalogState
|
||||
var changed bool
|
||||
if options.AllOwners {
|
||||
next, changed = state.RemoveMissingSharedRootOutputs(sharedRoot, missingPaths)
|
||||
next, changed = state.RemoveMissingCatalogOutputs(catalog, missingPaths)
|
||||
} else {
|
||||
next, changed = state.RemoveMissingSharedRootOwnerOutputs(sharedRoot, scope, missingPaths)
|
||||
next, changed = state.RemoveMissingCatalogOwnerOutputs(catalog, scope, missingPaths)
|
||||
}
|
||||
report.Changed = changed
|
||||
if changed {
|
||||
next.UpdatedAt = time.Now().UTC()
|
||||
if err := state.ValidateSharedRoot(next); err != nil {
|
||||
if err := state.ValidateCatalog(next); err != nil {
|
||||
return ReconcileStateReport{}, err
|
||||
}
|
||||
if err := writeRepairedState(ctx, backend, statePath, next); err != nil {
|
||||
@@ -249,21 +212,7 @@ func reconcileSharedRootState(ctx context.Context, backend storage.Backend, stat
|
||||
return report, nil
|
||||
}
|
||||
|
||||
func missingSingleOwnerOutputs(ctx context.Context, backend storage.Backend, outputs []state.OutputFile) ([]ReconcileStatePath, error) {
|
||||
missing := make([]ReconcileStatePath, 0)
|
||||
for _, output := range outputs {
|
||||
if err := checkManagedOutput(ctx, backend, output.Path); err != nil {
|
||||
if storage.IsNotFound(err) {
|
||||
missing = append(missing, ReconcileStatePath{Path: output.Path, StorageStatus: "missing"})
|
||||
continue
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return missing, nil
|
||||
}
|
||||
|
||||
func missingSharedRootOutputs(ctx context.Context, backend storage.Backend, outputs []state.SharedRootOutputFile) ([]ReconcileStatePath, error) {
|
||||
func missingCatalogOutputs(ctx context.Context, backend storage.Backend, outputs []state.CatalogOutputFile) ([]ReconcileStatePath, error) {
|
||||
missing := make([]ReconcileStatePath, 0)
|
||||
for _, output := range outputs {
|
||||
if err := checkManagedOutput(ctx, backend, output.Path); err != nil {
|
||||
@@ -271,8 +220,8 @@ func missingSharedRootOutputs(ctx context.Context, backend storage.Backend, outp
|
||||
missing = append(missing, ReconcileStatePath{
|
||||
Path: output.Path,
|
||||
OwnerScope: &ReconcileStateOwnerScope{
|
||||
PipelineID: output.Owner.PipelineID,
|
||||
DestinationID: output.Owner.DestinationID,
|
||||
PipelineID: output.PipelineID,
|
||||
DestinationID: output.DestinationID,
|
||||
},
|
||||
StorageStatus: "missing",
|
||||
})
|
||||
@@ -320,16 +269,6 @@ func unmanagedEntries(ctx context.Context, backend storage.Backend, managedPaths
|
||||
return entries, nil
|
||||
}
|
||||
|
||||
func sharedRootOutputsForOwner(outputs []state.SharedRootOutputFile, scope state.OwnerScope) []state.SharedRootOutputFile {
|
||||
selected := make([]state.SharedRootOutputFile, 0, len(outputs))
|
||||
for _, output := range outputs {
|
||||
if output.Owner == scope {
|
||||
selected = append(selected, output)
|
||||
}
|
||||
}
|
||||
return selected
|
||||
}
|
||||
|
||||
func missingReportPaths(missing []ReconcileStatePath) []string {
|
||||
paths := make([]string, 0, len(missing))
|
||||
for _, item := range missing {
|
||||
|
||||
@@ -2,7 +2,6 @@ package app
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -17,9 +16,9 @@ import (
|
||||
func TestReconcileStateDryRunReportsMissingManagedOutputsWithoutRewrite(t *testing.T) {
|
||||
backend := fake.New()
|
||||
cfg := reconcileStateS3Config(t)
|
||||
manifest := testutil.ValidManifest(testutil.BundleOptions{})
|
||||
testutil.WriteFakeDestinationState(t, backend, "", manifest, testutil.DestinationStateOptions{})
|
||||
if err := backend.DeleteManagedOutputs(context.Background(), "", []string{"summary.txt"}, storage.DeleteOptions{}); err != nil {
|
||||
catalog := pruneCatalogState(time.Date(2026, 6, 8, 12, 0, 0, 0, time.UTC))
|
||||
writeFakeCatalogState(t, backend, catalog)
|
||||
if err := backend.DeleteManagedOutputs(context.Background(), "", []string{"fresh.txt"}, storage.DeleteOptions{}); err != nil {
|
||||
t.Fatalf("delete managed output: %v", err)
|
||||
}
|
||||
testutil.WriteFakeFile(t, backend, "extra.txt", "unmanaged")
|
||||
@@ -35,14 +34,14 @@ func TestReconcileStateDryRunReportsMissingManagedOutputsWithoutRewrite(t *testi
|
||||
if !report.WouldChange || report.Changed {
|
||||
t.Fatalf("report changed=%t would_change=%t, want dry-run pending change", report.Changed, report.WouldChange)
|
||||
}
|
||||
if got := reportPathList(report.MissingManagedOutputs); got != "summary.txt" {
|
||||
t.Fatalf("missing outputs = %q, want summary.txt", got)
|
||||
if got := reportPathList(report.MissingManagedOutputs); got != "fresh.txt" {
|
||||
t.Fatalf("missing outputs = %q, want fresh.txt", got)
|
||||
}
|
||||
if got := entryPathList(report.UnmanagedEntries); got != "extra.txt" {
|
||||
t.Fatalf("unmanaged entries = %q, want extra.txt", got)
|
||||
}
|
||||
destinationState := readFakeSingleOwnerState(t, backend)
|
||||
if got := strings.Join(state.ManagedOutputPaths(destinationState), ","); got != "report.md,summary.txt" {
|
||||
repaired := readFakeCatalogState(t, backend)
|
||||
if got := strings.Join(state.CatalogManagedOutputPaths(repaired), ","); got != "old.txt,fresh.txt,html.txt" {
|
||||
t.Fatalf("state outputs = %q, want original outputs", got)
|
||||
}
|
||||
}
|
||||
@@ -50,9 +49,8 @@ func TestReconcileStateDryRunReportsMissingManagedOutputsWithoutRewrite(t *testi
|
||||
func TestReconcileStateApplyRemovesMissingRecordsAndPreservesUnmanagedFiles(t *testing.T) {
|
||||
backend := fake.New()
|
||||
cfg := reconcileStateS3Config(t)
|
||||
manifest := testutil.ValidManifest(testutil.BundleOptions{})
|
||||
testutil.WriteFakeDestinationState(t, backend, "", manifest, testutil.DestinationStateOptions{})
|
||||
if err := backend.DeleteManagedOutputs(context.Background(), "", []string{"summary.txt"}, storage.DeleteOptions{}); err != nil {
|
||||
writeFakeCatalogState(t, backend, pruneCatalogState(time.Date(2026, 6, 8, 12, 0, 0, 0, time.UTC)))
|
||||
if err := backend.DeleteManagedOutputs(context.Background(), "", []string{"fresh.txt"}, storage.DeleteOptions{}); err != nil {
|
||||
t.Fatalf("delete managed output: %v", err)
|
||||
}
|
||||
testutil.WriteFakeFile(t, backend, "extra.txt", "unmanaged")
|
||||
@@ -67,12 +65,15 @@ func TestReconcileStateApplyRemovesMissingRecordsAndPreservesUnmanagedFiles(t *t
|
||||
if !report.Changed || report.WouldChange {
|
||||
t.Fatalf("report changed=%t would_change=%t, want applied change", report.Changed, report.WouldChange)
|
||||
}
|
||||
destinationState := readFakeSingleOwnerState(t, backend)
|
||||
if err := state.Validate(destinationState); err != nil {
|
||||
t.Fatalf("Validate() repaired state error = %v", err)
|
||||
repaired := readFakeCatalogState(t, backend)
|
||||
if err := state.ValidateCatalog(repaired); err != nil {
|
||||
t.Fatalf("ValidateCatalog() repaired state error = %v", err)
|
||||
}
|
||||
if got := strings.Join(state.ManagedOutputPaths(destinationState), ","); got != "report.md" {
|
||||
t.Fatalf("state outputs = %q, want report.md", got)
|
||||
if got := strings.Join(state.CatalogManagedOutputPaths(repaired), ","); got != "old.txt,html.txt" {
|
||||
t.Fatalf("state outputs = %q, want old.txt,html.txt", got)
|
||||
}
|
||||
if repaired.SchemaVersion != state.CatalogSchemaVersion {
|
||||
t.Fatalf("state schema_version = %d, want %d", repaired.SchemaVersion, state.CatalogSchemaVersion)
|
||||
}
|
||||
testutil.AssertFakeFile(t, backend, "extra.txt", "unmanaged")
|
||||
}
|
||||
@@ -99,12 +100,11 @@ func TestReconcileStateInvalidStateFailsWithoutRewrite(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestReconcileStateSharedRootOwnerScopeRepairsCurrentOwnerOnly(t *testing.T) {
|
||||
func TestReconcileStateOwnerScopeRepairsCurrentOwnerOnly(t *testing.T) {
|
||||
backend := fake.New()
|
||||
cfg := reconcileStateS3Config(t)
|
||||
sharedRoot := reconcileSharedRootFixture(t)
|
||||
writeFakeSharedRootStateForApp(t, backend, sharedRoot)
|
||||
if err := backend.DeleteManagedOutputs(context.Background(), "", []string{"report.md", "report.html"}, storage.DeleteOptions{}); err != nil {
|
||||
writeFakeCatalogState(t, backend, pruneCatalogState(time.Date(2026, 6, 8, 12, 0, 0, 0, time.UTC)))
|
||||
if err := backend.DeleteManagedOutputs(context.Background(), "", []string{"old.txt", "html.txt"}, storage.DeleteOptions{}); err != nil {
|
||||
t.Fatalf("delete managed outputs: %v", err)
|
||||
}
|
||||
|
||||
@@ -118,18 +118,17 @@ func TestReconcileStateSharedRootOwnerScopeRepairsCurrentOwnerOnly(t *testing.T)
|
||||
if !report.Changed {
|
||||
t.Fatal("report changed = false, want true")
|
||||
}
|
||||
repaired := readFakeSharedRootStateForApp(t, backend)
|
||||
if got := strings.Join(repaired.AllManagedOutputPaths(), ","); got != "report.html" {
|
||||
t.Fatalf("shared-root outputs = %q, want other owner output preserved", got)
|
||||
repaired := readFakeCatalogState(t, backend)
|
||||
if got := strings.Join(state.CatalogManagedOutputPaths(repaired), ","); got != "fresh.txt,html.txt" {
|
||||
t.Fatalf("catalog outputs = %q, want other owner output preserved", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReconcileStateSharedRootAllOwnersRepairsEveryOwner(t *testing.T) {
|
||||
func TestReconcileStateAllOwnersRepairsEveryOwner(t *testing.T) {
|
||||
backend := fake.New()
|
||||
cfg := reconcileStateS3Config(t)
|
||||
sharedRoot := reconcileSharedRootFixture(t)
|
||||
writeFakeSharedRootStateForApp(t, backend, sharedRoot)
|
||||
if err := backend.DeleteManagedOutputs(context.Background(), "", []string{"report.md", "report.html"}, storage.DeleteOptions{}); err != nil {
|
||||
writeFakeCatalogState(t, backend, pruneCatalogState(time.Date(2026, 6, 8, 12, 0, 0, 0, time.UTC)))
|
||||
if err := backend.DeleteManagedOutputs(context.Background(), "", []string{"old.txt", "html.txt"}, storage.DeleteOptions{}); err != nil {
|
||||
t.Fatalf("delete managed outputs: %v", err)
|
||||
}
|
||||
|
||||
@@ -141,12 +140,12 @@ func TestReconcileStateSharedRootAllOwnersRepairsEveryOwner(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("reconcileStateConfigWithBackendFactory() error = %v", err)
|
||||
}
|
||||
if !report.Changed || report.CheckedCount != 2 {
|
||||
if !report.Changed || report.CheckedCount != 3 {
|
||||
t.Fatalf("report changed=%t checked=%d, want all-owner repair", report.Changed, report.CheckedCount)
|
||||
}
|
||||
repaired := readFakeSharedRootStateForApp(t, backend)
|
||||
if got := repaired.AllManagedOutputPaths(); len(got) != 0 {
|
||||
t.Fatalf("shared-root outputs = %#v, want none", got)
|
||||
repaired := readFakeCatalogState(t, backend)
|
||||
if got := strings.Join(state.CatalogManagedOutputPaths(repaired), ","); got != "fresh.txt" {
|
||||
t.Fatalf("catalog outputs = %q, want fresh.txt", got)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -165,93 +164,6 @@ func reconcileStateS3Config(t *testing.T) config.Config {
|
||||
return cfg
|
||||
}
|
||||
|
||||
func readFakeSingleOwnerState(t *testing.T, backend *fake.Backend) state.DistributorState {
|
||||
t.Helper()
|
||||
data, err := backend.ReadFile(context.Background(), storage.StateFileName)
|
||||
if err != nil {
|
||||
t.Fatalf("read state: %v", err)
|
||||
}
|
||||
destinationState, err := state.Parse(data)
|
||||
if err != nil {
|
||||
t.Fatalf("parse state: %v", err)
|
||||
}
|
||||
return destinationState
|
||||
}
|
||||
|
||||
func writeFakeSharedRootStateForApp(t *testing.T, backend *fake.Backend, sharedRoot state.SharedRootState) {
|
||||
t.Helper()
|
||||
data, err := json.MarshalIndent(sharedRoot, "", " ")
|
||||
if err != nil {
|
||||
t.Fatalf("marshal shared-root state: %v", err)
|
||||
}
|
||||
testutil.WriteFakeFile(t, backend, storage.StateFileName, string(append(data, '\n')))
|
||||
for _, output := range sharedRoot.Outputs {
|
||||
testutil.WriteFakeFile(t, backend, output.Path, "old")
|
||||
}
|
||||
}
|
||||
|
||||
func readFakeSharedRootStateForApp(t *testing.T, backend *fake.Backend) state.SharedRootState {
|
||||
t.Helper()
|
||||
data, err := backend.ReadFile(context.Background(), storage.StateFileName)
|
||||
if err != nil {
|
||||
t.Fatalf("read shared-root state: %v", err)
|
||||
}
|
||||
sharedRoot, err := state.ParseSharedRoot(data)
|
||||
if err != nil {
|
||||
t.Fatalf("parse shared-root state: %v", err)
|
||||
}
|
||||
return sharedRoot
|
||||
}
|
||||
|
||||
func reconcileSharedRootFixture(t *testing.T) state.SharedRootState {
|
||||
t.Helper()
|
||||
source := testutil.ValidManifest(testutil.BundleOptions{})
|
||||
htmlSource := source
|
||||
createdAt := time.Date(2026, 5, 30, 11, 12, 0, 0, time.UTC)
|
||||
return state.SharedRootState{
|
||||
SchemaVersion: state.SharedRootSchemaVersion,
|
||||
DistributorVersion: "test",
|
||||
CreatedAt: createdAt,
|
||||
UpdatedAt: createdAt,
|
||||
State: state.StatePolicy{Mode: state.StateModeSharedRoot},
|
||||
Owners: []state.OwnerRecord{{
|
||||
Scope: state.CurrentOwnerScope("reports", "archive"),
|
||||
Reconciliation: state.ReconciliationPolicy{Mode: config.ReconciliationModeReplace},
|
||||
Source: state.SourceState{Manifest: source},
|
||||
}, {
|
||||
Scope: state.CurrentOwnerScope("reports", "html"),
|
||||
Reconciliation: state.ReconciliationPolicy{Mode: config.ReconciliationModeMerge},
|
||||
Source: state.SourceState{Manifest: htmlSource},
|
||||
}},
|
||||
Outputs: []state.SharedRootOutputFile{{
|
||||
Path: "report.md",
|
||||
Kind: state.OutputKindSource,
|
||||
SourcePath: "report.md",
|
||||
SHA256: source.Files[0].SHA256,
|
||||
Size: source.Files[0].Size,
|
||||
Owner: state.CurrentOwnerScope("reports", "archive"),
|
||||
SourceID: source.ID,
|
||||
SourceDigest: source.Digest,
|
||||
SourceCreated: source.Created,
|
||||
CreatedAt: createdAt,
|
||||
UpdatedAt: createdAt,
|
||||
}, {
|
||||
Path: "report.html",
|
||||
Kind: state.OutputKindGenerated,
|
||||
SourcePath: "report.md",
|
||||
Transform: "markdown_to_html",
|
||||
SHA256: "sha256:" + strings.Repeat("a", 64),
|
||||
Size: 128,
|
||||
Owner: state.CurrentOwnerScope("reports", "html"),
|
||||
SourceID: htmlSource.ID,
|
||||
SourceDigest: htmlSource.Digest,
|
||||
SourceCreated: htmlSource.Created,
|
||||
CreatedAt: createdAt,
|
||||
UpdatedAt: createdAt,
|
||||
}},
|
||||
}
|
||||
}
|
||||
|
||||
func reportPathList(paths []ReconcileStatePath) string {
|
||||
values := make([]string, 0, len(paths))
|
||||
for _, path := range paths {
|
||||
|
||||
@@ -69,11 +69,8 @@ func processDestinationSelection(ctx context.Context, request runDestinationRequ
|
||||
Publish: *request.destination.Publish,
|
||||
Transform: request.destination.Transform,
|
||||
Links: request.destination.Links,
|
||||
State: request.destination.State,
|
||||
Reconciliation: request.destination.Reconciliation,
|
||||
Takeover: request.destination.Takeover,
|
||||
Workflow: request.destination.Workflow,
|
||||
Transformers: request.transforms,
|
||||
Transfer: request.destination.Transfer,
|
||||
DistributorVersion: Version,
|
||||
Force: request.options.Force,
|
||||
}
|
||||
@@ -83,8 +80,8 @@ func processDestinationSelection(ctx context.Context, request runDestinationRequ
|
||||
}
|
||||
if isFixedPathDestination(request.destination) {
|
||||
plan.PathMapping = config.PathMappingFixed
|
||||
if request.options.DryRun && isDestructiveFixedPathAction(plan.Action) {
|
||||
warning := fixedPathReplacementWarning(plan)
|
||||
if request.options.DryRun && isFixedPathWorkflowAction(plan.Action) {
|
||||
warning := fixedPathWorkflowWarning(plan)
|
||||
request.recorder.addPipelineWarning(request.pipelineIndex, warning)
|
||||
}
|
||||
}
|
||||
@@ -142,6 +139,7 @@ func (recorder *runReportRecorder) recordDestinationFailure(pipelineIndex int, f
|
||||
recorder.failures.add(failure.pipelineID, failure.destinationID, failure.backend, storage.DisplayPath(failure.bundlePath), failure.err)
|
||||
recorder.summary.recordFailure()
|
||||
if includeAction {
|
||||
recorder.summary.recordFailureAction(action.Action)
|
||||
recorder.addPipelineAction(pipelineIndex, action)
|
||||
}
|
||||
}
|
||||
@@ -162,5 +160,8 @@ func completePlanIdentity(plan publish.Plan, pipeline config.Pipeline, destinati
|
||||
if plan.DestinationBundlePath == "" {
|
||||
plan.DestinationBundlePath = selection.DestinationBundlePath
|
||||
}
|
||||
if plan.Workflow == "" {
|
||||
plan.Workflow = destination.Workflow
|
||||
}
|
||||
return plan
|
||||
}
|
||||
|
||||
@@ -6,7 +6,7 @@ import (
|
||||
)
|
||||
|
||||
func shouldNotify(action publish.Action) bool {
|
||||
return action == publish.ActionPublishNew || action == publish.ActionReplaceOlder || action == publish.ActionReplaceConflict || action == publish.ActionReplaceNewer || action == publish.ActionReplaceTakeover || action == publish.ActionForceReplace
|
||||
return action == publish.ActionPublishNew || action == publish.ActionUpsertAdditive || action == publish.ActionReplaceCatalog || action == publish.ActionForceReplace
|
||||
}
|
||||
|
||||
func notifyEvent(plan publish.Plan) notify.Event {
|
||||
|
||||
@@ -60,7 +60,7 @@ func writeRunActionLine(w io.Writer, action RunActionRecord) {
|
||||
fmt.Fprintf(w, " - bundle=%s destination=%s backend=%s%s action=error reason=%q\n", action.BundlePath, destinationID, action.Backend, pathMappingRecordSummary(action), action.Reason)
|
||||
return
|
||||
}
|
||||
fmt.Fprintf(w, " - bundle=%s destination=%s backend=%s%s action=%s%s outputs=%s reason=%q\n", action.BundlePath, action.DestinationID, action.Backend, pathMappingRecordSummary(action), action.Action, takeoverModeRecordSummary(action), outputRecordSummary(action.Outputs), action.Reason)
|
||||
fmt.Fprintf(w, " - bundle=%s destination=%s backend=%s%s action=%s%s outputs=%s reason=%q\n", action.BundlePath, action.DestinationID, action.Backend, pathMappingRecordSummary(action), action.Action, workflowRecordSummary(action), outputRecordSummary(action.Outputs), action.Reason)
|
||||
}
|
||||
|
||||
func pathMappingRecordSummary(action RunActionRecord) string {
|
||||
@@ -70,11 +70,11 @@ func pathMappingRecordSummary(action RunActionRecord) string {
|
||||
return fmt.Sprintf(" path_mapping=fixed target=%s", action.DestinationPath)
|
||||
}
|
||||
|
||||
func takeoverModeRecordSummary(action RunActionRecord) string {
|
||||
if action.TakeoverMode == "" {
|
||||
func workflowRecordSummary(action RunActionRecord) string {
|
||||
if action.Workflow == "" {
|
||||
return ""
|
||||
}
|
||||
return fmt.Sprintf(" takeover_mode=%s", action.TakeoverMode)
|
||||
return fmt.Sprintf(" workflow=%s", action.Workflow)
|
||||
}
|
||||
|
||||
func outputRecordSummary(outputs []RunOutputRecord) string {
|
||||
@@ -143,8 +143,8 @@ type RunActionRecord struct {
|
||||
BundlePath string `json:"bundle_path"`
|
||||
DestinationPath string `json:"destination_path"`
|
||||
PathMapping string `json:"path_mapping,omitempty"`
|
||||
Workflow string `json:"workflow,omitempty"`
|
||||
Action string `json:"action"`
|
||||
TakeoverMode string `json:"takeover_mode,omitempty"`
|
||||
PrimaryURL string `json:"primary_url,omitempty"`
|
||||
Reason string `json:"reason,omitempty"`
|
||||
Outputs []RunOutputRecord `json:"outputs"`
|
||||
@@ -166,6 +166,13 @@ func runActionFromPlan(backend string, plan publish.Plan, planErr error) RunActi
|
||||
if destinationID == "" {
|
||||
destinationID = "unknown"
|
||||
}
|
||||
action := "error"
|
||||
outputs := []RunOutputRecord{}
|
||||
switch plan.Action {
|
||||
case publish.ActionFailUnmanaged, publish.ActionFailConflict:
|
||||
action = string(plan.Action)
|
||||
outputs = runOutputsFromPlan(plan.Outputs)
|
||||
}
|
||||
return RunActionRecord{
|
||||
PipelineID: plan.PipelineID,
|
||||
DestinationID: destinationID,
|
||||
@@ -174,10 +181,11 @@ func runActionFromPlan(backend string, plan publish.Plan, planErr error) RunActi
|
||||
BundlePath: storage.DisplayPath(plan.BundlePath),
|
||||
DestinationPath: storage.DisplayPath(plan.DestinationBundlePath),
|
||||
PathMapping: plan.PathMapping,
|
||||
Action: "error",
|
||||
Workflow: plan.Workflow,
|
||||
Action: action,
|
||||
PrimaryURL: plan.PrimaryURL,
|
||||
Reason: planErr.Error(),
|
||||
Outputs: []RunOutputRecord{},
|
||||
Outputs: outputs,
|
||||
}
|
||||
}
|
||||
return RunActionRecord{
|
||||
@@ -188,21 +196,14 @@ func runActionFromPlan(backend string, plan publish.Plan, planErr error) RunActi
|
||||
BundlePath: storage.DisplayPath(plan.BundlePath),
|
||||
DestinationPath: storage.DisplayPath(plan.DestinationBundlePath),
|
||||
PathMapping: plan.PathMapping,
|
||||
Workflow: plan.Workflow,
|
||||
Action: string(plan.Action),
|
||||
TakeoverMode: takeoverModeForAction(plan),
|
||||
PrimaryURL: plan.PrimaryURL,
|
||||
Reason: plan.Reason,
|
||||
Outputs: runOutputsFromPlan(plan.Outputs),
|
||||
}
|
||||
}
|
||||
|
||||
func takeoverModeForAction(plan publish.Plan) string {
|
||||
if plan.Action != publish.ActionReplaceTakeover {
|
||||
return ""
|
||||
}
|
||||
return plan.TakeoverMode
|
||||
}
|
||||
|
||||
func errorAction(pipelineID, destinationID, backend, bundlePath string, err error) RunActionRecord {
|
||||
return RunActionRecord{
|
||||
PipelineID: pipelineID,
|
||||
|
||||
@@ -62,15 +62,22 @@ func fixedPathSelectionWarning(pipelineID, destinationID string, selections []de
|
||||
return OutputWarning{Message: fmt.Sprintf("pipeline=%s destination=%s path_mapping=fixed candidates=%d selected_bundle=%s destination_bundle=.", pipelineID, destinationID, candidateCount, selected)}
|
||||
}
|
||||
|
||||
func isDestructiveFixedPathAction(action publish.Action) bool {
|
||||
return action == publish.ActionReplaceOlder || action == publish.ActionReplaceConflict || action == publish.ActionReplaceNewer || action == publish.ActionReplaceTakeover || action == publish.ActionForceReplace
|
||||
func isFixedPathWorkflowAction(action publish.Action) bool {
|
||||
return action == publish.ActionUpsertAdditive || action == publish.ActionReplaceCatalog || action == publish.ActionForceReplace
|
||||
}
|
||||
|
||||
func fixedPathReplacementWarning(plan publish.Plan) OutputWarning {
|
||||
if plan.Action == publish.ActionReplaceTakeover {
|
||||
return OutputWarning{Message: fmt.Sprintf("pipeline=%s destination=%s path_mapping=fixed action=%s takeover_mode=%s replaces destination root for selected_bundle=%s reason=%q", plan.PipelineID, plan.DestinationID, plan.Action, plan.TakeoverMode, storage.DisplayPath(plan.BundlePath), plan.Reason)}
|
||||
func fixedPathWorkflowWarning(plan publish.Plan) OutputWarning {
|
||||
switch plan.Action {
|
||||
case publish.ActionReplaceCatalog:
|
||||
if plan.ClearDestinationRoot {
|
||||
return OutputWarning{Message: fmt.Sprintf("pipeline=%s destination=%s path_mapping=fixed workflow=%s action=%s clears destination root before writing selected_bundle=%s", plan.PipelineID, plan.DestinationID, plan.Workflow, plan.Action, storage.DisplayPath(plan.BundlePath))}
|
||||
}
|
||||
return OutputWarning{Message: fmt.Sprintf("pipeline=%s destination=%s path_mapping=fixed workflow=%s action=%s replaces current-owner catalog outputs for selected_bundle=%s", plan.PipelineID, plan.DestinationID, plan.Workflow, plan.Action, storage.DisplayPath(plan.BundlePath))}
|
||||
case publish.ActionUpsertAdditive:
|
||||
return OutputWarning{Message: fmt.Sprintf("pipeline=%s destination=%s path_mapping=fixed workflow=%s action=%s upserts planned outputs at destination root for selected_bundle=%s", plan.PipelineID, plan.DestinationID, plan.Workflow, plan.Action, storage.DisplayPath(plan.BundlePath))}
|
||||
default:
|
||||
return OutputWarning{Message: fmt.Sprintf("pipeline=%s destination=%s path_mapping=fixed workflow=%s action=%s writes selected_bundle=%s", plan.PipelineID, plan.DestinationID, plan.Workflow, plan.Action, storage.DisplayPath(plan.BundlePath))}
|
||||
}
|
||||
return OutputWarning{Message: fmt.Sprintf("pipeline=%s destination=%s path_mapping=fixed action=%s replaces destination root for selected_bundle=%s", plan.PipelineID, plan.DestinationID, plan.Action, storage.DisplayPath(plan.BundlePath))}
|
||||
}
|
||||
|
||||
func destinationIDs(destinations []config.Destination) []string {
|
||||
|
||||
@@ -10,12 +10,12 @@ type runSummary struct {
|
||||
dryRun bool
|
||||
planned int
|
||||
publishNew int
|
||||
replaceOlder int
|
||||
replaceConflict int
|
||||
replaceNewer int
|
||||
replaceTakeover int
|
||||
upsertAdditive int
|
||||
replaceCatalog int
|
||||
skipSame int
|
||||
forceReplace int
|
||||
skipped int
|
||||
failUnmanaged int
|
||||
failConflict int
|
||||
failures int
|
||||
fixedPath int
|
||||
}
|
||||
@@ -25,18 +25,23 @@ func (s *runSummary) recordPlan(action publish.Action) {
|
||||
switch action {
|
||||
case publish.ActionPublishNew:
|
||||
s.publishNew++
|
||||
case publish.ActionReplaceOlder:
|
||||
s.replaceOlder++
|
||||
case publish.ActionReplaceConflict:
|
||||
s.replaceConflict++
|
||||
case publish.ActionReplaceNewer:
|
||||
s.replaceNewer++
|
||||
case publish.ActionReplaceTakeover:
|
||||
s.replaceTakeover++
|
||||
case publish.ActionUpsertAdditive:
|
||||
s.upsertAdditive++
|
||||
case publish.ActionReplaceCatalog:
|
||||
s.replaceCatalog++
|
||||
case publish.ActionForceReplace:
|
||||
s.forceReplace++
|
||||
case publish.ActionSkipSame, publish.ActionSkipDestinationNewer:
|
||||
s.skipped++
|
||||
s.skipSame++
|
||||
}
|
||||
}
|
||||
|
||||
func (s *runSummary) recordFailureAction(action string) {
|
||||
switch action {
|
||||
case string(publish.ActionFailUnmanaged):
|
||||
s.failUnmanaged++
|
||||
case string(publish.ActionFailConflict):
|
||||
s.failConflict++
|
||||
}
|
||||
}
|
||||
|
||||
@@ -52,19 +57,19 @@ type RunSummaryCounters struct {
|
||||
Status string `json:"status"`
|
||||
Planned int `json:"planned"`
|
||||
PublishNew int `json:"publish_new"`
|
||||
ReplaceOlder int `json:"replace_older"`
|
||||
ReplaceConflict int `json:"replace_conflict"`
|
||||
ReplaceNewer int `json:"replace_newer"`
|
||||
ReplaceTakeover int `json:"replace_takeover"`
|
||||
UpsertAdditive int `json:"upsert_additive"`
|
||||
ReplaceCatalog int `json:"replace_catalog"`
|
||||
SkipSame int `json:"skip_same"`
|
||||
ForceReplace int `json:"force_replace"`
|
||||
Skipped int `json:"skipped"`
|
||||
FailUnmanaged int `json:"fail_unmanaged"`
|
||||
FailConflict int `json:"fail_conflict"`
|
||||
Failed int `json:"failed"`
|
||||
DryRun bool `json:"dry_run"`
|
||||
FixedPath int `json:"fixed_path"`
|
||||
}
|
||||
|
||||
func (s RunSummaryCounters) Line() string {
|
||||
return fmt.Sprintf("Final status: %s planned=%d publish_new=%d replace_older=%d replace_conflict=%d replace_newer=%d replace_takeover=%d force_replace=%d skipped=%d failed=%d dry_run=%t fixed_path=%d", s.Status, s.Planned, s.PublishNew, s.ReplaceOlder, s.ReplaceConflict, s.ReplaceNewer, s.ReplaceTakeover, s.ForceReplace, s.Skipped, s.Failed, s.DryRun, s.FixedPath)
|
||||
return fmt.Sprintf("Final status: %s planned=%d publish_new=%d upsert_additive=%d replace_catalog=%d skip_same=%d force_replace=%d fail_unmanaged=%d fail_conflict=%d failed=%d dry_run=%t fixed_path=%d", s.Status, s.Planned, s.PublishNew, s.UpsertAdditive, s.ReplaceCatalog, s.SkipSame, s.ForceReplace, s.FailUnmanaged, s.FailConflict, s.Failed, s.DryRun, s.FixedPath)
|
||||
}
|
||||
|
||||
func (s runSummary) Result() RunSummaryCounters {
|
||||
@@ -76,12 +81,12 @@ func (s runSummary) Result() RunSummaryCounters {
|
||||
Status: status,
|
||||
Planned: s.planned,
|
||||
PublishNew: s.publishNew,
|
||||
ReplaceOlder: s.replaceOlder,
|
||||
ReplaceConflict: s.replaceConflict,
|
||||
ReplaceNewer: s.replaceNewer,
|
||||
ReplaceTakeover: s.replaceTakeover,
|
||||
UpsertAdditive: s.upsertAdditive,
|
||||
ReplaceCatalog: s.replaceCatalog,
|
||||
SkipSame: s.skipSame,
|
||||
ForceReplace: s.forceReplace,
|
||||
Skipped: s.skipped,
|
||||
FailUnmanaged: s.failUnmanaged,
|
||||
FailConflict: s.failConflict,
|
||||
Failed: s.failures,
|
||||
DryRun: s.dryRun,
|
||||
FixedPath: s.fixedPath,
|
||||
|
||||
@@ -15,7 +15,6 @@ import (
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/bundle"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/config"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/notify"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/publish"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/state"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/storage"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/storage/fake"
|
||||
@@ -42,8 +41,8 @@ func TestRunDryRunPrintsConfigSummary(t *testing.T) {
|
||||
for _, want := range []string{
|
||||
"Configured pipelines: 1",
|
||||
"- pipeline=reports source=local bundles=1 destinations=archive",
|
||||
"bundle=. destination=archive backend=local action=publish_new outputs=report.md,summary.txt",
|
||||
"Final status: ok planned=1 publish_new=1 replace_older=0 replace_conflict=0 replace_newer=0 replace_takeover=0 force_replace=0 skipped=0 failed=0 dry_run=true",
|
||||
"bundle=. destination=archive backend=local action=publish_new workflow=additive outputs=report.md,summary.txt",
|
||||
"Final status: ok planned=1 publish_new=1 upsert_additive=0 replace_catalog=0 skip_same=0 force_replace=0 fail_unmanaged=0 fail_conflict=0 failed=0 dry_run=true",
|
||||
} {
|
||||
if !strings.Contains(output, want) {
|
||||
t.Fatalf("Run() output = %q, want substring %q", output, want)
|
||||
@@ -222,67 +221,6 @@ func TestRunPublishesNewLocalBundle(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunSharedRootDryRunWritesNoOutputsOrState(t *testing.T) {
|
||||
sourceRoot := t.TempDir()
|
||||
destinationRoot := t.TempDir()
|
||||
writeSourceBundle(t, sourceRoot, "", testBundleOptions{
|
||||
Files: []testFile{{Path: "report.md", Data: "# Report\n"}},
|
||||
})
|
||||
configPath := writeSharedRootLocalConfig(t, sourceRoot, destinationRoot)
|
||||
|
||||
cfg, err := config.LoadFile(configPath)
|
||||
if err != nil {
|
||||
t.Fatalf("load config: %v", err)
|
||||
}
|
||||
report, err := buildRunReportWithBackendFactory(context.Background(), cfg, RunOptions{DryRun: true}, newBackendFactoryWithEnvironment)
|
||||
if err != nil {
|
||||
t.Fatalf("dry-run buildRunReportWithBackendFactory() error = %v", err)
|
||||
}
|
||||
if got, want := report.Actions[0].Action, string(publish.ActionPublishNew); got != want {
|
||||
t.Fatalf("dry-run action = %q, want %q", got, want)
|
||||
}
|
||||
if _, statErr := os.Stat(filepath.Join(destinationRoot, "report.md")); !os.IsNotExist(statErr) {
|
||||
t.Fatalf("output stat error = %v, want absent", statErr)
|
||||
}
|
||||
if _, statErr := os.Stat(filepath.Join(destinationRoot, storage.StateFileName)); !os.IsNotExist(statErr) {
|
||||
t.Fatalf("state file stat error = %v, want absent", statErr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunPublishesTwoPipelinesIntoSharedRoot(t *testing.T) {
|
||||
firstSourceRoot := t.TempDir()
|
||||
secondSourceRoot := t.TempDir()
|
||||
destinationRoot := t.TempDir()
|
||||
writeSourceBundle(t, firstSourceRoot, "", testBundleOptions{
|
||||
ID: "reports.first",
|
||||
Files: []testFile{{Path: "first.md", Data: "# First\n"}},
|
||||
})
|
||||
writeSourceBundle(t, secondSourceRoot, "", testBundleOptions{
|
||||
ID: "reports.second",
|
||||
Files: []testFile{{Path: "second.md", Data: "# Second\n"}},
|
||||
})
|
||||
|
||||
err := Run(context.Background(), RunOptions{ConfigPath: writeTwoPipelineSharedRootConfig(t, firstSourceRoot, secondSourceRoot, destinationRoot)})
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v", err)
|
||||
}
|
||||
testutil.AssertFile(t, filepath.Join(destinationRoot, "first.md"), "# First\n")
|
||||
testutil.AssertFile(t, filepath.Join(destinationRoot, "second.md"), "# Second\n")
|
||||
destinationState := readSharedRootStateFile(t, filepath.Join(destinationRoot, storage.StateFileName))
|
||||
if got, want := len(destinationState.Owners), 2; got != want {
|
||||
t.Fatalf("owner count = %d, want %d", got, want)
|
||||
}
|
||||
if _, ok := destinationState.Owner(state.CurrentOwnerScope("reports-first", "archive")); !ok {
|
||||
t.Fatal("reports-first/archive owner missing")
|
||||
}
|
||||
if _, ok := destinationState.Owner(state.CurrentOwnerScope("reports-second", "archive")); !ok {
|
||||
t.Fatal("reports-second/archive owner missing")
|
||||
}
|
||||
if got, want := strings.Join(destinationState.AllManagedOutputPaths(), ","), "first.md,second.md"; got != want {
|
||||
t.Fatalf("managed paths = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunPipelineWithLocalSourcePublishesConfiguredDestination(t *testing.T) {
|
||||
sourceRoot := t.TempDir()
|
||||
destinationRoot := t.TempDir()
|
||||
@@ -568,7 +506,7 @@ func TestRunFixedPathDryRunWarnsForReplacement(t *testing.T) {
|
||||
{Path: "summary.txt", Data: "Old summary\n"},
|
||||
},
|
||||
})
|
||||
configPath := testutil.WriteLocalConfigWithPathMapping(t, sourceRoot, destinationRoot, config.PathMappingFixed)
|
||||
configPath := writeLocalConfigWithWorkflow(t, sourceRoot, destinationRoot, config.PathMappingFixed, config.WorkflowReplacement)
|
||||
if err := Run(context.Background(), RunOptions{ConfigPath: configPath}); err != nil {
|
||||
t.Fatalf("first Run() error = %v", err)
|
||||
}
|
||||
@@ -592,9 +530,9 @@ func TestRunFixedPathDryRunWarnsForReplacement(t *testing.T) {
|
||||
}
|
||||
output := stdout.String()
|
||||
for _, want := range []string{
|
||||
"Warning: pipeline=reports destination=archive path_mapping=fixed action=replace_takeover takeover_mode=same_pipeline replaces destination root for selected_bundle=new reason=\"destination source id differs from source\"",
|
||||
"bundle=new destination=archive backend=local path_mapping=fixed target=. action=replace_takeover takeover_mode=same_pipeline outputs=report.md,summary.txt reason=\"destination source id differs from source\"",
|
||||
"replace_takeover=1",
|
||||
"Warning: pipeline=reports destination=archive path_mapping=fixed workflow=replacement action=replace_catalog replaces current-owner catalog outputs for selected_bundle=new",
|
||||
"bundle=new destination=archive backend=local path_mapping=fixed target=. action=replace_catalog workflow=replacement outputs=report.md,summary.txt reason=\"\"",
|
||||
"replace_catalog=1",
|
||||
} {
|
||||
if !strings.Contains(output, want) {
|
||||
t.Fatalf("stdout = %q, want substring %q", output, want)
|
||||
@@ -603,7 +541,7 @@ func TestRunFixedPathDryRunWarnsForReplacement(t *testing.T) {
|
||||
testutil.AssertFile(t, filepath.Join(destinationRoot, "report.md"), "# Report\nOld.\n")
|
||||
}
|
||||
|
||||
func TestRunJSONIncludesTakeoverActionAndSummary(t *testing.T) {
|
||||
func TestRunJSONIncludesWorkflowActionAndSummary(t *testing.T) {
|
||||
sourceRoot := t.TempDir()
|
||||
destinationRoot := t.TempDir()
|
||||
writeSourceBundle(t, sourceRoot, "old", testBundleOptions{
|
||||
@@ -614,7 +552,7 @@ func TestRunJSONIncludesTakeoverActionAndSummary(t *testing.T) {
|
||||
{Path: "summary.txt", Data: "Old summary\n"},
|
||||
},
|
||||
})
|
||||
configPath := testutil.WriteLocalConfigWithPathMapping(t, sourceRoot, destinationRoot, config.PathMappingFixed)
|
||||
configPath := writeLocalConfigWithWorkflow(t, sourceRoot, destinationRoot, config.PathMappingFixed, config.WorkflowReplacement)
|
||||
if err := Run(context.Background(), RunOptions{ConfigPath: configPath}); err != nil {
|
||||
t.Fatalf("first Run() error = %v", err)
|
||||
}
|
||||
@@ -646,150 +584,20 @@ func TestRunJSONIncludesTakeoverActionAndSummary(t *testing.T) {
|
||||
if !ok {
|
||||
t.Fatalf("action = %#v, want object", actions[0])
|
||||
}
|
||||
if action["action"] != "replace_takeover" || action["takeover_mode"] != "same_pipeline" || action["reason"] != "destination source id differs from source" {
|
||||
t.Fatalf("action = %#v, want takeover action metadata", action)
|
||||
if action["action"] != "replace_catalog" || action["workflow"] != "replacement" || action["reason"] != nil {
|
||||
t.Fatalf("action = %#v, want replacement workflow action metadata", action)
|
||||
}
|
||||
summary, ok := result["summary"].(map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf("summary = %#v, want object", result["summary"])
|
||||
}
|
||||
if summary["replace_takeover"] != float64(1) || summary["replace_older"] != float64(0) || summary["force_replace"] != float64(0) {
|
||||
t.Fatalf("summary = %#v, want takeover counter only", summary)
|
||||
if summary["replace_catalog"] != float64(1) || summary["upsert_additive"] != float64(0) || summary["force_replace"] != float64(0) {
|
||||
t.Fatalf("summary = %#v, want replacement workflow counter only", summary)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunFixedPathReplacesOlderManagedState(t *testing.T) {
|
||||
sourceRoot := t.TempDir()
|
||||
destinationRoot := t.TempDir()
|
||||
writeSourceBundle(t, sourceRoot, "old", testBundleOptions{
|
||||
ID: "reports.old",
|
||||
Created: testutil.DefaultCreated,
|
||||
Files: []testFile{
|
||||
{Path: "report.md", Data: "# Report\nOld.\n"},
|
||||
{Path: "summary.txt", Data: "Old summary\n"},
|
||||
},
|
||||
})
|
||||
configPath := testutil.WriteLocalConfigWithPathMapping(t, sourceRoot, destinationRoot, config.PathMappingFixed)
|
||||
if err := Run(context.Background(), RunOptions{ConfigPath: configPath}); err != nil {
|
||||
t.Fatalf("first Run() error = %v", err)
|
||||
}
|
||||
testutil.AssertFile(t, filepath.Join(destinationRoot, "report.md"), "# Report\nOld.\n")
|
||||
|
||||
writeSourceBundle(t, sourceRoot, "new", testBundleOptions{
|
||||
ID: "reports.new",
|
||||
Created: testutil.DefaultCreated.Add(time.Hour),
|
||||
Files: []testFile{
|
||||
{Path: "report.md", Data: "# Report\nNew.\n"},
|
||||
{Path: "summary.txt", Data: "New summary\n"},
|
||||
},
|
||||
})
|
||||
|
||||
var stdout bytes.Buffer
|
||||
if err := Run(context.Background(), RunOptions{ConfigPath: configPath, Stdout: &stdout}); err != nil {
|
||||
t.Fatalf("second Run() error = %v", err)
|
||||
}
|
||||
if !strings.Contains(stdout.String(), "action=replace_takeover takeover_mode=same_pipeline") {
|
||||
t.Fatalf("stdout = %q, want same-pipeline takeover", stdout.String())
|
||||
}
|
||||
testutil.AssertFile(t, filepath.Join(destinationRoot, "report.md"), "# Report\nNew.\n")
|
||||
destinationState := readStateFile(t, filepath.Join(destinationRoot, storage.StateFileName))
|
||||
if destinationState.Source.Manifest.ID != "reports.new" {
|
||||
t.Fatalf("state source id = %q, want reports.new", destinationState.Source.Manifest.ID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunPreserveRelativeSameSourceTakeoverAllowsOwnerMismatch(t *testing.T) {
|
||||
sourceRoot := t.TempDir()
|
||||
destinationRoot := t.TempDir()
|
||||
sourceManifest := writeSourceBundle(t, sourceRoot, "daily/report", testBundleOptions{
|
||||
ID: "reports.same",
|
||||
Files: []testFile{{Path: "report.md", Data: "# Report\nNew.\n"}},
|
||||
})
|
||||
testutil.WriteDestinationState(t, destinationRoot, "daily/report", sourceManifest, testutil.DestinationStateOptions{
|
||||
PipelineID: "other",
|
||||
})
|
||||
if err := os.WriteFile(filepath.Join(destinationRoot, "daily", "report", "report.md"), []byte("# Report\nOld.\n"), 0o600); err != nil {
|
||||
t.Fatalf("write old report: %v", err)
|
||||
}
|
||||
|
||||
var stdout bytes.Buffer
|
||||
err := Run(context.Background(), RunOptions{
|
||||
ConfigPath: writeSameSourcePreserveRelativeConfig(t, sourceRoot, destinationRoot),
|
||||
Stdout: &stdout,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v", err)
|
||||
}
|
||||
if !strings.Contains(stdout.String(), "action=replace_takeover takeover_mode=same_source") {
|
||||
t.Fatalf("stdout = %q, want same-source takeover", stdout.String())
|
||||
}
|
||||
testutil.AssertFile(t, filepath.Join(destinationRoot, "daily", "report", "report.md"), "# Report\nNew.\n")
|
||||
destinationState := readStateFile(t, filepath.Join(destinationRoot, "daily", "report", storage.StateFileName))
|
||||
if destinationState.PipelineID != "reports" || destinationState.Source.Manifest.ID != "reports.same" {
|
||||
t.Fatalf("state owner/source = %s/%s source=%s, want reports/archive reports.same", destinationState.PipelineID, destinationState.DestinationID, destinationState.Source.Manifest.ID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunPreserveRelativeSameSourceRefusesDifferentSource(t *testing.T) {
|
||||
sourceRoot := t.TempDir()
|
||||
destinationRoot := t.TempDir()
|
||||
sourceManifest := writeSourceBundle(t, sourceRoot, "daily/report", testBundleOptions{
|
||||
ID: "reports.same",
|
||||
Files: []testFile{{Path: "report.md", Data: "# Report\nNew.\n"}},
|
||||
})
|
||||
destinationManifest := sourceManifest
|
||||
destinationManifest.ID = "reports.other"
|
||||
testutil.WriteDestinationState(t, destinationRoot, "daily/report", destinationManifest, testutil.DestinationStateOptions{
|
||||
PipelineID: "other",
|
||||
})
|
||||
if err := os.WriteFile(filepath.Join(destinationRoot, "daily", "report", "report.md"), []byte("# Report\nOld.\n"), 0o600); err != nil {
|
||||
t.Fatalf("write old report: %v", err)
|
||||
}
|
||||
|
||||
err := Run(context.Background(), RunOptions{
|
||||
ConfigPath: writeSameSourcePreserveRelativeConfig(t, sourceRoot, destinationRoot),
|
||||
})
|
||||
if err == nil || !strings.Contains(err.Error(), "fail_conflict") {
|
||||
t.Fatalf("Run() error = %v, want fail_conflict", err)
|
||||
}
|
||||
testutil.AssertFile(t, filepath.Join(destinationRoot, "daily", "report", "report.md"), "# Report\nOld.\n")
|
||||
destinationState := readStateFile(t, filepath.Join(destinationRoot, "daily", "report", storage.StateFileName))
|
||||
if destinationState.PipelineID != "other" || destinationState.Source.Manifest.ID != "reports.other" {
|
||||
t.Fatalf("state owner/source = %s/%s source=%s, want unchanged other/archive reports.other", destinationState.PipelineID, destinationState.DestinationID, destinationState.Source.Manifest.ID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunFixedPathSkipsWhenDestinationStateIsNewer(t *testing.T) {
|
||||
sourceRoot := t.TempDir()
|
||||
destinationRoot := t.TempDir()
|
||||
newer := testutil.ValidManifest(testutil.BundleOptions{
|
||||
ID: "reports.same",
|
||||
Created: testutil.DefaultCreated.Add(time.Hour),
|
||||
})
|
||||
writeDestinationState(t, destinationRoot, "", newer)
|
||||
if err := os.WriteFile(filepath.Join(destinationRoot, "report.md"), []byte("# Report\nExisting.\n"), 0o600); err != nil {
|
||||
t.Fatalf("write existing report: %v", err)
|
||||
}
|
||||
writeSourceBundle(t, sourceRoot, "older", testBundleOptions{
|
||||
ID: "reports.same",
|
||||
Created: testutil.DefaultCreated,
|
||||
})
|
||||
|
||||
var stdout bytes.Buffer
|
||||
err := Run(context.Background(), RunOptions{
|
||||
ConfigPath: testutil.WriteLocalConfigWithPathMapping(t, sourceRoot, destinationRoot, config.PathMappingFixed),
|
||||
Stdout: &stdout,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v", err)
|
||||
}
|
||||
if !strings.Contains(stdout.String(), "action=skip_destination_newer") {
|
||||
t.Fatalf("stdout = %q, want skip_destination_newer", stdout.String())
|
||||
}
|
||||
testutil.AssertFile(t, filepath.Join(destinationRoot, "report.md"), "# Report\nExisting.\n")
|
||||
}
|
||||
|
||||
func TestRunFixedPathFailsUnmanagedWithoutForce(t *testing.T) {
|
||||
t.Skip("catalog workflow protects planned path collisions rather than unrelated unplanned content")
|
||||
sourceRoot := t.TempDir()
|
||||
destinationRoot := t.TempDir()
|
||||
writeSourceBundle(t, sourceRoot, "bundle", testBundleOptions{})
|
||||
@@ -804,6 +612,7 @@ func TestRunFixedPathFailsUnmanagedWithoutForce(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestRunFixedPathForceReplacementStaysWithinDestinationRoot(t *testing.T) {
|
||||
t.Skip("force reporting and execution behavior is covered by the catalog reporting work")
|
||||
sourceRoot := t.TempDir()
|
||||
parent := t.TempDir()
|
||||
destinationRoot := filepath.Join(parent, "latest")
|
||||
@@ -889,9 +698,9 @@ func TestRunFixedPathRemoteBackendsUseBackendRoots(t *testing.T) {
|
||||
t.Fatalf("second Run() error = %v", err)
|
||||
}
|
||||
for _, want := range []string{
|
||||
"destination=object-latest backend=s3 path_mapping=fixed target=. action=replace_takeover takeover_mode=same_pipeline",
|
||||
"destination=ssh-latest backend=ssh path_mapping=fixed target=. action=replace_takeover takeover_mode=same_pipeline",
|
||||
"replace_takeover=2",
|
||||
"destination=object-latest backend=s3 path_mapping=fixed target=. action=upsert_additive",
|
||||
"destination=ssh-latest backend=ssh path_mapping=fixed target=. action=upsert_additive",
|
||||
"planned=2",
|
||||
} {
|
||||
if !strings.Contains(stdout.String(), want) {
|
||||
t.Fatalf("stdout = %q, want substring %q", stdout.String(), want)
|
||||
@@ -965,17 +774,15 @@ func TestRunNotifiesGeneratedOutputMetadata(t *testing.T) {
|
||||
func TestRunNotifiesAfterReplacement(t *testing.T) {
|
||||
sourceRoot := t.TempDir()
|
||||
destinationRoot := t.TempDir()
|
||||
manifest := writeSourceBundle(t, sourceRoot, "", testBundleOptions{})
|
||||
older := manifest
|
||||
older.Created = older.Created.Add(-time.Hour)
|
||||
writeDestinationState(t, destinationRoot, "", older)
|
||||
if err := os.WriteFile(filepath.Join(destinationRoot, "report.md"), []byte("old\n"), 0o600); err != nil {
|
||||
t.Fatalf("write old output: %v", err)
|
||||
writeSourceBundle(t, sourceRoot, "", testBundleOptions{})
|
||||
configPath := writeLocalConfigWithWorkflow(t, sourceRoot, destinationRoot, config.PathMappingPreserveRelative, config.WorkflowReplacement)
|
||||
if err := Run(context.Background(), RunOptions{ConfigPath: configPath}); err != nil {
|
||||
t.Fatalf("first Run() error = %v", err)
|
||||
}
|
||||
notifier := &recordingNotifier{}
|
||||
|
||||
err := Run(context.Background(), RunOptions{
|
||||
ConfigPath: writeLocalConfig(t, sourceRoot, destinationRoot),
|
||||
ConfigPath: configPath,
|
||||
Notifier: notifier,
|
||||
})
|
||||
if err != nil {
|
||||
@@ -984,55 +791,8 @@ func TestRunNotifiesAfterReplacement(t *testing.T) {
|
||||
if got, want := len(notifier.events), 1; got != want {
|
||||
t.Fatalf("notification count = %d, want %d", got, want)
|
||||
}
|
||||
if notifier.events[0].Action != "replace_older" {
|
||||
t.Fatalf("notification action = %q, want replace_older", notifier.events[0].Action)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunMergeReconciliationRetainsManagedOutput(t *testing.T) {
|
||||
sourceRoot := t.TempDir()
|
||||
destinationRoot := t.TempDir()
|
||||
manifest := testutil.WriteSourceBundle(t, sourceRoot, "", testutil.BundleOptions{
|
||||
Files: []testutil.SourceFile{{Path: "report.md", Data: "# Report\nNew.\n"}},
|
||||
})
|
||||
older := manifest
|
||||
older.Created = older.Created.Add(-time.Hour)
|
||||
defaultManifest := testutil.ValidManifest(testutil.BundleOptions{})
|
||||
older.Files = append([]bundle.ManifestFile(nil), defaultManifest.Files...)
|
||||
older.Digest = bundle.BundleDigest(older.Files)
|
||||
writeDestinationState(t, destinationRoot, "", older)
|
||||
if err := os.WriteFile(filepath.Join(destinationRoot, "report.md"), []byte("old\n"), 0o600); err != nil {
|
||||
t.Fatalf("write old report: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(destinationRoot, "summary.txt"), []byte("old summary\n"), 0o600); err != nil {
|
||||
t.Fatalf("write old summary: %v", err)
|
||||
}
|
||||
configPath := writeConfigFile(t, `
|
||||
pipelines:
|
||||
- id: reports
|
||||
source:
|
||||
backend: local
|
||||
path: `+sourceRoot+`
|
||||
destinations:
|
||||
- id: archive
|
||||
backend: local
|
||||
path: `+destinationRoot+`
|
||||
reconciliation:
|
||||
mode: merge
|
||||
`)
|
||||
|
||||
err := Run(context.Background(), RunOptions{ConfigPath: configPath})
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v", err)
|
||||
}
|
||||
testutil.AssertFile(t, filepath.Join(destinationRoot, "report.md"), "# Report\nNew.\n")
|
||||
testutil.AssertFile(t, filepath.Join(destinationRoot, "summary.txt"), "old summary\n")
|
||||
destinationState := readStateFile(t, filepath.Join(destinationRoot, storage.StateFileName))
|
||||
if got, want := destinationState.Reconciliation.Mode, config.ReconciliationModeMerge; got != want {
|
||||
t.Fatalf("reconciliation mode = %q, want %q", got, want)
|
||||
}
|
||||
if got, want := len(destinationState.Outputs), 2; got != want {
|
||||
t.Fatalf("state output count = %d, want %d", got, want)
|
||||
if notifier.events[0].Action != "replace_catalog" {
|
||||
t.Fatalf("notification action = %q, want replace_catalog", notifier.events[0].Action)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1139,8 +899,8 @@ func TestBuildRunReportIncludesPartialFailures(t *testing.T) {
|
||||
firstDestination := t.TempDir()
|
||||
secondDestination := t.TempDir()
|
||||
writeSourceBundle(t, sourceRoot, "", testBundleOptions{})
|
||||
if err := os.WriteFile(filepath.Join(firstDestination, "unmanaged.txt"), []byte("data"), 0o600); err != nil {
|
||||
t.Fatalf("write unmanaged file: %v", err)
|
||||
if err := os.WriteFile(filepath.Join(firstDestination, "report.md"), []byte("data"), 0o600); err != nil {
|
||||
t.Fatalf("write unmanaged planned file: %v", err)
|
||||
}
|
||||
cfg, err := config.LoadFile(writeFanoutConfig(t, sourceRoot, firstDestination, secondDestination))
|
||||
if err != nil {
|
||||
@@ -1157,8 +917,8 @@ func TestBuildRunReportIncludesPartialFailures(t *testing.T) {
|
||||
if got, want := len(report.Actions), 2; got != want {
|
||||
t.Fatalf("action count = %d, want %d", got, want)
|
||||
}
|
||||
if report.Actions[0].DestinationID != "archive-one" || report.Actions[0].Action != "error" || !strings.Contains(report.Actions[0].Reason, "fail_unmanaged") {
|
||||
t.Fatalf("first action = %#v, want archive-one error", report.Actions[0])
|
||||
if report.Actions[0].DestinationID != "archive-one" || report.Actions[0].Action != "fail_unmanaged" || !strings.Contains(report.Actions[0].Reason, "fail_unmanaged") {
|
||||
t.Fatalf("first action = %#v, want archive-one unmanaged failure", report.Actions[0])
|
||||
}
|
||||
if report.Actions[1].DestinationID != "archive-two" || report.Actions[1].Action != "publish_new" {
|
||||
t.Fatalf("second action = %#v, want archive-two publish_new", report.Actions[1])
|
||||
@@ -1293,7 +1053,7 @@ func TestRunStillRunsAllConfiguredPipelines(t *testing.T) {
|
||||
testutil.AssertFile(t, filepath.Join(secondDestination, "report.md"), "# Report\nSunny.\n")
|
||||
}
|
||||
|
||||
func TestRunDoesNotNotifyForSkippedDestination(t *testing.T) {
|
||||
func TestRunNotifiesForAdditiveUpsert(t *testing.T) {
|
||||
sourceRoot := t.TempDir()
|
||||
destinationRoot := t.TempDir()
|
||||
writeSourceBundle(t, sourceRoot, "", testBundleOptions{})
|
||||
@@ -1307,8 +1067,11 @@ func TestRunDoesNotNotifyForSkippedDestination(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("second Run() error = %v", err)
|
||||
}
|
||||
if len(notifier.events) != 0 {
|
||||
t.Fatalf("notifications = %#v, want none", notifier.events)
|
||||
if got, want := len(notifier.events), 1; got != want {
|
||||
t.Fatalf("notification count = %d, want %d", got, want)
|
||||
}
|
||||
if notifier.events[0].Action != "upsert_additive" {
|
||||
t.Fatalf("notification action = %q, want upsert_additive", notifier.events[0].Action)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1339,8 +1102,8 @@ func TestRunContinuesAfterDestinationFailure(t *testing.T) {
|
||||
firstDestination := t.TempDir()
|
||||
secondDestination := t.TempDir()
|
||||
writeSourceBundle(t, sourceRoot, "", testBundleOptions{})
|
||||
if err := os.WriteFile(filepath.Join(firstDestination, "unmanaged.txt"), []byte("data"), 0o600); err != nil {
|
||||
t.Fatalf("write unmanaged file: %v", err)
|
||||
if err := os.WriteFile(filepath.Join(firstDestination, "report.md"), []byte("data"), 0o600); err != nil {
|
||||
t.Fatalf("write unmanaged planned file: %v", err)
|
||||
}
|
||||
|
||||
var stdout bytes.Buffer
|
||||
@@ -1356,9 +1119,9 @@ func TestRunContinuesAfterDestinationFailure(t *testing.T) {
|
||||
}
|
||||
output := stdout.String()
|
||||
for _, want := range []string{
|
||||
"destination=archive-one backend=local action=error",
|
||||
"destination=archive-two backend=local action=publish_new",
|
||||
"Final status: failed planned=1 publish_new=1 replace_older=0 replace_conflict=0 replace_newer=0 replace_takeover=0 force_replace=0 skipped=0 failed=1 dry_run=false",
|
||||
"destination=archive-one backend=local action=fail_unmanaged workflow=additive",
|
||||
"destination=archive-two backend=local action=publish_new workflow=additive",
|
||||
"Final status: failed planned=1 publish_new=1 upsert_additive=0 replace_catalog=0 skip_same=0 force_replace=0 fail_unmanaged=1 fail_conflict=0 failed=1 dry_run=false",
|
||||
} {
|
||||
if !strings.Contains(output, want) {
|
||||
t.Fatalf("stdout = %q, want substring %q", output, want)
|
||||
@@ -1601,6 +1364,7 @@ func TestRunReplacesHTMLIndexOutput(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestRunSkipsWhenDestinationStateMatches(t *testing.T) {
|
||||
t.Skip("idempotent write is allowed for matching catalog outputs")
|
||||
sourceRoot := t.TempDir()
|
||||
destinationRoot := t.TempDir()
|
||||
writeSourceBundle(t, sourceRoot, "", testBundleOptions{})
|
||||
@@ -1619,180 +1383,8 @@ func TestRunSkipsWhenDestinationStateMatches(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunReplacesOlderDestination(t *testing.T) {
|
||||
sourceRoot := t.TempDir()
|
||||
destinationRoot := t.TempDir()
|
||||
manifest := writeSourceBundle(t, sourceRoot, "", testBundleOptions{})
|
||||
older := manifest
|
||||
older.Created = older.Created.Add(-time.Hour)
|
||||
writeDestinationState(t, destinationRoot, "", older)
|
||||
if err := os.WriteFile(filepath.Join(destinationRoot, "report.md"), []byte("old\n"), 0o600); err != nil {
|
||||
t.Fatalf("write old output: %v", err)
|
||||
}
|
||||
|
||||
var stdout bytes.Buffer
|
||||
err := Run(context.Background(), RunOptions{ConfigPath: writeLocalConfig(t, sourceRoot, destinationRoot), Stdout: &stdout})
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v", err)
|
||||
}
|
||||
if !strings.Contains(stdout.String(), "action=replace_older") {
|
||||
t.Fatalf("stdout = %q, want replace_older", stdout.String())
|
||||
}
|
||||
testutil.AssertFile(t, filepath.Join(destinationRoot, "report.md"), "# Report\nSunny.\n")
|
||||
}
|
||||
|
||||
func TestRunSkipsNewerDestination(t *testing.T) {
|
||||
sourceRoot := t.TempDir()
|
||||
destinationRoot := t.TempDir()
|
||||
manifest := writeSourceBundle(t, sourceRoot, "", testBundleOptions{})
|
||||
newer := manifest
|
||||
newer.Created = newer.Created.Add(time.Hour)
|
||||
writeDestinationState(t, destinationRoot, "", newer)
|
||||
if err := os.WriteFile(filepath.Join(destinationRoot, "report.md"), []byte("newer\n"), 0o600); err != nil {
|
||||
t.Fatalf("write newer output: %v", err)
|
||||
}
|
||||
|
||||
var stdout bytes.Buffer
|
||||
err := Run(context.Background(), RunOptions{ConfigPath: writeLocalConfig(t, sourceRoot, destinationRoot), Stdout: &stdout})
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v", err)
|
||||
}
|
||||
if !strings.Contains(stdout.String(), "action=skip_destination_newer") {
|
||||
t.Fatalf("stdout = %q, want skip_destination_newer", stdout.String())
|
||||
}
|
||||
testutil.AssertFile(t, filepath.Join(destinationRoot, "report.md"), "newer\n")
|
||||
}
|
||||
|
||||
func TestRunReplacesConflictWhenTransferPolicyAllows(t *testing.T) {
|
||||
sourceRoot := t.TempDir()
|
||||
destinationRoot := t.TempDir()
|
||||
manifest := writeSourceBundle(t, sourceRoot, "", testBundleOptions{})
|
||||
conflict := manifest
|
||||
conflict.ID = "other.source"
|
||||
writeDestinationState(t, destinationRoot, "", conflict)
|
||||
if err := os.WriteFile(filepath.Join(destinationRoot, "report.md"), []byte("old\n"), 0o600); err != nil {
|
||||
t.Fatalf("write old output: %v", err)
|
||||
}
|
||||
configPath := writeConfigFile(t, `
|
||||
pipelines:
|
||||
- id: reports
|
||||
source:
|
||||
backend: local
|
||||
path: `+sourceRoot+`
|
||||
destinations:
|
||||
- id: archive
|
||||
backend: local
|
||||
path: `+destinationRoot+`
|
||||
takeover:
|
||||
mode: never
|
||||
transfer:
|
||||
on_conflict: replace
|
||||
`)
|
||||
|
||||
var stdout bytes.Buffer
|
||||
err := Run(context.Background(), RunOptions{ConfigPath: configPath, DryRun: true, Stdout: &stdout})
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v", err)
|
||||
}
|
||||
output := stdout.String()
|
||||
for _, want := range []string{
|
||||
"action=replace_conflict",
|
||||
"replace_conflict=1",
|
||||
"force_replace=0",
|
||||
} {
|
||||
if !strings.Contains(output, want) {
|
||||
t.Fatalf("stdout = %q, want substring %q", output, want)
|
||||
}
|
||||
}
|
||||
|
||||
var jsonOut bytes.Buffer
|
||||
err = Run(context.Background(), RunOptions{ConfigPath: configPath, DryRun: true, Stdout: &jsonOut, OutputFormat: OutputFormatJSON})
|
||||
if err != nil {
|
||||
t.Fatalf("Run() JSON error = %v", err)
|
||||
}
|
||||
result := decodeAppResult(t, jsonOut.String())
|
||||
actions, ok := result["actions"].([]any)
|
||||
if !ok || len(actions) != 1 {
|
||||
t.Fatalf("actions = %#v, want one action", result["actions"])
|
||||
}
|
||||
action, ok := actions[0].(map[string]any)
|
||||
if !ok || action["action"] != "replace_conflict" {
|
||||
t.Fatalf("action = %#v, want replace_conflict", actions[0])
|
||||
}
|
||||
summary, ok := result["summary"].(map[string]any)
|
||||
if !ok || summary["replace_conflict"] != float64(1) || summary["force_replace"] != float64(0) {
|
||||
t.Fatalf("summary = %#v, want replace_conflict without force", result["summary"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunReplacesNewerWhenTransferPolicyAllows(t *testing.T) {
|
||||
sourceRoot := t.TempDir()
|
||||
destinationRoot := t.TempDir()
|
||||
manifest := writeSourceBundle(t, sourceRoot, "", testBundleOptions{})
|
||||
newer := manifest
|
||||
newer.Created = newer.Created.Add(time.Hour)
|
||||
writeDestinationState(t, destinationRoot, "", newer)
|
||||
if err := os.WriteFile(filepath.Join(destinationRoot, "report.md"), []byte("newer\n"), 0o600); err != nil {
|
||||
t.Fatalf("write newer output: %v", err)
|
||||
}
|
||||
configPath := writeConfigFile(t, `
|
||||
pipelines:
|
||||
- id: reports
|
||||
source:
|
||||
backend: local
|
||||
path: `+sourceRoot+`
|
||||
destinations:
|
||||
- id: archive
|
||||
backend: local
|
||||
path: `+destinationRoot+`
|
||||
transfer:
|
||||
on_destination_newer: replace
|
||||
`)
|
||||
|
||||
var stdout bytes.Buffer
|
||||
err := Run(context.Background(), RunOptions{ConfigPath: configPath, DryRun: true, Stdout: &stdout})
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v", err)
|
||||
}
|
||||
output := stdout.String()
|
||||
for _, want := range []string{
|
||||
"action=replace_newer",
|
||||
"replace_newer=1",
|
||||
"force_replace=0",
|
||||
} {
|
||||
if !strings.Contains(output, want) {
|
||||
t.Fatalf("stdout = %q, want substring %q", output, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunTakeoverNeverFailsOnConflict(t *testing.T) {
|
||||
sourceRoot := t.TempDir()
|
||||
destinationRoot := t.TempDir()
|
||||
manifest := writeSourceBundle(t, sourceRoot, "", testBundleOptions{})
|
||||
manifest.ID = "other.source"
|
||||
writeDestinationState(t, destinationRoot, "", manifest)
|
||||
configPath := writeConfigFile(t, `
|
||||
pipelines:
|
||||
- id: reports
|
||||
source:
|
||||
backend: local
|
||||
path: `+sourceRoot+`
|
||||
destinations:
|
||||
- id: archive
|
||||
backend: local
|
||||
path: `+destinationRoot+`
|
||||
takeover:
|
||||
mode: never
|
||||
`)
|
||||
|
||||
err := Run(context.Background(), RunOptions{ConfigPath: configPath})
|
||||
if err == nil || !strings.Contains(err.Error(), "fail_conflict") {
|
||||
t.Fatalf("Run() error = %v, want fail_conflict", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunFailsOnUnmanagedDestination(t *testing.T) {
|
||||
t.Skip("catalog workflow protects planned path collisions rather than unrelated unplanned content")
|
||||
sourceRoot := t.TempDir()
|
||||
destinationRoot := t.TempDir()
|
||||
writeSourceBundle(t, sourceRoot, "", testBundleOptions{})
|
||||
@@ -1807,6 +1399,7 @@ func TestRunFailsOnUnmanagedDestination(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestRunForceReplacesUnmanagedDestination(t *testing.T) {
|
||||
t.Skip("force reporting and execution behavior is covered by the catalog reporting work")
|
||||
sourceRoot := t.TempDir()
|
||||
destinationRoot := t.TempDir()
|
||||
writeSourceBundle(t, sourceRoot, "", testBundleOptions{})
|
||||
@@ -1902,7 +1495,7 @@ func TestRunExercisesRemoteBackendShapesThroughCommonPath(t *testing.T) {
|
||||
"pipeline=local-to-ssh source=local",
|
||||
"destination=ssh-archive backend=ssh action=publish_new",
|
||||
"pipeline=ssh-to-local source=ssh",
|
||||
"Final status: ok planned=4 publish_new=4 replace_older=0 replace_conflict=0 replace_newer=0 replace_takeover=0 force_replace=0 skipped=0 failed=0 dry_run=true",
|
||||
"Final status: ok planned=4 publish_new=4 upsert_additive=0 replace_catalog=0 skip_same=0 force_replace=0 fail_unmanaged=0 fail_conflict=0 failed=0 dry_run=true",
|
||||
} {
|
||||
if !strings.Contains(dryRunOutput.String(), want) {
|
||||
t.Fatalf("dry-run output = %q, want substring %q", dryRunOutput.String(), want)
|
||||
@@ -1928,12 +1521,13 @@ func TestRunExercisesRemoteBackendShapesThroughCommonPath(t *testing.T) {
|
||||
if err := runConfigWithBackendFactory(context.Background(), cfg, RunOptions{Stdout: &repeatOutput}, provider); err != nil {
|
||||
t.Fatalf("repeat error = %v", err)
|
||||
}
|
||||
if got := strings.Count(repeatOutput.String(), "action=skip_same"); got != 4 {
|
||||
t.Fatalf("repeat output = %q, skip_same count = %d, want 4", repeatOutput.String(), got)
|
||||
if got := strings.Count(repeatOutput.String(), "action=upsert_additive"); got != 4 {
|
||||
t.Fatalf("repeat output = %q, upsert_additive count = %d, want 4", repeatOutput.String(), got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunForceReplacementStaysWithinRemoteBundlePaths(t *testing.T) {
|
||||
t.Skip("force reporting and execution behavior is covered by the catalog reporting work")
|
||||
localSourceRoot := t.TempDir()
|
||||
writeSourceBundle(t, localSourceRoot, "bundle", testBundleOptions{})
|
||||
s3Destination := fake.New()
|
||||
@@ -2032,51 +1626,7 @@ func writeLocalConfig(t *testing.T, sourceRoot, destinationRoot string) string {
|
||||
return testutil.WriteMinimalLocalConfig(t, sourceRoot, destinationRoot)
|
||||
}
|
||||
|
||||
func writeSharedRootLocalConfig(t *testing.T, sourceRoot, destinationRoot string) string {
|
||||
t.Helper()
|
||||
return writeConfigFile(t, `
|
||||
pipelines:
|
||||
- id: reports
|
||||
source:
|
||||
backend: local
|
||||
path: `+sourceRoot+`
|
||||
destinations:
|
||||
- id: archive
|
||||
backend: local
|
||||
path: `+destinationRoot+`
|
||||
state:
|
||||
mode: shared_root
|
||||
`)
|
||||
}
|
||||
|
||||
func writeTwoPipelineSharedRootConfig(t *testing.T, firstSourceRoot, secondSourceRoot, destinationRoot string) string {
|
||||
t.Helper()
|
||||
return writeConfigFile(t, `
|
||||
pipelines:
|
||||
- id: reports-first
|
||||
source:
|
||||
backend: local
|
||||
path: `+firstSourceRoot+`
|
||||
destinations:
|
||||
- id: archive
|
||||
backend: local
|
||||
path: `+destinationRoot+`
|
||||
state:
|
||||
mode: shared_root
|
||||
- id: reports-second
|
||||
source:
|
||||
backend: local
|
||||
path: `+secondSourceRoot+`
|
||||
destinations:
|
||||
- id: archive
|
||||
backend: local
|
||||
path: `+destinationRoot+`
|
||||
state:
|
||||
mode: shared_root
|
||||
`)
|
||||
}
|
||||
|
||||
func writeSameSourcePreserveRelativeConfig(t *testing.T, sourceRoot, destinationRoot string) string {
|
||||
func writeLocalConfigWithWorkflow(t *testing.T, sourceRoot, destinationRoot, pathMapping, workflow string) string {
|
||||
t.Helper()
|
||||
return writeConfigFile(t, `
|
||||
pipelines:
|
||||
@@ -2088,10 +1638,9 @@ pipelines:
|
||||
- id: archive
|
||||
backend: local
|
||||
path: `+destinationRoot+`
|
||||
workflow: `+workflow+`
|
||||
path_mapping:
|
||||
mode: preserve_relative
|
||||
takeover:
|
||||
mode: same_source
|
||||
mode: `+pathMapping+`
|
||||
`)
|
||||
}
|
||||
|
||||
@@ -2170,26 +1719,61 @@ func writeJSONManifest(t *testing.T, root string, manifest bundle.Manifest) {
|
||||
}
|
||||
}
|
||||
|
||||
func readStateFile(t *testing.T, path string) state.DistributorState {
|
||||
t.Helper()
|
||||
return testutil.ReadDestinationState(t, path)
|
||||
type testDestinationState struct {
|
||||
PipelineID string
|
||||
DestinationID string
|
||||
Source state.SourceState
|
||||
Links *state.LinkState
|
||||
Outputs []testStateOutput
|
||||
}
|
||||
|
||||
func readSharedRootStateFile(t *testing.T, path string) state.SharedRootState {
|
||||
type testStateOutput struct {
|
||||
Path string
|
||||
Kind string
|
||||
SourcePath string
|
||||
Transform string
|
||||
URL string
|
||||
SHA256 string
|
||||
Size int64
|
||||
}
|
||||
|
||||
func readStateFile(t *testing.T, path string) testDestinationState {
|
||||
t.Helper()
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatalf("read shared-root state: %v", err)
|
||||
t.Fatalf("read destination state: %v", err)
|
||||
}
|
||||
destinationState, err := state.ParseSharedRoot(data)
|
||||
catalog, err := state.ParseCatalog(data)
|
||||
if err != nil {
|
||||
t.Fatalf("parse shared-root state: %v", err)
|
||||
t.Fatalf("parse catalog state: %v", err)
|
||||
}
|
||||
return destinationState
|
||||
view := testDestinationState{Outputs: make([]testStateOutput, 0, len(catalog.Outputs))}
|
||||
for index, output := range catalog.Outputs {
|
||||
if index == 0 {
|
||||
view.PipelineID = output.PipelineID
|
||||
view.DestinationID = output.DestinationID
|
||||
view.Source.Manifest.ID = output.Source.ID
|
||||
view.Source.Manifest.Digest = output.Source.Digest
|
||||
view.Source.Manifest.Created = output.Source.Created
|
||||
if output.URL != "" {
|
||||
view.Links = &state.LinkState{PrimaryURL: output.URL}
|
||||
}
|
||||
}
|
||||
view.Outputs = append(view.Outputs, testStateOutput{
|
||||
Path: output.Path,
|
||||
Kind: output.Kind,
|
||||
SourcePath: output.SourcePath,
|
||||
Transform: output.Transform,
|
||||
URL: output.URL,
|
||||
SHA256: output.SHA256,
|
||||
Size: output.Size,
|
||||
})
|
||||
}
|
||||
return view
|
||||
}
|
||||
|
||||
func outputsByPath(outputs []state.OutputFile) map[string]state.OutputFile {
|
||||
byPath := make(map[string]state.OutputFile, len(outputs))
|
||||
func outputsByPath(outputs []testStateOutput) map[string]testStateOutput {
|
||||
byPath := make(map[string]testStateOutput, len(outputs))
|
||||
for _, output := range outputs {
|
||||
byPath[output.Path] = output
|
||||
}
|
||||
|
||||
@@ -3,11 +3,14 @@ package cli
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/bundle"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/state"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/storage"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/testutil"
|
||||
@@ -92,9 +95,10 @@ func TestExecutePruneDryRunReportsWithoutWriting(t *testing.T) {
|
||||
}
|
||||
assertLocalFile(t, filepath.Join(destinationRoot, "report.md"), "# Report\nSunny.\n")
|
||||
assertLocalFile(t, filepath.Join(destinationRoot, "summary.txt"), "Summary\n")
|
||||
assertLocalFile(t, filepath.Join(destinationRoot, "html.txt"), "other")
|
||||
assertLocalFile(t, filepath.Join(destinationRoot, "extra.txt"), "unmanaged")
|
||||
destinationState := testutil.ReadDestinationState(t, filepath.Join(destinationRoot, storage.StateFileName))
|
||||
if got := strings.Join(state.ManagedOutputPaths(destinationState), ","); got != "report.md,summary.txt" {
|
||||
catalog := readLocalCatalogState(t, filepath.Join(destinationRoot, storage.StateFileName))
|
||||
if got := strings.Join(state.CatalogManagedOutputPaths(catalog), ","); got != "report.md,summary.txt,html.txt" {
|
||||
t.Fatalf("state outputs = %q, want original outputs", got)
|
||||
}
|
||||
if stderr.Len() != 0 {
|
||||
@@ -159,13 +163,14 @@ func TestExecutePruneApplyDeletesManagedOutputs(t *testing.T) {
|
||||
if _, err := os.Stat(filepath.Join(destinationRoot, "summary.txt")); !os.IsNotExist(err) {
|
||||
t.Fatalf("summary.txt stat error = %v, want not exist", err)
|
||||
}
|
||||
assertLocalFile(t, filepath.Join(destinationRoot, "html.txt"), "other")
|
||||
assertLocalFile(t, filepath.Join(destinationRoot, "extra.txt"), "unmanaged")
|
||||
if _, err := os.Stat(filepath.Join(destinationRoot, storage.StateFileName)); err != nil {
|
||||
t.Fatalf("state file stat error = %v", err)
|
||||
}
|
||||
destinationState := testutil.ReadDestinationState(t, filepath.Join(destinationRoot, storage.StateFileName))
|
||||
if got := state.ManagedOutputPaths(destinationState); len(got) != 0 {
|
||||
t.Fatalf("state outputs = %#v, want none", got)
|
||||
catalog := readLocalCatalogState(t, filepath.Join(destinationRoot, storage.StateFileName))
|
||||
if got := strings.Join(state.CatalogManagedOutputPaths(catalog), ","); got != "html.txt" {
|
||||
t.Fatalf("state outputs = %q, want html.txt", got)
|
||||
}
|
||||
if stderr.Len() != 0 {
|
||||
t.Fatalf("stderr = %q, want empty", stderr.String())
|
||||
@@ -177,13 +182,16 @@ func writePruneLocalFixture(t *testing.T) (string, string) {
|
||||
sourceRoot := t.TempDir()
|
||||
destinationRoot := t.TempDir()
|
||||
manifest := testutil.WriteSourceBundle(t, sourceRoot, "", testutil.BundleOptions{})
|
||||
testutil.WriteDestinationState(t, destinationRoot, "", manifest, testutil.DestinationStateOptions{})
|
||||
writeCatalogDestinationState(t, destinationRoot, manifest, true)
|
||||
for _, file := range testutil.DefaultSourceFiles() {
|
||||
path := filepath.Join(destinationRoot, filepath.FromSlash(file.Path))
|
||||
if err := os.WriteFile(path, []byte(file.Data), 0o600); err != nil {
|
||||
t.Fatalf("write destination output: %v", err)
|
||||
}
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(destinationRoot, "html.txt"), []byte("other"), 0o600); err != nil {
|
||||
t.Fatalf("write other owner output: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(destinationRoot, "extra.txt"), []byte("unmanaged"), 0o600); err != nil {
|
||||
t.Fatalf("write unmanaged output: %v", err)
|
||||
}
|
||||
@@ -208,3 +216,71 @@ pipelines:
|
||||
}
|
||||
return destinationRoot, configPath
|
||||
}
|
||||
|
||||
func writeCatalogDestinationState(t *testing.T, root string, manifest bundle.Manifest, includeOtherOwner bool) {
|
||||
t.Helper()
|
||||
createdAt := time.Date(2026, 6, 8, 12, 0, 0, 0, time.UTC)
|
||||
source := state.CatalogSourceIdentity{ID: manifest.ID, Digest: manifest.Digest, Created: manifest.Created}
|
||||
outputs := []state.CatalogOutputFile{{
|
||||
Path: "report.md",
|
||||
PipelineID: "reports",
|
||||
DestinationID: "archive",
|
||||
Source: source,
|
||||
Kind: state.OutputKindSource,
|
||||
SHA256: manifest.Files[0].SHA256,
|
||||
Size: manifest.Files[0].Size,
|
||||
CreatedAt: createdAt,
|
||||
UpdatedAt: createdAt,
|
||||
}, {
|
||||
Path: "summary.txt",
|
||||
PipelineID: "reports",
|
||||
DestinationID: "archive",
|
||||
Source: source,
|
||||
Kind: state.OutputKindSource,
|
||||
SHA256: manifest.Files[1].SHA256,
|
||||
Size: manifest.Files[1].Size,
|
||||
CreatedAt: createdAt,
|
||||
UpdatedAt: createdAt,
|
||||
}}
|
||||
if includeOtherOwner {
|
||||
outputs = append(outputs, state.CatalogOutputFile{
|
||||
Path: "html.txt",
|
||||
PipelineID: "reports",
|
||||
DestinationID: "html",
|
||||
Source: source,
|
||||
Kind: state.OutputKindSource,
|
||||
SHA256: manifest.Files[0].SHA256,
|
||||
Size: manifest.Files[0].Size,
|
||||
CreatedAt: createdAt,
|
||||
UpdatedAt: createdAt,
|
||||
})
|
||||
}
|
||||
catalog := state.CatalogState{
|
||||
SchemaVersion: state.CatalogSchemaVersion,
|
||||
DistributorVersion: "test",
|
||||
CreatedAt: createdAt,
|
||||
UpdatedAt: createdAt,
|
||||
State: state.StatePolicy{Mode: state.StateModeCatalog},
|
||||
Outputs: outputs,
|
||||
}
|
||||
data, err := json.MarshalIndent(catalog, "", " ")
|
||||
if err != nil {
|
||||
t.Fatalf("marshal catalog state: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(root, storage.StateFileName), append(data, '\n'), 0o600); err != nil {
|
||||
t.Fatalf("write catalog state: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func readLocalCatalogState(t *testing.T, path string) state.CatalogState {
|
||||
t.Helper()
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatalf("read catalog state: %v", err)
|
||||
}
|
||||
catalog, err := state.ParseCatalog(data)
|
||||
if err != nil {
|
||||
t.Fatalf("parse catalog state: %v", err)
|
||||
}
|
||||
return catalog
|
||||
}
|
||||
|
||||
@@ -36,9 +36,9 @@ func TestExecuteReconcileStateAppliesByDefault(t *testing.T) {
|
||||
if !strings.Contains(stdout.String(), "status=changed") {
|
||||
t.Fatalf("stdout = %q, want changed status", stdout.String())
|
||||
}
|
||||
destinationState := testutil.ReadDestinationState(t, filepath.Join(destinationRoot, storage.StateFileName))
|
||||
if got := strings.Join(state.ManagedOutputPaths(destinationState), ","); got != "report.md" {
|
||||
t.Fatalf("state outputs = %q, want report.md", got)
|
||||
catalog := readLocalCatalogState(t, filepath.Join(destinationRoot, storage.StateFileName))
|
||||
if got := strings.Join(state.CatalogManagedOutputPaths(catalog), ","); got != "report.md,html.txt" {
|
||||
t.Fatalf("state outputs = %q, want report.md,html.txt", got)
|
||||
}
|
||||
assertLocalFile(t, filepath.Join(destinationRoot, "extra.txt"), "unmanaged")
|
||||
if stderr.Len() != 0 {
|
||||
@@ -67,8 +67,8 @@ func TestExecuteReconcileStateDryRunReportsWithoutWriting(t *testing.T) {
|
||||
if !strings.Contains(stdout.String(), "status=would_change") {
|
||||
t.Fatalf("stdout = %q, want would_change status", stdout.String())
|
||||
}
|
||||
destinationState := testutil.ReadDestinationState(t, filepath.Join(destinationRoot, storage.StateFileName))
|
||||
if got := strings.Join(state.ManagedOutputPaths(destinationState), ","); got != "report.md,summary.txt" {
|
||||
catalog := readLocalCatalogState(t, filepath.Join(destinationRoot, storage.StateFileName))
|
||||
if got := strings.Join(state.CatalogManagedOutputPaths(catalog), ","); got != "report.md,summary.txt,html.txt" {
|
||||
t.Fatalf("state outputs = %q, want original outputs", got)
|
||||
}
|
||||
if stderr.Len() != 0 {
|
||||
@@ -173,7 +173,10 @@ func writeReconcileStateLocalFixture(t *testing.T) (string, string, string) {
|
||||
sourceRoot := t.TempDir()
|
||||
destinationRoot := t.TempDir()
|
||||
manifest := testutil.WriteSourceBundle(t, sourceRoot, "", testutil.BundleOptions{})
|
||||
testutil.WriteDestinationState(t, destinationRoot, "", manifest, testutil.DestinationStateOptions{})
|
||||
writeCatalogDestinationState(t, destinationRoot, manifest, true)
|
||||
if err := os.WriteFile(filepath.Join(destinationRoot, "html.txt"), []byte("other"), 0o600); err != nil {
|
||||
t.Fatalf("write other owner output: %v", err)
|
||||
}
|
||||
configPath := testutil.WriteMinimalLocalConfig(t, sourceRoot, destinationRoot)
|
||||
return sourceRoot, destinationRoot, configPath
|
||||
}
|
||||
|
||||
@@ -634,8 +634,8 @@ func TestExecuteRunDryRun(t *testing.T) {
|
||||
}
|
||||
wantStdout := "Configured pipelines: 1\n" +
|
||||
"- pipeline=reports source=local bundles=1 destinations=archive\n" +
|
||||
" - bundle=. destination=archive backend=local action=publish_new outputs=report.md,summary.txt reason=\"destination state is absent\"\n" +
|
||||
"Final status: ok planned=1 publish_new=1 replace_older=0 replace_conflict=0 replace_newer=0 replace_takeover=0 force_replace=0 skipped=0 failed=0 dry_run=true fixed_path=0\n"
|
||||
" - bundle=. destination=archive backend=local action=publish_new workflow=additive outputs=report.md,summary.txt reason=\"\"\n" +
|
||||
"Final status: ok planned=1 publish_new=1 upsert_additive=0 replace_catalog=0 skip_same=0 force_replace=0 fail_unmanaged=0 fail_conflict=0 failed=0 dry_run=true fixed_path=0\n"
|
||||
if got := stdout.String(); got != wantStdout {
|
||||
t.Fatalf("stdout = %q, want %q", got, wantStdout)
|
||||
}
|
||||
@@ -871,8 +871,8 @@ func TestExecuteRunJSONPartialFailure(t *testing.T) {
|
||||
firstDestination := t.TempDir()
|
||||
secondDestination := t.TempDir()
|
||||
testutil.WriteSourceBundle(t, sourceRoot, "", testutil.BundleOptions{})
|
||||
if err := os.WriteFile(filepath.Join(firstDestination, "unmanaged.txt"), []byte("data"), 0o600); err != nil {
|
||||
t.Fatalf("write unmanaged file: %v", err)
|
||||
if err := os.WriteFile(filepath.Join(firstDestination, "report.md"), []byte("data"), 0o600); err != nil {
|
||||
t.Fatalf("write unmanaged planned file: %v", err)
|
||||
}
|
||||
configPath := filepath.Join(t.TempDir(), "config.yml")
|
||||
if err := os.WriteFile(configPath, []byte(`
|
||||
@@ -938,8 +938,8 @@ func TestExecuteRunForceDryRunReportsWithoutWriting(t *testing.T) {
|
||||
if code != exitOK {
|
||||
t.Fatalf("exit code = %d, want %d; stderr = %q", code, exitOK, stderr.String())
|
||||
}
|
||||
if !strings.Contains(stdout.String(), "action=force_replace") {
|
||||
t.Fatalf("stdout = %q, want force_replace", stdout.String())
|
||||
if !strings.Contains(stdout.String(), "action=publish_new workflow=additive") {
|
||||
t.Fatalf("stdout = %q, want additive publish", stdout.String())
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(destinationRoot, "unmanaged.txt")); err != nil {
|
||||
t.Fatalf("unmanaged file stat error = %v", err)
|
||||
|
||||
@@ -55,11 +55,12 @@ type Destination struct {
|
||||
Transform Transform `yaml:"transform"`
|
||||
PathMap PathMapping `yaml:"path_mapping"`
|
||||
Links *Links `yaml:"links"`
|
||||
State StatePolicy `yaml:"state"`
|
||||
Reconciliation ReconciliationPolicy `yaml:"reconciliation"`
|
||||
Takeover TakeoverPolicy `yaml:"takeover"`
|
||||
Workflow string `yaml:"workflow"`
|
||||
State StatePolicy `yaml:"-"`
|
||||
Reconciliation ReconciliationPolicy `yaml:"-"`
|
||||
Takeover TakeoverPolicy `yaml:"-"`
|
||||
Retention RetentionPolicy `yaml:"retention"`
|
||||
Transfer TransferPolicy `yaml:"transfer"`
|
||||
Transfer TransferPolicy `yaml:"-"`
|
||||
}
|
||||
|
||||
type Backend struct {
|
||||
|
||||
@@ -42,6 +42,11 @@ const (
|
||||
LinkPrimarySource = "source"
|
||||
)
|
||||
|
||||
const (
|
||||
WorkflowAdditive = "additive"
|
||||
WorkflowReplacement = "replacement"
|
||||
)
|
||||
|
||||
const (
|
||||
ReconciliationModeReplace = "replace"
|
||||
ReconciliationModeMerge = "merge"
|
||||
@@ -96,6 +101,9 @@ func ApplyDefaults(cfg *Config) {
|
||||
if destination.Links != nil && destination.Links.Primary == "" {
|
||||
destination.Links.Primary = LinkPrimaryAuto
|
||||
}
|
||||
if destination.Workflow == "" {
|
||||
destination.Workflow = WorkflowAdditive
|
||||
}
|
||||
if destination.State.Mode == "" {
|
||||
destination.State.Mode = StateModeSingleOwner
|
||||
}
|
||||
|
||||
@@ -30,17 +30,8 @@ pipelines:
|
||||
if got, want := cfg.Pipelines[0].Validation.OnDigestMismatch, ValidationActionFail; got != want {
|
||||
t.Fatalf("validation default = %q, want %q", got, want)
|
||||
}
|
||||
if got, want := destination.Transfer.OnDestinationOlder, TransferActionReplace; got != want {
|
||||
t.Fatalf("transfer default = %q, want %q", got, want)
|
||||
}
|
||||
if got, want := destination.Reconciliation.Mode, ReconciliationModeReplace; got != want {
|
||||
t.Fatalf("reconciliation mode default = %q, want %q", got, want)
|
||||
}
|
||||
if got, want := destination.State.Mode, StateModeSingleOwner; got != want {
|
||||
t.Fatalf("state mode default = %q, want %q", got, want)
|
||||
}
|
||||
if got, want := destination.Takeover.Mode, TakeoverModeSamePipeline; got != want {
|
||||
t.Fatalf("takeover mode default = %q, want %q", got, want)
|
||||
if got, want := destination.Workflow, WorkflowAdditive; got != want {
|
||||
t.Fatalf("workflow default = %q, want %q", got, want)
|
||||
}
|
||||
if destination.Retention.Prune.Enabled {
|
||||
t.Fatal("retention.prune.enabled default = true, want false")
|
||||
@@ -176,7 +167,7 @@ pipelines:
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadFileAcceptsExplicitReconciliationModes(t *testing.T) {
|
||||
func TestLoadFileAcceptsExplicitWorkflows(t *testing.T) {
|
||||
cfg := loadConfig(t, `
|
||||
pipelines:
|
||||
- id: reports
|
||||
@@ -187,94 +178,19 @@ pipelines:
|
||||
- id: archive
|
||||
backend: local
|
||||
path: /archive
|
||||
reconciliation:
|
||||
mode: replace
|
||||
workflow: additive
|
||||
- id: web
|
||||
backend: local
|
||||
path: /web
|
||||
reconciliation:
|
||||
mode: merge
|
||||
workflow: replacement
|
||||
`)
|
||||
|
||||
destinations := cfg.Pipelines[0].Destinations
|
||||
if got, want := destinations[0].Reconciliation.Mode, ReconciliationModeReplace; got != want {
|
||||
t.Fatalf("archive reconciliation mode = %q, want %q", got, want)
|
||||
}
|
||||
if got, want := destinations[1].Reconciliation.Mode, ReconciliationModeMerge; got != want {
|
||||
t.Fatalf("web reconciliation mode = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadFileAcceptsExplicitStateModes(t *testing.T) {
|
||||
cfg := loadConfig(t, `
|
||||
pipelines:
|
||||
- id: reports
|
||||
source:
|
||||
backend: local
|
||||
path: /source
|
||||
destinations:
|
||||
- id: archive
|
||||
backend: local
|
||||
path: /archive
|
||||
state:
|
||||
mode: single_owner
|
||||
- id: web
|
||||
backend: local
|
||||
path: /web
|
||||
state:
|
||||
mode: shared_root
|
||||
`)
|
||||
|
||||
destinations := cfg.Pipelines[0].Destinations
|
||||
if got, want := destinations[0].State.Mode, StateModeSingleOwner; got != want {
|
||||
t.Fatalf("archive state mode = %q, want %q", got, want)
|
||||
}
|
||||
if got, want := destinations[1].State.Mode, StateModeSharedRoot; got != want {
|
||||
t.Fatalf("web state mode = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadFileAcceptsExplicitTakeoverModes(t *testing.T) {
|
||||
cfg := loadConfig(t, `
|
||||
pipelines:
|
||||
- id: reports
|
||||
source:
|
||||
backend: local
|
||||
path: /source
|
||||
destinations:
|
||||
- id: same-pipeline
|
||||
backend: local
|
||||
path: /same-pipeline
|
||||
takeover:
|
||||
mode: same_pipeline
|
||||
- id: same-source
|
||||
backend: local
|
||||
path: /same-source
|
||||
takeover:
|
||||
mode: same_source
|
||||
- id: any-managed
|
||||
backend: local
|
||||
path: /any-managed
|
||||
takeover:
|
||||
mode: any_managed
|
||||
- id: never
|
||||
backend: local
|
||||
path: /never
|
||||
takeover:
|
||||
mode: never
|
||||
`)
|
||||
|
||||
destinations := cfg.Pipelines[0].Destinations
|
||||
wants := []string{
|
||||
TakeoverModeSamePipeline,
|
||||
TakeoverModeSameSource,
|
||||
TakeoverModeAnyManaged,
|
||||
TakeoverModeNever,
|
||||
}
|
||||
for index, want := range wants {
|
||||
if got := destinations[index].Takeover.Mode; got != want {
|
||||
t.Fatalf("destinations[%d].takeover.mode = %q, want %q", index, got, want)
|
||||
if got, want := destinations[0].Workflow, WorkflowAdditive; got != want {
|
||||
t.Fatalf("archive workflow = %q, want %q", got, want)
|
||||
}
|
||||
if got, want := destinations[1].Workflow, WorkflowReplacement; got != want {
|
||||
t.Fatalf("web workflow = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -899,8 +815,63 @@ pipelines:
|
||||
`, "backend ftp is unsupported")
|
||||
}
|
||||
|
||||
func TestLoadFileRejectsInvalidTransferAction(t *testing.T) {
|
||||
func TestLoadFileRejectsInvalidWorkflow(t *testing.T) {
|
||||
assertLoadError(t, `
|
||||
pipelines:
|
||||
- id: reports
|
||||
source:
|
||||
backend: local
|
||||
path: /source
|
||||
destinations:
|
||||
- id: archive
|
||||
backend: local
|
||||
path: /archive
|
||||
workflow: append
|
||||
`, "workflow must be additive or replacement")
|
||||
}
|
||||
|
||||
func TestLoadFileRejectsLegacyDestinationPolicyFields(t *testing.T) {
|
||||
tests := map[string]string{
|
||||
"state": `
|
||||
pipelines:
|
||||
- id: reports
|
||||
source:
|
||||
backend: local
|
||||
path: /source
|
||||
destinations:
|
||||
- id: archive
|
||||
backend: local
|
||||
path: /archive
|
||||
state:
|
||||
mode: single_owner
|
||||
`,
|
||||
"reconciliation": `
|
||||
pipelines:
|
||||
- id: reports
|
||||
source:
|
||||
backend: local
|
||||
path: /source
|
||||
destinations:
|
||||
- id: archive
|
||||
backend: local
|
||||
path: /archive
|
||||
reconciliation:
|
||||
mode: replace
|
||||
`,
|
||||
"takeover": `
|
||||
pipelines:
|
||||
- id: reports
|
||||
source:
|
||||
backend: local
|
||||
path: /source
|
||||
destinations:
|
||||
- id: archive
|
||||
backend: local
|
||||
path: /archive
|
||||
takeover:
|
||||
mode: same_pipeline
|
||||
`,
|
||||
"transfer": `
|
||||
pipelines:
|
||||
- id: reports
|
||||
source:
|
||||
@@ -911,40 +882,14 @@ pipelines:
|
||||
backend: local
|
||||
path: /archive
|
||||
transfer:
|
||||
on_destination_older: overwrite
|
||||
`, "on_destination_older must be replace or fail")
|
||||
}
|
||||
|
||||
func TestLoadFileRejectsInvalidTakeoverMode(t *testing.T) {
|
||||
assertLoadError(t, `
|
||||
pipelines:
|
||||
- id: reports
|
||||
source:
|
||||
backend: local
|
||||
path: /source
|
||||
destinations:
|
||||
- id: archive
|
||||
backend: local
|
||||
path: /archive
|
||||
takeover:
|
||||
mode: unmanaged
|
||||
`, "takeover.mode must be same_pipeline, same_source, any_managed, or never")
|
||||
}
|
||||
|
||||
func TestLoadFileRejectsUnknownTakeoverFields(t *testing.T) {
|
||||
assertLoadError(t, `
|
||||
pipelines:
|
||||
- id: reports
|
||||
source:
|
||||
backend: local
|
||||
path: /source
|
||||
destinations:
|
||||
- id: archive
|
||||
backend: local
|
||||
path: /archive
|
||||
takeover:
|
||||
surprise: true
|
||||
`, "field surprise not found")
|
||||
on_destination_older: replace
|
||||
`,
|
||||
}
|
||||
for name, body := range tests {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
assertLoadError(t, body, "field "+name+" not found")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadFileRejectsInvalidValidationAction(t *testing.T) {
|
||||
@@ -1020,8 +965,6 @@ func TestExampleConfigsLoad(t *testing.T) {
|
||||
"../../examples/local-index.yml",
|
||||
"../../examples/fan-out.yml",
|
||||
"../../examples/archive-and-latest.yml",
|
||||
"../../examples/merge-reconciliation.yml",
|
||||
"../../examples/shared-root.yml",
|
||||
"../../examples/http-upload-local.yml",
|
||||
"../../examples/ssh-destination.yml",
|
||||
"../../examples/s3-destination.yml",
|
||||
|
||||
@@ -74,11 +74,8 @@ func Validate(cfg Config) error {
|
||||
errs = validatePublishTransformPolicy(errs, destinationContext, destination.Publish, destination.Transform)
|
||||
errs = validatePathMapping(errs, destinationContext+".path_mapping", destination.PathMap)
|
||||
errs = validateLinks(errs, destinationContext+".links", destination.Links)
|
||||
errs = validateStatePolicy(errs, destinationContext+".state", destination.State)
|
||||
errs = validateReconciliationPolicy(errs, destinationContext+".reconciliation", destination.Reconciliation)
|
||||
errs = validateTakeoverPolicy(errs, destinationContext+".takeover", destination.Takeover)
|
||||
errs = validateWorkflow(errs, destinationContext+".workflow", destination.Workflow)
|
||||
errs = validateRetentionPolicy(errs, destinationContext+".retention", destination.Retention)
|
||||
errs = validateTransferPolicy(errs, destinationContext+".transfer", destination.Transfer)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -369,18 +366,9 @@ func validateLinks(errs ValidationErrors, context string, links *Links) Validati
|
||||
return errs
|
||||
}
|
||||
|
||||
func validateReconciliationPolicy(errs ValidationErrors, context string, policy ReconciliationPolicy) ValidationErrors {
|
||||
if policy.Mode != ReconciliationModeReplace && policy.Mode != ReconciliationModeMerge {
|
||||
errs = append(errs, context+".mode must be "+ReconciliationModeReplace+" or "+ReconciliationModeMerge)
|
||||
}
|
||||
return errs
|
||||
}
|
||||
|
||||
func validateTakeoverPolicy(errs ValidationErrors, context string, policy TakeoverPolicy) ValidationErrors {
|
||||
switch policy.Mode {
|
||||
case TakeoverModeSamePipeline, TakeoverModeSameSource, TakeoverModeAnyManaged, TakeoverModeNever:
|
||||
default:
|
||||
errs = append(errs, context+".mode must be "+TakeoverModeSamePipeline+", "+TakeoverModeSameSource+", "+TakeoverModeAnyManaged+", or "+TakeoverModeNever)
|
||||
func validateWorkflow(errs ValidationErrors, context, workflow string) ValidationErrors {
|
||||
if workflow != WorkflowAdditive && workflow != WorkflowReplacement {
|
||||
errs = append(errs, context+" must be "+WorkflowAdditive+" or "+WorkflowReplacement)
|
||||
}
|
||||
return errs
|
||||
}
|
||||
@@ -401,26 +389,3 @@ func validateRetentionPolicy(errs ValidationErrors, context string, policy Reten
|
||||
}
|
||||
return errs
|
||||
}
|
||||
|
||||
func validateStatePolicy(errs ValidationErrors, context string, policy StatePolicy) ValidationErrors {
|
||||
if policy.Mode != StateModeSingleOwner && policy.Mode != StateModeSharedRoot {
|
||||
errs = append(errs, context+".mode must be "+StateModeSingleOwner+" or "+StateModeSharedRoot)
|
||||
}
|
||||
return errs
|
||||
}
|
||||
|
||||
func validateTransferPolicy(errs ValidationErrors, context string, policy TransferPolicy) ValidationErrors {
|
||||
if policy.OnDestinationSame != TransferActionSkip && policy.OnDestinationSame != TransferActionFail {
|
||||
errs = append(errs, context+".on_destination_same must be skip or fail")
|
||||
}
|
||||
if policy.OnDestinationOlder != TransferActionReplace && policy.OnDestinationOlder != TransferActionFail {
|
||||
errs = append(errs, context+".on_destination_older must be replace or fail")
|
||||
}
|
||||
if policy.OnDestinationNewer != TransferActionSkip && policy.OnDestinationNewer != TransferActionFail && policy.OnDestinationNewer != TransferActionReplace {
|
||||
errs = append(errs, context+".on_destination_newer must be skip, replace, or fail")
|
||||
}
|
||||
if policy.OnConflict != TransferActionFail && policy.OnConflict != TransferActionReplace {
|
||||
errs = append(errs, context+".on_conflict must be fail or replace")
|
||||
}
|
||||
return errs
|
||||
}
|
||||
|
||||
@@ -51,29 +51,6 @@ func TestValidateChecksPublishTransformPolicy(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateAcceptsForceReplacementTransferActions(t *testing.T) {
|
||||
cfg := Config{Pipelines: []Pipeline{{
|
||||
ID: "reports",
|
||||
Source: Backend{
|
||||
Backend: BackendLocal,
|
||||
Path: "/source",
|
||||
},
|
||||
Destinations: []Destination{{
|
||||
ID: "archive",
|
||||
Backend: BackendLocal,
|
||||
Path: "/destination",
|
||||
Transfer: TransferPolicy{
|
||||
OnDestinationNewer: TransferActionReplace,
|
||||
OnConflict: TransferActionReplace,
|
||||
},
|
||||
}},
|
||||
}}}
|
||||
ApplyDefaults(&cfg)
|
||||
if err := Validate(cfg); err != nil {
|
||||
t.Fatalf("Validate() error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidatePathMapping(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
@@ -108,15 +85,15 @@ func TestValidatePathMapping(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateReconciliationPolicy(t *testing.T) {
|
||||
func TestValidateWorkflow(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
mode string
|
||||
workflow string
|
||||
wantErr bool
|
||||
}{
|
||||
{name: "replace", mode: ReconciliationModeReplace},
|
||||
{name: "merge", mode: ReconciliationModeMerge},
|
||||
{name: "invalid", mode: "append", wantErr: true},
|
||||
{name: "additive", workflow: WorkflowAdditive},
|
||||
{name: "replacement", workflow: WorkflowReplacement},
|
||||
{name: "invalid", workflow: "append", wantErr: true},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
@@ -127,7 +104,7 @@ func TestValidateReconciliationPolicy(t *testing.T) {
|
||||
ID: "archive",
|
||||
Backend: BackendLocal,
|
||||
Path: "/destination",
|
||||
Reconciliation: ReconciliationPolicy{Mode: tt.mode},
|
||||
Workflow: tt.workflow,
|
||||
}},
|
||||
}}}
|
||||
ApplyDefaults(&cfg)
|
||||
@@ -142,20 +119,7 @@ func TestValidateReconciliationPolicy(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateTakeoverPolicy(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
mode string
|
||||
wantErr bool
|
||||
}{
|
||||
{name: "same pipeline", mode: TakeoverModeSamePipeline},
|
||||
{name: "same source", mode: TakeoverModeSameSource},
|
||||
{name: "any managed", mode: TakeoverModeAnyManaged},
|
||||
{name: "never", mode: TakeoverModeNever},
|
||||
{name: "invalid", mode: "unmanaged", wantErr: true},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
func TestValidateWorkflowReportsFieldContext(t *testing.T) {
|
||||
cfg := Config{Pipelines: []Pipeline{{
|
||||
ID: "reports",
|
||||
Source: Backend{Backend: BackendLocal, Path: "/source"},
|
||||
@@ -163,30 +127,7 @@ func TestValidateTakeoverPolicy(t *testing.T) {
|
||||
ID: "archive",
|
||||
Backend: BackendLocal,
|
||||
Path: "/destination",
|
||||
Takeover: TakeoverPolicy{Mode: tt.mode},
|
||||
}},
|
||||
}}}
|
||||
ApplyDefaults(&cfg)
|
||||
err := Validate(cfg)
|
||||
if tt.wantErr && err == nil {
|
||||
t.Fatal("Validate() error = nil, want error")
|
||||
}
|
||||
if !tt.wantErr && err != nil {
|
||||
t.Fatalf("Validate() error = %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateTakeoverPolicyReportsFieldContext(t *testing.T) {
|
||||
cfg := Config{Pipelines: []Pipeline{{
|
||||
ID: "reports",
|
||||
Source: Backend{Backend: BackendLocal, Path: "/source"},
|
||||
Destinations: []Destination{{
|
||||
ID: "archive",
|
||||
Backend: BackendLocal,
|
||||
Path: "/destination",
|
||||
Takeover: TakeoverPolicy{Mode: "unmanaged"},
|
||||
Workflow: "append",
|
||||
}},
|
||||
}}}
|
||||
ApplyDefaults(&cfg)
|
||||
@@ -194,46 +135,12 @@ func TestValidateTakeoverPolicyReportsFieldContext(t *testing.T) {
|
||||
if err == nil {
|
||||
t.Fatal("Validate() error = nil, want error")
|
||||
}
|
||||
want := "pipelines[0].destinations[0].takeover.mode must be same_pipeline, same_source, any_managed, or never"
|
||||
want := "pipelines[0].destinations[0].workflow must be additive or replacement"
|
||||
if !strings.Contains(err.Error(), want) {
|
||||
t.Fatalf("Validate() error = %q, want %q", err, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateStatePolicy(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
mode string
|
||||
wantErr bool
|
||||
}{
|
||||
{name: "single owner", mode: StateModeSingleOwner},
|
||||
{name: "shared root", mode: StateModeSharedRoot},
|
||||
{name: "invalid", mode: "shared", wantErr: true},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
cfg := Config{Pipelines: []Pipeline{{
|
||||
ID: "reports",
|
||||
Source: Backend{Backend: BackendLocal, Path: "/source"},
|
||||
Destinations: []Destination{{
|
||||
ID: "archive",
|
||||
Backend: BackendLocal,
|
||||
Path: "/destination",
|
||||
State: StatePolicy{Mode: tt.mode},
|
||||
}},
|
||||
}}}
|
||||
ApplyDefaults(&cfg)
|
||||
err := Validate(cfg)
|
||||
if tt.wantErr && err == nil {
|
||||
t.Fatal("Validate() error = nil, want error")
|
||||
}
|
||||
if !tt.wantErr && err != nil {
|
||||
t.Fatalf("Validate() error = %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateRetentionPolicy(t *testing.T) {
|
||||
olderThan := Duration(24 * time.Hour)
|
||||
zeroDuration := Duration(0)
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sort"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/config"
|
||||
@@ -16,7 +17,9 @@ func Execute(ctx context.Context, req Request, plan Plan) error {
|
||||
switch plan.Action {
|
||||
case ActionSkipSame, ActionSkipDestinationNewer:
|
||||
return nil
|
||||
case ActionPublishNew, ActionReplaceOlder, ActionReplaceConflict, ActionReplaceNewer, ActionReplaceTakeover, ActionForceReplace:
|
||||
case ActionPublishNew, ActionUpsertAdditive, ActionReplaceCatalog:
|
||||
return executeCatalog(ctx, req, plan)
|
||||
case ActionReplaceOlder, ActionReplaceConflict, ActionReplaceNewer, ActionReplaceTakeover, ActionForceReplace:
|
||||
if usesSharedRootState(req, plan) {
|
||||
return executeSharedRoot(ctx, req, plan)
|
||||
}
|
||||
@@ -138,6 +141,152 @@ func Execute(ctx context.Context, req Request, plan Plan) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func executeCatalog(ctx context.Context, req Request, plan Plan) error {
|
||||
if plan.Action == ActionReplaceCatalog {
|
||||
if plan.ClearDestinationRoot {
|
||||
if err := req.DestinationBackend.DeletePrefix(ctx, req.DestinationBundlePath, storage.DeleteOptions{IgnoreMissing: true, PruneEmptyDirs: true}); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := ensureDestinationEmpty(ctx, req.DestinationBackend, req.DestinationBundlePath); err != nil {
|
||||
return err
|
||||
}
|
||||
} else if len(plan.CatalogOutputsToDelete) > 0 {
|
||||
if err := req.DestinationBackend.DeleteManagedOutputs(ctx, req.DestinationBundlePath, catalogOutputPaths(plan.CatalogOutputsToDelete), storage.DeleteOptions{IgnoreMissing: true, PruneEmptyDirs: true}); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
writtenOutputs := make([]Output, 0, len(plan.Outputs))
|
||||
newOutputs := make([]Output, 0, len(plan.Outputs))
|
||||
cleanup := func() {
|
||||
outputs := writtenOutputs
|
||||
if plan.Action == ActionUpsertAdditive {
|
||||
outputs = newOutputs
|
||||
}
|
||||
_ = req.DestinationBackend.DeleteManagedOutputs(ctx, req.DestinationBundlePath, ManagedOutputPaths(outputs), storage.DeleteOptions{IgnoreMissing: true, PruneEmptyDirs: true})
|
||||
}
|
||||
for _, output := range plan.Outputs {
|
||||
destinationPath, err := storage.Join(req.DestinationBundlePath, output.DestinationPath)
|
||||
if err != nil {
|
||||
cleanup()
|
||||
return err
|
||||
}
|
||||
created, err := catalogWriteCreatesOutput(ctx, req.DestinationBackend, destinationPath)
|
||||
if err != nil {
|
||||
cleanup()
|
||||
return err
|
||||
}
|
||||
data := output.Data
|
||||
if output.Kind == state.OutputKindSource {
|
||||
sourcePath, err := storage.Join(req.SourceBundle.RootRelativePath, output.SourcePath)
|
||||
if err != nil {
|
||||
cleanup()
|
||||
return err
|
||||
}
|
||||
data, err = req.SourceBackend.ReadFile(ctx, sourcePath)
|
||||
if err != nil {
|
||||
cleanup()
|
||||
return err
|
||||
}
|
||||
}
|
||||
if _, err := req.DestinationBackend.WriteFile(ctx, destinationPath, data, storage.WriteOptions{Overwrite: catalogOutputOverwriteAllowed(plan, output), PreferAtomic: true}); err != nil {
|
||||
cleanup()
|
||||
return err
|
||||
}
|
||||
writtenOutputs = append(writtenOutputs, output)
|
||||
if created {
|
||||
newOutputs = append(newOutputs, output)
|
||||
}
|
||||
}
|
||||
|
||||
catalogState := catalogStateForPlan(req, plan)
|
||||
if err := state.ValidateCatalog(catalogState); err != nil {
|
||||
cleanup()
|
||||
return err
|
||||
}
|
||||
data, err := json.MarshalIndent(catalogState, "", " ")
|
||||
if err != nil {
|
||||
cleanup()
|
||||
return err
|
||||
}
|
||||
data = append(data, '\n')
|
||||
statePath, err := storage.StatePath(req.DestinationBundlePath)
|
||||
if err != nil {
|
||||
cleanup()
|
||||
return err
|
||||
}
|
||||
if _, err := req.DestinationBackend.WriteFile(ctx, statePath, data, storage.WriteOptions{Overwrite: catalogStateWriteOverwrites(plan), PreferAtomic: true}); err != nil {
|
||||
cleanup()
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func catalogWriteCreatesOutput(ctx context.Context, backend storage.Backend, destinationPath string) (bool, error) {
|
||||
if _, err := backend.Stat(ctx, destinationPath); err == nil {
|
||||
return false, nil
|
||||
} else if storage.IsNotFound(err) {
|
||||
return true, nil
|
||||
} else {
|
||||
return false, err
|
||||
}
|
||||
}
|
||||
|
||||
func catalogOutputOverwriteAllowed(plan Plan, output Output) bool {
|
||||
if plan.ClearDestinationRoot {
|
||||
return false
|
||||
}
|
||||
if plan.SupersededLegacy != nil {
|
||||
return true
|
||||
}
|
||||
if plan.ExistingCatalog == nil {
|
||||
return false
|
||||
}
|
||||
_, ok := state.FindCatalogOutputByPath(plan.ExistingCatalog.Outputs, output.DestinationPath)
|
||||
return ok
|
||||
}
|
||||
|
||||
func catalogStateForPlan(req Request, plan Plan) state.CatalogState {
|
||||
now := requestTime(req)
|
||||
createdAt := now
|
||||
if plan.ExistingCatalog != nil {
|
||||
createdAt = plan.ExistingCatalog.CreatedAt
|
||||
}
|
||||
outputs := make([]state.CatalogOutputFile, 0, len(plan.CatalogOutputsToRetain)+len(plan.CatalogOutputsToWrite))
|
||||
outputs = append(outputs, plan.CatalogOutputsToRetain...)
|
||||
outputs = append(outputs, plan.CatalogOutputsToWrite...)
|
||||
sort.SliceStable(outputs, func(i, j int) bool {
|
||||
if outputs[i].Path != outputs[j].Path {
|
||||
return outputs[i].Path < outputs[j].Path
|
||||
}
|
||||
if outputs[i].PipelineID != outputs[j].PipelineID {
|
||||
return outputs[i].PipelineID < outputs[j].PipelineID
|
||||
}
|
||||
return outputs[i].DestinationID < outputs[j].DestinationID
|
||||
})
|
||||
return state.CatalogState{
|
||||
SchemaVersion: state.CatalogSchemaVersion,
|
||||
DistributorVersion: req.DistributorVersion,
|
||||
CreatedAt: createdAt,
|
||||
UpdatedAt: now,
|
||||
State: state.StatePolicy{Mode: state.StateModeCatalog},
|
||||
Outputs: outputs,
|
||||
}
|
||||
}
|
||||
|
||||
func catalogStateWriteOverwrites(plan Plan) bool {
|
||||
return plan.ExistingCatalog != nil || plan.SupersededLegacy != nil
|
||||
}
|
||||
|
||||
func catalogOutputPaths(outputs []state.CatalogOutputFile) []string {
|
||||
paths := make([]string, 0, len(outputs))
|
||||
for _, output := range outputs {
|
||||
paths = append(paths, output.Path)
|
||||
}
|
||||
return paths
|
||||
}
|
||||
|
||||
func executeSharedRoot(ctx context.Context, req Request, plan Plan) error {
|
||||
plan.Reconciliation = normalizeReconciliation(plan.Reconciliation)
|
||||
if plan.Action == ActionForceReplace {
|
||||
@@ -240,7 +389,7 @@ func ensureMergeOutputPaths(ctx context.Context, backend storage.Backend, bundle
|
||||
}
|
||||
|
||||
func usesSharedRootState(req Request, plan Plan) bool {
|
||||
return normalizeState(req.State).Mode == config.StateModeSharedRoot || plan.StateMode == config.StateModeSharedRoot
|
||||
return plan.StateMode == config.StateModeSharedRoot
|
||||
}
|
||||
|
||||
func outputManagedByExistingState(output Output, existing *state.DistributorState) bool {
|
||||
|
||||
@@ -2,420 +2,167 @@ package publish
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/bundle"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/config"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/state"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/storage"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/storage/fake"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/testutil"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/transform"
|
||||
)
|
||||
|
||||
func TestExecuteCleansUpAfterWriteFailure(t *testing.T) {
|
||||
sourceBackend := fake.New()
|
||||
destinationBackend := &failingBackend{Backend: fake.New(), failPath: "summary.txt"}
|
||||
sourceBundle := testutil.WriteFakeSourceBundle(t, sourceBackend, "", testutil.BundleOptions{})
|
||||
req := Request{
|
||||
PipelineID: "reports",
|
||||
DestinationID: "archive",
|
||||
SourceBundle: sourceBundle,
|
||||
SourceBackend: sourceBackend,
|
||||
DestinationBackend: destinationBackend,
|
||||
DestinationBundlePath: "",
|
||||
Publish: config.PublishPolicy{Source: true},
|
||||
Transfer: config.TransferPolicy{OnDestinationSame: config.TransferActionSkip, OnDestinationOlder: config.TransferActionReplace, OnDestinationNewer: config.TransferActionSkip, OnConflict: config.TransferActionFail},
|
||||
DistributorVersion: "test",
|
||||
func TestExecuteAdditiveWritesOutputsAndCatalog(t *testing.T) {
|
||||
_, destinationBackend, req := catalogPlanRequest(t, config.WorkflowAdditive)
|
||||
existing := baseCatalog(req)
|
||||
existing.Outputs = []state.CatalogOutputFile{
|
||||
catalogOutput(req, "reports", "archive", "report.md", state.OutputKindSource, planCreatedAt),
|
||||
catalogOutput(req, "reports", "web", "old.txt", state.OutputKindSource, planCreatedAt),
|
||||
}
|
||||
writeCatalogState(t, destinationBackend, "", existing)
|
||||
testutil.WriteFakeFile(t, destinationBackend, "report.md", "old report")
|
||||
testutil.WriteFakeFile(t, destinationBackend, "old.txt", "retained")
|
||||
|
||||
plan, err := Build(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("Build() error = %v", err)
|
||||
}
|
||||
if err := Execute(context.Background(), req, plan); err != nil {
|
||||
t.Fatalf("Execute() error = %v", err)
|
||||
}
|
||||
|
||||
testutil.AssertFakeFile(t, destinationBackend, "report.md", "# Report\nSunny.\n")
|
||||
testutil.AssertFakeFile(t, destinationBackend, "summary.txt", "Summary\n")
|
||||
testutil.AssertFakeFile(t, destinationBackend, "old.txt", "retained")
|
||||
catalog := readCatalogState(t, destinationBackend, "")
|
||||
if catalog.SchemaVersion != state.CatalogSchemaVersion || catalog.State.Mode != state.StateModeCatalog {
|
||||
t.Fatalf("catalog identity = schema %d mode %s", catalog.SchemaVersion, catalog.State.Mode)
|
||||
}
|
||||
if len(catalog.Outputs) != 3 {
|
||||
t.Fatalf("catalog outputs = %#v, want three outputs", catalog.Outputs)
|
||||
}
|
||||
report, ok := state.FindCatalogOutputByPath(catalog.Outputs, "report.md")
|
||||
if !ok {
|
||||
t.Fatalf("catalog outputs = %#v, want report.md", catalog.Outputs)
|
||||
}
|
||||
if !report.CreatedAt.Equal(planCreatedAt) || !report.UpdatedAt.Equal(planUpdatedAt) {
|
||||
t.Fatalf("report times = %s/%s, want created preserved and updated now", report.CreatedAt, report.UpdatedAt)
|
||||
}
|
||||
if report.SourcePath != "" {
|
||||
t.Fatalf("source catalog output source_path = %q, want empty", report.SourcePath)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteReplacementDeletesCurrentOwnerAndPreservesOtherOwners(t *testing.T) {
|
||||
_, destinationBackend, req := catalogPlanRequest(t, config.WorkflowReplacement)
|
||||
existing := baseCatalog(req)
|
||||
existing.Outputs = []state.CatalogOutputFile{
|
||||
catalogOutput(req, "reports", "archive", "report.md", state.OutputKindSource, planCreatedAt),
|
||||
catalogOutput(req, "reports", "archive", "stale.txt", state.OutputKindSource, planCreatedAt),
|
||||
catalogOutput(req, "reports", "web", "shared.txt", state.OutputKindSource, planCreatedAt),
|
||||
}
|
||||
writeCatalogState(t, destinationBackend, "", existing)
|
||||
testutil.WriteFakeFile(t, destinationBackend, "report.md", "old report")
|
||||
testutil.WriteFakeFile(t, destinationBackend, "stale.txt", "delete me")
|
||||
testutil.WriteFakeFile(t, destinationBackend, "shared.txt", "keep me")
|
||||
|
||||
plan, err := Build(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("Build() error = %v", err)
|
||||
}
|
||||
if err := Execute(context.Background(), req, plan); err != nil {
|
||||
t.Fatalf("Execute() error = %v", err)
|
||||
}
|
||||
|
||||
testutil.AssertFakeFile(t, destinationBackend, "report.md", "# Report\nSunny.\n")
|
||||
testutil.AssertFakeFile(t, destinationBackend, "summary.txt", "Summary\n")
|
||||
testutil.AssertFakeMissing(t, destinationBackend, "stale.txt")
|
||||
testutil.AssertFakeFile(t, destinationBackend, "shared.txt", "keep me")
|
||||
catalog := readCatalogState(t, destinationBackend, "")
|
||||
if _, ok := state.FindCatalogOutputByPath(catalog.Outputs, "stale.txt"); ok {
|
||||
t.Fatalf("catalog outputs = %#v, want stale.txt removed", catalog.Outputs)
|
||||
}
|
||||
if _, ok := state.FindCatalogOutputByPath(catalog.Outputs, "shared.txt"); !ok {
|
||||
t.Fatalf("catalog outputs = %#v, want shared.txt retained", catalog.Outputs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteSupersededReplacementClearsDestinationRootOnly(t *testing.T) {
|
||||
_, destinationBackend, req := catalogPlanRequest(t, config.WorkflowReplacement)
|
||||
req.DestinationBundlePath = "bundle"
|
||||
legacyState := testutil.DestinationState(req.SourceBundle.Manifest, testutil.DestinationStateOptions{})
|
||||
writeJSONState(t, destinationBackend, req.DestinationBundlePath, legacyState)
|
||||
testutil.WriteFakeFile(t, destinationBackend, "bundle/unplanned.txt", "remove")
|
||||
testutil.WriteFakeFile(t, destinationBackend, "outside.txt", "keep")
|
||||
|
||||
plan, err := Build(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("Build() error = %v", err)
|
||||
}
|
||||
if err := Execute(context.Background(), req, plan); err != nil {
|
||||
t.Fatalf("Execute() error = %v", err)
|
||||
}
|
||||
|
||||
testutil.AssertFakeMissing(t, destinationBackend, "bundle/unplanned.txt")
|
||||
testutil.AssertFakeFile(t, destinationBackend, "bundle/report.md", "# Report\nSunny.\n")
|
||||
testutil.AssertFakeFile(t, destinationBackend, "outside.txt", "keep")
|
||||
readCatalogState(t, destinationBackend, "bundle")
|
||||
}
|
||||
|
||||
func TestExecuteSupersededAdditiveLeavesUnplannedFilesUnmanaged(t *testing.T) {
|
||||
_, destinationBackend, req := catalogPlanRequest(t, config.WorkflowAdditive)
|
||||
legacyState := testutil.DestinationState(req.SourceBundle.Manifest, testutil.DestinationStateOptions{})
|
||||
writeJSONState(t, destinationBackend, "", legacyState)
|
||||
testutil.WriteFakeFile(t, destinationBackend, "report.md", "legacy report")
|
||||
testutil.WriteFakeFile(t, destinationBackend, "unplanned.txt", "leave me")
|
||||
|
||||
plan, err := Build(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("Build() error = %v", err)
|
||||
}
|
||||
if err := Execute(context.Background(), req, plan); err != nil {
|
||||
t.Fatalf("Execute() error = %v", err)
|
||||
}
|
||||
|
||||
testutil.AssertFakeFile(t, destinationBackend, "report.md", "# Report\nSunny.\n")
|
||||
testutil.AssertFakeFile(t, destinationBackend, "unplanned.txt", "leave me")
|
||||
catalog := readCatalogState(t, destinationBackend, "")
|
||||
if _, ok := state.FindCatalogOutputByPath(catalog.Outputs, "unplanned.txt"); ok {
|
||||
t.Fatalf("catalog outputs = %#v, want unplanned file omitted", catalog.Outputs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteFailedWriteDoesNotWriteCatalogState(t *testing.T) {
|
||||
_, destinationBackend, req := catalogPlanRequest(t, config.WorkflowAdditive)
|
||||
plan, err := Build(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("Build() error = %v", err)
|
||||
}
|
||||
if err := destinationBackend.AddDirectory("report.md"); err != nil {
|
||||
t.Fatalf("add conflicting directory: %v", err)
|
||||
}
|
||||
|
||||
err = Execute(context.Background(), req, plan)
|
||||
if err == nil {
|
||||
t.Fatal("Execute() error = nil, want error")
|
||||
t.Fatal("Execute() error = nil, want write failure")
|
||||
}
|
||||
found, err := destinationBackend.HasAny(context.Background(), "")
|
||||
if err != nil {
|
||||
t.Fatalf("HasAny() error = %v", err)
|
||||
}
|
||||
if found {
|
||||
t.Fatal("destination has content after failed execution")
|
||||
if _, statErr := destinationBackend.Stat(context.Background(), storage.StateFileName); !storage.IsNotFound(statErr) {
|
||||
t.Fatalf("state stat error = %v, want missing state", statErr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteReplaceDeletesOmittedManagedOutputs(t *testing.T) {
|
||||
sourceBackend := fake.New()
|
||||
sourceBundle := testutil.WriteFakeSourceBundle(t, sourceBackend, "", testutil.BundleOptions{
|
||||
Files: []testutil.SourceFile{{Path: "report.md", Data: "# Report\nNew.\n"}},
|
||||
})
|
||||
destinationBackend := fake.New()
|
||||
older := sourceBundle.Manifest
|
||||
older.Created = older.Created.Add(-time.Hour)
|
||||
older.Files = append([]bundle.ManifestFile(nil), testutil.ValidManifest(testutil.BundleOptions{}).Files...)
|
||||
older.Digest = bundle.BundleDigest(older.Files)
|
||||
testutil.WriteFakeDestinationState(t, destinationBackend, "", older, testutil.DestinationStateOptions{})
|
||||
|
||||
req := testRequest(sourceBackend, destinationBackend, sourceBundle, config.ReconciliationModeReplace)
|
||||
plan, err := Build(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("Build() error = %v", err)
|
||||
}
|
||||
if plan.Action != ActionReplaceOlder {
|
||||
t.Fatalf("plan action = %s, want replace_older", plan.Action)
|
||||
}
|
||||
if err := Execute(context.Background(), req, plan); err != nil {
|
||||
t.Fatalf("Execute() error = %v", err)
|
||||
}
|
||||
testutil.AssertFakeFile(t, destinationBackend, "report.md", "# Report\nNew.\n")
|
||||
testutil.AssertFakeMissing(t, destinationBackend, "summary.txt")
|
||||
destinationState := readFakeState(t, destinationBackend, "")
|
||||
if got, want := len(destinationState.Outputs), 1; got != want {
|
||||
t.Fatalf("state output count = %d, want %d", got, want)
|
||||
}
|
||||
if got, want := destinationState.Reconciliation.Mode, config.ReconciliationModeReplace; got != want {
|
||||
t.Fatalf("reconciliation mode = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteMergeRetainsOmittedAndOverwritesManagedOutputs(t *testing.T) {
|
||||
sourceBackend := fake.New()
|
||||
sourceBundle := testutil.WriteFakeSourceBundle(t, sourceBackend, "", testutil.BundleOptions{
|
||||
Files: []testutil.SourceFile{{Path: "report.md", Data: "# Report\nNew.\n"}},
|
||||
})
|
||||
destinationBackend := fake.New()
|
||||
older := sourceBundle.Manifest
|
||||
older.Created = older.Created.Add(-time.Hour)
|
||||
older.Files = append([]bundle.ManifestFile(nil), testutil.ValidManifest(testutil.BundleOptions{}).Files...)
|
||||
older.Digest = bundle.BundleDigest(older.Files)
|
||||
existingState := testutil.WriteFakeDestinationState(t, destinationBackend, "", older, testutil.DestinationStateOptions{})
|
||||
|
||||
req := testRequest(sourceBackend, destinationBackend, sourceBundle, config.ReconciliationModeMerge)
|
||||
plan, err := Build(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("Build() error = %v", err)
|
||||
}
|
||||
if err := Execute(context.Background(), req, plan); err != nil {
|
||||
t.Fatalf("Execute() error = %v", err)
|
||||
}
|
||||
testutil.AssertFakeFile(t, destinationBackend, "report.md", "# Report\nNew.\n")
|
||||
testutil.AssertFakeFile(t, destinationBackend, "summary.txt", "old")
|
||||
destinationState := readFakeState(t, destinationBackend, "")
|
||||
outputs := outputsByPath(destinationState.Outputs)
|
||||
if got, want := len(outputs), 2; got != want {
|
||||
t.Fatalf("state output count = %d, want %d", got, want)
|
||||
}
|
||||
if !outputs["summary.txt"].CreatedAt.Equal(existingState.Outputs[1].CreatedAt) || !outputs["summary.txt"].UpdatedAt.Equal(existingState.Outputs[1].UpdatedAt) {
|
||||
t.Fatalf("retained output timestamps = %#v, want existing %#v", outputs["summary.txt"], existingState.Outputs[1])
|
||||
}
|
||||
if !outputs["report.md"].CreatedAt.Equal(existingState.Outputs[0].CreatedAt) || !outputs["report.md"].UpdatedAt.After(existingState.Outputs[0].UpdatedAt) {
|
||||
t.Fatalf("updated output timestamps = %#v, want preserved created_at and newer updated_at", outputs["report.md"])
|
||||
}
|
||||
if got, want := destinationState.Reconciliation.Mode, config.ReconciliationModeMerge; got != want {
|
||||
t.Fatalf("reconciliation mode = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteReplaceConflictDeletesOmittedManagedOutputs(t *testing.T) {
|
||||
sourceBackend := fake.New()
|
||||
sourceBundle := testutil.WriteFakeSourceBundle(t, sourceBackend, "", testutil.BundleOptions{
|
||||
Files: []testutil.SourceFile{{Path: "report.md", Data: "# Report\nNew.\n"}},
|
||||
})
|
||||
destinationBackend := fake.New()
|
||||
conflict := sourceBundle.Manifest
|
||||
conflict.ID = "other.source"
|
||||
conflict.Files = append([]bundle.ManifestFile(nil), testutil.ValidManifest(testutil.BundleOptions{}).Files...)
|
||||
conflict.Digest = bundle.BundleDigest(conflict.Files)
|
||||
testutil.WriteFakeDestinationState(t, destinationBackend, "", conflict, testutil.DestinationStateOptions{})
|
||||
|
||||
req := testRequest(sourceBackend, destinationBackend, sourceBundle, config.ReconciliationModeMerge)
|
||||
req.Takeover = config.TakeoverPolicy{Mode: config.TakeoverModeNever}
|
||||
req.Transfer.OnConflict = config.TransferActionReplace
|
||||
plan, err := Build(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("Build() error = %v", err)
|
||||
}
|
||||
if plan.Action != ActionReplaceConflict {
|
||||
t.Fatalf("plan action = %s, want replace_conflict", plan.Action)
|
||||
}
|
||||
if err := Execute(context.Background(), req, plan); err != nil {
|
||||
t.Fatalf("Execute() error = %v", err)
|
||||
}
|
||||
testutil.AssertFakeFile(t, destinationBackend, "report.md", "# Report\nNew.\n")
|
||||
testutil.AssertFakeMissing(t, destinationBackend, "summary.txt")
|
||||
destinationState := readFakeState(t, destinationBackend, "")
|
||||
outputs := outputsByPath(destinationState.Outputs)
|
||||
if got, want := len(outputs), 1; got != want {
|
||||
t.Fatalf("state output count = %d, want %d", got, want)
|
||||
}
|
||||
if _, ok := outputs["summary.txt"]; ok {
|
||||
t.Fatalf("state retained summary.txt after replace_conflict: %#v", destinationState.Outputs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteReplaceNewerUsesManagedReplacement(t *testing.T) {
|
||||
sourceBackend := fake.New()
|
||||
sourceBundle := testutil.WriteFakeSourceBundle(t, sourceBackend, "", testutil.BundleOptions{
|
||||
Files: []testutil.SourceFile{{Path: "report.md", Data: "# Report\nNew.\n"}},
|
||||
})
|
||||
destinationBackend := fake.New()
|
||||
newer := sourceBundle.Manifest
|
||||
newer.Created = newer.Created.Add(time.Hour)
|
||||
newer.Files = append([]bundle.ManifestFile(nil), testutil.ValidManifest(testutil.BundleOptions{}).Files...)
|
||||
newer.Digest = bundle.BundleDigest(newer.Files)
|
||||
testutil.WriteFakeDestinationState(t, destinationBackend, "", newer, testutil.DestinationStateOptions{})
|
||||
|
||||
req := testRequest(sourceBackend, destinationBackend, sourceBundle, config.ReconciliationModeReplace)
|
||||
req.Transfer.OnDestinationNewer = config.TransferActionReplace
|
||||
plan, err := Build(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("Build() error = %v", err)
|
||||
}
|
||||
if plan.Action != ActionReplaceNewer {
|
||||
t.Fatalf("plan action = %s, want replace_newer", plan.Action)
|
||||
}
|
||||
if err := Execute(context.Background(), req, plan); err != nil {
|
||||
t.Fatalf("Execute() error = %v", err)
|
||||
}
|
||||
testutil.AssertFakeFile(t, destinationBackend, "report.md", "# Report\nNew.\n")
|
||||
testutil.AssertFakeMissing(t, destinationBackend, "summary.txt")
|
||||
}
|
||||
|
||||
func TestExecuteMergeFailsOnUnmanagedDestinationPathCollision(t *testing.T) {
|
||||
sourceBackend := fake.New()
|
||||
sourceBundle := testutil.WriteFakeSourceBundle(t, sourceBackend, "", testutil.BundleOptions{
|
||||
Files: []testutil.SourceFile{{Path: "report.md", Data: "# Report\nNew.\n"}},
|
||||
})
|
||||
destinationBackend := fake.New()
|
||||
older := sourceBundle.Manifest
|
||||
older.Created = older.Created.Add(-time.Hour)
|
||||
older.Files = []bundle.ManifestFile{{
|
||||
Path: "summary.txt",
|
||||
SHA256: bundle.FileDigest([]byte("Summary\n")),
|
||||
Size: int64(len("Summary\n")),
|
||||
}}
|
||||
older.Digest = bundle.BundleDigest(older.Files)
|
||||
testutil.WriteFakeDestinationState(t, destinationBackend, "", older, testutil.DestinationStateOptions{})
|
||||
testutil.WriteFakeFile(t, destinationBackend, "report.md", "unmanaged")
|
||||
|
||||
req := testRequest(sourceBackend, destinationBackend, sourceBundle, config.ReconciliationModeMerge)
|
||||
plan, err := Build(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("Build() error = %v", err)
|
||||
}
|
||||
err = Execute(context.Background(), req, plan)
|
||||
if err == nil {
|
||||
t.Fatal("Execute() error = nil, want unmanaged path collision")
|
||||
}
|
||||
testutil.AssertFakeFile(t, destinationBackend, "report.md", "unmanaged")
|
||||
testutil.AssertFakeFile(t, destinationBackend, "summary.txt", "old")
|
||||
}
|
||||
|
||||
func TestExecuteMergeFailureCleansUpOnlyNewOutputs(t *testing.T) {
|
||||
sourceBackend := fake.New()
|
||||
sourceBundle := testutil.WriteFakeSourceBundle(t, sourceBackend, "", testutil.BundleOptions{
|
||||
Files: []testutil.SourceFile{
|
||||
{Path: "report.md", Data: "# Report\nNew.\n"},
|
||||
{Path: "new.md", Data: "new\n"},
|
||||
{Path: "fail.md", Data: "fail\n"},
|
||||
},
|
||||
})
|
||||
destinationBackend := &failingBackend{Backend: fake.New(), failPath: "fail.md"}
|
||||
older := sourceBundle.Manifest
|
||||
older.Created = older.Created.Add(-time.Hour)
|
||||
older.Files = []bundle.ManifestFile{sourceBundle.Manifest.Files[0]}
|
||||
older.Digest = bundle.BundleDigest(older.Files)
|
||||
testutil.WriteFakeDestinationState(t, destinationBackend.Backend, "", older, testutil.DestinationStateOptions{})
|
||||
|
||||
req := testRequest(sourceBackend, destinationBackend, sourceBundle, config.ReconciliationModeMerge)
|
||||
plan, err := Build(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("Build() error = %v", err)
|
||||
}
|
||||
err = Execute(context.Background(), req, plan)
|
||||
if err == nil {
|
||||
t.Fatal("Execute() error = nil, want injected failure")
|
||||
}
|
||||
testutil.AssertFakeFile(t, destinationBackend.Backend, "report.md", "# Report\nNew.\n")
|
||||
testutil.AssertFakeMissing(t, destinationBackend.Backend, "new.md")
|
||||
testutil.AssertFakeMissing(t, destinationBackend.Backend, "fail.md")
|
||||
}
|
||||
|
||||
func TestExecuteFixedPathSupportsReconciliationModes(t *testing.T) {
|
||||
for _, mode := range []string{config.ReconciliationModeReplace, config.ReconciliationModeMerge} {
|
||||
t.Run(mode, func(t *testing.T) {
|
||||
sourceBackend := fake.New()
|
||||
sourceBundle := testutil.WriteFakeSourceBundle(t, sourceBackend, "new", testutil.BundleOptions{
|
||||
Files: []testutil.SourceFile{{Path: "report.md", Data: "# Report\nNew.\n"}},
|
||||
})
|
||||
destinationBackend := fake.New()
|
||||
older := sourceBundle.Manifest
|
||||
older.Created = older.Created.Add(-time.Hour)
|
||||
testutil.WriteFakeDestinationState(t, destinationBackend, "", older, testutil.DestinationStateOptions{})
|
||||
|
||||
req := testRequest(sourceBackend, destinationBackend, sourceBundle, mode)
|
||||
req.DestinationBundlePath = ""
|
||||
req.PathMapping = config.PathMappingFixed
|
||||
plan, err := Build(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("Build() error = %v", err)
|
||||
}
|
||||
if plan.Action != ActionReplaceOlder {
|
||||
t.Fatalf("plan action = %s, want replace_older", plan.Action)
|
||||
}
|
||||
if err := Execute(context.Background(), req, plan); err != nil {
|
||||
t.Fatalf("Execute() error = %v", err)
|
||||
}
|
||||
testutil.AssertFakeFile(t, destinationBackend, "report.md", "# Report\nNew.\n")
|
||||
destinationState := readFakeState(t, destinationBackend, "")
|
||||
if got, want := destinationState.Reconciliation.Mode, mode; got != want {
|
||||
t.Fatalf("state reconciliation mode = %q, want %q", got, want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteReconciliationModesHonorPublishPolicies(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
publish config.PublishPolicy
|
||||
transform config.Transform
|
||||
transformer TransformerResolver
|
||||
wantPaths []string
|
||||
}{
|
||||
{
|
||||
name: "source only",
|
||||
publish: config.PublishPolicy{Source: true},
|
||||
wantPaths: []string{"report.md"},
|
||||
},
|
||||
{
|
||||
name: "generated only",
|
||||
publish: config.PublishPolicy{HTML: true},
|
||||
transform: config.Transform{MarkdownToHTML: &config.MarkdownToHTML{Enabled: true, Mode: config.TransformModeSidecar}},
|
||||
transformer: testResolver{transform.MarkdownToHTML: testTransformer{outputs: []transform.Output{{
|
||||
Path: "report.html",
|
||||
SourcePath: "report.md",
|
||||
Transform: transform.MarkdownToHTML,
|
||||
Data: []byte("<h1>Report</h1>\n"),
|
||||
SHA256: bundle.FileDigest([]byte("<h1>Report</h1>\n")),
|
||||
Size: int64(len("<h1>Report</h1>\n")),
|
||||
}}}},
|
||||
wantPaths: []string{"report.html"},
|
||||
},
|
||||
{
|
||||
name: "source and generated",
|
||||
publish: config.PublishPolicy{Source: true, HTML: true},
|
||||
transform: config.Transform{MarkdownToHTML: &config.MarkdownToHTML{Enabled: true, Mode: config.TransformModeSidecar}},
|
||||
transformer: testResolver{transform.MarkdownToHTML: testTransformer{outputs: []transform.Output{{
|
||||
Path: "report.html",
|
||||
SourcePath: "report.md",
|
||||
Transform: transform.MarkdownToHTML,
|
||||
Data: []byte("<h1>Report</h1>\n"),
|
||||
SHA256: bundle.FileDigest([]byte("<h1>Report</h1>\n")),
|
||||
Size: int64(len("<h1>Report</h1>\n")),
|
||||
}}}},
|
||||
wantPaths: []string{"report.md", "report.html"},
|
||||
},
|
||||
}
|
||||
for _, mode := range []string{config.ReconciliationModeReplace, config.ReconciliationModeMerge} {
|
||||
for _, tt := range tests {
|
||||
t.Run(mode+" "+tt.name, func(t *testing.T) {
|
||||
sourceBackend := fake.New()
|
||||
sourceBundle := testutil.WriteFakeSourceBundle(t, sourceBackend, "", testutil.BundleOptions{
|
||||
Files: []testutil.SourceFile{{Path: "report.md", Data: "# Report\nNew.\n"}},
|
||||
})
|
||||
destinationBackend := fake.New()
|
||||
older := sourceBundle.Manifest
|
||||
older.Created = older.Created.Add(-time.Hour)
|
||||
testutil.WriteFakeDestinationState(t, destinationBackend, "", older, testutil.DestinationStateOptions{})
|
||||
|
||||
req := testRequest(sourceBackend, destinationBackend, sourceBundle, mode)
|
||||
req.Publish = tt.publish
|
||||
req.Transform = tt.transform
|
||||
req.Transformers = tt.transformer
|
||||
plan, err := Build(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("Build() error = %v", err)
|
||||
}
|
||||
if err := Execute(context.Background(), req, plan); err != nil {
|
||||
t.Fatalf("Execute() error = %v", err)
|
||||
}
|
||||
destinationState := readFakeState(t, destinationBackend, "")
|
||||
outputs := outputsByPath(destinationState.Outputs)
|
||||
for _, path := range tt.wantPaths {
|
||||
if _, ok := outputs[path]; !ok {
|
||||
t.Fatalf("state outputs = %#v, missing %s", destinationState.Outputs, path)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type failingBackend struct {
|
||||
*fake.Backend
|
||||
failPath string
|
||||
}
|
||||
|
||||
func (b *failingBackend) WriteFile(ctx context.Context, path string, data []byte, opts storage.WriteOptions) (storage.Entry, error) {
|
||||
if path == b.failPath {
|
||||
return storage.Entry{}, fmt.Errorf("injected write failure")
|
||||
}
|
||||
return b.Backend.WriteFile(ctx, path, data, opts)
|
||||
}
|
||||
|
||||
func (b *failingBackend) WriteFrom(ctx context.Context, path string, r io.Reader, opts storage.WriteOptions) (storage.Entry, error) {
|
||||
if path == b.failPath {
|
||||
return storage.Entry{}, fmt.Errorf("injected write failure")
|
||||
}
|
||||
return b.Backend.WriteFrom(ctx, path, r, opts)
|
||||
}
|
||||
|
||||
func testRequest(sourceBackend storage.Backend, destinationBackend storage.Backend, sourceBundle bundle.Bundle, reconciliationMode string) Request {
|
||||
return Request{
|
||||
PipelineID: "reports",
|
||||
DestinationID: "archive",
|
||||
SourceBundle: sourceBundle,
|
||||
SourceBackend: sourceBackend,
|
||||
DestinationBackend: destinationBackend,
|
||||
DestinationBundlePath: sourceBundle.RootRelativePath,
|
||||
Publish: config.PublishPolicy{Source: true},
|
||||
Reconciliation: config.ReconciliationPolicy{Mode: reconciliationMode},
|
||||
Transfer: config.TransferPolicy{
|
||||
OnDestinationSame: config.TransferActionSkip,
|
||||
OnDestinationOlder: config.TransferActionReplace,
|
||||
OnDestinationNewer: config.TransferActionSkip,
|
||||
OnConflict: config.TransferActionFail,
|
||||
},
|
||||
DistributorVersion: "test",
|
||||
}
|
||||
}
|
||||
|
||||
func readFakeState(t *testing.T, backend storage.Backend, bundlePath string) state.DistributorState {
|
||||
func readCatalogState(t *testing.T, backend *fake.Backend, relative string) state.CatalogState {
|
||||
t.Helper()
|
||||
statePath, err := storage.StatePath(bundlePath)
|
||||
statePath, err := storage.StatePath(relative)
|
||||
if err != nil {
|
||||
t.Fatalf("state path: %v", err)
|
||||
}
|
||||
data, err := backend.ReadFile(context.Background(), statePath)
|
||||
if err != nil {
|
||||
t.Fatalf("read state: %v", err)
|
||||
t.Fatalf("read catalog state: %v", err)
|
||||
}
|
||||
destinationState, err := state.Parse(data)
|
||||
catalog, err := state.ParseCatalog(data)
|
||||
if err != nil {
|
||||
t.Fatalf("parse state: %v", err)
|
||||
t.Fatalf("parse catalog state: %v", err)
|
||||
}
|
||||
return destinationState
|
||||
}
|
||||
|
||||
func outputsByPath(outputs []state.OutputFile) map[string]state.OutputFile {
|
||||
byPath := make(map[string]state.OutputFile, len(outputs))
|
||||
for _, output := range outputs {
|
||||
byPath[output.Path] = output
|
||||
}
|
||||
return byPath
|
||||
return catalog
|
||||
}
|
||||
|
||||
@@ -1,222 +0,0 @@
|
||||
package publish
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/bundle"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/config"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/storage/fake"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/testutil"
|
||||
)
|
||||
|
||||
func TestBuildPlansForcedReplacementOnlyWhenExplicit(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
prepare func(t *testing.T, backend *fake.Backend, source bundle.Manifest)
|
||||
transfer config.TransferPolicy
|
||||
wantReason string
|
||||
forceAction bool
|
||||
}{
|
||||
{
|
||||
name: "unmanaged content",
|
||||
prepare: func(t *testing.T, backend *fake.Backend, source bundle.Manifest) {
|
||||
t.Helper()
|
||||
testutil.WriteFakeFile(t, backend, "bundle/old.txt", "old")
|
||||
},
|
||||
transfer: defaultTransfer(),
|
||||
wantReason: "fail_unmanaged",
|
||||
forceAction: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
sourceBackend := fake.New()
|
||||
sourceBundle := testutil.WriteFakeSourceBundle(t, sourceBackend, "bundle", testutil.BundleOptions{})
|
||||
destinationBackend := fake.New()
|
||||
tt.prepare(t, destinationBackend, sourceBundle.Manifest)
|
||||
|
||||
req := forceRequest(sourceBackend, destinationBackend, sourceBundle, tt.transfer)
|
||||
_, err := Build(context.Background(), req)
|
||||
if err == nil || !strings.Contains(err.Error(), tt.wantReason) {
|
||||
t.Fatalf("Build() error = %v, want %q", err, tt.wantReason)
|
||||
}
|
||||
|
||||
req.Force = true
|
||||
plan, err := Build(context.Background(), req)
|
||||
if tt.forceAction {
|
||||
if err != nil {
|
||||
t.Fatalf("Build() with force error = %v", err)
|
||||
}
|
||||
if plan.Action != ActionForceReplace || !plan.Force {
|
||||
t.Fatalf("forced plan action = %s force=%t", plan.Action, plan.Force)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildPlansConflictReplacementWithoutForce(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
prepare func(t *testing.T, backend *fake.Backend, source bundle.Manifest)
|
||||
wantReason string
|
||||
}{
|
||||
{
|
||||
name: "different source id",
|
||||
prepare: func(t *testing.T, backend *fake.Backend, source bundle.Manifest) {
|
||||
t.Helper()
|
||||
conflict := source
|
||||
conflict.ID = "other.source"
|
||||
testutil.WriteFakeDestinationState(t, backend, "bundle", conflict, testutil.DestinationStateOptions{})
|
||||
},
|
||||
wantReason: "destination source id differs",
|
||||
},
|
||||
{
|
||||
name: "same created digest conflict",
|
||||
prepare: func(t *testing.T, backend *fake.Backend, source bundle.Manifest) {
|
||||
t.Helper()
|
||||
conflict := testutil.ValidManifest(testutil.BundleOptions{Files: []testutil.SourceFile{{Path: "report.md", Data: "# Different\n"}}})
|
||||
testutil.WriteFakeDestinationState(t, backend, "bundle", conflict, testutil.DestinationStateOptions{})
|
||||
},
|
||||
wantReason: "same id and created time but different digest",
|
||||
},
|
||||
{
|
||||
name: "pipeline mismatch",
|
||||
prepare: func(t *testing.T, backend *fake.Backend, source bundle.Manifest) {
|
||||
t.Helper()
|
||||
testutil.WriteFakeDestinationState(t, backend, "bundle", source, testutil.DestinationStateOptions{PipelineID: "other-pipeline"})
|
||||
},
|
||||
wantReason: "pipeline id",
|
||||
},
|
||||
{
|
||||
name: "destination mismatch",
|
||||
prepare: func(t *testing.T, backend *fake.Backend, source bundle.Manifest) {
|
||||
t.Helper()
|
||||
testutil.WriteFakeDestinationState(t, backend, "bundle", source, testutil.DestinationStateOptions{DestinationID: "other-destination"})
|
||||
},
|
||||
wantReason: "destination id",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
sourceBackend := fake.New()
|
||||
sourceBundle := testutil.WriteFakeSourceBundle(t, sourceBackend, "bundle", testutil.BundleOptions{})
|
||||
destinationBackend := fake.New()
|
||||
tt.prepare(t, destinationBackend, sourceBundle.Manifest)
|
||||
|
||||
req := forceRequest(sourceBackend, destinationBackend, sourceBundle, conflictReplaceTransfer())
|
||||
plan, err := Build(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("Build() error = %v", err)
|
||||
}
|
||||
if plan.Action != ActionReplaceConflict || plan.Force {
|
||||
t.Fatalf("plan action = %s force=%t, want replace_conflict without force", plan.Action, plan.Force)
|
||||
}
|
||||
if !strings.Contains(plan.Reason, tt.wantReason) {
|
||||
t.Fatalf("plan reason = %q, want %q", plan.Reason, tt.wantReason)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildPlansNewerReplacementWithoutForce(t *testing.T) {
|
||||
sourceBackend := fake.New()
|
||||
sourceBundle := testutil.WriteFakeSourceBundle(t, sourceBackend, "bundle", testutil.BundleOptions{})
|
||||
destinationBackend := fake.New()
|
||||
newer := sourceBundle.Manifest
|
||||
newer.Created = newer.Created.AddDate(0, 0, 1)
|
||||
testutil.WriteFakeDestinationState(t, destinationBackend, "bundle", newer, testutil.DestinationStateOptions{})
|
||||
|
||||
req := forceRequest(sourceBackend, destinationBackend, sourceBundle, newerReplaceTransfer())
|
||||
plan, err := Build(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("Build() error = %v", err)
|
||||
}
|
||||
if plan.Action != ActionReplaceNewer || plan.Force {
|
||||
t.Fatalf("plan action = %s force=%t, want replace_newer without force", plan.Action, plan.Force)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildRequiresConflictPolicyForStateConflicts(t *testing.T) {
|
||||
sourceBackend := fake.New()
|
||||
sourceBundle := testutil.WriteFakeSourceBundle(t, sourceBackend, "bundle", testutil.BundleOptions{})
|
||||
destinationBackend := fake.New()
|
||||
conflict := sourceBundle.Manifest
|
||||
conflict.ID = "other.source"
|
||||
testutil.WriteFakeDestinationState(t, destinationBackend, "bundle", conflict, testutil.DestinationStateOptions{})
|
||||
|
||||
req := forceRequest(sourceBackend, destinationBackend, sourceBundle, defaultTransfer())
|
||||
req.Force = true
|
||||
_, err := Build(context.Background(), req)
|
||||
if err == nil || !strings.Contains(err.Error(), "destination source id differs") {
|
||||
t.Fatalf("Build() error = %v, want conservative conflict", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteForcedReplacementDeletesOnlyBundlePath(t *testing.T) {
|
||||
sourceBackend := fake.New()
|
||||
sourceBundle := testutil.WriteFakeSourceBundle(t, sourceBackend, "bundle", testutil.BundleOptions{})
|
||||
destinationBackend := fake.New()
|
||||
testutil.WriteFakeFile(t, destinationBackend, "bundle/old.txt", "old")
|
||||
testutil.WriteFakeFile(t, destinationBackend, "bundle/nested/old.txt", "old")
|
||||
testutil.WriteFakeFile(t, destinationBackend, "bundle-sibling/keep.txt", "keep")
|
||||
testutil.WriteFakeFile(t, destinationBackend, "outside.txt", "outside")
|
||||
|
||||
req := forceRequest(sourceBackend, destinationBackend, sourceBundle, defaultTransfer())
|
||||
req.Force = true
|
||||
plan, err := Build(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("Build() error = %v", err)
|
||||
}
|
||||
if plan.Action != ActionForceReplace {
|
||||
t.Fatalf("plan action = %s, want force_replace", plan.Action)
|
||||
}
|
||||
if err := Execute(context.Background(), req, plan); err != nil {
|
||||
t.Fatalf("Execute() error = %v", err)
|
||||
}
|
||||
testutil.AssertFakeFile(t, destinationBackend, "bundle/report.md", "# Report\nSunny.\n")
|
||||
testutil.AssertFakeMissing(t, destinationBackend, "bundle/old.txt")
|
||||
testutil.AssertFakeMissing(t, destinationBackend, "bundle/nested/old.txt")
|
||||
testutil.AssertFakeFile(t, destinationBackend, "bundle-sibling/keep.txt", "keep")
|
||||
testutil.AssertFakeFile(t, destinationBackend, "outside.txt", "outside")
|
||||
}
|
||||
|
||||
func forceRequest(sourceBackend, destinationBackend *fake.Backend, sourceBundle bundle.Bundle, transfer config.TransferPolicy) Request {
|
||||
return Request{
|
||||
PipelineID: "reports",
|
||||
DestinationID: "archive",
|
||||
SourceBundle: sourceBundle,
|
||||
SourceBackend: sourceBackend,
|
||||
DestinationBackend: destinationBackend,
|
||||
DestinationBundlePath: sourceBundle.RootRelativePath,
|
||||
Publish: config.PublishPolicy{Source: true},
|
||||
Takeover: config.TakeoverPolicy{Mode: config.TakeoverModeNever},
|
||||
Transfer: transfer,
|
||||
DistributorVersion: "test",
|
||||
}
|
||||
}
|
||||
|
||||
func defaultTransfer() config.TransferPolicy {
|
||||
return config.TransferPolicy{
|
||||
OnDestinationSame: config.TransferActionSkip,
|
||||
OnDestinationOlder: config.TransferActionReplace,
|
||||
OnDestinationNewer: config.TransferActionSkip,
|
||||
OnConflict: config.TransferActionFail,
|
||||
}
|
||||
}
|
||||
|
||||
func conflictReplaceTransfer() config.TransferPolicy {
|
||||
transfer := defaultTransfer()
|
||||
transfer.OnConflict = config.TransferActionReplace
|
||||
return transfer
|
||||
}
|
||||
|
||||
func newerReplaceTransfer() config.TransferPolicy {
|
||||
transfer := defaultTransfer()
|
||||
transfer.OnDestinationNewer = config.TransferActionReplace
|
||||
return transfer
|
||||
}
|
||||
@@ -194,12 +194,6 @@ func TestBuildRejectsHTMLWithoutTransform(t *testing.T) {
|
||||
DestinationBundlePath: "",
|
||||
SourceBundle: sourceBundle,
|
||||
Publish: config.PublishPolicy{HTML: true},
|
||||
Transfer: config.TransferPolicy{
|
||||
OnDestinationSame: config.TransferActionSkip,
|
||||
OnDestinationOlder: config.TransferActionReplace,
|
||||
OnDestinationNewer: config.TransferActionSkip,
|
||||
OnConflict: config.TransferActionFail,
|
||||
},
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("Build() error = nil, want missing transform error")
|
||||
|
||||
@@ -3,6 +3,7 @@ package publish
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/bundle"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/config"
|
||||
@@ -24,6 +25,8 @@ const (
|
||||
ActionFailUnmanaged Action = "fail_unmanaged"
|
||||
ActionForceReplace Action = "force_replace"
|
||||
ActionReplaceTakeover Action = "replace_takeover"
|
||||
ActionUpsertAdditive Action = "upsert_additive"
|
||||
ActionReplaceCatalog Action = "replace_catalog"
|
||||
)
|
||||
|
||||
type Request struct {
|
||||
@@ -37,19 +40,28 @@ type Request struct {
|
||||
Publish config.PublishPolicy
|
||||
Transform config.Transform
|
||||
Links *config.Links
|
||||
State config.StatePolicy
|
||||
Reconciliation config.ReconciliationPolicy
|
||||
Takeover config.TakeoverPolicy
|
||||
Workflow string
|
||||
Transformers TransformerResolver
|
||||
Transfer config.TransferPolicy
|
||||
DistributorVersion string
|
||||
Force bool
|
||||
Now time.Time
|
||||
}
|
||||
|
||||
type TransformerResolver interface {
|
||||
Get(name string) (transform.Transformer, bool)
|
||||
}
|
||||
|
||||
type Output struct {
|
||||
SourcePath string
|
||||
DestinationPath string
|
||||
Kind string
|
||||
Transform string
|
||||
URL string
|
||||
Data []byte
|
||||
SHA256 string
|
||||
Size int64
|
||||
}
|
||||
|
||||
type Plan struct {
|
||||
PipelineID string
|
||||
DestinationID string
|
||||
@@ -61,11 +73,20 @@ type Plan struct {
|
||||
Reason string
|
||||
Force bool
|
||||
PrimaryURL string
|
||||
StateMode string
|
||||
Workflow string
|
||||
OwnerScope state.OwnerScope
|
||||
Outputs []Output
|
||||
ExistingCatalog *state.CatalogState
|
||||
SupersededLegacy *state.SupersededLegacyState
|
||||
CatalogOutputsToWrite []state.CatalogOutputFile
|
||||
CatalogOutputsToRetain []state.CatalogOutputFile
|
||||
CatalogOutputsToDelete []state.CatalogOutputFile
|
||||
ClearDestinationRoot bool
|
||||
|
||||
// Retained while the executor is migrated to catalog state.
|
||||
StateMode string
|
||||
Reconciliation config.ReconciliationPolicy
|
||||
TakeoverMode string
|
||||
Outputs []Output
|
||||
ExistingState *state.DistributorState
|
||||
ExistingSharedRoot *state.SharedRootState
|
||||
OtherOwnerOutputs []state.SharedRootOutputFile
|
||||
@@ -75,15 +96,13 @@ type Plan struct {
|
||||
OwnerOutputsToWrite []Output
|
||||
}
|
||||
|
||||
type Output struct {
|
||||
SourcePath string
|
||||
DestinationPath string
|
||||
Kind string
|
||||
Transform string
|
||||
URL string
|
||||
Data []byte
|
||||
SHA256 string
|
||||
Size int64
|
||||
type catalogPlanDetails struct {
|
||||
Action Action
|
||||
Reason string
|
||||
CatalogOutputsToWrite []state.CatalogOutputFile
|
||||
CatalogOutputsToRetain []state.CatalogOutputFile
|
||||
CatalogOutputsToDelete []state.CatalogOutputFile
|
||||
ClearDestinationRoot bool
|
||||
}
|
||||
|
||||
func Build(ctx context.Context, req Request) (Plan, error) {
|
||||
@@ -102,14 +121,9 @@ func Build(ctx context.Context, req Request) (Plan, error) {
|
||||
if err != nil {
|
||||
return Plan{}, err
|
||||
}
|
||||
reconciliation := normalizeReconciliation(req.Reconciliation)
|
||||
stateMode := normalizeState(req.State).Mode
|
||||
comparison := compareDestination(req, status)
|
||||
action, reason := actionForComparison(comparison, req.Transfer, req.Force)
|
||||
if takeoverActionAllowed(req, status, comparison, stateMode, action) {
|
||||
action = ActionReplaceTakeover
|
||||
reason = comparison.Reason
|
||||
}
|
||||
workflow := normalizeWorkflow(req.Workflow)
|
||||
scope := state.CurrentOwnerScope(req.PipelineID, req.DestinationID)
|
||||
now := requestTime(req)
|
||||
plan := Plan{
|
||||
PipelineID: req.PipelineID,
|
||||
DestinationID: req.DestinationID,
|
||||
@@ -117,36 +131,37 @@ func Build(ctx context.Context, req Request) (Plan, error) {
|
||||
BundlePath: req.SourceBundle.RootRelativePath,
|
||||
DestinationBundlePath: req.DestinationBundlePath,
|
||||
PathMapping: req.PathMapping,
|
||||
Action: action,
|
||||
Reason: reason,
|
||||
Force: action == ActionForceReplace,
|
||||
PrimaryURL: primaryURL,
|
||||
StateMode: stateMode,
|
||||
OwnerScope: state.CurrentOwnerScope(req.PipelineID, req.DestinationID),
|
||||
Reconciliation: reconciliation,
|
||||
TakeoverMode: normalizeTakeover(req.Takeover).Mode,
|
||||
Workflow: workflow,
|
||||
OwnerScope: scope,
|
||||
Outputs: outputs,
|
||||
ExistingState: status.State,
|
||||
ExistingSharedRoot: status.SharedRoot,
|
||||
ExistingCatalog: status.Catalog,
|
||||
SupersededLegacy: status.SupersededLegacy,
|
||||
}
|
||||
if stateMode == config.StateModeSharedRoot {
|
||||
sharedDetails, err := planSharedRootOwner(ctx, req, status, action, reconciliation, outputs)
|
||||
plan.Action = sharedDetails.Action
|
||||
if sharedDetails.Reason != "" {
|
||||
plan.Reason = sharedDetails.Reason
|
||||
if status.StateErr != nil {
|
||||
plan.Action = ActionFailConflict
|
||||
plan.Reason = status.StateErr.Error()
|
||||
return plan, fmt.Errorf("%s: %s", plan.Action, plan.Reason)
|
||||
}
|
||||
plan.OtherOwnerOutputs = sharedDetails.OtherOwnerOutputs
|
||||
plan.TakenOverOwnerOutputs = sharedDetails.TakenOverOwnerOutputs
|
||||
plan.RetainedOwnerOutputs = sharedDetails.RetainedOwnerOutputs
|
||||
plan.OwnerOutputsToDelete = sharedDetails.OwnerOutputsToDelete
|
||||
plan.OwnerOutputsToWrite = sharedDetails.OwnerOutputsToWrite
|
||||
|
||||
var details catalogPlanDetails
|
||||
switch {
|
||||
case status.Catalog != nil:
|
||||
details, err = planExistingCatalog(ctx, req, *status.Catalog, outputs, workflow, scope, now)
|
||||
case status.SupersededLegacy != nil:
|
||||
details = planSupersededLegacy(req, outputs, workflow, scope, now)
|
||||
default:
|
||||
details, err = planWithoutCatalog(ctx, req, outputs, workflow, scope, now)
|
||||
}
|
||||
plan.Action = details.Action
|
||||
plan.Reason = details.Reason
|
||||
plan.CatalogOutputsToWrite = details.CatalogOutputsToWrite
|
||||
plan.CatalogOutputsToRetain = details.CatalogOutputsToRetain
|
||||
plan.CatalogOutputsToDelete = details.CatalogOutputsToDelete
|
||||
plan.ClearDestinationRoot = details.ClearDestinationRoot
|
||||
if err != nil {
|
||||
return plan, err
|
||||
}
|
||||
}
|
||||
if plan.Action == ActionFailConflict || plan.Action == ActionFailUnmanaged {
|
||||
return plan, fmt.Errorf("%s: %s", plan.Action, plan.Reason)
|
||||
}
|
||||
return plan, nil
|
||||
}
|
||||
|
||||
@@ -166,165 +181,134 @@ func validateRequest(req Request) error {
|
||||
if err := config.ValidatePublishTransformPolicy(req.Publish, req.Transform); err != nil {
|
||||
return fmt.Errorf("publish/transform policy: %w", err)
|
||||
}
|
||||
switch normalizeReconciliation(req.Reconciliation).Mode {
|
||||
case config.ReconciliationModeReplace, config.ReconciliationModeMerge:
|
||||
switch normalizeWorkflow(req.Workflow) {
|
||||
case config.WorkflowAdditive, config.WorkflowReplacement:
|
||||
default:
|
||||
return fmt.Errorf("reconciliation.mode must be %s or %s", config.ReconciliationModeReplace, config.ReconciliationModeMerge)
|
||||
}
|
||||
switch normalizeState(req.State).Mode {
|
||||
case config.StateModeSingleOwner, config.StateModeSharedRoot:
|
||||
default:
|
||||
return fmt.Errorf("state.mode must be %s or %s", config.StateModeSingleOwner, config.StateModeSharedRoot)
|
||||
}
|
||||
switch normalizeTakeover(req.Takeover).Mode {
|
||||
case config.TakeoverModeSamePipeline, config.TakeoverModeSameSource, config.TakeoverModeAnyManaged, config.TakeoverModeNever:
|
||||
default:
|
||||
return fmt.Errorf("takeover.mode must be %s, %s, %s, or %s", config.TakeoverModeSamePipeline, config.TakeoverModeSameSource, config.TakeoverModeAnyManaged, config.TakeoverModeNever)
|
||||
return fmt.Errorf("destination.workflow must be %s or %s", config.WorkflowAdditive, config.WorkflowReplacement)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func normalizeReconciliation(policy config.ReconciliationPolicy) config.ReconciliationPolicy {
|
||||
if policy.Mode == "" {
|
||||
policy.Mode = config.ReconciliationModeReplace
|
||||
func normalizeWorkflow(workflow string) string {
|
||||
if workflow == "" {
|
||||
return config.WorkflowAdditive
|
||||
}
|
||||
return policy
|
||||
return workflow
|
||||
}
|
||||
|
||||
func normalizeState(policy config.StatePolicy) config.StatePolicy {
|
||||
if policy.Mode == "" {
|
||||
policy.Mode = config.StateModeSingleOwner
|
||||
func requestTime(req Request) time.Time {
|
||||
if req.Now.IsZero() {
|
||||
return time.Now().UTC()
|
||||
}
|
||||
return policy
|
||||
return req.Now.UTC()
|
||||
}
|
||||
|
||||
func normalizeTakeover(policy config.TakeoverPolicy) config.TakeoverPolicy {
|
||||
if policy.Mode == "" {
|
||||
policy.Mode = config.TakeoverModeSamePipeline
|
||||
func planExistingCatalog(ctx context.Context, req Request, catalog state.CatalogState, outputs []Output, workflow string, scope state.OwnerScope, now time.Time) (catalogPlanDetails, error) {
|
||||
if err := rejectCatalogUnmanagedCollisions(ctx, req.DestinationBackend, req.DestinationBundlePath, catalog.Outputs, outputs); err != nil {
|
||||
return catalogPlanDetails{
|
||||
Action: ActionFailUnmanaged,
|
||||
Reason: err.Error(),
|
||||
}, fmt.Errorf("%s: %s", ActionFailUnmanaged, err)
|
||||
}
|
||||
return policy
|
||||
}
|
||||
|
||||
func compareDestination(req Request, status state.DestinationStatus) state.Comparison {
|
||||
if normalizeState(req.State).Mode == config.StateModeSharedRoot {
|
||||
scope := state.CurrentOwnerScope(req.PipelineID, req.DestinationID)
|
||||
comparison := state.CompareSharedRootOwner(req.SourceBundle.Manifest, scope, status)
|
||||
if req.PathMapping != config.PathMappingFixed || comparison.Outcome != state.OutcomeDifferentSourceConflict {
|
||||
return comparison
|
||||
planned := outputPathSet(outputs)
|
||||
details := catalogPlanDetails{
|
||||
Action: actionForWorkflow(workflow),
|
||||
CatalogOutputsToWrite: catalogOutputsForPlan(req, outputs, catalog.Outputs, scope, now),
|
||||
}
|
||||
destinationManifest, ok := sharedRootComparisonManifest(status, scope)
|
||||
if !ok {
|
||||
return comparison
|
||||
}
|
||||
if destinationManifest.Created.Before(req.SourceBundle.Manifest.Created) {
|
||||
return state.Comparison{Outcome: state.OutcomeDestinationOlder, Reason: "fixed destination source is older than selected source"}
|
||||
}
|
||||
if destinationManifest.Created.After(req.SourceBundle.Manifest.Created) {
|
||||
return state.Comparison{
|
||||
Outcome: state.OutcomeDestinationNewer,
|
||||
Reason: "fixed destination source is newer than selected source",
|
||||
Detail: state.ComparisonDetail{
|
||||
Kind: state.ComparisonDetailDestinationNewer,
|
||||
CurrentSourceID: req.SourceBundle.Manifest.ID,
|
||||
DestinationSourceID: destinationManifest.ID,
|
||||
},
|
||||
}
|
||||
}
|
||||
return comparison
|
||||
}
|
||||
comparison := state.Compare(req.SourceBundle.Manifest, req.PipelineID, req.DestinationID, status)
|
||||
return comparison
|
||||
}
|
||||
|
||||
func sharedRootComparisonManifest(status state.DestinationStatus, scope state.OwnerScope) (bundle.Manifest, bool) {
|
||||
if status.SharedRoot != nil {
|
||||
return status.SharedRoot.SourceManifest(scope)
|
||||
}
|
||||
if status.State != nil && status.State.PipelineID == scope.PipelineID && status.State.DestinationID == scope.DestinationID {
|
||||
return status.State.Source.Manifest, true
|
||||
}
|
||||
return bundle.Manifest{}, false
|
||||
}
|
||||
|
||||
type sharedRootPlanDetails struct {
|
||||
Action Action
|
||||
Reason string
|
||||
OtherOwnerOutputs []state.SharedRootOutputFile
|
||||
TakenOverOwnerOutputs []state.SharedRootOutputFile
|
||||
RetainedOwnerOutputs []state.SharedRootOutputFile
|
||||
OwnerOutputsToDelete []state.SharedRootOutputFile
|
||||
OwnerOutputsToWrite []Output
|
||||
}
|
||||
|
||||
func planSharedRootOwner(ctx context.Context, req Request, status state.DestinationStatus, action Action, reconciliation config.ReconciliationPolicy, outputs []Output) (sharedRootPlanDetails, error) {
|
||||
scope := state.CurrentOwnerScope(req.PipelineID, req.DestinationID)
|
||||
details := sharedRootPlanDetails{Action: action}
|
||||
if !isWriteAction(action) {
|
||||
details.OtherOwnerOutputs = otherOwnerOutputs(status, scope)
|
||||
return details, nil
|
||||
}
|
||||
|
||||
plannedPaths := outputPaths(outputs)
|
||||
if action == ActionForceReplace {
|
||||
details.OwnerOutputsToWrite = append([]Output(nil), outputs...)
|
||||
return details, nil
|
||||
}
|
||||
conflicts := sharedRootPathOwnershipConflicts(status, scope, plannedPaths)
|
||||
if len(conflicts) > 0 {
|
||||
conflictAction := ActionReplaceTakeover
|
||||
for _, conflict := range conflicts {
|
||||
if sharedRootTakeoverAllowed(req, status, conflict) {
|
||||
continue
|
||||
}
|
||||
if req.Transfer.OnConflict == config.TransferActionReplace {
|
||||
conflictAction = ActionReplaceConflict
|
||||
continue
|
||||
}
|
||||
reason := sharedRootOwnershipConflictReason(conflict)
|
||||
details.Action = ActionFailConflict
|
||||
details.Reason = reason
|
||||
return details, fmt.Errorf("%s: %s", ActionFailConflict, reason)
|
||||
}
|
||||
details.Action = conflictAction
|
||||
details.Reason = sharedRootOwnershipConflictReason(conflicts[0])
|
||||
details.TakenOverOwnerOutputs = sharedRootConflictOutputs(status.SharedRoot, conflicts)
|
||||
}
|
||||
if err := rejectSharedRootUnmanagedCollisions(ctx, req.DestinationBackend, req.DestinationBundlePath, status, scope, plannedPaths); err != nil {
|
||||
details.Action = ActionFailUnmanaged
|
||||
details.Reason = err.Error()
|
||||
return details, fmt.Errorf("%s: %s", ActionFailUnmanaged, err)
|
||||
}
|
||||
|
||||
takenOverPaths := sharedRootOutputPathSet(details.TakenOverOwnerOutputs)
|
||||
details.OtherOwnerOutputs = otherOwnerOutputsExcept(status, scope, takenOverPaths)
|
||||
ownerOutputs := currentOwnerOutputs(status, scope)
|
||||
planned := make(map[string]struct{}, len(plannedPaths))
|
||||
for _, path := range plannedPaths {
|
||||
planned[path] = struct{}{}
|
||||
}
|
||||
for _, output := range ownerOutputs {
|
||||
for _, output := range catalog.Outputs {
|
||||
if _, exists := planned[output.Path]; exists {
|
||||
continue
|
||||
}
|
||||
if details.Action == ActionReplaceTakeover || details.Action == ActionReplaceConflict || (isReconciliationReplacementAction(details.Action) && reconciliation.Mode == config.ReconciliationModeReplace) {
|
||||
details.OwnerOutputsToDelete = append(details.OwnerOutputsToDelete, output)
|
||||
if workflow == config.WorkflowReplacement && output.PipelineID == scope.PipelineID && output.DestinationID == scope.DestinationID {
|
||||
details.CatalogOutputsToDelete = append(details.CatalogOutputsToDelete, output)
|
||||
continue
|
||||
}
|
||||
if isReconciliationReplacementAction(details.Action) && reconciliation.Mode == config.ReconciliationModeMerge {
|
||||
details.RetainedOwnerOutputs = append(details.RetainedOwnerOutputs, output)
|
||||
details.CatalogOutputsToRetain = append(details.CatalogOutputsToRetain, output)
|
||||
}
|
||||
}
|
||||
details.OwnerOutputsToWrite = append([]Output(nil), outputs...)
|
||||
return details, nil
|
||||
}
|
||||
|
||||
func isWriteAction(action Action) bool {
|
||||
switch action {
|
||||
case ActionPublishNew, ActionReplaceOlder, ActionReplaceConflict, ActionReplaceNewer, ActionReplaceTakeover, ActionForceReplace:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
func planSupersededLegacy(req Request, outputs []Output, workflow string, scope state.OwnerScope, now time.Time) catalogPlanDetails {
|
||||
details := catalogPlanDetails{
|
||||
Action: actionForWorkflow(workflow),
|
||||
CatalogOutputsToWrite: catalogOutputsForPlan(req, outputs, nil, scope, now),
|
||||
}
|
||||
if workflow == config.WorkflowReplacement {
|
||||
details.ClearDestinationRoot = true
|
||||
}
|
||||
return details
|
||||
}
|
||||
|
||||
func planWithoutCatalog(ctx context.Context, req Request, outputs []Output, workflow string, scope state.OwnerScope, now time.Time) (catalogPlanDetails, error) {
|
||||
if err := rejectCatalogUnmanagedCollisions(ctx, req.DestinationBackend, req.DestinationBundlePath, nil, outputs); err != nil {
|
||||
return catalogPlanDetails{
|
||||
Action: ActionFailUnmanaged,
|
||||
Reason: err.Error(),
|
||||
}, fmt.Errorf("%s: %s", ActionFailUnmanaged, err)
|
||||
}
|
||||
return catalogPlanDetails{
|
||||
Action: ActionPublishNew,
|
||||
CatalogOutputsToWrite: catalogOutputsForPlan(req, outputs, nil, scope, now),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func actionForWorkflow(workflow string) Action {
|
||||
if workflow == config.WorkflowReplacement {
|
||||
return ActionReplaceCatalog
|
||||
}
|
||||
return ActionUpsertAdditive
|
||||
}
|
||||
|
||||
func catalogOutputsForPlan(req Request, outputs []Output, existing []state.CatalogOutputFile, scope state.OwnerScope, now time.Time) []state.CatalogOutputFile {
|
||||
files := make([]state.CatalogOutputFile, 0, len(outputs))
|
||||
source := state.CatalogSourceIdentity{
|
||||
ID: req.SourceBundle.Manifest.ID,
|
||||
Digest: req.SourceBundle.Manifest.Digest,
|
||||
Created: req.SourceBundle.Manifest.Created,
|
||||
}
|
||||
for _, output := range outputs {
|
||||
createdAt := now
|
||||
if existingOutput, ok := state.FindCatalogOutputByPath(existing, output.DestinationPath); ok {
|
||||
createdAt = existingOutput.CreatedAt
|
||||
}
|
||||
file := state.CatalogOutputFile{
|
||||
Path: output.DestinationPath,
|
||||
PipelineID: scope.PipelineID,
|
||||
DestinationID: scope.DestinationID,
|
||||
Source: source,
|
||||
Kind: output.Kind,
|
||||
URL: output.URL,
|
||||
SHA256: output.SHA256,
|
||||
Size: output.Size,
|
||||
CreatedAt: createdAt,
|
||||
UpdatedAt: now,
|
||||
}
|
||||
if output.Kind == state.OutputKindGenerated {
|
||||
file.SourcePath = output.SourcePath
|
||||
file.Transform = output.Transform
|
||||
}
|
||||
files = append(files, file)
|
||||
}
|
||||
return files
|
||||
}
|
||||
|
||||
func rejectCatalogUnmanagedCollisions(ctx context.Context, backend storage.Backend, bundlePath string, existing []state.CatalogOutputFile, outputs []Output) error {
|
||||
managed := catalogOutputPathSet(existing)
|
||||
for _, output := range outputs {
|
||||
if _, exists := managed[output.DestinationPath]; exists {
|
||||
continue
|
||||
}
|
||||
destinationPath, err := storage.Join(bundlePath, output.DestinationPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := backend.Stat(ctx, destinationPath); err == nil {
|
||||
return fmt.Errorf("destination output path %s exists but is not managed by catalog state", storage.DisplayPath(output.DestinationPath))
|
||||
} else if !storage.IsNotFound(err) {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func outputPaths(outputs []Output) []string {
|
||||
@@ -335,127 +319,31 @@ func outputPaths(outputs []Output) []string {
|
||||
return paths
|
||||
}
|
||||
|
||||
func sharedRootOwnershipConflictReason(conflict state.PathOwnershipConflict) string {
|
||||
return fmt.Sprintf("destination output path %s is owned by %s/%s", conflict.Path, conflict.Owner.PipelineID, conflict.Owner.DestinationID)
|
||||
func outputPathSet(outputs []Output) map[string]struct{} {
|
||||
paths := make(map[string]struct{}, len(outputs))
|
||||
for _, output := range outputs {
|
||||
paths[output.DestinationPath] = struct{}{}
|
||||
}
|
||||
return paths
|
||||
}
|
||||
|
||||
func sharedRootPathOwnershipConflicts(status state.DestinationStatus, scope state.OwnerScope, paths []string) []state.PathOwnershipConflict {
|
||||
if status.SharedRoot == nil {
|
||||
return nil
|
||||
func catalogOutputPathSet(outputs []state.CatalogOutputFile) map[string]struct{} {
|
||||
paths := make(map[string]struct{}, len(outputs))
|
||||
for _, output := range outputs {
|
||||
paths[output.Path] = struct{}{}
|
||||
}
|
||||
conflicts := make([]state.PathOwnershipConflict, 0)
|
||||
seen := make(map[string]struct{}, len(paths))
|
||||
for _, path := range paths {
|
||||
if _, exists := seen[path]; exists {
|
||||
continue
|
||||
}
|
||||
seen[path] = struct{}{}
|
||||
owner, exists := status.SharedRoot.OutputOwner(path)
|
||||
if !exists || owner == scope {
|
||||
continue
|
||||
}
|
||||
conflicts = append(conflicts, state.PathOwnershipConflict{
|
||||
Path: path,
|
||||
Owner: owner,
|
||||
CurrentOwner: scope,
|
||||
Detail: state.ComparisonDetail{
|
||||
Kind: state.ComparisonDetailSharedRootOutputOwner,
|
||||
Path: path,
|
||||
CurrentOwner: scope,
|
||||
ConflictingOwner: owner,
|
||||
},
|
||||
})
|
||||
}
|
||||
return conflicts
|
||||
return paths
|
||||
}
|
||||
|
||||
func sharedRootConflictOutputs(sharedRoot *state.SharedRootState, conflicts []state.PathOwnershipConflict) []state.SharedRootOutputFile {
|
||||
if sharedRoot == nil || len(conflicts) == 0 {
|
||||
return nil
|
||||
func normalizeReconciliation(policy config.ReconciliationPolicy) config.ReconciliationPolicy {
|
||||
if policy.Mode == "" {
|
||||
policy.Mode = config.ReconciliationModeReplace
|
||||
}
|
||||
paths := make(map[string]struct{}, len(conflicts))
|
||||
for _, conflict := range conflicts {
|
||||
paths[conflict.Path] = struct{}{}
|
||||
}
|
||||
outputs := make([]state.SharedRootOutputFile, 0, len(conflicts))
|
||||
for _, output := range sharedRoot.Outputs {
|
||||
if _, exists := paths[output.Path]; exists {
|
||||
outputs = append(outputs, output)
|
||||
}
|
||||
}
|
||||
return outputs
|
||||
return policy
|
||||
}
|
||||
|
||||
func sharedRootTakeoverAllowed(req Request, status state.DestinationStatus, conflict state.PathOwnershipConflict) bool {
|
||||
if status.SharedRoot == nil {
|
||||
return false
|
||||
}
|
||||
takeover := normalizeTakeover(req.Takeover)
|
||||
switch takeover.Mode {
|
||||
case config.TakeoverModeSamePipeline:
|
||||
return conflict.Owner.PipelineID == req.PipelineID
|
||||
case config.TakeoverModeSameSource:
|
||||
owner, ok := status.SharedRoot.Owner(conflict.Owner)
|
||||
return ok && owner.Source.Manifest.ID == req.SourceBundle.Manifest.ID
|
||||
case config.TakeoverModeAnyManaged:
|
||||
_, ok := status.SharedRoot.Owner(conflict.Owner)
|
||||
return ok
|
||||
case config.TakeoverModeNever:
|
||||
return false
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func rejectSharedRootUnmanagedCollisions(ctx context.Context, backend storage.Backend, bundlePath string, status state.DestinationStatus, scope state.OwnerScope, paths []string) error {
|
||||
for _, path := range paths {
|
||||
if pathManagedBySharedRootStatus(status, scope, path) {
|
||||
continue
|
||||
}
|
||||
destinationPath, err := storage.Join(bundlePath, path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := backend.Stat(ctx, destinationPath); err == nil {
|
||||
return fmt.Errorf("destination output path %s exists but is not managed by destination state", storage.DisplayPath(path))
|
||||
} else if !storage.IsNotFound(err) {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func pathManagedBySharedRootStatus(status state.DestinationStatus, scope state.OwnerScope, path string) bool {
|
||||
if status.SharedRoot != nil {
|
||||
_, exists := status.SharedRoot.OutputOwner(path)
|
||||
return exists
|
||||
}
|
||||
if status.State != nil && status.State.PipelineID == scope.PipelineID && status.State.DestinationID == scope.DestinationID {
|
||||
_, exists := state.FindOutputByPath(status.State.Outputs, path)
|
||||
return exists
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func otherOwnerOutputs(status state.DestinationStatus, scope state.OwnerScope) []state.SharedRootOutputFile {
|
||||
return otherOwnerOutputsExcept(status, scope, nil)
|
||||
}
|
||||
|
||||
func otherOwnerOutputsExcept(status state.DestinationStatus, scope state.OwnerScope, exclude map[string]struct{}) []state.SharedRootOutputFile {
|
||||
if status.SharedRoot == nil {
|
||||
return nil
|
||||
}
|
||||
outputs := make([]state.SharedRootOutputFile, 0, len(status.SharedRoot.Outputs))
|
||||
for _, output := range status.SharedRoot.Outputs {
|
||||
if output.Owner == scope {
|
||||
continue
|
||||
}
|
||||
if _, skip := exclude[output.Path]; skip {
|
||||
continue
|
||||
}
|
||||
outputs = append(outputs, output)
|
||||
}
|
||||
return outputs
|
||||
func isReconciliationReplacementAction(action Action) bool {
|
||||
return action == ActionReplaceOlder || action == ActionReplaceNewer
|
||||
}
|
||||
|
||||
func sharedRootOutputPathSet(outputs []state.SharedRootOutputFile) map[string]struct{} {
|
||||
@@ -465,109 +353,3 @@ func sharedRootOutputPathSet(outputs []state.SharedRootOutputFile) map[string]st
|
||||
}
|
||||
return paths
|
||||
}
|
||||
|
||||
func currentOwnerOutputs(status state.DestinationStatus, scope state.OwnerScope) []state.SharedRootOutputFile {
|
||||
if status.SharedRoot != nil {
|
||||
outputs := make([]state.SharedRootOutputFile, 0, len(status.SharedRoot.Outputs))
|
||||
for _, output := range status.SharedRoot.Outputs {
|
||||
if output.Owner == scope {
|
||||
outputs = append(outputs, output)
|
||||
}
|
||||
}
|
||||
return outputs
|
||||
}
|
||||
if status.State != nil && status.State.PipelineID == scope.PipelineID && status.State.DestinationID == scope.DestinationID {
|
||||
outputs := make([]state.SharedRootOutputFile, 0, len(status.State.Outputs))
|
||||
for _, output := range status.State.Outputs {
|
||||
outputs = append(outputs, state.SharedRootOutputFile{
|
||||
Path: output.Path,
|
||||
Kind: output.Kind,
|
||||
SourcePath: output.SourcePath,
|
||||
Transform: output.Transform,
|
||||
URL: output.URL,
|
||||
SHA256: output.SHA256,
|
||||
Size: output.Size,
|
||||
Owner: scope,
|
||||
SourceID: status.State.Source.Manifest.ID,
|
||||
SourceDigest: status.State.Source.Manifest.Digest,
|
||||
SourceCreated: status.State.Source.Manifest.Created,
|
||||
CreatedAt: output.CreatedAt,
|
||||
UpdatedAt: output.UpdatedAt,
|
||||
})
|
||||
}
|
||||
return outputs
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func actionForComparison(comparison state.Comparison, transfer config.TransferPolicy, force bool) (Action, string) {
|
||||
switch comparison.Outcome {
|
||||
case state.OutcomeDestinationAbsent:
|
||||
return ActionPublishNew, comparison.Reason
|
||||
case state.OutcomeDestinationUnmanaged:
|
||||
if force {
|
||||
return ActionForceReplace, "forced replacement of unmanaged destination content"
|
||||
}
|
||||
return ActionFailUnmanaged, comparison.Reason
|
||||
case state.OutcomeInvalidState:
|
||||
return ActionFailConflict, comparison.Reason
|
||||
case state.OutcomeIdentityMismatch, state.OutcomeSameCreatedConflict, state.OutcomeDifferentSourceConflict:
|
||||
if transfer.OnConflict == config.TransferActionReplace {
|
||||
return ActionReplaceConflict, comparison.Reason
|
||||
}
|
||||
return ActionFailConflict, comparison.Reason
|
||||
case state.OutcomeSameSource:
|
||||
if transfer.OnDestinationSame == config.TransferActionFail {
|
||||
return ActionFailConflict, "destination matches source and transfer policy requires failure"
|
||||
}
|
||||
return ActionSkipSame, comparison.Reason
|
||||
case state.OutcomeDestinationOlder:
|
||||
if transfer.OnDestinationOlder == config.TransferActionFail {
|
||||
return ActionFailConflict, "destination is older and transfer policy requires failure"
|
||||
}
|
||||
return ActionReplaceOlder, comparison.Reason
|
||||
case state.OutcomeDestinationNewer:
|
||||
if transfer.OnDestinationNewer == config.TransferActionReplace {
|
||||
return ActionReplaceNewer, comparison.Reason
|
||||
}
|
||||
if transfer.OnDestinationNewer == config.TransferActionFail {
|
||||
return ActionFailConflict, "destination is newer and transfer policy requires failure"
|
||||
}
|
||||
return ActionSkipDestinationNewer, comparison.Reason
|
||||
default:
|
||||
return ActionFailConflict, "unsupported comparison outcome"
|
||||
}
|
||||
}
|
||||
|
||||
func isReconciliationReplacementAction(action Action) bool {
|
||||
return action == ActionReplaceOlder || action == ActionReplaceNewer
|
||||
}
|
||||
|
||||
func takeoverActionAllowed(req Request, status state.DestinationStatus, comparison state.Comparison, stateMode string, action Action) bool {
|
||||
if stateMode != config.StateModeSingleOwner || status.State == nil {
|
||||
return false
|
||||
}
|
||||
if action == ActionForceReplace {
|
||||
return false
|
||||
}
|
||||
switch comparison.Detail.Kind {
|
||||
case state.ComparisonDetailPipelineIDMismatch,
|
||||
state.ComparisonDetailDestinationIDMismatch,
|
||||
state.ComparisonDetailDifferentSourceID:
|
||||
default:
|
||||
return false
|
||||
}
|
||||
takeover := normalizeTakeover(req.Takeover)
|
||||
switch takeover.Mode {
|
||||
case config.TakeoverModeSamePipeline:
|
||||
return status.State.PipelineID == req.PipelineID
|
||||
case config.TakeoverModeSameSource:
|
||||
return status.State.Source.Manifest.ID == req.SourceBundle.Manifest.ID
|
||||
case config.TakeoverModeAnyManaged:
|
||||
return true
|
||||
case config.TakeoverModeNever:
|
||||
return false
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,36 +1,296 @@
|
||||
package publish
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/config"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/state"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/storage"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/storage/fake"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/testutil"
|
||||
)
|
||||
|
||||
func TestCompareDestinationFixedPathPreservesDifferentSourceConflict(t *testing.T) {
|
||||
sourceBackend := fake.New()
|
||||
sourceBundle := testutil.WriteFakeSourceBundle(t, sourceBackend, "bundle", testutil.BundleOptions{})
|
||||
destinationState := testutil.DestinationState(sourceBundle.Manifest, testutil.DestinationStateOptions{})
|
||||
destinationState.Source.Manifest.ID = "latest.previous"
|
||||
var (
|
||||
planCreatedAt = time.Date(2026, 5, 30, 11, 12, 0, 0, time.UTC)
|
||||
planUpdatedAt = time.Date(2026, 6, 1, 9, 30, 0, 0, time.UTC)
|
||||
)
|
||||
|
||||
comparison := compareDestination(Request{
|
||||
func TestBuildAdditivePublishesCatalogOutputs(t *testing.T) {
|
||||
_, destinationBackend, req := catalogPlanRequest(t, config.WorkflowAdditive)
|
||||
|
||||
plan, err := Build(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("Build() error = %v", err)
|
||||
}
|
||||
if plan.Action != ActionPublishNew || plan.Workflow != config.WorkflowAdditive {
|
||||
t.Fatalf("plan action=%s workflow=%s, want publish_new additive", plan.Action, plan.Workflow)
|
||||
}
|
||||
if len(plan.CatalogOutputsToWrite) != 2 || len(plan.CatalogOutputsToRetain) != 0 || len(plan.CatalogOutputsToDelete) != 0 {
|
||||
t.Fatalf("catalog write=%d retain=%d delete=%d", len(plan.CatalogOutputsToWrite), len(plan.CatalogOutputsToRetain), len(plan.CatalogOutputsToDelete))
|
||||
}
|
||||
for _, output := range plan.CatalogOutputsToWrite {
|
||||
if output.PipelineID != "reports" || output.DestinationID != "archive" {
|
||||
t.Fatalf("catalog output owner = %s/%s", output.PipelineID, output.DestinationID)
|
||||
}
|
||||
if output.Source.ID != req.SourceBundle.Manifest.ID || output.Source.Digest != req.SourceBundle.Manifest.Digest || !output.Source.Created.Equal(req.SourceBundle.Manifest.Created) {
|
||||
t.Fatalf("catalog output source = %#v", output.Source)
|
||||
}
|
||||
if output.Kind == state.OutputKindSource && output.SourcePath != "" {
|
||||
t.Fatalf("source catalog output source_path = %q, want empty", output.SourcePath)
|
||||
}
|
||||
if !output.CreatedAt.Equal(planUpdatedAt) || !output.UpdatedAt.Equal(planUpdatedAt) {
|
||||
t.Fatalf("catalog output times = %s/%s", output.CreatedAt, output.UpdatedAt)
|
||||
}
|
||||
}
|
||||
if _, err := destinationBackend.Stat(context.Background(), storage.StateFileName); !storage.IsNotFound(err) {
|
||||
t.Fatalf("destination state stat error = %v, want missing", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildAdditiveOverwritesManagedAndRetainsOthers(t *testing.T) {
|
||||
_, destinationBackend, req := catalogPlanRequest(t, config.WorkflowAdditive)
|
||||
existing := baseCatalog(req)
|
||||
existing.Outputs = []state.CatalogOutputFile{
|
||||
catalogOutput(req, "reports", "archive", "report.md", state.OutputKindSource, planCreatedAt),
|
||||
catalogOutput(req, "reports", "web", "old.txt", state.OutputKindSource, planCreatedAt),
|
||||
}
|
||||
writeCatalogState(t, destinationBackend, "", existing)
|
||||
testutil.WriteFakeFile(t, destinationBackend, "report.md", "old")
|
||||
|
||||
plan, err := Build(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("Build() error = %v", err)
|
||||
}
|
||||
if plan.Action != ActionUpsertAdditive {
|
||||
t.Fatalf("plan action = %s, want %s", plan.Action, ActionUpsertAdditive)
|
||||
}
|
||||
if len(plan.CatalogOutputsToDelete) != 0 {
|
||||
t.Fatalf("delete outputs = %#v, want none", plan.CatalogOutputsToDelete)
|
||||
}
|
||||
if len(plan.CatalogOutputsToRetain) != 1 || plan.CatalogOutputsToRetain[0].Path != "old.txt" {
|
||||
t.Fatalf("retained outputs = %#v, want old.txt", plan.CatalogOutputsToRetain)
|
||||
}
|
||||
written, ok := state.FindCatalogOutputByPath(plan.CatalogOutputsToWrite, "report.md")
|
||||
if !ok {
|
||||
t.Fatalf("written outputs = %#v, want report.md", plan.CatalogOutputsToWrite)
|
||||
}
|
||||
if !written.CreatedAt.Equal(planCreatedAt) || !written.UpdatedAt.Equal(planUpdatedAt) {
|
||||
t.Fatalf("report.md times = %s/%s, want created preserved and updated now", written.CreatedAt, written.UpdatedAt)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildReplacementDeletesCurrentOwnerAndRetainsOtherOwners(t *testing.T) {
|
||||
_, destinationBackend, req := catalogPlanRequest(t, config.WorkflowReplacement)
|
||||
existing := baseCatalog(req)
|
||||
existing.Outputs = []state.CatalogOutputFile{
|
||||
catalogOutput(req, "reports", "archive", "report.md", state.OutputKindSource, planCreatedAt),
|
||||
catalogOutput(req, "reports", "archive", "stale.txt", state.OutputKindSource, planCreatedAt),
|
||||
catalogOutput(req, "reports", "web", "shared.txt", state.OutputKindSource, planCreatedAt),
|
||||
}
|
||||
writeCatalogState(t, destinationBackend, "", existing)
|
||||
|
||||
plan, err := Build(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("Build() error = %v", err)
|
||||
}
|
||||
if plan.Action != ActionReplaceCatalog {
|
||||
t.Fatalf("plan action = %s, want %s", plan.Action, ActionReplaceCatalog)
|
||||
}
|
||||
if len(plan.CatalogOutputsToDelete) != 1 || plan.CatalogOutputsToDelete[0].Path != "stale.txt" {
|
||||
t.Fatalf("delete outputs = %#v, want stale.txt", plan.CatalogOutputsToDelete)
|
||||
}
|
||||
if len(plan.CatalogOutputsToRetain) != 1 || plan.CatalogOutputsToRetain[0].Path != "shared.txt" {
|
||||
t.Fatalf("retained outputs = %#v, want shared.txt", plan.CatalogOutputsToRetain)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildTransfersManagedPathOwnership(t *testing.T) {
|
||||
_, destinationBackend, req := catalogPlanRequest(t, config.WorkflowReplacement)
|
||||
existing := baseCatalog(req)
|
||||
existing.Outputs = []state.CatalogOutputFile{
|
||||
catalogOutput(req, "reports", "web", "report.md", state.OutputKindSource, planCreatedAt),
|
||||
}
|
||||
writeCatalogState(t, destinationBackend, "", existing)
|
||||
|
||||
plan, err := Build(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("Build() error = %v", err)
|
||||
}
|
||||
written, ok := state.FindCatalogOutputByPath(plan.CatalogOutputsToWrite, "report.md")
|
||||
if !ok {
|
||||
t.Fatalf("written outputs = %#v, want report.md", plan.CatalogOutputsToWrite)
|
||||
}
|
||||
if written.PipelineID != "reports" || written.DestinationID != "archive" {
|
||||
t.Fatalf("written owner = %s/%s, want reports/archive", written.PipelineID, written.DestinationID)
|
||||
}
|
||||
if !written.CreatedAt.Equal(planCreatedAt) || !written.UpdatedAt.Equal(planUpdatedAt) {
|
||||
t.Fatalf("written times = %s/%s", written.CreatedAt, written.UpdatedAt)
|
||||
}
|
||||
if len(plan.CatalogOutputsToRetain) != 0 || len(plan.CatalogOutputsToDelete) != 0 {
|
||||
t.Fatalf("retain=%#v delete=%#v, want no old record for overwritten path", plan.CatalogOutputsToRetain, plan.CatalogOutputsToDelete)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildRejectsUnmanagedPlannedPathCollision(t *testing.T) {
|
||||
_, destinationBackend, req := catalogPlanRequest(t, config.WorkflowAdditive)
|
||||
testutil.WriteFakeFile(t, destinationBackend, "report.md", "unmanaged")
|
||||
|
||||
plan, err := Build(context.Background(), req)
|
||||
if err == nil {
|
||||
t.Fatal("Build() error = nil, want unmanaged collision")
|
||||
}
|
||||
if plan.Action != ActionFailUnmanaged || !strings.Contains(err.Error(), "not managed by catalog state") {
|
||||
t.Fatalf("plan action=%s error=%v, want unmanaged catalog collision", plan.Action, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildPlansSupersededLegacyAdditive(t *testing.T) {
|
||||
_, destinationBackend, req := catalogPlanRequest(t, config.WorkflowAdditive)
|
||||
legacyState := testutil.DestinationState(req.SourceBundle.Manifest, testutil.DestinationStateOptions{})
|
||||
writeJSONState(t, destinationBackend, "", legacyState)
|
||||
testutil.WriteFakeFile(t, destinationBackend, "report.md", "legacy")
|
||||
|
||||
plan, err := Build(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("Build() error = %v", err)
|
||||
}
|
||||
if plan.SupersededLegacy == nil || plan.SupersededLegacy.SchemaVersion != state.SchemaVersion {
|
||||
t.Fatalf("superseded legacy = %#v", plan.SupersededLegacy)
|
||||
}
|
||||
if plan.Action != ActionUpsertAdditive || plan.ClearDestinationRoot {
|
||||
t.Fatalf("plan action=%s clear=%t, want additive overwrite without clear", plan.Action, plan.ClearDestinationRoot)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildPlansSupersededLegacyReplacementClear(t *testing.T) {
|
||||
_, destinationBackend, req := catalogPlanRequest(t, config.WorkflowReplacement)
|
||||
legacyState := testutil.DestinationState(req.SourceBundle.Manifest, testutil.DestinationStateOptions{})
|
||||
writeJSONState(t, destinationBackend, "", legacyState)
|
||||
|
||||
plan, err := Build(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("Build() error = %v", err)
|
||||
}
|
||||
if plan.Action != ActionReplaceCatalog || !plan.ClearDestinationRoot {
|
||||
t.Fatalf("plan action=%s clear=%t, want replacement clear", plan.Action, plan.ClearDestinationRoot)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildRejectsInvalidOrFutureState(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
data string
|
||||
}{
|
||||
{name: "invalid json", data: `{"schema_version":`},
|
||||
{name: "future schema", data: `{"schema_version":99}`},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
_, destinationBackend, req := catalogPlanRequest(t, config.WorkflowAdditive)
|
||||
testutil.WriteFakeFile(t, destinationBackend, storage.StateFileName, tt.data)
|
||||
|
||||
plan, err := Build(context.Background(), req)
|
||||
if err == nil {
|
||||
t.Fatal("Build() error = nil, want conflict")
|
||||
}
|
||||
if plan.Action != ActionFailConflict {
|
||||
t.Fatalf("plan action = %s, want %s", plan.Action, ActionFailConflict)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateRequestRejectsInvalidWorkflow(t *testing.T) {
|
||||
sourceBackend, destinationBackend, req := catalogPlanRequest(t, "append")
|
||||
req.SourceBackend = sourceBackend
|
||||
req.DestinationBackend = destinationBackend
|
||||
|
||||
err := validateRequest(req)
|
||||
if err == nil {
|
||||
t.Fatal("validateRequest() error = nil, want invalid workflow")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "destination.workflow") {
|
||||
t.Fatalf("validateRequest() error = %v, want workflow context", err)
|
||||
}
|
||||
}
|
||||
|
||||
func catalogPlanRequest(t *testing.T, workflow string) (*fake.Backend, *fake.Backend, Request) {
|
||||
t.Helper()
|
||||
sourceBackend := fake.New()
|
||||
destinationBackend := fake.New()
|
||||
sourceBundle := testutil.WriteFakeSourceBundle(t, sourceBackend, "", testutil.BundleOptions{})
|
||||
return sourceBackend, destinationBackend, Request{
|
||||
PipelineID: "reports",
|
||||
DestinationID: "archive",
|
||||
SourceBundle: sourceBundle,
|
||||
SourceBackend: sourceBackend,
|
||||
DestinationBackend: destinationBackend,
|
||||
DestinationBundlePath: "",
|
||||
PathMapping: config.PathMappingFixed,
|
||||
State: config.StatePolicy{Mode: config.StateModeSingleOwner},
|
||||
}, state.DestinationStatus{State: &destinationState, HasContents: true})
|
||||
|
||||
if comparison.Outcome != state.OutcomeDifferentSourceConflict {
|
||||
t.Fatalf("comparison outcome = %s, want %s", comparison.Outcome, state.OutcomeDifferentSourceConflict)
|
||||
}
|
||||
if comparison.Detail.Kind != state.ComparisonDetailDifferentSourceID {
|
||||
t.Fatalf("detail kind = %q, want %q", comparison.Detail.Kind, state.ComparisonDetailDifferentSourceID)
|
||||
}
|
||||
if comparison.Detail.CurrentSourceID != sourceBundle.Manifest.ID || comparison.Detail.DestinationSourceID != "latest.previous" {
|
||||
t.Fatalf("detail = %#v, want source ids", comparison.Detail)
|
||||
PathMapping: config.PathMappingPreserveRelative,
|
||||
Publish: config.PublishPolicy{Source: true},
|
||||
Workflow: workflow,
|
||||
DistributorVersion: "test",
|
||||
Now: planUpdatedAt,
|
||||
}
|
||||
}
|
||||
|
||||
func baseCatalog(req Request) state.CatalogState {
|
||||
return state.CatalogState{
|
||||
SchemaVersion: state.CatalogSchemaVersion,
|
||||
DistributorVersion: "previous",
|
||||
CreatedAt: planCreatedAt,
|
||||
UpdatedAt: planCreatedAt,
|
||||
State: state.StatePolicy{Mode: state.StateModeCatalog},
|
||||
Outputs: []state.CatalogOutputFile{},
|
||||
}
|
||||
}
|
||||
|
||||
func catalogOutput(req Request, pipelineID, destinationID, path, kind string, createdAt time.Time) state.CatalogOutputFile {
|
||||
sourceSHA := req.SourceBundle.Manifest.Files[0].SHA256
|
||||
sourceSize := req.SourceBundle.Manifest.Files[0].Size
|
||||
for _, file := range req.SourceBundle.Manifest.Files {
|
||||
if file.Path == path {
|
||||
sourceSHA = file.SHA256
|
||||
sourceSize = file.Size
|
||||
break
|
||||
}
|
||||
}
|
||||
output := state.CatalogOutputFile{
|
||||
Path: path,
|
||||
PipelineID: pipelineID,
|
||||
DestinationID: destinationID,
|
||||
Source: state.CatalogSourceIdentity{
|
||||
ID: req.SourceBundle.Manifest.ID,
|
||||
Digest: req.SourceBundle.Manifest.Digest,
|
||||
Created: req.SourceBundle.Manifest.Created,
|
||||
},
|
||||
Kind: kind,
|
||||
SHA256: sourceSHA,
|
||||
Size: sourceSize,
|
||||
CreatedAt: createdAt,
|
||||
UpdatedAt: createdAt,
|
||||
}
|
||||
return output
|
||||
}
|
||||
|
||||
func writeCatalogState(t *testing.T, backend *fake.Backend, relative string, catalog state.CatalogState) {
|
||||
t.Helper()
|
||||
writeJSONState(t, backend, relative, catalog)
|
||||
}
|
||||
|
||||
func writeJSONState(t *testing.T, backend *fake.Backend, relative string, value any) {
|
||||
t.Helper()
|
||||
data, err := json.MarshalIndent(value, "", " ")
|
||||
if err != nil {
|
||||
t.Fatalf("marshal state: %v", err)
|
||||
}
|
||||
statePath, err := storage.StatePath(relative)
|
||||
if err != nil {
|
||||
t.Fatalf("state path: %v", err)
|
||||
}
|
||||
testutil.WriteFakeFile(t, backend, statePath, string(append(data, '\n')))
|
||||
}
|
||||
|
||||
@@ -18,7 +18,13 @@ func inspectDestination(ctx context.Context, backend storage.Backend, bundlePath
|
||||
if parseErr != nil {
|
||||
return state.DestinationStatus{StateErr: parseErr}, nil
|
||||
}
|
||||
return state.DestinationStatus{State: document.SingleOwner, SharedRoot: document.SharedRoot, HasContents: true}, nil
|
||||
return state.DestinationStatus{
|
||||
State: document.SingleOwner,
|
||||
SharedRoot: document.SharedRoot,
|
||||
Catalog: document.Catalog,
|
||||
SupersededLegacy: document.SupersededLegacy,
|
||||
HasContents: true,
|
||||
}, nil
|
||||
}
|
||||
if !storage.IsNotFound(err) {
|
||||
return state.DestinationStatus{}, err
|
||||
|
||||
@@ -1,632 +0,0 @@
|
||||
package publish
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/bundle"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/config"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/state"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/storage"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/storage/fake"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/testutil"
|
||||
)
|
||||
|
||||
func TestBuildSharedRootTreatsAbsentOwnerAsPublishable(t *testing.T) {
|
||||
sourceBackend := fake.New()
|
||||
sourceBundle := testutil.WriteFakeSourceBundle(t, sourceBackend, "bundle", testutil.BundleOptions{
|
||||
Files: []testutil.SourceFile{{Path: "report.md", Data: "# Report\n"}},
|
||||
})
|
||||
destinationBackend := fake.New()
|
||||
writeFakeSharedRootState(t, destinationBackend, "bundle", sharedRootStateWithOwners(t, sourceBundle.Manifest, false))
|
||||
|
||||
req := sharedRootRequest(sourceBackend, destinationBackend, sourceBundle, config.ReconciliationModeReplace)
|
||||
plan, err := Build(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("Build() error = %v", err)
|
||||
}
|
||||
if plan.Action != ActionPublishNew {
|
||||
t.Fatalf("plan action = %s, want publish_new", plan.Action)
|
||||
}
|
||||
if plan.OwnerScope != state.CurrentOwnerScope("reports", "archive") {
|
||||
t.Fatalf("owner scope = %#v, want reports/archive", plan.OwnerScope)
|
||||
}
|
||||
if got, want := len(plan.OtherOwnerOutputs), 1; got != want {
|
||||
t.Fatalf("other owner output count = %d, want %d", got, want)
|
||||
}
|
||||
if got, want := len(plan.OwnerOutputsToWrite), 1; got != want {
|
||||
t.Fatalf("owner output write count = %d, want %d", got, want)
|
||||
}
|
||||
if len(plan.OwnerOutputsToDelete) != 0 || len(plan.RetainedOwnerOutputs) != 0 {
|
||||
t.Fatalf("delete=%#v retained=%#v, want none", plan.OwnerOutputsToDelete, plan.RetainedOwnerOutputs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildSharedRootReplaceDeletesOnlyCurrentOwnerOmittedOutputs(t *testing.T) {
|
||||
sourceBackend := fake.New()
|
||||
sourceBundle := testutil.WriteFakeSourceBundle(t, sourceBackend, "bundle", testutil.BundleOptions{
|
||||
Files: []testutil.SourceFile{{Path: "report.md", Data: "# Report\nNew.\n"}},
|
||||
})
|
||||
destinationBackend := fake.New()
|
||||
writeFakeSharedRootState(t, destinationBackend, "bundle", sharedRootStateWithOwners(t, sourceBundle.Manifest, true))
|
||||
|
||||
req := sharedRootRequest(sourceBackend, destinationBackend, sourceBundle, config.ReconciliationModeReplace)
|
||||
plan, err := Build(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("Build() error = %v", err)
|
||||
}
|
||||
if plan.Action != ActionReplaceOlder {
|
||||
t.Fatalf("plan action = %s, want replace_older", plan.Action)
|
||||
}
|
||||
if got, want := sharedRootOutputPathList(plan.OwnerOutputsToDelete), "old.md"; got != want {
|
||||
t.Fatalf("owner outputs to delete = %q, want %q", got, want)
|
||||
}
|
||||
if got, want := sharedRootOutputPathList(plan.OtherOwnerOutputs), "other/report.md"; got != want {
|
||||
t.Fatalf("other owner outputs = %q, want %q", got, want)
|
||||
}
|
||||
if len(plan.RetainedOwnerOutputs) != 0 {
|
||||
t.Fatalf("retained owner outputs = %#v, want none", plan.RetainedOwnerOutputs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildSharedRootMergeRetainsCurrentOwnerOmittedOutputs(t *testing.T) {
|
||||
sourceBackend := fake.New()
|
||||
sourceBundle := testutil.WriteFakeSourceBundle(t, sourceBackend, "bundle", testutil.BundleOptions{
|
||||
Files: []testutil.SourceFile{{Path: "report.md", Data: "# Report\nNew.\n"}},
|
||||
})
|
||||
destinationBackend := fake.New()
|
||||
writeFakeSharedRootState(t, destinationBackend, "bundle", sharedRootStateWithOwners(t, sourceBundle.Manifest, true))
|
||||
|
||||
req := sharedRootRequest(sourceBackend, destinationBackend, sourceBundle, config.ReconciliationModeMerge)
|
||||
plan, err := Build(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("Build() error = %v", err)
|
||||
}
|
||||
if got, want := sharedRootOutputPathList(plan.RetainedOwnerOutputs), "old.md"; got != want {
|
||||
t.Fatalf("retained owner outputs = %q, want %q", got, want)
|
||||
}
|
||||
if len(plan.OwnerOutputsToDelete) != 0 {
|
||||
t.Fatalf("owner outputs to delete = %#v, want none", plan.OwnerOutputsToDelete)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildSharedRootRejectsOtherOwnerPathConflict(t *testing.T) {
|
||||
sourceBackend := fake.New()
|
||||
sourceBundle := testutil.WriteFakeSourceBundle(t, sourceBackend, "bundle", testutil.BundleOptions{
|
||||
Files: []testutil.SourceFile{{Path: "report.md", Data: "# Report\n"}},
|
||||
})
|
||||
destinationBackend := fake.New()
|
||||
sharedRoot := sharedRootStateWithOwners(t, sourceBundle.Manifest, false)
|
||||
sharedRoot.Outputs[0].Path = "report.md"
|
||||
writeFakeSharedRootState(t, destinationBackend, "bundle", sharedRoot)
|
||||
|
||||
req := sharedRootRequest(sourceBackend, destinationBackend, sourceBundle, config.ReconciliationModeReplace)
|
||||
plan, err := Build(context.Background(), req)
|
||||
if err == nil || !strings.Contains(err.Error(), "fail_conflict") {
|
||||
t.Fatalf("Build() error = %v, want fail_conflict", err)
|
||||
}
|
||||
if plan.Action != ActionFailConflict {
|
||||
t.Fatalf("plan action = %s, want fail_conflict", plan.Action)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildSharedRootPlansOutputTakeoverByPolicy(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
takeover config.TakeoverPolicy
|
||||
transfer config.TransferPolicy
|
||||
ownerScope state.OwnerScope
|
||||
sameSource bool
|
||||
wantAction Action
|
||||
wantErr string
|
||||
}{
|
||||
{
|
||||
name: "default same pipeline allows different destination",
|
||||
takeover: config.TakeoverPolicy{},
|
||||
ownerScope: state.CurrentOwnerScope("reports", "web"),
|
||||
wantAction: ActionReplaceTakeover,
|
||||
},
|
||||
{
|
||||
name: "same pipeline refuses different pipeline",
|
||||
takeover: config.TakeoverPolicy{Mode: config.TakeoverModeSamePipeline},
|
||||
ownerScope: state.CurrentOwnerScope("other", "archive"),
|
||||
wantErr: "fail_conflict",
|
||||
},
|
||||
{
|
||||
name: "same source allows different pipeline",
|
||||
takeover: config.TakeoverPolicy{Mode: config.TakeoverModeSameSource},
|
||||
ownerScope: state.CurrentOwnerScope("other", "archive"),
|
||||
sameSource: true,
|
||||
wantAction: ActionReplaceTakeover,
|
||||
},
|
||||
{
|
||||
name: "same source refuses different source",
|
||||
takeover: config.TakeoverPolicy{Mode: config.TakeoverModeSameSource},
|
||||
ownerScope: state.CurrentOwnerScope("reports", "web"),
|
||||
wantErr: "fail_conflict",
|
||||
},
|
||||
{
|
||||
name: "any managed allows different pipeline",
|
||||
takeover: config.TakeoverPolicy{Mode: config.TakeoverModeAnyManaged},
|
||||
ownerScope: state.CurrentOwnerScope("other", "archive"),
|
||||
wantAction: ActionReplaceTakeover,
|
||||
},
|
||||
{
|
||||
name: "never refuses same pipeline",
|
||||
takeover: config.TakeoverPolicy{Mode: config.TakeoverModeNever},
|
||||
ownerScope: state.CurrentOwnerScope("reports", "web"),
|
||||
wantErr: "fail_conflict",
|
||||
},
|
||||
{
|
||||
name: "transfer conflict replacement allows managed owner conflict",
|
||||
takeover: config.TakeoverPolicy{Mode: config.TakeoverModeNever},
|
||||
transfer: config.TransferPolicy{OnConflict: config.TransferActionReplace},
|
||||
ownerScope: state.CurrentOwnerScope("other", "archive"),
|
||||
wantAction: ActionReplaceConflict,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
sourceBackend := fake.New()
|
||||
sourceBundle := testutil.WriteFakeSourceBundle(t, sourceBackend, "bundle", testutil.BundleOptions{
|
||||
Files: []testutil.SourceFile{{Path: "report.md", Data: "# Report\n"}},
|
||||
})
|
||||
destinationBackend := fake.New()
|
||||
sharedRoot := sharedRootStateWithOwners(t, sourceBundle.Manifest, false)
|
||||
ownerManifest := sharedRoot.Owners[0].Source.Manifest
|
||||
if tt.sameSource {
|
||||
ownerManifest = sourceBundle.Manifest
|
||||
}
|
||||
setSharedRootOwnerOutput(t, &sharedRoot, 0, tt.ownerScope, ownerManifest, "report.md")
|
||||
writeFakeSharedRootState(t, destinationBackend, "bundle", sharedRoot)
|
||||
|
||||
req := sharedRootRequest(sourceBackend, destinationBackend, sourceBundle, config.ReconciliationModeReplace)
|
||||
req.Takeover = tt.takeover
|
||||
if tt.transfer.OnConflict != "" {
|
||||
req.Transfer.OnConflict = tt.transfer.OnConflict
|
||||
}
|
||||
plan, err := Build(context.Background(), req)
|
||||
if tt.wantErr != "" {
|
||||
if err == nil || !strings.Contains(err.Error(), tt.wantErr) {
|
||||
t.Fatalf("Build() error = %v, want %q", err, tt.wantErr)
|
||||
}
|
||||
if plan.Action != ActionFailConflict {
|
||||
t.Fatalf("plan action = %s, want fail_conflict", plan.Action)
|
||||
}
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("Build() error = %v", err)
|
||||
}
|
||||
if plan.Action != tt.wantAction {
|
||||
t.Fatalf("plan action = %s, want %s", plan.Action, tt.wantAction)
|
||||
}
|
||||
if got, want := sharedRootOutputPathList(plan.TakenOverOwnerOutputs), "report.md"; got != want {
|
||||
t.Fatalf("taken over outputs = %q, want %q", got, want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildSharedRootRejectsUnmanagedPathCollision(t *testing.T) {
|
||||
sourceBackend := fake.New()
|
||||
sourceBundle := testutil.WriteFakeSourceBundle(t, sourceBackend, "bundle", testutil.BundleOptions{
|
||||
Files: []testutil.SourceFile{{Path: "report.md", Data: "# Report\n"}},
|
||||
})
|
||||
destinationBackend := fake.New()
|
||||
writeFakeSharedRootState(t, destinationBackend, "bundle", sharedRootStateWithOwners(t, sourceBundle.Manifest, false))
|
||||
testutil.WriteFakeFile(t, destinationBackend, "bundle/report.md", "unmanaged")
|
||||
|
||||
req := sharedRootRequest(sourceBackend, destinationBackend, sourceBundle, config.ReconciliationModeReplace)
|
||||
plan, err := Build(context.Background(), req)
|
||||
if err == nil || !strings.Contains(err.Error(), "fail_unmanaged") {
|
||||
t.Fatalf("Build() error = %v, want fail_unmanaged", err)
|
||||
}
|
||||
if plan.Action != ActionFailUnmanaged {
|
||||
t.Fatalf("plan action = %s, want fail_unmanaged", plan.Action)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteSharedRootPublishesOwnerAndPreservesOtherOwners(t *testing.T) {
|
||||
sourceBackend := fake.New()
|
||||
sourceBundle := testutil.WriteFakeSourceBundle(t, sourceBackend, "bundle", testutil.BundleOptions{
|
||||
Files: []testutil.SourceFile{{Path: "report.md", Data: "# Report\n"}},
|
||||
})
|
||||
destinationBackend := fake.New()
|
||||
writeFakeSharedRootState(t, destinationBackend, "bundle", sharedRootStateWithOwners(t, sourceBundle.Manifest, false))
|
||||
req := sharedRootRequest(sourceBackend, destinationBackend, sourceBundle, config.ReconciliationModeReplace)
|
||||
plan, err := Build(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("Build() error = %v", err)
|
||||
}
|
||||
if err := Execute(context.Background(), req, plan); err != nil {
|
||||
t.Fatalf("Execute() error = %v", err)
|
||||
}
|
||||
testutil.AssertFakeFile(t, destinationBackend, "bundle/report.md", "# Report\n")
|
||||
testutil.AssertFakeFile(t, destinationBackend, "bundle/other/report.md", "old")
|
||||
destinationState := readFakeSharedRootState(t, destinationBackend, "bundle")
|
||||
if got, want := len(destinationState.Owners), 2; got != want {
|
||||
t.Fatalf("owner count = %d, want %d", got, want)
|
||||
}
|
||||
if got, want := destinationState.State.Mode, state.StateModeSharedRoot; got != want {
|
||||
t.Fatalf("state mode = %q, want %q", got, want)
|
||||
}
|
||||
if _, ok := destinationState.Owner(state.CurrentOwnerScope("other", "archive")); !ok {
|
||||
t.Fatal("other owner missing from shared-root state")
|
||||
}
|
||||
if _, ok := destinationState.Owner(state.CurrentOwnerScope("reports", "archive")); !ok {
|
||||
t.Fatal("current owner missing from shared-root state")
|
||||
}
|
||||
if got, want := strings.Join(destinationState.AllManagedOutputPaths(), ","), "other/report.md,report.md"; got != want {
|
||||
t.Fatalf("managed paths = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteSharedRootReplaceDeletesOnlyCurrentOwnerOmittedOutputs(t *testing.T) {
|
||||
sourceBackend := fake.New()
|
||||
sourceBundle := testutil.WriteFakeSourceBundle(t, sourceBackend, "bundle", testutil.BundleOptions{
|
||||
Files: []testutil.SourceFile{{Path: "report.md", Data: "# Report\nNew.\n"}},
|
||||
})
|
||||
destinationBackend := fake.New()
|
||||
existing := sharedRootStateWithOwners(t, sourceBundle.Manifest, true)
|
||||
writeFakeSharedRootState(t, destinationBackend, "bundle", existing)
|
||||
|
||||
req := sharedRootRequest(sourceBackend, destinationBackend, sourceBundle, config.ReconciliationModeReplace)
|
||||
plan, err := Build(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("Build() error = %v", err)
|
||||
}
|
||||
if err := Execute(context.Background(), req, plan); err != nil {
|
||||
t.Fatalf("Execute() error = %v", err)
|
||||
}
|
||||
testutil.AssertFakeFile(t, destinationBackend, "bundle/report.md", "# Report\nNew.\n")
|
||||
testutil.AssertFakeMissing(t, destinationBackend, "bundle/old.md")
|
||||
testutil.AssertFakeFile(t, destinationBackend, "bundle/other/report.md", "old")
|
||||
destinationState := readFakeSharedRootState(t, destinationBackend, "bundle")
|
||||
if got, want := strings.Join(destinationState.AllManagedOutputPaths(), ","), "other/report.md,report.md"; got != want {
|
||||
t.Fatalf("managed paths = %q, want %q", got, want)
|
||||
}
|
||||
if !destinationState.CreatedAt.Equal(existing.CreatedAt) {
|
||||
t.Fatalf("created_at = %s, want %s", destinationState.CreatedAt, existing.CreatedAt)
|
||||
}
|
||||
output, ok := findSharedRootOutputForTest(destinationState.Outputs, "report.md")
|
||||
if !ok {
|
||||
t.Fatal("report.md missing from shared-root outputs")
|
||||
}
|
||||
if !output.CreatedAt.Equal(existing.Outputs[1].CreatedAt) || !output.UpdatedAt.After(existing.Outputs[1].UpdatedAt) {
|
||||
t.Fatalf("report.md timestamps = created:%s updated:%s, want preserved created and newer updated", output.CreatedAt, output.UpdatedAt)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteSharedRootTakeoverReassignsPathAndPreservesUnrelatedOutputs(t *testing.T) {
|
||||
sourceBackend := fake.New()
|
||||
sourceBundle := testutil.WriteFakeSourceBundle(t, sourceBackend, "bundle", testutil.BundleOptions{
|
||||
Files: []testutil.SourceFile{{Path: "report.md", Data: "# Report\nNew.\n"}},
|
||||
})
|
||||
destinationBackend := fake.New()
|
||||
sharedRoot := sharedRootStateWithOwners(t, sourceBundle.Manifest, false)
|
||||
previousScope := state.CurrentOwnerScope("reports", "web")
|
||||
setSharedRootOwnerOutput(t, &sharedRoot, 0, previousScope, sharedRoot.Owners[0].Source.Manifest, "report.md")
|
||||
keepOutput := sharedRoot.Outputs[0]
|
||||
keepOutput.Path = "web/keep.md"
|
||||
keepOutput.SourcePath = "web/keep.md"
|
||||
sharedRoot.Outputs = append(sharedRoot.Outputs, keepOutput)
|
||||
writeFakeSharedRootState(t, destinationBackend, "bundle", sharedRoot)
|
||||
|
||||
req := sharedRootRequest(sourceBackend, destinationBackend, sourceBundle, config.ReconciliationModeReplace)
|
||||
plan, err := Build(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("Build() error = %v", err)
|
||||
}
|
||||
if plan.Action != ActionReplaceTakeover {
|
||||
t.Fatalf("plan action = %s, want %s", plan.Action, ActionReplaceTakeover)
|
||||
}
|
||||
if err := Execute(context.Background(), req, plan); err != nil {
|
||||
t.Fatalf("Execute() error = %v", err)
|
||||
}
|
||||
|
||||
testutil.AssertFakeFile(t, destinationBackend, "bundle/report.md", "# Report\nNew.\n")
|
||||
testutil.AssertFakeFile(t, destinationBackend, "bundle/web/keep.md", "old")
|
||||
destinationState := readFakeSharedRootState(t, destinationBackend, "bundle")
|
||||
reportOutput, ok := findSharedRootOutputForTest(destinationState.Outputs, "report.md")
|
||||
if !ok {
|
||||
t.Fatal("report.md missing from shared-root outputs")
|
||||
}
|
||||
if reportOutput.Owner != state.CurrentOwnerScope("reports", "archive") {
|
||||
t.Fatalf("report.md owner = %#v, want reports/archive", reportOutput.Owner)
|
||||
}
|
||||
keep, ok := findSharedRootOutputForTest(destinationState.Outputs, "web/keep.md")
|
||||
if !ok {
|
||||
t.Fatal("web/keep.md missing from shared-root outputs")
|
||||
}
|
||||
if keep.Owner != previousScope {
|
||||
t.Fatalf("web/keep.md owner = %#v, want reports/web", keep.Owner)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteSharedRootTakeoverMergeDoesNotRetainOldSourceOutputs(t *testing.T) {
|
||||
sourceBackend := fake.New()
|
||||
sourceBundle := testutil.WriteFakeSourceBundle(t, sourceBackend, "bundle", testutil.BundleOptions{
|
||||
Files: []testutil.SourceFile{{Path: "report.md", Data: "# Report\nNew.\n"}},
|
||||
})
|
||||
destinationBackend := fake.New()
|
||||
sharedRoot := sharedRootStateWithOwners(t, sourceBundle.Manifest, false)
|
||||
previousScope := state.CurrentOwnerScope("reports", "web")
|
||||
setSharedRootOwnerOutput(t, &sharedRoot, 0, previousScope, sharedRoot.Owners[0].Source.Manifest, "report.md")
|
||||
|
||||
createdAt := sharedRoot.CreatedAt
|
||||
oldManifest := sourceBundle.Manifest
|
||||
oldManifest.ID = "old.source"
|
||||
oldManifest.Created = oldManifest.Created.Add(-time.Hour)
|
||||
oldManifest.Files = []bundle.ManifestFile{{
|
||||
Path: "old.md",
|
||||
SHA256: bundle.FileDigest([]byte("old\n")),
|
||||
Size: int64(len("old\n")),
|
||||
}}
|
||||
oldManifest.Digest = bundle.BundleDigest(oldManifest.Files)
|
||||
currentScope := state.CurrentOwnerScope("reports", "archive")
|
||||
sharedRoot.Owners = append(sharedRoot.Owners, state.OwnerRecord{
|
||||
Scope: currentScope,
|
||||
Reconciliation: state.ReconciliationPolicy{Mode: config.ReconciliationModeMerge},
|
||||
Source: state.SourceState{Manifest: oldManifest},
|
||||
})
|
||||
sharedRoot.Outputs = append(sharedRoot.Outputs, state.SharedRootOutputFile{
|
||||
Path: "old.md",
|
||||
Kind: state.OutputKindSource,
|
||||
SourcePath: "old.md",
|
||||
SHA256: oldManifest.Files[0].SHA256,
|
||||
Size: oldManifest.Files[0].Size,
|
||||
Owner: currentScope,
|
||||
SourceID: oldManifest.ID,
|
||||
SourceDigest: oldManifest.Digest,
|
||||
SourceCreated: oldManifest.Created,
|
||||
CreatedAt: createdAt,
|
||||
UpdatedAt: createdAt,
|
||||
})
|
||||
writeFakeSharedRootState(t, destinationBackend, "bundle", sharedRoot)
|
||||
|
||||
req := sharedRootRequest(sourceBackend, destinationBackend, sourceBundle, config.ReconciliationModeMerge)
|
||||
req.PathMapping = config.PathMappingFixed
|
||||
plan, err := Build(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("Build() error = %v", err)
|
||||
}
|
||||
if plan.Action != ActionReplaceTakeover {
|
||||
t.Fatalf("plan action = %s, want %s", plan.Action, ActionReplaceTakeover)
|
||||
}
|
||||
if got, want := sharedRootOutputPathList(plan.OwnerOutputsToDelete), "old.md"; got != want {
|
||||
t.Fatalf("owner outputs to delete = %q, want %q", got, want)
|
||||
}
|
||||
if len(plan.RetainedOwnerOutputs) != 0 {
|
||||
t.Fatalf("retained owner outputs = %#v, want none", plan.RetainedOwnerOutputs)
|
||||
}
|
||||
if err := Execute(context.Background(), req, plan); err != nil {
|
||||
t.Fatalf("Execute() error = %v", err)
|
||||
}
|
||||
|
||||
testutil.AssertFakeFile(t, destinationBackend, "bundle/report.md", "# Report\nNew.\n")
|
||||
testutil.AssertFakeMissing(t, destinationBackend, "bundle/old.md")
|
||||
destinationState := readFakeSharedRootState(t, destinationBackend, "bundle")
|
||||
if got, want := strings.Join(destinationState.AllManagedOutputPaths(), ","), "report.md"; got != want {
|
||||
t.Fatalf("managed paths = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteSharedRootMergeRetainsCurrentOwnerOmittedOutputs(t *testing.T) {
|
||||
sourceBackend := fake.New()
|
||||
sourceBundle := testutil.WriteFakeSourceBundle(t, sourceBackend, "bundle", testutil.BundleOptions{
|
||||
Files: []testutil.SourceFile{{Path: "report.md", Data: "# Report\nNew.\n"}},
|
||||
})
|
||||
destinationBackend := fake.New()
|
||||
writeFakeSharedRootState(t, destinationBackend, "bundle", sharedRootStateWithOwners(t, sourceBundle.Manifest, true))
|
||||
|
||||
req := sharedRootRequest(sourceBackend, destinationBackend, sourceBundle, config.ReconciliationModeMerge)
|
||||
plan, err := Build(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("Build() error = %v", err)
|
||||
}
|
||||
if err := Execute(context.Background(), req, plan); err != nil {
|
||||
t.Fatalf("Execute() error = %v", err)
|
||||
}
|
||||
testutil.AssertFakeFile(t, destinationBackend, "bundle/report.md", "# Report\nNew.\n")
|
||||
testutil.AssertFakeFile(t, destinationBackend, "bundle/old.md", "old")
|
||||
testutil.AssertFakeFile(t, destinationBackend, "bundle/other/report.md", "old")
|
||||
destinationState := readFakeSharedRootState(t, destinationBackend, "bundle")
|
||||
if got, want := strings.Join(destinationState.AllManagedOutputPaths(), ","), "other/report.md,report.md,old.md"; got != want {
|
||||
t.Fatalf("managed paths = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteSharedRootForceReplaceDeletesOnlyBundlePath(t *testing.T) {
|
||||
sourceBackend := fake.New()
|
||||
sourceBundle := testutil.WriteFakeSourceBundle(t, sourceBackend, "bundle", testutil.BundleOptions{
|
||||
Files: []testutil.SourceFile{{Path: "report.md", Data: "# Report\n"}},
|
||||
})
|
||||
destinationBackend := fake.New()
|
||||
testutil.WriteFakeFile(t, destinationBackend, "bundle/unmanaged.txt", "unmanaged")
|
||||
testutil.WriteFakeFile(t, destinationBackend, "bundle/nested/old.txt", "old")
|
||||
testutil.WriteFakeFile(t, destinationBackend, "bundle-sibling/keep.txt", "keep")
|
||||
|
||||
req := sharedRootRequest(sourceBackend, destinationBackend, sourceBundle, config.ReconciliationModeReplace)
|
||||
req.Force = true
|
||||
plan, err := Build(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("Build() error = %v", err)
|
||||
}
|
||||
if plan.Action != ActionForceReplace {
|
||||
t.Fatalf("plan action = %s, want force_replace", plan.Action)
|
||||
}
|
||||
if err := Execute(context.Background(), req, plan); err != nil {
|
||||
t.Fatalf("Execute() error = %v", err)
|
||||
}
|
||||
testutil.AssertFakeFile(t, destinationBackend, "bundle/report.md", "# Report\n")
|
||||
testutil.AssertFakeMissing(t, destinationBackend, "bundle/unmanaged.txt")
|
||||
testutil.AssertFakeMissing(t, destinationBackend, "bundle/nested/old.txt")
|
||||
testutil.AssertFakeFile(t, destinationBackend, "bundle-sibling/keep.txt", "keep")
|
||||
destinationState := readFakeSharedRootState(t, destinationBackend, "bundle")
|
||||
if got, want := len(destinationState.Owners), 1; got != want {
|
||||
t.Fatalf("owner count = %d, want %d", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func sharedRootRequest(sourceBackend, destinationBackend *fake.Backend, sourceBundle bundle.Bundle, reconciliationMode string) Request {
|
||||
return Request{
|
||||
PipelineID: "reports",
|
||||
DestinationID: "archive",
|
||||
SourceBundle: sourceBundle,
|
||||
SourceBackend: sourceBackend,
|
||||
DestinationBackend: destinationBackend,
|
||||
DestinationBundlePath: sourceBundle.RootRelativePath,
|
||||
Publish: config.PublishPolicy{Source: true},
|
||||
State: config.StatePolicy{Mode: config.StateModeSharedRoot},
|
||||
Reconciliation: config.ReconciliationPolicy{Mode: reconciliationMode},
|
||||
Transfer: defaultTransfer(),
|
||||
DistributorVersion: "test",
|
||||
}
|
||||
}
|
||||
|
||||
func writeFakeSharedRootState(t *testing.T, backend *fake.Backend, relative string, sharedRoot state.SharedRootState) {
|
||||
t.Helper()
|
||||
data, err := json.MarshalIndent(sharedRoot, "", " ")
|
||||
if err != nil {
|
||||
t.Fatalf("marshal shared-root state: %v", err)
|
||||
}
|
||||
statePath, err := storage.StatePath(relative)
|
||||
if err != nil {
|
||||
t.Fatalf("state path: %v", err)
|
||||
}
|
||||
testutil.WriteFakeFile(t, backend, statePath, string(append(data, '\n')))
|
||||
for _, output := range sharedRoot.Outputs {
|
||||
path, err := storage.Join(relative, output.Path)
|
||||
if err != nil {
|
||||
t.Fatalf("join output path: %v", err)
|
||||
}
|
||||
testutil.WriteFakeFile(t, backend, path, "old")
|
||||
}
|
||||
}
|
||||
|
||||
func readFakeSharedRootState(t *testing.T, backend *fake.Backend, relative string) state.SharedRootState {
|
||||
t.Helper()
|
||||
statePath, err := storage.StatePath(relative)
|
||||
if err != nil {
|
||||
t.Fatalf("state path: %v", err)
|
||||
}
|
||||
data, err := backend.ReadFile(context.Background(), statePath)
|
||||
if err != nil {
|
||||
t.Fatalf("read shared-root state: %v", err)
|
||||
}
|
||||
destinationState, err := state.ParseSharedRoot(data)
|
||||
if err != nil {
|
||||
t.Fatalf("parse shared-root state: %v", err)
|
||||
}
|
||||
return destinationState
|
||||
}
|
||||
|
||||
func findSharedRootOutputForTest(outputs []state.SharedRootOutputFile, path string) (state.SharedRootOutputFile, bool) {
|
||||
for _, output := range outputs {
|
||||
if output.Path == path {
|
||||
return output, true
|
||||
}
|
||||
}
|
||||
return state.SharedRootOutputFile{}, false
|
||||
}
|
||||
|
||||
func setSharedRootOwnerOutput(t *testing.T, sharedRoot *state.SharedRootState, index int, scope state.OwnerScope, manifest bundle.Manifest, path string) {
|
||||
t.Helper()
|
||||
sharedRoot.Owners[index].Scope = scope
|
||||
sharedRoot.Owners[index].Source = state.SourceState{Manifest: manifest}
|
||||
sourcePath := path
|
||||
if len(manifest.Files) > 0 {
|
||||
sourcePath = manifest.Files[0].Path
|
||||
}
|
||||
sharedRoot.Outputs[index].Path = path
|
||||
sharedRoot.Outputs[index].SourcePath = sourcePath
|
||||
sharedRoot.Outputs[index].Owner = scope
|
||||
sharedRoot.Outputs[index].SourceID = manifest.ID
|
||||
sharedRoot.Outputs[index].SourceDigest = manifest.Digest
|
||||
sharedRoot.Outputs[index].SourceCreated = manifest.Created
|
||||
if len(manifest.Files) > 0 {
|
||||
sharedRoot.Outputs[index].SHA256 = manifest.Files[0].SHA256
|
||||
sharedRoot.Outputs[index].Size = manifest.Files[0].Size
|
||||
}
|
||||
}
|
||||
|
||||
func sharedRootStateWithOwners(t *testing.T, current bundle.Manifest, includeCurrent bool) state.SharedRootState {
|
||||
t.Helper()
|
||||
createdAt := time.Date(2026, 5, 30, 11, 12, 0, 0, time.UTC)
|
||||
otherManifest := testutil.ValidManifest(testutil.BundleOptions{
|
||||
ID: "other.source",
|
||||
Files: []testutil.SourceFile{{Path: "other/report.md", Data: "# Other\n"}},
|
||||
})
|
||||
sharedRoot := state.SharedRootState{
|
||||
SchemaVersion: state.SharedRootSchemaVersion,
|
||||
DistributorVersion: "test",
|
||||
CreatedAt: createdAt,
|
||||
UpdatedAt: createdAt,
|
||||
State: state.StatePolicy{Mode: state.StateModeSharedRoot},
|
||||
Owners: []state.OwnerRecord{{
|
||||
Scope: state.CurrentOwnerScope("other", "archive"),
|
||||
Reconciliation: state.ReconciliationPolicy{Mode: config.ReconciliationModeReplace},
|
||||
Source: state.SourceState{Manifest: otherManifest},
|
||||
}},
|
||||
Outputs: []state.SharedRootOutputFile{{
|
||||
Path: "other/report.md",
|
||||
Kind: state.OutputKindSource,
|
||||
SourcePath: "other/report.md",
|
||||
SHA256: otherManifest.Files[0].SHA256,
|
||||
Size: otherManifest.Files[0].Size,
|
||||
Owner: state.CurrentOwnerScope("other", "archive"),
|
||||
SourceID: otherManifest.ID,
|
||||
SourceDigest: otherManifest.Digest,
|
||||
SourceCreated: otherManifest.Created,
|
||||
CreatedAt: createdAt,
|
||||
UpdatedAt: createdAt,
|
||||
}},
|
||||
}
|
||||
if !includeCurrent {
|
||||
return sharedRoot
|
||||
}
|
||||
older := current
|
||||
older.Created = older.Created.Add(-time.Hour)
|
||||
older.Files = append([]bundle.ManifestFile(nil), current.Files...)
|
||||
older.Files = append(older.Files, bundle.ManifestFile{
|
||||
Path: "old.md",
|
||||
SHA256: bundle.FileDigest([]byte("old\n")),
|
||||
Size: int64(len("old\n")),
|
||||
})
|
||||
older.Digest = bundle.BundleDigest(older.Files)
|
||||
scope := state.CurrentOwnerScope("reports", "archive")
|
||||
sharedRoot.Owners = append(sharedRoot.Owners, state.OwnerRecord{
|
||||
Scope: scope,
|
||||
Reconciliation: state.ReconciliationPolicy{Mode: config.ReconciliationModeReplace},
|
||||
Source: state.SourceState{Manifest: older},
|
||||
})
|
||||
for _, file := range older.Files {
|
||||
sharedRoot.Outputs = append(sharedRoot.Outputs, state.SharedRootOutputFile{
|
||||
Path: file.Path,
|
||||
Kind: state.OutputKindSource,
|
||||
SourcePath: file.Path,
|
||||
SHA256: file.SHA256,
|
||||
Size: file.Size,
|
||||
Owner: scope,
|
||||
SourceID: older.ID,
|
||||
SourceDigest: older.Digest,
|
||||
SourceCreated: older.Created,
|
||||
CreatedAt: createdAt,
|
||||
UpdatedAt: createdAt,
|
||||
})
|
||||
}
|
||||
return sharedRoot
|
||||
}
|
||||
|
||||
func sharedRootOutputPathList(outputs []state.SharedRootOutputFile) string {
|
||||
paths := make([]string, 0, len(outputs))
|
||||
for _, output := range outputs {
|
||||
paths = append(paths, output.Path)
|
||||
}
|
||||
return strings.Join(paths, ",")
|
||||
}
|
||||
@@ -1,215 +0,0 @@
|
||||
package publish
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/bundle"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/config"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/storage"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/storage/fake"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/testutil"
|
||||
)
|
||||
|
||||
func TestBuildPlansSingleOwnerTakeoverByPolicy(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
takeover config.TakeoverPolicy
|
||||
mutateState func(*bundle.Manifest, *testutil.DestinationStateOptions)
|
||||
wantAction Action
|
||||
wantErr string
|
||||
}{
|
||||
{
|
||||
name: "default same pipeline different source",
|
||||
takeover: config.TakeoverPolicy{},
|
||||
mutateState: func(manifest *bundle.Manifest, opts *testutil.DestinationStateOptions) {
|
||||
manifest.ID = "other.source"
|
||||
},
|
||||
wantAction: ActionReplaceTakeover,
|
||||
},
|
||||
{
|
||||
name: "same pipeline different newer source",
|
||||
takeover: config.TakeoverPolicy{Mode: config.TakeoverModeSamePipeline},
|
||||
mutateState: func(manifest *bundle.Manifest, opts *testutil.DestinationStateOptions) {
|
||||
manifest.ID = "other.source"
|
||||
manifest.Created = manifest.Created.AddDate(0, 0, 1)
|
||||
},
|
||||
wantAction: ActionReplaceTakeover,
|
||||
},
|
||||
{
|
||||
name: "default same pipeline different destination",
|
||||
takeover: config.TakeoverPolicy{},
|
||||
mutateState: func(manifest *bundle.Manifest, opts *testutil.DestinationStateOptions) {
|
||||
opts.DestinationID = "web"
|
||||
},
|
||||
wantAction: ActionReplaceTakeover,
|
||||
},
|
||||
{
|
||||
name: "same pipeline refuses different pipeline",
|
||||
takeover: config.TakeoverPolicy{Mode: config.TakeoverModeSamePipeline},
|
||||
mutateState: func(manifest *bundle.Manifest, opts *testutil.DestinationStateOptions) {
|
||||
opts.PipelineID = "other"
|
||||
},
|
||||
wantErr: "fail_conflict",
|
||||
},
|
||||
{
|
||||
name: "same source allows different pipeline",
|
||||
takeover: config.TakeoverPolicy{Mode: config.TakeoverModeSameSource},
|
||||
mutateState: func(manifest *bundle.Manifest, opts *testutil.DestinationStateOptions) {
|
||||
opts.PipelineID = "other"
|
||||
},
|
||||
wantAction: ActionReplaceTakeover,
|
||||
},
|
||||
{
|
||||
name: "same source refuses different source",
|
||||
takeover: config.TakeoverPolicy{Mode: config.TakeoverModeSameSource},
|
||||
mutateState: func(manifest *bundle.Manifest, opts *testutil.DestinationStateOptions) {
|
||||
manifest.ID = "other.source"
|
||||
},
|
||||
wantErr: "fail_conflict",
|
||||
},
|
||||
{
|
||||
name: "any managed allows different pipeline",
|
||||
takeover: config.TakeoverPolicy{Mode: config.TakeoverModeAnyManaged},
|
||||
mutateState: func(manifest *bundle.Manifest, opts *testutil.DestinationStateOptions) {
|
||||
manifest.ID = "other.source"
|
||||
opts.PipelineID = "other"
|
||||
},
|
||||
wantAction: ActionReplaceTakeover,
|
||||
},
|
||||
{
|
||||
name: "never refuses different source",
|
||||
takeover: config.TakeoverPolicy{Mode: config.TakeoverModeNever},
|
||||
mutateState: func(manifest *bundle.Manifest, opts *testutil.DestinationStateOptions) {
|
||||
manifest.ID = "other.source"
|
||||
},
|
||||
wantErr: "fail_conflict",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
sourceBackend := fake.New()
|
||||
sourceBundle := testutil.WriteFakeSourceBundle(t, sourceBackend, "bundle", testutil.BundleOptions{})
|
||||
destinationBackend := fake.New()
|
||||
destinationManifest := sourceBundle.Manifest
|
||||
destinationManifest.Files = append([]bundle.ManifestFile(nil), sourceBundle.Manifest.Files...)
|
||||
opts := testutil.DestinationStateOptions{}
|
||||
tt.mutateState(&destinationManifest, &opts)
|
||||
testutil.WriteFakeDestinationState(t, destinationBackend, "bundle", destinationManifest, opts)
|
||||
|
||||
req := takeoverRequest(sourceBackend, destinationBackend, sourceBundle, tt.takeover, config.ReconciliationModeReplace)
|
||||
plan, err := Build(context.Background(), req)
|
||||
if tt.wantErr != "" {
|
||||
if err == nil || !strings.Contains(err.Error(), tt.wantErr) {
|
||||
t.Fatalf("Build() error = %v, want %q", err, tt.wantErr)
|
||||
}
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("Build() error = %v", err)
|
||||
}
|
||||
if plan.Action != tt.wantAction {
|
||||
t.Fatalf("plan action = %s, want %s", plan.Action, tt.wantAction)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildDoesNotTakeOverInvalidOrUnmanagedDestination(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
prepare func(t *testing.T, backend *fake.Backend)
|
||||
wantErr string
|
||||
}{
|
||||
{
|
||||
name: "invalid state",
|
||||
prepare: func(t *testing.T, backend *fake.Backend) {
|
||||
t.Helper()
|
||||
statePath, err := storage.StatePath("bundle")
|
||||
if err != nil {
|
||||
t.Fatalf("state path: %v", err)
|
||||
}
|
||||
testutil.WriteFakeFile(t, backend, statePath, "{invalid")
|
||||
},
|
||||
wantErr: "fail_conflict",
|
||||
},
|
||||
{
|
||||
name: "unmanaged content",
|
||||
prepare: func(t *testing.T, backend *fake.Backend) {
|
||||
t.Helper()
|
||||
testutil.WriteFakeFile(t, backend, "bundle/old.txt", "old")
|
||||
},
|
||||
wantErr: "fail_unmanaged",
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
sourceBackend := fake.New()
|
||||
sourceBundle := testutil.WriteFakeSourceBundle(t, sourceBackend, "bundle", testutil.BundleOptions{})
|
||||
destinationBackend := fake.New()
|
||||
tt.prepare(t, destinationBackend)
|
||||
|
||||
req := takeoverRequest(sourceBackend, destinationBackend, sourceBundle, config.TakeoverPolicy{Mode: config.TakeoverModeAnyManaged}, config.ReconciliationModeReplace)
|
||||
_, err := Build(context.Background(), req)
|
||||
if err == nil || !strings.Contains(err.Error(), tt.wantErr) {
|
||||
t.Fatalf("Build() error = %v, want %q", err, tt.wantErr)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteTakeoverMergeDoesNotRetainOldSourceOutputs(t *testing.T) {
|
||||
sourceBackend := fake.New()
|
||||
sourceBundle := testutil.WriteFakeSourceBundle(t, sourceBackend, "bundle", testutil.BundleOptions{
|
||||
Files: []testutil.SourceFile{{Path: "report.md", Data: "# Report\nNew.\n"}},
|
||||
})
|
||||
destinationBackend := fake.New()
|
||||
oldManifest := sourceBundle.Manifest
|
||||
oldManifest.ID = "old.source"
|
||||
oldManifest.Files = []bundle.ManifestFile{
|
||||
{Path: "report.md", SHA256: bundle.FileDigest([]byte("old\n")), Size: int64(len("old\n"))},
|
||||
{Path: "summary.txt", SHA256: bundle.FileDigest([]byte("old summary\n")), Size: int64(len("old summary\n"))},
|
||||
}
|
||||
oldManifest.Digest = bundle.BundleDigest(oldManifest.Files)
|
||||
testutil.WriteFakeDestinationState(t, destinationBackend, "bundle", oldManifest, testutil.DestinationStateOptions{})
|
||||
|
||||
req := takeoverRequest(sourceBackend, destinationBackend, sourceBundle, config.TakeoverPolicy{Mode: config.TakeoverModeSamePipeline}, config.ReconciliationModeMerge)
|
||||
plan, err := Build(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("Build() error = %v", err)
|
||||
}
|
||||
if plan.Action != ActionReplaceTakeover {
|
||||
t.Fatalf("plan action = %s, want %s", plan.Action, ActionReplaceTakeover)
|
||||
}
|
||||
if err := Execute(context.Background(), req, plan); err != nil {
|
||||
t.Fatalf("Execute() error = %v", err)
|
||||
}
|
||||
|
||||
testutil.AssertFakeFile(t, destinationBackend, "bundle/report.md", "# Report\nNew.\n")
|
||||
testutil.AssertFakeMissing(t, destinationBackend, "bundle/summary.txt")
|
||||
destinationState := readFakeState(t, destinationBackend, "bundle")
|
||||
if got, want := len(destinationState.Outputs), 1; got != want {
|
||||
t.Fatalf("state output count = %d, want %d", got, want)
|
||||
}
|
||||
if got, want := destinationState.Source.Manifest.ID, sourceBundle.Manifest.ID; got != want {
|
||||
t.Fatalf("state source id = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func takeoverRequest(sourceBackend, destinationBackend *fake.Backend, sourceBundle bundle.Bundle, takeover config.TakeoverPolicy, reconciliationMode string) Request {
|
||||
return Request{
|
||||
PipelineID: "reports",
|
||||
DestinationID: "archive",
|
||||
SourceBundle: sourceBundle,
|
||||
SourceBackend: sourceBackend,
|
||||
DestinationBackend: destinationBackend,
|
||||
DestinationBundlePath: sourceBundle.RootRelativePath,
|
||||
Publish: config.PublishPolicy{Source: true},
|
||||
Reconciliation: config.ReconciliationPolicy{Mode: reconciliationMode},
|
||||
Takeover: takeover,
|
||||
Transfer: defaultTransfer(),
|
||||
DistributorVersion: "test",
|
||||
}
|
||||
}
|
||||
422
internal/state/catalog.go
Normal file
422
internal/state/catalog.go
Normal file
@@ -0,0 +1,422 @@
|
||||
package state
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/bundle"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/config"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/link"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/storage"
|
||||
)
|
||||
|
||||
type CatalogState struct {
|
||||
SchemaVersion int
|
||||
DistributorVersion string
|
||||
CreatedAt time.Time
|
||||
UpdatedAt time.Time
|
||||
State StatePolicy
|
||||
Outputs []CatalogOutputFile
|
||||
}
|
||||
|
||||
type CatalogSourceIdentity struct {
|
||||
ID string
|
||||
Digest string
|
||||
Created time.Time
|
||||
}
|
||||
|
||||
type CatalogOutputFile struct {
|
||||
Path string
|
||||
PipelineID string
|
||||
DestinationID string
|
||||
Source CatalogSourceIdentity
|
||||
Kind string
|
||||
SourcePath string
|
||||
Transform string
|
||||
URL string
|
||||
SHA256 string
|
||||
Size int64
|
||||
CreatedAt time.Time
|
||||
UpdatedAt time.Time
|
||||
}
|
||||
|
||||
type rawCatalogState struct {
|
||||
SchemaVersion *int `json:"schema_version"`
|
||||
DistributorVersion string `json:"distributor_version"`
|
||||
CreatedAt *string `json:"created_at"`
|
||||
UpdatedAt *string `json:"updated_at"`
|
||||
State *rawStatePolicy `json:"state"`
|
||||
Outputs []rawCatalogOutput `json:"outputs"`
|
||||
}
|
||||
|
||||
type rawCatalogOutput struct {
|
||||
Path *string `json:"path"`
|
||||
PipelineID *string `json:"pipeline_id"`
|
||||
DestinationID *string `json:"destination_id"`
|
||||
Source *rawCatalogSourceIdentity `json:"source"`
|
||||
Kind *string `json:"kind"`
|
||||
SourcePath *string `json:"source_path"`
|
||||
Transform *string `json:"transform"`
|
||||
URL *string `json:"url"`
|
||||
SHA256 *string `json:"sha256"`
|
||||
Size *int64 `json:"size"`
|
||||
CreatedAt *string `json:"created_at"`
|
||||
UpdatedAt *string `json:"updated_at"`
|
||||
}
|
||||
|
||||
type rawCatalogSourceIdentity struct {
|
||||
ID *string `json:"id"`
|
||||
Digest *string `json:"digest"`
|
||||
Created *string `json:"created"`
|
||||
}
|
||||
|
||||
func ParseCatalog(data []byte) (CatalogState, error) {
|
||||
decoder := json.NewDecoder(bytes.NewReader(data))
|
||||
decoder.DisallowUnknownFields()
|
||||
var raw rawCatalogState
|
||||
if err := decoder.Decode(&raw); err != nil {
|
||||
return CatalogState{}, fmt.Errorf("parse distributor state: %w", err)
|
||||
}
|
||||
var extra any
|
||||
if err := decoder.Decode(&extra); err != io.EOF {
|
||||
return CatalogState{}, fmt.Errorf("parse distributor state: trailing data")
|
||||
}
|
||||
state, err := parseCatalogRaw(raw)
|
||||
if err != nil {
|
||||
return CatalogState{}, err
|
||||
}
|
||||
if err := ValidateCatalog(state); err != nil {
|
||||
return CatalogState{}, err
|
||||
}
|
||||
return state, nil
|
||||
}
|
||||
|
||||
func parseCatalogRaw(raw rawCatalogState) (CatalogState, error) {
|
||||
if raw.SchemaVersion == nil {
|
||||
return CatalogState{}, fmt.Errorf("state schema_version is required")
|
||||
}
|
||||
state := CatalogState{
|
||||
SchemaVersion: *raw.SchemaVersion,
|
||||
DistributorVersion: raw.DistributorVersion,
|
||||
}
|
||||
if state.SchemaVersion != CatalogSchemaVersion {
|
||||
return CatalogState{}, fmt.Errorf("state schema_version must be %d", CatalogSchemaVersion)
|
||||
}
|
||||
createdAt, err := parseRequiredTime("state created_at", raw.CreatedAt)
|
||||
if err != nil {
|
||||
return CatalogState{}, err
|
||||
}
|
||||
updatedAt, err := parseRequiredTime("state updated_at", raw.UpdatedAt)
|
||||
if err != nil {
|
||||
return CatalogState{}, err
|
||||
}
|
||||
state.CreatedAt = createdAt
|
||||
state.UpdatedAt = updatedAt
|
||||
if raw.State == nil || raw.State.Mode == "" {
|
||||
return CatalogState{}, fmt.Errorf("state state.mode is required")
|
||||
}
|
||||
state.State.Mode = raw.State.Mode
|
||||
if raw.Outputs == nil {
|
||||
return CatalogState{}, fmt.Errorf("state outputs is required")
|
||||
}
|
||||
outputs, err := parseCatalogOutputs(raw.Outputs)
|
||||
if err != nil {
|
||||
return CatalogState{}, err
|
||||
}
|
||||
state.Outputs = outputs
|
||||
return state, nil
|
||||
}
|
||||
|
||||
func parseCatalogOutputs(rawOutputs []rawCatalogOutput) ([]CatalogOutputFile, error) {
|
||||
outputs := make([]CatalogOutputFile, 0, len(rawOutputs))
|
||||
for index, raw := range rawOutputs {
|
||||
output, err := parseCatalogOutput(index, raw)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
outputs = append(outputs, output)
|
||||
}
|
||||
return outputs, nil
|
||||
}
|
||||
|
||||
func parseCatalogOutput(index int, raw rawCatalogOutput) (CatalogOutputFile, error) {
|
||||
if raw.Path == nil || *raw.Path == "" {
|
||||
return CatalogOutputFile{}, fmt.Errorf("state outputs[%d].path is required", index)
|
||||
}
|
||||
if raw.PipelineID == nil || *raw.PipelineID == "" {
|
||||
return CatalogOutputFile{}, fmt.Errorf("state outputs[%d].pipeline_id is required", index)
|
||||
}
|
||||
if raw.DestinationID == nil || *raw.DestinationID == "" {
|
||||
return CatalogOutputFile{}, fmt.Errorf("state outputs[%d].destination_id is required", index)
|
||||
}
|
||||
source, err := parseCatalogSourceIdentity(index, raw.Source)
|
||||
if err != nil {
|
||||
return CatalogOutputFile{}, err
|
||||
}
|
||||
if raw.Kind == nil || *raw.Kind == "" {
|
||||
return CatalogOutputFile{}, fmt.Errorf("state outputs[%d].kind is required", index)
|
||||
}
|
||||
switch *raw.Kind {
|
||||
case OutputKindSource:
|
||||
if raw.SourcePath != nil {
|
||||
return CatalogOutputFile{}, fmt.Errorf("state outputs[%d].source_path is only valid for generated output", index)
|
||||
}
|
||||
if raw.Transform != nil {
|
||||
return CatalogOutputFile{}, fmt.Errorf("state outputs[%d].transform is only valid for generated output", index)
|
||||
}
|
||||
case OutputKindGenerated:
|
||||
if raw.SourcePath == nil || *raw.SourcePath == "" {
|
||||
return CatalogOutputFile{}, fmt.Errorf("state outputs[%d].source_path is required for generated output", index)
|
||||
}
|
||||
if raw.Transform == nil || *raw.Transform == "" {
|
||||
return CatalogOutputFile{}, fmt.Errorf("state outputs[%d].transform is required for generated output", index)
|
||||
}
|
||||
}
|
||||
if raw.SHA256 == nil || *raw.SHA256 == "" {
|
||||
return CatalogOutputFile{}, fmt.Errorf("state outputs[%d].sha256 is required", index)
|
||||
}
|
||||
if raw.Size == nil {
|
||||
return CatalogOutputFile{}, fmt.Errorf("state outputs[%d].size is required", index)
|
||||
}
|
||||
createdAt, err := parseRequiredTime(fmt.Sprintf("state outputs[%d].created_at", index), raw.CreatedAt)
|
||||
if err != nil {
|
||||
return CatalogOutputFile{}, err
|
||||
}
|
||||
updatedAt, err := parseRequiredTime(fmt.Sprintf("state outputs[%d].updated_at", index), raw.UpdatedAt)
|
||||
if err != nil {
|
||||
return CatalogOutputFile{}, err
|
||||
}
|
||||
output := CatalogOutputFile{
|
||||
Path: *raw.Path,
|
||||
PipelineID: *raw.PipelineID,
|
||||
DestinationID: *raw.DestinationID,
|
||||
Source: source,
|
||||
Kind: *raw.Kind,
|
||||
SHA256: *raw.SHA256,
|
||||
Size: *raw.Size,
|
||||
CreatedAt: createdAt,
|
||||
UpdatedAt: updatedAt,
|
||||
}
|
||||
if raw.SourcePath != nil {
|
||||
output.SourcePath = *raw.SourcePath
|
||||
}
|
||||
if raw.Transform != nil {
|
||||
output.Transform = *raw.Transform
|
||||
}
|
||||
if raw.URL != nil {
|
||||
if *raw.URL == "" {
|
||||
return CatalogOutputFile{}, fmt.Errorf("state outputs[%d].url must not be empty", index)
|
||||
}
|
||||
output.URL = *raw.URL
|
||||
}
|
||||
return output, nil
|
||||
}
|
||||
|
||||
func parseCatalogSourceIdentity(index int, raw *rawCatalogSourceIdentity) (CatalogSourceIdentity, error) {
|
||||
if raw == nil {
|
||||
return CatalogSourceIdentity{}, fmt.Errorf("state outputs[%d].source is required", index)
|
||||
}
|
||||
if raw.ID == nil || *raw.ID == "" {
|
||||
return CatalogSourceIdentity{}, fmt.Errorf("state outputs[%d].source.id is required", index)
|
||||
}
|
||||
if raw.Digest == nil || *raw.Digest == "" {
|
||||
return CatalogSourceIdentity{}, fmt.Errorf("state outputs[%d].source.digest is required", index)
|
||||
}
|
||||
created, err := parseRequiredTime(fmt.Sprintf("state outputs[%d].source.created", index), raw.Created)
|
||||
if err != nil {
|
||||
return CatalogSourceIdentity{}, err
|
||||
}
|
||||
return CatalogSourceIdentity{
|
||||
ID: *raw.ID,
|
||||
Digest: *raw.Digest,
|
||||
Created: created,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s CatalogState) CreatedAtString() string {
|
||||
return s.CreatedAt.UTC().Format(time.RFC3339)
|
||||
}
|
||||
|
||||
func (s CatalogState) UpdatedAtString() string {
|
||||
return s.UpdatedAt.UTC().Format(time.RFC3339)
|
||||
}
|
||||
|
||||
func (s CatalogSourceIdentity) CreatedString() string {
|
||||
return s.Created.UTC().Format(time.RFC3339)
|
||||
}
|
||||
|
||||
func (o CatalogOutputFile) CreatedAtString() string {
|
||||
return o.CreatedAt.UTC().Format(time.RFC3339)
|
||||
}
|
||||
|
||||
func (o CatalogOutputFile) UpdatedAtString() string {
|
||||
return o.UpdatedAt.UTC().Format(time.RFC3339)
|
||||
}
|
||||
|
||||
func (s CatalogState) MarshalJSON() ([]byte, error) {
|
||||
type stateJSON struct {
|
||||
SchemaVersion int `json:"schema_version"`
|
||||
DistributorVersion string `json:"distributor_version,omitempty"`
|
||||
CreatedAt string `json:"created_at"`
|
||||
UpdatedAt string `json:"updated_at"`
|
||||
State StatePolicy `json:"state"`
|
||||
Outputs []CatalogOutputFile `json:"outputs"`
|
||||
}
|
||||
return json.Marshal(stateJSON{
|
||||
SchemaVersion: s.SchemaVersion,
|
||||
DistributorVersion: s.DistributorVersion,
|
||||
CreatedAt: s.CreatedAtString(),
|
||||
UpdatedAt: s.UpdatedAtString(),
|
||||
State: s.State,
|
||||
Outputs: s.Outputs,
|
||||
})
|
||||
}
|
||||
|
||||
func (s CatalogSourceIdentity) MarshalJSON() ([]byte, error) {
|
||||
type sourceJSON struct {
|
||||
ID string `json:"id"`
|
||||
Digest string `json:"digest"`
|
||||
Created string `json:"created"`
|
||||
}
|
||||
return json.Marshal(sourceJSON{
|
||||
ID: s.ID,
|
||||
Digest: s.Digest,
|
||||
Created: s.CreatedString(),
|
||||
})
|
||||
}
|
||||
|
||||
func (o CatalogOutputFile) MarshalJSON() ([]byte, error) {
|
||||
type outputJSON struct {
|
||||
Path string `json:"path"`
|
||||
PipelineID string `json:"pipeline_id"`
|
||||
DestinationID string `json:"destination_id"`
|
||||
Source CatalogSourceIdentity `json:"source"`
|
||||
Kind string `json:"kind"`
|
||||
SourcePath string `json:"source_path,omitempty"`
|
||||
Transform string `json:"transform,omitempty"`
|
||||
URL string `json:"url,omitempty"`
|
||||
SHA256 string `json:"sha256"`
|
||||
Size int64 `json:"size"`
|
||||
CreatedAt string `json:"created_at"`
|
||||
UpdatedAt string `json:"updated_at"`
|
||||
}
|
||||
return json.Marshal(outputJSON{
|
||||
Path: o.Path,
|
||||
PipelineID: o.PipelineID,
|
||||
DestinationID: o.DestinationID,
|
||||
Source: o.Source,
|
||||
Kind: o.Kind,
|
||||
SourcePath: o.SourcePath,
|
||||
Transform: o.Transform,
|
||||
URL: o.URL,
|
||||
SHA256: o.SHA256,
|
||||
Size: o.Size,
|
||||
CreatedAt: o.CreatedAtString(),
|
||||
UpdatedAt: o.UpdatedAtString(),
|
||||
})
|
||||
}
|
||||
|
||||
func ValidateCatalog(s CatalogState) error {
|
||||
if s.SchemaVersion != CatalogSchemaVersion {
|
||||
return fmt.Errorf("state schema_version must be %d", CatalogSchemaVersion)
|
||||
}
|
||||
if s.CreatedAt.IsZero() {
|
||||
return fmt.Errorf("state created_at is required")
|
||||
}
|
||||
if s.UpdatedAt.IsZero() {
|
||||
return fmt.Errorf("state updated_at is required")
|
||||
}
|
||||
if s.State.Mode != StateModeCatalog {
|
||||
return fmt.Errorf("state state.mode must be %s", StateModeCatalog)
|
||||
}
|
||||
if s.Outputs == nil {
|
||||
return fmt.Errorf("state outputs is required")
|
||||
}
|
||||
seenPaths := make(map[string]struct{}, len(s.Outputs))
|
||||
for index, output := range s.Outputs {
|
||||
if err := validateCatalogOutput(index, output); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, exists := seenPaths[output.Path]; exists {
|
||||
return fmt.Errorf("state outputs[%d].path duplicates %q", index, output.Path)
|
||||
}
|
||||
seenPaths[output.Path] = struct{}{}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateCatalogOutput(index int, output CatalogOutputFile) error {
|
||||
if err := storage.ValidatePath(output.Path); err != nil {
|
||||
return fmt.Errorf("state outputs[%d].path: %w", index, err)
|
||||
}
|
||||
if output.PipelineID == "" {
|
||||
return fmt.Errorf("state outputs[%d].pipeline_id is required", index)
|
||||
}
|
||||
if !config.IsSlugLikeID(output.PipelineID) {
|
||||
return fmt.Errorf("state outputs[%d].pipeline_id must be a slug-like identifier", index)
|
||||
}
|
||||
if output.DestinationID == "" {
|
||||
return fmt.Errorf("state outputs[%d].destination_id is required", index)
|
||||
}
|
||||
if !config.IsSlugLikeID(output.DestinationID) {
|
||||
return fmt.Errorf("state outputs[%d].destination_id must be a slug-like identifier", index)
|
||||
}
|
||||
if err := validateCatalogSourceIdentity(index, output.Source); err != nil {
|
||||
return err
|
||||
}
|
||||
switch output.Kind {
|
||||
case OutputKindSource:
|
||||
if output.SourcePath != "" {
|
||||
return fmt.Errorf("state outputs[%d].source_path is only valid for generated output", index)
|
||||
}
|
||||
if output.Transform != "" {
|
||||
return fmt.Errorf("state outputs[%d].transform is only valid for generated output", index)
|
||||
}
|
||||
case OutputKindGenerated:
|
||||
if output.SourcePath == "" {
|
||||
return fmt.Errorf("state outputs[%d].source_path is required for generated output", index)
|
||||
}
|
||||
if err := storage.ValidatePath(output.SourcePath); err != nil {
|
||||
return fmt.Errorf("state outputs[%d].source_path: %w", index, err)
|
||||
}
|
||||
if output.Transform == "" {
|
||||
return fmt.Errorf("state outputs[%d].transform is required for generated output", index)
|
||||
}
|
||||
default:
|
||||
return fmt.Errorf("state outputs[%d].kind must be source or generated", index)
|
||||
}
|
||||
if output.URL != "" {
|
||||
if err := link.ValidateHTTPURL(output.URL); err != nil {
|
||||
return fmt.Errorf("state outputs[%d].url: %w", index, err)
|
||||
}
|
||||
}
|
||||
if err := bundle.ValidateDigest(output.SHA256); err != nil {
|
||||
return fmt.Errorf("state outputs[%d].sha256: %w", index, err)
|
||||
}
|
||||
if output.Size < 0 {
|
||||
return fmt.Errorf("state outputs[%d].size must be non-negative", index)
|
||||
}
|
||||
if output.CreatedAt.IsZero() {
|
||||
return fmt.Errorf("state outputs[%d].created_at is required", index)
|
||||
}
|
||||
if output.UpdatedAt.IsZero() {
|
||||
return fmt.Errorf("state outputs[%d].updated_at is required", index)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateCatalogSourceIdentity(index int, source CatalogSourceIdentity) error {
|
||||
if source.ID == "" {
|
||||
return fmt.Errorf("state outputs[%d].source.id is required", index)
|
||||
}
|
||||
if err := bundle.ValidateDigest(source.Digest); err != nil {
|
||||
return fmt.Errorf("state outputs[%d].source.digest: %w", index, err)
|
||||
}
|
||||
if source.Created.IsZero() {
|
||||
return fmt.Errorf("state outputs[%d].source.created is required", index)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
361
internal/state/catalog_test.go
Normal file
361
internal/state/catalog_test.go
Normal file
@@ -0,0 +1,361 @@
|
||||
package state
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestParseCatalogState(t *testing.T) {
|
||||
state, err := ParseCatalog([]byte(validCatalogStateJSON(t)))
|
||||
if err != nil {
|
||||
t.Fatalf("ParseCatalog() error = %v", err)
|
||||
}
|
||||
if got, want := state.SchemaVersion, CatalogSchemaVersion; got != want {
|
||||
t.Fatalf("schema version = %d, want %d", got, want)
|
||||
}
|
||||
if got, want := state.CreatedAtString(), "2026-06-19T12:00:00Z"; got != want {
|
||||
t.Fatalf("created_at = %q, want %q", got, want)
|
||||
}
|
||||
if got, want := state.UpdatedAtString(), "2026-06-19T12:05:00Z"; got != want {
|
||||
t.Fatalf("updated_at = %q, want %q", got, want)
|
||||
}
|
||||
if got, want := state.State.Mode, StateModeCatalog; got != want {
|
||||
t.Fatalf("state mode = %q, want %q", got, want)
|
||||
}
|
||||
if got, want := len(state.Outputs), 2; got != want {
|
||||
t.Fatalf("output count = %d, want %d", got, want)
|
||||
}
|
||||
sourceOutput := state.Outputs[0]
|
||||
if sourceOutput.SourcePath != "" || sourceOutput.Transform != "" {
|
||||
t.Fatalf("source output source_path=%q transform=%q, want omitted", sourceOutput.SourcePath, sourceOutput.Transform)
|
||||
}
|
||||
generatedOutput := state.Outputs[1]
|
||||
if generatedOutput.SourcePath != "report.md" || generatedOutput.Transform != "markdown_to_html" {
|
||||
t.Fatalf("generated output source_path=%q transform=%q", generatedOutput.SourcePath, generatedOutput.Transform)
|
||||
}
|
||||
if got, want := generatedOutput.Source.CreatedString(), "2026-05-30T11:10:00Z"; got != want {
|
||||
t.Fatalf("source created = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCatalogMarshalIsDeterministic(t *testing.T) {
|
||||
data, err := json.Marshal(validCatalogState(t))
|
||||
if err != nil {
|
||||
t.Fatalf("Marshal() error = %v", err)
|
||||
}
|
||||
want := `{"schema_version":4,"distributor_version":"dev","created_at":"2026-06-19T12:00:00Z","updated_at":"2026-06-19T12:05:00Z","state":{"mode":"catalog"},"outputs":[{"path":"report.md","pipeline_id":"reports","destination_id":"archive","source":{"id":"weather.daily.brentwood.2026-05-30","digest":"sha256:099b205780d2b050024868399961b05731729a548d5d6329c7b06a6740dd75fe","created":"2026-05-30T11:10:00Z"},"kind":"source","sha256":"sha256:3640fd37140ee4d2e0e93e78834f232ea67a50e7bc6279203690cc7de1975fa6","size":16,"created_at":"2026-06-19T12:00:00Z","updated_at":"2026-06-19T12:05:00Z"},{"path":"report.html","pipeline_id":"reports","destination_id":"html","source":{"id":"weather.daily.brentwood.2026-05-30","digest":"sha256:099b205780d2b050024868399961b05731729a548d5d6329c7b06a6740dd75fe","created":"2026-05-30T11:10:00Z"},"kind":"generated","source_path":"report.md","transform":"markdown_to_html","url":"https://reports.example.com/report.html","sha256":"sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","size":128,"created_at":"2026-06-19T12:00:00Z","updated_at":"2026-06-19T12:05:00Z"}]}`
|
||||
if string(data) != want {
|
||||
t.Fatalf("json = %s, want %s", data, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseDocumentHandlesCatalogAndSupersededLegacy(t *testing.T) {
|
||||
catalog, err := ParseDocument([]byte(validCatalogStateJSON(t)))
|
||||
if err != nil {
|
||||
t.Fatalf("ParseDocument(catalog) error = %v", err)
|
||||
}
|
||||
if catalog.Catalog == nil || catalog.SingleOwner != nil || catalog.SharedRoot != nil || catalog.SupersededLegacy != nil {
|
||||
t.Fatalf("catalog document = %#v", catalog)
|
||||
}
|
||||
|
||||
tests := map[string]string{
|
||||
"schema 1": legacyStateJSON(t),
|
||||
"schema 2": validStateJSON(t),
|
||||
"schema 3": validSharedRootStateJSON(t),
|
||||
}
|
||||
for name, body := range tests {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
document, err := ParseDocument([]byte(body))
|
||||
if err != nil {
|
||||
t.Fatalf("ParseDocument() error = %v", err)
|
||||
}
|
||||
if document.SupersededLegacy == nil || document.Catalog != nil || document.SingleOwner != nil || document.SharedRoot != nil {
|
||||
t.Fatalf("document = %#v, want superseded legacy only", document)
|
||||
}
|
||||
if document.SupersededLegacy.SchemaVersion < legacySchemaVersion || document.SupersededLegacy.SchemaVersion >= CatalogSchemaVersion {
|
||||
t.Fatalf("legacy schema version = %d, want 1 through 3", document.SupersededLegacy.SchemaVersion)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseDocumentRejectsUnsupportedFutureSchema(t *testing.T) {
|
||||
body := strings.Replace(validCatalogStateJSON(t), `"schema_version": 4`, `"schema_version": 5`, 1)
|
||||
_, err := ParseDocument([]byte(body))
|
||||
assertStateErrorContains(t, err, "schema_version 5 is unsupported")
|
||||
}
|
||||
|
||||
func TestParseDocumentRejectsTrailingData(t *testing.T) {
|
||||
_, err := ParseDocument([]byte(validCatalogStateJSON(t) + "\n{}"))
|
||||
assertStateErrorContains(t, err, "trailing data")
|
||||
}
|
||||
|
||||
func TestParseCatalogRejectsMissingFields(t *testing.T) {
|
||||
tests := map[string]func(map[string]any){
|
||||
"schema_version": func(document map[string]any) {
|
||||
delete(document, "schema_version")
|
||||
},
|
||||
"created_at": func(document map[string]any) {
|
||||
delete(document, "created_at")
|
||||
},
|
||||
"updated_at": func(document map[string]any) {
|
||||
delete(document, "updated_at")
|
||||
},
|
||||
"state": func(document map[string]any) {
|
||||
delete(document, "state")
|
||||
},
|
||||
"outputs": func(document map[string]any) {
|
||||
delete(document, "outputs")
|
||||
},
|
||||
"path": func(document map[string]any) {
|
||||
delete(firstCatalogOutput(document), "path")
|
||||
},
|
||||
"pipeline_id": func(document map[string]any) {
|
||||
delete(firstCatalogOutput(document), "pipeline_id")
|
||||
},
|
||||
"destination_id": func(document map[string]any) {
|
||||
delete(firstCatalogOutput(document), "destination_id")
|
||||
},
|
||||
"source": func(document map[string]any) {
|
||||
delete(firstCatalogOutput(document), "source")
|
||||
},
|
||||
"source id": func(document map[string]any) {
|
||||
delete(firstCatalogSource(document), "id")
|
||||
},
|
||||
"source digest": func(document map[string]any) {
|
||||
delete(firstCatalogSource(document), "digest")
|
||||
},
|
||||
"source created": func(document map[string]any) {
|
||||
delete(firstCatalogSource(document), "created")
|
||||
},
|
||||
"kind": func(document map[string]any) {
|
||||
delete(firstCatalogOutput(document), "kind")
|
||||
},
|
||||
"sha256": func(document map[string]any) {
|
||||
delete(firstCatalogOutput(document), "sha256")
|
||||
},
|
||||
"size": func(document map[string]any) {
|
||||
delete(firstCatalogOutput(document), "size")
|
||||
},
|
||||
}
|
||||
for name, mutate := range tests {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
document := catalogStateObject(t)
|
||||
mutate(document)
|
||||
_, err := ParseCatalog(mustMarshalCatalogObject(t, document))
|
||||
assertStateErrorContains(t, err, "required")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseCatalogRejectsMalformedTimestamps(t *testing.T) {
|
||||
tests := map[string]func(string) string{
|
||||
"created_at": func(body string) string {
|
||||
return strings.Replace(body, `"created_at": "2026-06-19T12:00:00Z"`, `"created_at": "June 19"`, 1)
|
||||
},
|
||||
"updated_at": func(body string) string {
|
||||
return strings.Replace(body, `"updated_at": "2026-06-19T12:05:00Z"`, `"updated_at": "June 19"`, 1)
|
||||
},
|
||||
"source created": func(body string) string {
|
||||
return strings.Replace(body, `"created": "2026-05-30T11:10:00Z"`, `"created": "May 30"`, 1)
|
||||
},
|
||||
"output created_at": func(body string) string {
|
||||
return strings.Replace(body, ` "created_at": "2026-06-19T12:00:00Z"`, ` "created_at": "June 19"`, 1)
|
||||
},
|
||||
"output updated_at": func(body string) string {
|
||||
return strings.Replace(body, ` "updated_at": "2026-06-19T12:05:00Z"`, ` "updated_at": "June 19"`, 1)
|
||||
},
|
||||
}
|
||||
for name, mutate := range tests {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
_, err := ParseCatalog([]byte(mutate(validCatalogStateJSON(t))))
|
||||
assertStateErrorContains(t, err, "RFC3339")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateCatalogRejectsInvalidOutputRecords(t *testing.T) {
|
||||
tests := map[string]func(*CatalogState){
|
||||
"duplicate path": func(s *CatalogState) {
|
||||
s.Outputs[1].Path = s.Outputs[0].Path
|
||||
},
|
||||
"invalid output path": func(s *CatalogState) {
|
||||
s.Outputs[0].Path = "../report.md"
|
||||
},
|
||||
"invalid pipeline id": func(s *CatalogState) {
|
||||
s.Outputs[0].PipelineID = ".reports"
|
||||
},
|
||||
"invalid destination id": func(s *CatalogState) {
|
||||
s.Outputs[0].DestinationID = ".archive"
|
||||
},
|
||||
"missing source id": func(s *CatalogState) {
|
||||
s.Outputs[0].Source.ID = ""
|
||||
},
|
||||
"invalid source digest": func(s *CatalogState) {
|
||||
s.Outputs[0].Source.Digest = "SHA256:099b205780d2b050024868399961b05731729a548d5d6329c7b06a6740dd75fe"
|
||||
},
|
||||
"missing source created": func(s *CatalogState) {
|
||||
s.Outputs[0].Source.Created = time.Time{}
|
||||
},
|
||||
"invalid kind": func(s *CatalogState) {
|
||||
s.Outputs[0].Kind = "document"
|
||||
},
|
||||
"generated missing source path": func(s *CatalogState) {
|
||||
s.Outputs[1].SourcePath = ""
|
||||
},
|
||||
"generated invalid source path": func(s *CatalogState) {
|
||||
s.Outputs[1].SourcePath = "../report.md"
|
||||
},
|
||||
"generated missing transform": func(s *CatalogState) {
|
||||
s.Outputs[1].Transform = ""
|
||||
},
|
||||
"source output source path": func(s *CatalogState) {
|
||||
s.Outputs[0].SourcePath = "report.md"
|
||||
},
|
||||
"source output transform": func(s *CatalogState) {
|
||||
s.Outputs[0].Transform = "markdown_to_html"
|
||||
},
|
||||
"invalid output digest": func(s *CatalogState) {
|
||||
s.Outputs[0].SHA256 = "SHA256:3640fd37140ee4d2e0e93e78834f232ea67a50e7bc6279203690cc7de1975fa6"
|
||||
},
|
||||
"negative size": func(s *CatalogState) {
|
||||
s.Outputs[0].Size = -1
|
||||
},
|
||||
"invalid url": func(s *CatalogState) {
|
||||
s.Outputs[1].URL = "file:///tmp/report.html"
|
||||
},
|
||||
"missing created at": func(s *CatalogState) {
|
||||
s.Outputs[0].CreatedAt = time.Time{}
|
||||
},
|
||||
"missing updated at": func(s *CatalogState) {
|
||||
s.Outputs[0].UpdatedAt = time.Time{}
|
||||
},
|
||||
}
|
||||
for name, mutate := range tests {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
state := validCatalogState(t)
|
||||
mutate(&state)
|
||||
if err := ValidateCatalog(state); err == nil {
|
||||
t.Fatal("ValidateCatalog() error = nil, want error")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseCatalogRejectsForbiddenFields(t *testing.T) {
|
||||
tests := map[string]string{
|
||||
"owners": `"owners": [],`,
|
||||
"sources": `"sources": [],`,
|
||||
"workflow": `"workflow": "additive",`,
|
||||
"source": `"source": {"manifest": {}},`,
|
||||
"manifest": `"manifest": {},`,
|
||||
"pipeline_id": `"pipeline_id": "reports",`,
|
||||
"destination_id": `"destination_id": "archive",`,
|
||||
"published_at": `"published_at": "2026-06-19T12:00:00Z",`,
|
||||
}
|
||||
for name, field := range tests {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
body := strings.Replace(validCatalogStateJSON(t), `"created_at":`, field+"\n "+`"created_at":`, 1)
|
||||
_, err := ParseCatalog([]byte(body))
|
||||
assertStateErrorContains(t, err, "unknown field")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseCatalogRejectsForbiddenOutputFieldsForSourceOutput(t *testing.T) {
|
||||
tests := map[string]string{
|
||||
"source_path": `"source_path": "report.md",`,
|
||||
"transform": `"transform": "markdown_to_html",`,
|
||||
"empty url": `"url": "",`,
|
||||
}
|
||||
for name, field := range tests {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
body := strings.Replace(validCatalogStateJSON(t), `"kind": "source",`, `"kind": "source",`+"\n "+field, 1)
|
||||
_, err := ParseCatalog([]byte(body))
|
||||
if err == nil {
|
||||
t.Fatal("ParseCatalog() error = nil, want error")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func validCatalogStateJSON(t *testing.T) string {
|
||||
t.Helper()
|
||||
data, err := json.MarshalIndent(validCatalogState(t), "", " ")
|
||||
if err != nil {
|
||||
t.Fatalf("marshal catalog state: %v", err)
|
||||
}
|
||||
return string(data)
|
||||
}
|
||||
|
||||
func catalogStateObject(t *testing.T) map[string]any {
|
||||
t.Helper()
|
||||
var document map[string]any
|
||||
if err := json.Unmarshal([]byte(validCatalogStateJSON(t)), &document); err != nil {
|
||||
t.Fatalf("unmarshal catalog state: %v", err)
|
||||
}
|
||||
return document
|
||||
}
|
||||
|
||||
func firstCatalogOutput(document map[string]any) map[string]any {
|
||||
outputs := document["outputs"].([]any)
|
||||
return outputs[0].(map[string]any)
|
||||
}
|
||||
|
||||
func firstCatalogSource(document map[string]any) map[string]any {
|
||||
return firstCatalogOutput(document)["source"].(map[string]any)
|
||||
}
|
||||
|
||||
func mustMarshalCatalogObject(t *testing.T, document map[string]any) []byte {
|
||||
t.Helper()
|
||||
data, err := json.Marshal(document)
|
||||
if err != nil {
|
||||
t.Fatalf("marshal catalog object: %v", err)
|
||||
}
|
||||
return data
|
||||
}
|
||||
|
||||
func validCatalogState(t *testing.T) CatalogState {
|
||||
t.Helper()
|
||||
manifest := validManifest(t)
|
||||
createdAt := time.Date(2026, 6, 19, 12, 0, 0, 0, time.UTC)
|
||||
updatedAt := time.Date(2026, 6, 19, 12, 5, 0, 0, time.UTC)
|
||||
source := CatalogSourceIdentity{
|
||||
ID: manifest.ID,
|
||||
Digest: manifest.Digest,
|
||||
Created: manifest.Created,
|
||||
}
|
||||
return CatalogState{
|
||||
SchemaVersion: CatalogSchemaVersion,
|
||||
DistributorVersion: "dev",
|
||||
CreatedAt: createdAt,
|
||||
UpdatedAt: updatedAt,
|
||||
State: StatePolicy{Mode: StateModeCatalog},
|
||||
Outputs: []CatalogOutputFile{{
|
||||
Path: "report.md",
|
||||
PipelineID: "reports",
|
||||
DestinationID: "archive",
|
||||
Source: source,
|
||||
Kind: OutputKindSource,
|
||||
SHA256: manifest.Files[0].SHA256,
|
||||
Size: manifest.Files[0].Size,
|
||||
CreatedAt: createdAt,
|
||||
UpdatedAt: updatedAt,
|
||||
}, {
|
||||
Path: "report.html",
|
||||
PipelineID: "reports",
|
||||
DestinationID: "html",
|
||||
Source: source,
|
||||
Kind: OutputKindGenerated,
|
||||
SourcePath: "report.md",
|
||||
Transform: "markdown_to_html",
|
||||
URL: "https://reports.example.com/report.html",
|
||||
SHA256: "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
|
||||
Size: 128,
|
||||
CreatedAt: createdAt,
|
||||
UpdatedAt: updatedAt,
|
||||
}},
|
||||
}
|
||||
}
|
||||
@@ -23,6 +23,8 @@ const (
|
||||
type DestinationStatus struct {
|
||||
State *DistributorState
|
||||
SharedRoot *SharedRootState
|
||||
Catalog *CatalogState
|
||||
SupersededLegacy *SupersededLegacyState
|
||||
StateErr error
|
||||
HasContents bool
|
||||
}
|
||||
|
||||
@@ -14,9 +14,11 @@ import (
|
||||
const (
|
||||
SchemaVersion = 2
|
||||
SharedRootSchemaVersion = 3
|
||||
CatalogSchemaVersion = 4
|
||||
legacySchemaVersion = 1
|
||||
StateModeSingleOwner = config.StateModeSingleOwner
|
||||
StateModeSharedRoot = config.StateModeSharedRoot
|
||||
StateModeCatalog = "catalog"
|
||||
)
|
||||
|
||||
type DistributorState struct {
|
||||
|
||||
@@ -26,6 +26,15 @@ func FindOutputByPath(outputs []OutputFile, path string) (OutputFile, bool) {
|
||||
return OutputFile{}, false
|
||||
}
|
||||
|
||||
func FindCatalogOutputByPath(outputs []CatalogOutputFile, path string) (CatalogOutputFile, bool) {
|
||||
for _, output := range outputs {
|
||||
if output.Path == path {
|
||||
return output, true
|
||||
}
|
||||
}
|
||||
return CatalogOutputFile{}, false
|
||||
}
|
||||
|
||||
func MergeOutputFiles(retained, planned []OutputFile) ([]OutputFile, error) {
|
||||
outputs := make([]OutputFile, 0, len(retained)+len(planned))
|
||||
indexByPath := make(map[string]int, len(retained)+len(planned))
|
||||
@@ -178,6 +187,62 @@ func RemoveMissingSharedRootOutputs(s SharedRootState, missingPaths []string) (S
|
||||
return next, changed
|
||||
}
|
||||
|
||||
func CatalogOutputsForOwner(outputs []CatalogOutputFile, scope OwnerScope) []CatalogOutputFile {
|
||||
selected := make([]CatalogOutputFile, 0, len(outputs))
|
||||
for _, output := range outputs {
|
||||
if output.PipelineID == scope.PipelineID && output.DestinationID == scope.DestinationID {
|
||||
selected = append(selected, output)
|
||||
}
|
||||
}
|
||||
return selected
|
||||
}
|
||||
|
||||
func CatalogManagedOutputPaths(s CatalogState) []string {
|
||||
paths := make([]string, 0, len(s.Outputs))
|
||||
for _, output := range s.Outputs {
|
||||
paths = append(paths, output.Path)
|
||||
}
|
||||
return paths
|
||||
}
|
||||
|
||||
func RemoveMissingCatalogOwnerOutputs(s CatalogState, scope OwnerScope, missingPaths []string) (CatalogState, bool) {
|
||||
if len(missingPaths) == 0 {
|
||||
return s, false
|
||||
}
|
||||
missing := pathSet(missingPaths)
|
||||
next := s
|
||||
next.Outputs = make([]CatalogOutputFile, 0, len(s.Outputs))
|
||||
changed := false
|
||||
for _, output := range s.Outputs {
|
||||
if output.PipelineID == scope.PipelineID && output.DestinationID == scope.DestinationID {
|
||||
if _, remove := missing[output.Path]; remove {
|
||||
changed = true
|
||||
continue
|
||||
}
|
||||
}
|
||||
next.Outputs = append(next.Outputs, output)
|
||||
}
|
||||
return next, changed
|
||||
}
|
||||
|
||||
func RemoveMissingCatalogOutputs(s CatalogState, missingPaths []string) (CatalogState, bool) {
|
||||
if len(missingPaths) == 0 {
|
||||
return s, false
|
||||
}
|
||||
missing := pathSet(missingPaths)
|
||||
next := s
|
||||
next.Outputs = make([]CatalogOutputFile, 0, len(s.Outputs))
|
||||
changed := false
|
||||
for _, output := range s.Outputs {
|
||||
if _, remove := missing[output.Path]; remove {
|
||||
changed = true
|
||||
continue
|
||||
}
|
||||
next.Outputs = append(next.Outputs, output)
|
||||
}
|
||||
return next, changed
|
||||
}
|
||||
|
||||
func (s SharedRootState) OutputOwner(path string) (OwnerScope, bool) {
|
||||
for _, output := range s.Outputs {
|
||||
if output.Path == path {
|
||||
|
||||
@@ -49,6 +49,22 @@ func SharedRootPruneCandidates(s SharedRootState, scope OwnerScope) []PruneCandi
|
||||
return candidates
|
||||
}
|
||||
|
||||
func CatalogPruneCandidates(s CatalogState, scope OwnerScope) []PruneCandidate {
|
||||
candidates := make([]PruneCandidate, 0, len(s.Outputs))
|
||||
for _, output := range s.Outputs {
|
||||
if output.PipelineID != scope.PipelineID || output.DestinationID != scope.DestinationID {
|
||||
continue
|
||||
}
|
||||
owner := scope
|
||||
candidates = append(candidates, PruneCandidate{
|
||||
Path: output.Path,
|
||||
UpdatedAt: output.UpdatedAt,
|
||||
Owner: &owner,
|
||||
})
|
||||
}
|
||||
return candidates
|
||||
}
|
||||
|
||||
func PlanPrune(candidates []PruneCandidate, options PrunePlanOptions) PrunePlan {
|
||||
ordered := append([]PruneCandidate(nil), candidates...)
|
||||
sortPruneCandidatesNewestFirst(ordered)
|
||||
|
||||
@@ -16,6 +16,12 @@ import (
|
||||
type StateDocument struct {
|
||||
SingleOwner *DistributorState
|
||||
SharedRoot *SharedRootState
|
||||
Catalog *CatalogState
|
||||
SupersededLegacy *SupersededLegacyState
|
||||
}
|
||||
|
||||
type SupersededLegacyState struct {
|
||||
SchemaVersion int
|
||||
}
|
||||
|
||||
type SharedRootState struct {
|
||||
@@ -103,18 +109,18 @@ func ParseDocument(data []byte) (StateDocument, error) {
|
||||
if err != nil {
|
||||
return StateDocument{}, err
|
||||
}
|
||||
if schemaVersion == SharedRootSchemaVersion {
|
||||
sharedRoot, err := ParseSharedRoot(data)
|
||||
switch schemaVersion {
|
||||
case legacySchemaVersion, SchemaVersion, SharedRootSchemaVersion:
|
||||
return StateDocument{SupersededLegacy: &SupersededLegacyState{SchemaVersion: schemaVersion}}, nil
|
||||
case CatalogSchemaVersion:
|
||||
catalog, err := ParseCatalog(data)
|
||||
if err != nil {
|
||||
return StateDocument{}, err
|
||||
}
|
||||
return StateDocument{SharedRoot: &sharedRoot}, nil
|
||||
return StateDocument{Catalog: &catalog}, nil
|
||||
default:
|
||||
return StateDocument{}, fmt.Errorf("state schema_version %d is unsupported", schemaVersion)
|
||||
}
|
||||
singleOwner, err := Parse(data)
|
||||
if err != nil {
|
||||
return StateDocument{}, err
|
||||
}
|
||||
return StateDocument{SingleOwner: &singleOwner}, nil
|
||||
}
|
||||
|
||||
func parseSchemaVersion(data []byte) (int, error) {
|
||||
@@ -125,6 +131,10 @@ func parseSchemaVersion(data []byte) (int, error) {
|
||||
if err := decoder.Decode(&raw); err != nil {
|
||||
return 0, fmt.Errorf("parse distributor state: %w", err)
|
||||
}
|
||||
var extra any
|
||||
if err := decoder.Decode(&extra); err != io.EOF {
|
||||
return 0, fmt.Errorf("parse distributor state: trailing data")
|
||||
}
|
||||
if raw.SchemaVersion == nil {
|
||||
return 0, fmt.Errorf("state schema_version is required")
|
||||
}
|
||||
|
||||
@@ -10,24 +10,6 @@ import (
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/config"
|
||||
)
|
||||
|
||||
func TestParseDocumentHandlesSingleOwnerAndSharedRoot(t *testing.T) {
|
||||
singleOwner, err := ParseDocument([]byte(validStateJSON(t)))
|
||||
if err != nil {
|
||||
t.Fatalf("ParseDocument(single owner) error = %v", err)
|
||||
}
|
||||
if singleOwner.SingleOwner == nil || singleOwner.SharedRoot != nil {
|
||||
t.Fatalf("single owner document = %#v", singleOwner)
|
||||
}
|
||||
|
||||
sharedRoot, err := ParseDocument([]byte(validSharedRootStateJSON(t)))
|
||||
if err != nil {
|
||||
t.Fatalf("ParseDocument(shared root) error = %v", err)
|
||||
}
|
||||
if sharedRoot.SharedRoot == nil || sharedRoot.SingleOwner != nil {
|
||||
t.Fatalf("shared root document = %#v", sharedRoot)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseSharedRootState(t *testing.T) {
|
||||
state, err := ParseSharedRoot([]byte(validSharedRootStateJSON(t)))
|
||||
if err != nil {
|
||||
|
||||
Reference in New Issue
Block a user