eric/distributor
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 7 Wiki Activity

Add destination workflow config

Browse Source
This commit is contained in:
Eric Rakestraw
2026-06-19 15:18:14 +00:00
parent d23c624179
commit a95662226f
6 changed files with 101 additions and 652 deletions
Download Patch File Download Diff File Expand all files Collapse all files

375
internal/app/run_test.go
View File

@@ -15,7 +15,6 @@ import (
"gitea.maximumdirect.net/eric/distributor/internal/bundle"
"gitea.maximumdirect.net/eric/distributor/internal/config"
"gitea.maximumdirect.net/eric/distributor/internal/notify"
"gitea.maximumdirect.net/eric/distributor/internal/publish"
"gitea.maximumdirect.net/eric/distributor/internal/state"
"gitea.maximumdirect.net/eric/distributor/internal/storage"
"gitea.maximumdirect.net/eric/distributor/internal/storage/fake"
@@ -222,67 +221,6 @@ func TestRunPublishesNewLocalBundle(t *testing.T) {
}
}
func TestRunSharedRootDryRunWritesNoOutputsOrState(t *testing.T) {
sourceRoot := t.TempDir()
destinationRoot := t.TempDir()
writeSourceBundle(t, sourceRoot, "", testBundleOptions{
Files: []testFile{{Path: "report.md", Data: "# Report\n"}},
})
configPath := writeSharedRootLocalConfig(t, sourceRoot, destinationRoot)
cfg, err := config.LoadFile(configPath)
if err != nil {
t.Fatalf("load config: %v", err)
}
report, err := buildRunReportWithBackendFactory(context.Background(), cfg, RunOptions{DryRun: true}, newBackendFactoryWithEnvironment)
if err != nil {
t.Fatalf("dry-run buildRunReportWithBackendFactory() error = %v", err)
}
if got, want := report.Actions[0].Action, string(publish.ActionPublishNew); got != want {
t.Fatalf("dry-run action = %q, want %q", got, want)
}
if _, statErr := os.Stat(filepath.Join(destinationRoot, "report.md")); !os.IsNotExist(statErr) {
t.Fatalf("output stat error = %v, want absent", statErr)
}
if _, statErr := os.Stat(filepath.Join(destinationRoot, storage.StateFileName)); !os.IsNotExist(statErr) {
t.Fatalf("state file stat error = %v, want absent", statErr)
}
}
func TestRunPublishesTwoPipelinesIntoSharedRoot(t *testing.T) {
firstSourceRoot := t.TempDir()
secondSourceRoot := t.TempDir()
destinationRoot := t.TempDir()
writeSourceBundle(t, firstSourceRoot, "", testBundleOptions{
ID: "reports.first",
Files: []testFile{{Path: "first.md", Data: "# First\n"}},
})
writeSourceBundle(t, secondSourceRoot, "", testBundleOptions{
ID: "reports.second",
Files: []testFile{{Path: "second.md", Data: "# Second\n"}},
})
err := Run(context.Background(), RunOptions{ConfigPath: writeTwoPipelineSharedRootConfig(t, firstSourceRoot, secondSourceRoot, destinationRoot)})
if err != nil {
t.Fatalf("Run() error = %v", err)
}
testutil.AssertFile(t, filepath.Join(destinationRoot, "first.md"), "# First\n")
testutil.AssertFile(t, filepath.Join(destinationRoot, "second.md"), "# Second\n")
destinationState := readSharedRootStateFile(t, filepath.Join(destinationRoot, storage.StateFileName))
if got, want := len(destinationState.Owners), 2; got != want {
t.Fatalf("owner count = %d, want %d", got, want)
}
if _, ok := destinationState.Owner(state.CurrentOwnerScope("reports-first", "archive")); !ok {
t.Fatal("reports-first/archive owner missing")
}
if _, ok := destinationState.Owner(state.CurrentOwnerScope("reports-second", "archive")); !ok {
t.Fatal("reports-second/archive owner missing")
}
if got, want := strings.Join(destinationState.AllManagedOutputPaths(), ","), "first.md,second.md"; got != want {
t.Fatalf("managed paths = %q, want %q", got, want)
}
}
func TestRunPipelineWithLocalSourcePublishesConfiguredDestination(t *testing.T) {
sourceRoot := t.TempDir()
destinationRoot := t.TempDir()
@@ -698,67 +636,6 @@ func TestRunFixedPathReplacesOlderManagedState(t *testing.T) {
}
}
func TestRunPreserveRelativeSameSourceTakeoverAllowsOwnerMismatch(t *testing.T) {
sourceRoot := t.TempDir()
destinationRoot := t.TempDir()
sourceManifest := writeSourceBundle(t, sourceRoot, "daily/report", testBundleOptions{
ID: "reports.same",
Files: []testFile{{Path: "report.md", Data: "# Report\nNew.\n"}},
})
testutil.WriteDestinationState(t, destinationRoot, "daily/report", sourceManifest, testutil.DestinationStateOptions{
PipelineID: "other",
})
if err := os.WriteFile(filepath.Join(destinationRoot, "daily", "report", "report.md"), []byte("# Report\nOld.\n"), 0o600); err != nil {
t.Fatalf("write old report: %v", err)
}
var stdout bytes.Buffer
err := Run(context.Background(), RunOptions{
ConfigPath: writeSameSourcePreserveRelativeConfig(t, sourceRoot, destinationRoot),
Stdout: &stdout,
})
if err != nil {
t.Fatalf("Run() error = %v", err)
}
if !strings.Contains(stdout.String(), "action=replace_takeover takeover_mode=same_source") {
t.Fatalf("stdout = %q, want same-source takeover", stdout.String())
}
testutil.AssertFile(t, filepath.Join(destinationRoot, "daily", "report", "report.md"), "# Report\nNew.\n")
destinationState := readStateFile(t, filepath.Join(destinationRoot, "daily", "report", storage.StateFileName))
if destinationState.PipelineID != "reports" || destinationState.Source.Manifest.ID != "reports.same" {
t.Fatalf("state owner/source = %s/%s source=%s, want reports/archive reports.same", destinationState.PipelineID, destinationState.DestinationID, destinationState.Source.Manifest.ID)
}
}
func TestRunPreserveRelativeSameSourceRefusesDifferentSource(t *testing.T) {
sourceRoot := t.TempDir()
destinationRoot := t.TempDir()
sourceManifest := writeSourceBundle(t, sourceRoot, "daily/report", testBundleOptions{
ID: "reports.same",
Files: []testFile{{Path: "report.md", Data: "# Report\nNew.\n"}},
})
destinationManifest := sourceManifest
destinationManifest.ID = "reports.other"
testutil.WriteDestinationState(t, destinationRoot, "daily/report", destinationManifest, testutil.DestinationStateOptions{
PipelineID: "other",
})
if err := os.WriteFile(filepath.Join(destinationRoot, "daily", "report", "report.md"), []byte("# Report\nOld.\n"), 0o600); err != nil {
t.Fatalf("write old report: %v", err)
}
err := Run(context.Background(), RunOptions{
ConfigPath: writeSameSourcePreserveRelativeConfig(t, sourceRoot, destinationRoot),
})
if err == nil || !strings.Contains(err.Error(), "fail_conflict") {
t.Fatalf("Run() error = %v, want fail_conflict", err)
}
testutil.AssertFile(t, filepath.Join(destinationRoot, "daily", "report", "report.md"), "# Report\nOld.\n")
destinationState := readStateFile(t, filepath.Join(destinationRoot, "daily", "report", storage.StateFileName))
if destinationState.PipelineID != "other" || destinationState.Source.Manifest.ID != "reports.other" {
t.Fatalf("state owner/source = %s/%s source=%s, want unchanged other/archive reports.other", destinationState.PipelineID, destinationState.DestinationID, destinationState.Source.Manifest.ID)
}
}
func TestRunFixedPathSkipsWhenDestinationStateIsNewer(t *testing.T) {
sourceRoot := t.TempDir()
destinationRoot := t.TempDir()
@@ -989,53 +866,6 @@ func TestRunNotifiesAfterReplacement(t *testing.T) {
}
}
func TestRunMergeReconciliationRetainsManagedOutput(t *testing.T) {
sourceRoot := t.TempDir()
destinationRoot := t.TempDir()
manifest := testutil.WriteSourceBundle(t, sourceRoot, "", testutil.BundleOptions{
Files: []testutil.SourceFile{{Path: "report.md", Data: "# Report\nNew.\n"}},
})
older := manifest
older.Created = older.Created.Add(-time.Hour)
defaultManifest := testutil.ValidManifest(testutil.BundleOptions{})
older.Files = append([]bundle.ManifestFile(nil), defaultManifest.Files...)
older.Digest = bundle.BundleDigest(older.Files)
writeDestinationState(t, destinationRoot, "", older)
if err := os.WriteFile(filepath.Join(destinationRoot, "report.md"), []byte("old\n"), 0o600); err != nil {
t.Fatalf("write old report: %v", err)
}
if err := os.WriteFile(filepath.Join(destinationRoot, "summary.txt"), []byte("old summary\n"), 0o600); err != nil {
t.Fatalf("write old summary: %v", err)
}
configPath := writeConfigFile(t, `
pipelines:
- id: reports
source:
backend: local
path: `+sourceRoot+`
destinations:
- id: archive
backend: local
path: `+destinationRoot+`
reconciliation:
mode: merge
`)
err := Run(context.Background(), RunOptions{ConfigPath: configPath})
if err != nil {
t.Fatalf("Run() error = %v", err)
}
testutil.AssertFile(t, filepath.Join(destinationRoot, "report.md"), "# Report\nNew.\n")
testutil.AssertFile(t, filepath.Join(destinationRoot, "summary.txt"), "old summary\n")
destinationState := readStateFile(t, filepath.Join(destinationRoot, storage.StateFileName))
if got, want := destinationState.Reconciliation.Mode, config.ReconciliationModeMerge; got != want {
t.Fatalf("reconciliation mode = %q, want %q", got, want)
}
if got, want := len(destinationState.Outputs), 2; got != want {
t.Fatalf("state output count = %d, want %d", got, want)
}
}
func TestRunJSONIncludesGeneratedOutputMetadata(t *testing.T) {
sourceRoot := t.TempDir()
destinationRoot := t.TempDir()
@@ -1663,135 +1493,6 @@ func TestRunSkipsNewerDestination(t *testing.T) {
testutil.AssertFile(t, filepath.Join(destinationRoot, "report.md"), "newer\n")
}
func TestRunReplacesConflictWhenTransferPolicyAllows(t *testing.T) {
sourceRoot := t.TempDir()
destinationRoot := t.TempDir()
manifest := writeSourceBundle(t, sourceRoot, "", testBundleOptions{})
conflict := manifest
conflict.ID = "other.source"
writeDestinationState(t, destinationRoot, "", conflict)
if err := os.WriteFile(filepath.Join(destinationRoot, "report.md"), []byte("old\n"), 0o600); err != nil {
t.Fatalf("write old output: %v", err)
}
configPath := writeConfigFile(t, `
pipelines:
- id: reports
source:
backend: local
path: `+sourceRoot+`
destinations:
- id: archive
backend: local
path: `+destinationRoot+`
takeover:
mode: never
transfer:
on_conflict: replace
`)
var stdout bytes.Buffer
err := Run(context.Background(), RunOptions{ConfigPath: configPath, DryRun: true, Stdout: &stdout})
if err != nil {
t.Fatalf("Run() error = %v", err)
}
output := stdout.String()
for _, want := range []string{
"action=replace_conflict",
"replace_conflict=1",
"force_replace=0",
} {
if !strings.Contains(output, want) {
t.Fatalf("stdout = %q, want substring %q", output, want)
}
}
var jsonOut bytes.Buffer
err = Run(context.Background(), RunOptions{ConfigPath: configPath, DryRun: true, Stdout: &jsonOut, OutputFormat: OutputFormatJSON})
if err != nil {
t.Fatalf("Run() JSON error = %v", err)
}
result := decodeAppResult(t, jsonOut.String())
actions, ok := result["actions"].([]any)
if !ok || len(actions) != 1 {
t.Fatalf("actions = %#v, want one action", result["actions"])
}
action, ok := actions[0].(map[string]any)
if !ok || action["action"] != "replace_conflict" {
t.Fatalf("action = %#v, want replace_conflict", actions[0])
}
summary, ok := result["summary"].(map[string]any)
if !ok || summary["replace_conflict"] != float64(1) || summary["force_replace"] != float64(0) {
t.Fatalf("summary = %#v, want replace_conflict without force", result["summary"])
}
}
func TestRunReplacesNewerWhenTransferPolicyAllows(t *testing.T) {
sourceRoot := t.TempDir()
destinationRoot := t.TempDir()
manifest := writeSourceBundle(t, sourceRoot, "", testBundleOptions{})
newer := manifest
newer.Created = newer.Created.Add(time.Hour)
writeDestinationState(t, destinationRoot, "", newer)
if err := os.WriteFile(filepath.Join(destinationRoot, "report.md"), []byte("newer\n"), 0o600); err != nil {
t.Fatalf("write newer output: %v", err)
}
configPath := writeConfigFile(t, `
pipelines:
- id: reports
source:
backend: local
path: `+sourceRoot+`
destinations:
- id: archive
backend: local
path: `+destinationRoot+`
transfer:
on_destination_newer: replace
`)
var stdout bytes.Buffer
err := Run(context.Background(), RunOptions{ConfigPath: configPath, DryRun: true, Stdout: &stdout})
if err != nil {
t.Fatalf("Run() error = %v", err)
}
output := stdout.String()
for _, want := range []string{
"action=replace_newer",
"replace_newer=1",
"force_replace=0",
} {
if !strings.Contains(output, want) {
t.Fatalf("stdout = %q, want substring %q", output, want)
}
}
}
func TestRunTakeoverNeverFailsOnConflict(t *testing.T) {
sourceRoot := t.TempDir()
destinationRoot := t.TempDir()
manifest := writeSourceBundle(t, sourceRoot, "", testBundleOptions{})
manifest.ID = "other.source"
writeDestinationState(t, destinationRoot, "", manifest)
configPath := writeConfigFile(t, `
pipelines:
- id: reports
source:
backend: local
path: `+sourceRoot+`
destinations:
- id: archive
backend: local
path: `+destinationRoot+`
takeover:
mode: never
`)
err := Run(context.Background(), RunOptions{ConfigPath: configPath})
if err == nil || !strings.Contains(err.Error(), "fail_conflict") {
t.Fatalf("Run() error = %v, want fail_conflict", err)
}
}
func TestRunFailsOnUnmanagedDestination(t *testing.T) {
sourceRoot := t.TempDir()
destinationRoot := t.TempDir()
@@ -2032,69 +1733,6 @@ func writeLocalConfig(t *testing.T, sourceRoot, destinationRoot string) string {
return testutil.WriteMinimalLocalConfig(t, sourceRoot, destinationRoot)
}
func writeSharedRootLocalConfig(t *testing.T, sourceRoot, destinationRoot string) string {
t.Helper()
return writeConfigFile(t, `
pipelines:
- id: reports
source:
backend: local
path: `+sourceRoot+`
destinations:
- id: archive
backend: local
path: `+destinationRoot+`
state:
mode: shared_root
`)
}
func writeTwoPipelineSharedRootConfig(t *testing.T, firstSourceRoot, secondSourceRoot, destinationRoot string) string {
t.Helper()
return writeConfigFile(t, `
pipelines:
- id: reports-first
source:
backend: local
path: `+firstSourceRoot+`
destinations:
- id: archive
backend: local
path: `+destinationRoot+`
state:
mode: shared_root
- id: reports-second
source:
backend: local
path: `+secondSourceRoot+`
destinations:
- id: archive
backend: local
path: `+destinationRoot+`
state:
mode: shared_root
`)
}
func writeSameSourcePreserveRelativeConfig(t *testing.T, sourceRoot, destinationRoot string) string {
t.Helper()
return writeConfigFile(t, `
pipelines:
- id: reports
source:
backend: local
path: `+sourceRoot+`
destinations:
- id: archive
backend: local
path: `+destinationRoot+`
path_mapping:
mode: preserve_relative
takeover:
mode: same_source
`)
}
func writeFanoutConfig(t *testing.T, sourceRoot, firstDestination, secondDestination string) string {
t.Helper()
return testutil.WriteFanoutLocalConfig(t, sourceRoot, firstDestination, secondDestination)
@@ -2175,19 +1813,6 @@ func readStateFile(t *testing.T, path string) state.DistributorState {
return testutil.ReadDestinationState(t, path)
}
func readSharedRootStateFile(t *testing.T, path string) state.SharedRootState {
t.Helper()
data, err := os.ReadFile(path)
if err != nil {
t.Fatalf("read shared-root state: %v", err)
}
destinationState, err := state.ParseSharedRoot(data)
if err != nil {
t.Fatalf("parse shared-root state: %v", err)
}
return destinationState
}
func outputsByPath(outputs []state.OutputFile) map[string]state.OutputFile {
byPath := make(map[string]state.OutputFile, len(outputs))
for _, output := range outputs {