eric/distributor
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 7 Wiki Activity

Add fixed destination path mapping

Browse Source
This commit is contained in:
Eric Rakestraw
2026-06-01 21:33:12 +00:00
parent 1a52fdce6f
commit a8564035d3
16 changed files with 816 additions and 67 deletions
Download Patch File Download Diff File Expand all files Collapse all files

227
internal/app/run.go
View File

@@ -5,6 +5,7 @@ import (
"errors"
"fmt"
"io"
"sort"
"strings"
"gitea.maximumdirect.net/eric/distributor/internal/bundle"
@@ -112,33 +113,55 @@ func runConfigWithBackendFactory(ctx context.Context, cfg config.Config, options
return err
}
}
for _, sourceBundle := range bundles {
for _, destination := range pipeline.Destinations {
destinationBackend, err := backends.openDestination(ctx, destination)
if err != nil {
failures.add(pipeline.ID, destination.ID, destination.Backend, storage.DisplayPath(sourceBundle.RootRelativePath), err)
for _, destination := range pipeline.Destinations {
selections := selectDestinationBundles(destination, bundles)
if isFixedPathDestination(destination) {
summary.recordFixedPath()
if options.DryRun {
warning := fixedPathSelectionWarning(pipeline.ID, destination.ID, selections, len(bundles))
if jsonOutput {
warnings = append(warnings, warning)
} else if options.Stdout != nil {
if err := writeWarnings(options.Stdout, []OutputWarning{warning}); err != nil {
closeBackend(sourceBackend)
return err
}
}
}
}
if len(selections) == 0 {
continue
}
destinationBackend, err := backends.openDestination(ctx, destination)
if err != nil {
for _, selection := range selections {
failures.add(pipeline.ID, destination.ID, destination.Backend, storage.DisplayPath(selection.SourceBundle.RootRelativePath), err)
summary.recordFailure()
if jsonOutput {
result.Actions = append(result.Actions, errorAction(pipeline.ID, destination.ID, destination.Backend, sourceBundle.RootRelativePath, err))
result.Actions = append(result.Actions, errorAction(pipeline.ID, destination.ID, destination.Backend, selection.SourceBundle.RootRelativePath, err))
} else if options.Stdout != nil {
writeErrorLine(options.Stdout, sourceBundle.RootRelativePath, destination.ID, destination.Backend, err)
}
continue
}
closeDestination := true
deferCloseDestination := func() {
if closeDestination {
closeBackend(destinationBackend)
closeDestination = false
writeErrorLine(options.Stdout, selection.SourceBundle.RootRelativePath, destination.ID, destination.Backend, err)
}
}
continue
}
closeDestination := true
deferCloseDestination := func() {
if closeDestination {
closeBackend(destinationBackend)
closeDestination = false
}
}
for _, selection := range selections {
sourceBundle := selection.SourceBundle
req := publish.Request{
PipelineID: pipeline.ID,
DestinationID: destination.ID,
SourceBundle: sourceBundle,
SourceBackend: sourceBackend,
DestinationBackend: destinationBackend,
DestinationBundlePath: sourceBundle.RootRelativePath,
DestinationBundlePath: selection.DestinationBundlePath,
PathMapping: destination.PathMap.Mode,
Publish: *destination.Publish,
Transform: destination.Transform,
Transformers: transforms,
@@ -160,6 +183,24 @@ func runConfigWithBackendFactory(ctx context.Context, cfg config.Config, options
if plan.BundlePath == "" {
plan.BundlePath = sourceBundle.RootRelativePath
}
if plan.DestinationBundlePath == "" {
plan.DestinationBundlePath = selection.DestinationBundlePath
}
}
if isFixedPathDestination(destination) {
plan.PathMapping = config.PathMappingFixed
if options.DryRun && isDestructiveFixedPathAction(plan.Action) {
warning := fixedPathReplacementWarning(plan)
if jsonOutput {
warnings = append(warnings, warning)
} else if options.Stdout != nil {
if err := writeWarnings(options.Stdout, []OutputWarning{warning}); err != nil {
deferCloseDestination()
closeBackend(sourceBackend)
return err
}
}
}
}
if jsonOutput {
result.Actions = append(result.Actions, runActionFromPlan(destination.Backend, plan, err))
@@ -167,7 +208,6 @@ func runConfigWithBackendFactory(ctx context.Context, cfg config.Config, options
writePlanLine(options.Stdout, destination.Backend, plan, err)
}
if err != nil {
deferCloseDestination()
failures.add(pipeline.ID, destination.ID, destination.Backend, storage.DisplayPath(sourceBundle.RootRelativePath), err)
summary.recordFailure()
continue
@@ -175,22 +215,20 @@ func runConfigWithBackendFactory(ctx context.Context, cfg config.Config, options
summary.recordPlan(plan.Action)
if !options.DryRun {
if err := publish.Execute(ctx, req, plan); err != nil {
deferCloseDestination()
failures.add(pipeline.ID, destination.ID, destination.Backend, storage.DisplayPath(sourceBundle.RootRelativePath), err)
summary.recordFailure()
continue
}
if shouldNotify(plan.Action) {
if err := notifier.Notify(ctx, notifyEvent(plan)); err != nil {
deferCloseDestination()
failures.add(pipeline.ID, destination.ID, destination.Backend, storage.DisplayPath(sourceBundle.RootRelativePath), err)
summary.recordFailure()
continue
}
}
}
deferCloseDestination()
}
deferCloseDestination()
}
closeBackend(sourceBackend)
}
@@ -231,10 +269,17 @@ func writePlanLine(w io.Writer, backend string, plan publish.Plan, planErr error
if destinationID == "" {
destinationID = "unknown"
}
fmt.Fprintf(w, " - bundle=%s destination=%s backend=%s action=error reason=%q\n", storage.DisplayPath(plan.BundlePath), destinationID, backend, planErr.Error())
fmt.Fprintf(w, " - bundle=%s destination=%s backend=%s%s action=error reason=%q\n", storage.DisplayPath(plan.BundlePath), destinationID, backend, pathMappingSummary(plan), planErr.Error())
return
}
fmt.Fprintf(w, " - bundle=%s destination=%s backend=%s action=%s outputs=%s reason=%q\n", storage.DisplayPath(plan.BundlePath), plan.DestinationID, backend, plan.Action, outputSummary(plan.Outputs), plan.Reason)
fmt.Fprintf(w, " - bundle=%s destination=%s backend=%s%s action=%s outputs=%s reason=%q\n", storage.DisplayPath(plan.BundlePath), plan.DestinationID, backend, pathMappingSummary(plan), plan.Action, outputSummary(plan.Outputs), plan.Reason)
}
func pathMappingSummary(plan publish.Plan) string {
if plan.PathMapping != config.PathMappingFixed {
return ""
}
return fmt.Sprintf(" path_mapping=fixed target=%s", storage.DisplayPath(plan.DestinationBundlePath))
}
func writeErrorLine(w io.Writer, bundlePath, destinationID, backend string, err error) {
@@ -255,6 +300,66 @@ func outputSummary(outputs []publish.Output) string {
return strings.Join(paths, ",")
}
type destinationBundleSelection struct {
SourceBundle bundle.Bundle
DestinationBundlePath string
}
func selectDestinationBundles(destination config.Destination, bundles []bundle.Bundle) []destinationBundleSelection {
if !isFixedPathDestination(destination) {
selections := make([]destinationBundleSelection, 0, len(bundles))
for _, sourceBundle := range bundles {
selections = append(selections, destinationBundleSelection{
SourceBundle: sourceBundle,
DestinationBundlePath: sourceBundle.RootRelativePath,
})
}
return selections
}
if len(bundles) == 0 {
return nil
}
sourceBundle := newestBundle(bundles)
return []destinationBundleSelection{{
SourceBundle: sourceBundle,
DestinationBundlePath: "",
}}
}
func newestBundle(bundles []bundle.Bundle) bundle.Bundle {
if len(bundles) == 0 {
return bundle.Bundle{}
}
sorted := append([]bundle.Bundle(nil), bundles...)
sort.Slice(sorted, func(i, j int) bool {
if sorted[i].Manifest.Created.Equal(sorted[j].Manifest.Created) {
return sorted[i].RootRelativePath < sorted[j].RootRelativePath
}
return sorted[i].Manifest.Created.After(sorted[j].Manifest.Created)
})
return sorted[0]
}
func isFixedPathDestination(destination config.Destination) bool {
return destination.PathMap.Mode == config.PathMappingFixed
}
func fixedPathSelectionWarning(pipelineID, destinationID string, selections []destinationBundleSelection, candidateCount int) OutputWarning {
selected := "none"
if len(selections) > 0 {
selected = storage.DisplayPath(selections[0].SourceBundle.RootRelativePath)
}
return OutputWarning{Message: fmt.Sprintf("pipeline=%s destination=%s path_mapping=fixed candidates=%d selected_bundle=%s destination_bundle=.", pipelineID, destinationID, candidateCount, selected)}
}
func isDestructiveFixedPathAction(action publish.Action) bool {
return action == publish.ActionReplaceOlder || action == publish.ActionForceReplace
}
func fixedPathReplacementWarning(plan publish.Plan) OutputWarning {
return OutputWarning{Message: fmt.Sprintf("pipeline=%s destination=%s path_mapping=fixed action=%s replaces destination root for selected_bundle=%s", plan.PipelineID, plan.DestinationID, plan.Action, storage.DisplayPath(plan.BundlePath))}
}
func destinationIDs(destinations []config.Destination) []string {
ids := make([]string, 0, len(destinations))
for _, destination := range destinations {
@@ -362,14 +467,16 @@ type runPipelineResult struct {
}
type runActionResult struct {
PipelineID string `json:"pipeline_id,omitempty"`
DestinationID string `json:"destination_id"`
Backend string `json:"backend"`
BundleID string `json:"bundle_id,omitempty"`
BundlePath string `json:"bundle_path"`
Action string `json:"action"`
Reason string `json:"reason,omitempty"`
Outputs []runOutputResult `json:"outputs"`
PipelineID string `json:"pipeline_id,omitempty"`
DestinationID string `json:"destination_id"`
Backend string `json:"backend"`
BundleID string `json:"bundle_id,omitempty"`
BundlePath string `json:"bundle_path"`
DestinationPath string `json:"destination_path"`
PathMapping string `json:"path_mapping,omitempty"`
Action string `json:"action"`
Reason string `json:"reason,omitempty"`
Outputs []runOutputResult `json:"outputs"`
}
type runOutputResult struct {
@@ -388,37 +495,42 @@ func runActionFromPlan(backend string, plan publish.Plan, planErr error) runActi
destinationID = "unknown"
}
return runActionResult{
PipelineID: plan.PipelineID,
DestinationID: destinationID,
Backend: backend,
BundleID: plan.BundleID,
BundlePath: storage.DisplayPath(plan.BundlePath),
Action: "error",
Reason: planErr.Error(),
Outputs: []runOutputResult{},
PipelineID: plan.PipelineID,
DestinationID: destinationID,
Backend: backend,
BundleID: plan.BundleID,
BundlePath: storage.DisplayPath(plan.BundlePath),
DestinationPath: storage.DisplayPath(plan.DestinationBundlePath),
PathMapping: plan.PathMapping,
Action: "error",
Reason: planErr.Error(),
Outputs: []runOutputResult{},
}
}
return runActionResult{
PipelineID: plan.PipelineID,
DestinationID: plan.DestinationID,
Backend: backend,
BundleID: plan.BundleID,
BundlePath: storage.DisplayPath(plan.BundlePath),
Action: string(plan.Action),
Reason: plan.Reason,
Outputs: runOutputsFromPlan(plan.Outputs),
PipelineID: plan.PipelineID,
DestinationID: plan.DestinationID,
Backend: backend,
BundleID: plan.BundleID,
BundlePath: storage.DisplayPath(plan.BundlePath),
DestinationPath: storage.DisplayPath(plan.DestinationBundlePath),
PathMapping: plan.PathMapping,
Action: string(plan.Action),
Reason: plan.Reason,
Outputs: runOutputsFromPlan(plan.Outputs),
}
}
func errorAction(pipelineID, destinationID, backend, bundlePath string, err error) runActionResult {
return runActionResult{
PipelineID: pipelineID,
DestinationID: destinationID,
Backend: backend,
BundlePath: storage.DisplayPath(bundlePath),
Action: "error",
Reason: err.Error(),
Outputs: []runOutputResult{},
PipelineID: pipelineID,
DestinationID: destinationID,
Backend: backend,
BundlePath: storage.DisplayPath(bundlePath),
DestinationPath: storage.DisplayPath(bundlePath),
Action: "error",
Reason: err.Error(),
Outputs: []runOutputResult{},
}
}
@@ -445,6 +557,7 @@ type runSummary struct {
forceReplace int
skipped int
failures int
fixedPath int
}
func (s *runSummary) recordPlan(action publish.Action) {
@@ -465,12 +578,16 @@ func (s *runSummary) recordFailure() {
s.failures++
}
func (s *runSummary) recordFixedPath() {
s.fixedPath++
}
func (s runSummary) Line() string {
status := "ok"
if s.failures > 0 {
status = "failed"
}
return fmt.Sprintf("Final status: %s planned=%d publish_new=%d replace_older=%d force_replace=%d skipped=%d failed=%d dry_run=%t", status, s.planned, s.publishNew, s.replaceOlder, s.forceReplace, s.skipped, s.failures, s.dryRun)
return fmt.Sprintf("Final status: %s planned=%d publish_new=%d replace_older=%d force_replace=%d skipped=%d failed=%d dry_run=%t fixed_path=%d", status, s.planned, s.publishNew, s.replaceOlder, s.forceReplace, s.skipped, s.failures, s.dryRun, s.fixedPath)
}
type runSummaryResult struct {
@@ -482,6 +599,7 @@ type runSummaryResult struct {
Skipped int `json:"skipped"`
Failed int `json:"failed"`
DryRun bool `json:"dry_run"`
FixedPath int `json:"fixed_path"`
}
func (s runSummary) Result() runSummaryResult {
@@ -498,6 +616,7 @@ func (s runSummary) Result() runSummaryResult {
Skipped: s.skipped,
Failed: s.failures,
DryRun: s.dryRun,
FixedPath: s.fixedPath,
}
}

340
internal/app/run_test.go
View File

@@ -217,6 +217,329 @@ func TestRunPublishesNewLocalBundle(t *testing.T) {
}
}
func TestRunExplicitPreserveRelativePathMappingMatchesDefault(t *testing.T) {
sourceRoot := t.TempDir()
destinationRoot := t.TempDir()
writeSourceBundle(t, sourceRoot, "daily/report", testBundleOptions{})
err := Run(context.Background(), RunOptions{ConfigPath: writeLocalConfigWithPathMapping(t, sourceRoot, destinationRoot, config.PathMappingPreserveRelative)})
if err != nil {
t.Fatalf("Run() error = %v", err)
}
assertFile(t, filepath.Join(destinationRoot, "daily", "report", "report.md"), "# Report\nSunny.\n")
if _, err := os.Stat(filepath.Join(destinationRoot, "report.md")); !os.IsNotExist(err) {
t.Fatalf("root report.md stat error = %v, want not exist", err)
}
}
func TestRunFixedPathPublishesNewestBundleAtDestinationRoot(t *testing.T) {
sourceRoot := t.TempDir()
destinationRoot := t.TempDir()
writeSourceBundle(t, sourceRoot, "old", testBundleOptions{
ID: "reports.old",
Created: testutil.DefaultCreated,
Files: []testFile{
{Path: "report.md", Data: "# Report\nOld.\n"},
{Path: "summary.txt", Data: "Old summary\n"},
},
})
writeSourceBundle(t, sourceRoot, "new", testBundleOptions{
ID: "reports.new",
Created: testutil.DefaultCreated.Add(time.Hour),
Files: []testFile{
{Path: "report.md", Data: "# Report\nNew.\n"},
{Path: "summary.txt", Data: "New summary\n"},
},
})
err := Run(context.Background(), RunOptions{ConfigPath: writeLocalConfigWithPathMapping(t, sourceRoot, destinationRoot, config.PathMappingFixed)})
if err != nil {
t.Fatalf("Run() error = %v", err)
}
assertFile(t, filepath.Join(destinationRoot, "report.md"), "# Report\nNew.\n")
if _, err := os.Stat(filepath.Join(destinationRoot, "new", "report.md")); !os.IsNotExist(err) {
t.Fatalf("nested new report stat error = %v, want not exist", err)
}
destinationState := readStateFile(t, filepath.Join(destinationRoot, storage.StateFileName))
if destinationState.Source.Manifest.ID != "reports.new" {
t.Fatalf("state source id = %q, want reports.new", destinationState.Source.Manifest.ID)
}
}
func TestRunFixedPathTieBreaksByBundlePath(t *testing.T) {
sourceRoot := t.TempDir()
destinationRoot := t.TempDir()
writeSourceBundle(t, sourceRoot, "b", testBundleOptions{
ID: "reports.b",
Created: testutil.DefaultCreated,
Files: []testFile{
{Path: "report.md", Data: "# Report\nB.\n"},
{Path: "summary.txt", Data: "B summary\n"},
},
})
writeSourceBundle(t, sourceRoot, "a", testBundleOptions{
ID: "reports.a",
Created: testutil.DefaultCreated,
Files: []testFile{
{Path: "report.md", Data: "# Report\nA.\n"},
{Path: "summary.txt", Data: "A summary\n"},
},
})
err := Run(context.Background(), RunOptions{ConfigPath: writeLocalConfigWithPathMapping(t, sourceRoot, destinationRoot, config.PathMappingFixed)})
if err != nil {
t.Fatalf("Run() error = %v", err)
}
destinationState := readStateFile(t, filepath.Join(destinationRoot, storage.StateFileName))
if destinationState.Source.Manifest.ID != "reports.a" {
t.Fatalf("state source id = %q, want reports.a", destinationState.Source.Manifest.ID)
}
}
func TestRunFixedPathDryRunReportsSelection(t *testing.T) {
sourceRoot := t.TempDir()
destinationRoot := t.TempDir()
writeSourceBundle(t, sourceRoot, "old", testBundleOptions{ID: "reports.old", Created: testutil.DefaultCreated})
writeSourceBundle(t, sourceRoot, "new", testBundleOptions{ID: "reports.new", Created: testutil.DefaultCreated.Add(time.Hour)})
var stdout bytes.Buffer
err := Run(context.Background(), RunOptions{
ConfigPath: writeLocalConfigWithPathMapping(t, sourceRoot, destinationRoot, config.PathMappingFixed),
DryRun: true,
Stdout: &stdout,
})
if err != nil {
t.Fatalf("Run() error = %v", err)
}
output := stdout.String()
for _, want := range []string{
"Warning: pipeline=reports destination=archive path_mapping=fixed candidates=2 selected_bundle=new destination_bundle=.",
"bundle=new destination=archive backend=local path_mapping=fixed target=. action=publish_new",
"fixed_path=1",
} {
if !strings.Contains(output, want) {
t.Fatalf("stdout = %q, want substring %q", output, want)
}
}
if strings.Contains(output, "bundle=old destination=archive") {
t.Fatalf("stdout = %q, older fixed candidate was planned", output)
}
if entries, err := os.ReadDir(destinationRoot); err != nil || len(entries) != 0 {
t.Fatalf("destination entries = %v err=%v, want empty", entries, err)
}
}
func TestRunFixedPathDryRunWarnsForReplacement(t *testing.T) {
sourceRoot := t.TempDir()
destinationRoot := t.TempDir()
writeSourceBundle(t, sourceRoot, "old", testBundleOptions{
ID: "reports.old",
Created: testutil.DefaultCreated,
Files: []testFile{
{Path: "report.md", Data: "# Report\nOld.\n"},
{Path: "summary.txt", Data: "Old summary\n"},
},
})
configPath := writeLocalConfigWithPathMapping(t, sourceRoot, destinationRoot, config.PathMappingFixed)
if err := Run(context.Background(), RunOptions{ConfigPath: configPath}); err != nil {
t.Fatalf("first Run() error = %v", err)
}
writeSourceBundle(t, sourceRoot, "new", testBundleOptions{
ID: "reports.new",
Created: testutil.DefaultCreated.Add(time.Hour),
Files: []testFile{
{Path: "report.md", Data: "# Report\nNew.\n"},
{Path: "summary.txt", Data: "New summary\n"},
},
})
var stdout bytes.Buffer
err := Run(context.Background(), RunOptions{
ConfigPath: configPath,
DryRun: true,
Stdout: &stdout,
})
if err != nil {
t.Fatalf("Run() error = %v", err)
}
output := stdout.String()
for _, want := range []string{
"Warning: pipeline=reports destination=archive path_mapping=fixed action=replace_older replaces destination root for selected_bundle=new",
"bundle=new destination=archive backend=local path_mapping=fixed target=. action=replace_older",
} {
if !strings.Contains(output, want) {
t.Fatalf("stdout = %q, want substring %q", output, want)
}
}
assertFile(t, filepath.Join(destinationRoot, "report.md"), "# Report\nOld.\n")
}
func TestRunFixedPathReplacesOlderManagedState(t *testing.T) {
sourceRoot := t.TempDir()
destinationRoot := t.TempDir()
writeSourceBundle(t, sourceRoot, "old", testBundleOptions{
ID: "reports.old",
Created: testutil.DefaultCreated,
Files: []testFile{
{Path: "report.md", Data: "# Report\nOld.\n"},
{Path: "summary.txt", Data: "Old summary\n"},
},
})
configPath := writeLocalConfigWithPathMapping(t, sourceRoot, destinationRoot, config.PathMappingFixed)
if err := Run(context.Background(), RunOptions{ConfigPath: configPath}); err != nil {
t.Fatalf("first Run() error = %v", err)
}
assertFile(t, filepath.Join(destinationRoot, "report.md"), "# Report\nOld.\n")
writeSourceBundle(t, sourceRoot, "new", testBundleOptions{
ID: "reports.new",
Created: testutil.DefaultCreated.Add(time.Hour),
Files: []testFile{
{Path: "report.md", Data: "# Report\nNew.\n"},
{Path: "summary.txt", Data: "New summary\n"},
},
})
if err := Run(context.Background(), RunOptions{ConfigPath: configPath}); err != nil {
t.Fatalf("second Run() error = %v", err)
}
assertFile(t, filepath.Join(destinationRoot, "report.md"), "# Report\nNew.\n")
destinationState := readStateFile(t, filepath.Join(destinationRoot, storage.StateFileName))
if destinationState.Source.Manifest.ID != "reports.new" {
t.Fatalf("state source id = %q, want reports.new", destinationState.Source.Manifest.ID)
}
}
func TestRunFixedPathSkipsWhenDestinationStateIsNewer(t *testing.T) {
sourceRoot := t.TempDir()
destinationRoot := t.TempDir()
newer := testutil.ValidManifest(testutil.BundleOptions{
ID: "reports.newer",
Created: testutil.DefaultCreated.Add(time.Hour),
})
writeDestinationState(t, destinationRoot, "", newer)
if err := os.WriteFile(filepath.Join(destinationRoot, "report.md"), []byte("# Report\nExisting.\n"), 0o600); err != nil {
t.Fatalf("write existing report: %v", err)
}
writeSourceBundle(t, sourceRoot, "older", testBundleOptions{
ID: "reports.older",
Created: testutil.DefaultCreated,
})
var stdout bytes.Buffer
err := Run(context.Background(), RunOptions{
ConfigPath: writeLocalConfigWithPathMapping(t, sourceRoot, destinationRoot, config.PathMappingFixed),
Stdout: &stdout,
})
if err != nil {
t.Fatalf("Run() error = %v", err)
}
if !strings.Contains(stdout.String(), "action=skip_destination_newer") {
t.Fatalf("stdout = %q, want skip_destination_newer", stdout.String())
}
assertFile(t, filepath.Join(destinationRoot, "report.md"), "# Report\nExisting.\n")
}
func TestRunFixedPathFailsUnmanagedWithoutForce(t *testing.T) {
sourceRoot := t.TempDir()
destinationRoot := t.TempDir()
writeSourceBundle(t, sourceRoot, "bundle", testBundleOptions{})
if err := os.WriteFile(filepath.Join(destinationRoot, "unmanaged.txt"), []byte("data"), 0o600); err != nil {
t.Fatalf("write unmanaged file: %v", err)
}
err := Run(context.Background(), RunOptions{ConfigPath: writeLocalConfigWithPathMapping(t, sourceRoot, destinationRoot, config.PathMappingFixed)})
if err == nil || !strings.Contains(err.Error(), "fail_unmanaged") {
t.Fatalf("Run() error = %v, want unmanaged failure", err)
}
}
func TestRunFixedPathForceReplacementStaysWithinDestinationRoot(t *testing.T) {
sourceRoot := t.TempDir()
parent := t.TempDir()
destinationRoot := filepath.Join(parent, "latest")
if err := os.MkdirAll(destinationRoot, 0o755); err != nil {
t.Fatalf("mkdir destination: %v", err)
}
if err := os.WriteFile(filepath.Join(parent, "keep.txt"), []byte("keep"), 0o600); err != nil {
t.Fatalf("write sibling: %v", err)
}
if err := os.WriteFile(filepath.Join(destinationRoot, "unmanaged.txt"), []byte("old"), 0o600); err != nil {
t.Fatalf("write unmanaged: %v", err)
}
writeSourceBundle(t, sourceRoot, "bundle", testBundleOptions{})
err := Run(context.Background(), RunOptions{
ConfigPath: writeLocalConfigWithPathMapping(t, sourceRoot, destinationRoot, config.PathMappingFixed),
Force: true,
})
if err != nil {
t.Fatalf("Run() error = %v", err)
}
assertFile(t, filepath.Join(destinationRoot, "report.md"), "# Report\nSunny.\n")
assertFile(t, filepath.Join(parent, "keep.txt"), "keep")
if _, err := os.Stat(filepath.Join(destinationRoot, "unmanaged.txt")); !os.IsNotExist(err) {
t.Fatalf("unmanaged stat error = %v, want removed", err)
}
}
func TestRunFixedPathRemoteBackendsUseBackendRoots(t *testing.T) {
localSourceRoot := t.TempDir()
writeSourceBundle(t, localSourceRoot, "old", testBundleOptions{
ID: "reports.old",
Created: testutil.DefaultCreated,
Files: []testFile{
{Path: "report.md", Data: "# Report\nOld.\n"},
{Path: "summary.txt", Data: "Old summary\n"},
},
})
writeSourceBundle(t, localSourceRoot, "new", testBundleOptions{
ID: "reports.new",
Created: testutil.DefaultCreated.Add(time.Hour),
Files: []testFile{
{Path: "report.md", Data: "# Report\nNew.\n"},
{Path: "summary.txt", Data: "New summary\n"},
},
})
s3Destination := fake.New()
sshDestination := fake.New()
cfg := config.Config{Pipelines: []config.Pipeline{{
ID: "reports",
Source: config.Backend{Backend: config.BackendLocal, Path: localSourceRoot},
Destinations: []config.Destination{
{
ID: "object-latest",
Backend: config.BackendS3,
Endpoint: "http://s3.test",
Bucket: "destination-bucket",
PathMap: config.PathMapping{Mode: config.PathMappingFixed},
},
{
ID: "ssh-latest",
Backend: config.BackendSSH,
Host: "ssh.test",
Path: "/latest",
PathMap: config.PathMapping{Mode: config.PathMappingFixed},
},
},
}}}
config.ApplyDefaults(&cfg)
provider := fakeBackendFactoryProvider(t, map[string]storage.Backend{
"s3:destination-bucket": s3Destination,
"ssh:/latest": sshDestination,
})
if err := runConfigWithBackendFactory(context.Background(), cfg, RunOptions{}, provider); err != nil {
t.Fatalf("Run() error = %v", err)
}
assertFakeFile(t, s3Destination, "report.md", "# Report\nNew.\n")
assertFakeFile(t, s3Destination, "summary.txt", "New summary\n")
assertFakeMissing(t, s3Destination, "new/report.md")
assertFakeFile(t, sshDestination, "report.md", "# Report\nNew.\n")
assertFakeFile(t, sshDestination, "summary.txt", "New summary\n")
assertFakeMissing(t, sshDestination, "new/report.md")
}
func TestRunNotifiesAfterPublication(t *testing.T) {
sourceRoot := t.TempDir()
destinationRoot := t.TempDir()
@@ -924,6 +1247,23 @@ pipelines:
`)
}
func writeLocalConfigWithPathMapping(t *testing.T, sourceRoot, destinationRoot, mode string) string {
t.Helper()
return writeConfigFile(t, `
pipelines:
- id: reports
source:
backend: local
path: `+sourceRoot+`
destinations:
- id: archive
backend: local
path: `+destinationRoot+`
path_mapping:
mode: `+mode+`
`)
}
func writeLocalConfigWithMarkdownTransform(t *testing.T, sourceRoot, destinationRoot string, publishSource, publishHTML bool, mode, input string) string {
t.Helper()
enabled := publishHTML