package app import ( "bytes" "context" "encoding/json" "fmt" "os" "path/filepath" "strings" "testing" "time" "gitea.maximumdirect.net/eric/distributor/internal/adapters/local" "gitea.maximumdirect.net/eric/distributor/internal/bundle" "gitea.maximumdirect.net/eric/distributor/internal/config" "gitea.maximumdirect.net/eric/distributor/internal/notify" "gitea.maximumdirect.net/eric/distributor/internal/publish" "gitea.maximumdirect.net/eric/distributor/internal/state" "gitea.maximumdirect.net/eric/distributor/internal/storage" "gitea.maximumdirect.net/eric/distributor/internal/storage/fake" "gitea.maximumdirect.net/eric/distributor/internal/testutil" ) func TestRunDryRunPrintsConfigSummary(t *testing.T) { sourceRoot := t.TempDir() destinationRoot := t.TempDir() writeSourceBundle(t, sourceRoot, "", testBundleOptions{}) configPath := writeLocalConfig(t, sourceRoot, destinationRoot) var stdout bytes.Buffer err := Run(context.Background(), RunOptions{ ConfigPath: configPath, DryRun: true, Stdout: &stdout, }) if err != nil { t.Fatalf("Run() error = %v", err) } output := stdout.String() for _, want := range []string{ "Configured pipelines: 1", "- pipeline=reports source=local bundles=1 destinations=archive", "bundle=. destination=archive backend=local action=publish_new outputs=report.md,summary.txt", "Final status: ok planned=1 publish_new=1 replace_older=0 replace_takeover=0 force_replace=0 skipped=0 failed=0 dry_run=true", } { if !strings.Contains(output, want) { t.Fatalf("Run() output = %q, want substring %q", output, want) } } } func TestRunDryRunUsesReadOnlySSHKnownHosts(t *testing.T) { sourceRoot := t.TempDir() writeSourceBundle(t, sourceRoot, "", testBundleOptions{}) cfg := config.Config{Pipelines: []config.Pipeline{{ ID: "reports", Source: config.Backend{Backend: config.BackendLocal, Path: sourceRoot}, Destinations: []config.Destination{{ ID: "archive", Backend: config.BackendSSH, Host: "destination.example.com", Path: "/archive", }}, }}} config.ApplyDefaults(&cfg) var got storage.OpenConfig provider := func(environment config.Environment) *backendFactory { registry := storage.NewRegistry() if err := registry.Register(config.BackendLocal, func(ctx context.Context, cfg storage.OpenConfig) (storage.Backend, error) { return local.New(cfg[storagePathKey]) }); err != nil { t.Fatalf("register local backend: %v", err) } if err := registry.Register(config.BackendSSH, func(ctx context.Context, cfg storage.OpenConfig) (storage.Backend, error) { got = cfg return fake.New(), nil }); err != nil { t.Fatalf("register ssh backend: %v", err) } return &backendFactory{registry: registry, environment: environment} } if err := runConfigWithBackendFactory(context.Background(), cfg, RunOptions{DryRun: true}, provider); err != nil { t.Fatalf("runConfigWithBackendFactory() error = %v", err) } if got[sshReadOnlyHostsKey] != "true" { t.Fatalf("open config %s = %q, want true", sshReadOnlyHostsKey, got[sshReadOnlyHostsKey]) } } func TestRunLoadsSecretsBeforeOpeningBackends(t *testing.T) { sourceRoot := filepath.Join(t.TempDir(), "missing-source") destinationRoot := t.TempDir() configPath := writeConfigFile(t, ` secrets: directory: `+filepath.Join(t.TempDir(), "missing-secrets")+` pipelines: - id: reports source: backend: local path: `+sourceRoot+` destinations: - id: archive backend: local path: `+destinationRoot+` `) err := Run(context.Background(), RunOptions{ConfigPath: configPath}) if err == nil { t.Fatal("Run() error = nil, want secrets directory error") } if !strings.Contains(err.Error(), "load secrets directory") { t.Fatalf("Run() error = %v, want secrets directory error", err) } if strings.Contains(err.Error(), "missing-source") { t.Fatalf("Run() error = %v, opened source before loading secrets", err) } } func TestRunPrintsSecretConflictWarningWithoutValues(t *testing.T) { name := "DISTRIBUTOR_TEST_RUN_SECRET" t.Setenv(name, "process-value") sourceRoot := t.TempDir() destinationRoot := t.TempDir() secretsRoot := t.TempDir() if err := os.WriteFile(filepath.Join(secretsRoot, name), []byte("secret-value\n"), 0o600); err != nil { t.Fatalf("write secret: %v", err) } writeSourceBundle(t, sourceRoot, "", testBundleOptions{}) configPath := writeConfigFile(t, ` secrets: directory: `+secretsRoot+` pipelines: - id: reports source: backend: local path: `+sourceRoot+` destinations: - id: archive backend: local path: `+destinationRoot+` `) var stdout bytes.Buffer err := Run(context.Background(), RunOptions{ ConfigPath: configPath, DryRun: true, Stdout: &stdout, }) if err != nil { t.Fatalf("Run() error = %v", err) } output := stdout.String() if !strings.Contains(output, "secret "+name+" ignored because the real environment already has that variable") { t.Fatalf("stdout = %q, want secret conflict warning", output) } if strings.Contains(output, "process-value") || strings.Contains(output, "secret-value") { t.Fatalf("stdout exposed secret values: %q", output) } } func TestSSHWarningsReportInsecureHostKeyPolicy(t *testing.T) { var stdout bytes.Buffer err := writeWarnings(&stdout, sshWarnings(config.Pipeline{ ID: "reports", Source: config.Backend{ Backend: config.BackendSSH, SSH: config.SSH{HostKeyPolicy: config.HostKeyPolicyOff}, }, Destinations: []config.Destination{{ ID: "archive", Backend: config.BackendSSH, SSH: config.SSH{HostKeyPolicy: config.HostKeyPolicyOff}, }}, })) if err != nil { t.Fatalf("writeWarnings() error = %v", err) } output := stdout.String() for _, want := range []string{ "pipeline=reports source host_key_policy=off disables SSH host key checking", "pipeline=reports destination=archive host_key_policy=off disables SSH host key checking", } { if !strings.Contains(output, want) { t.Fatalf("output = %q, want substring %q", output, want) } } } func TestRunPublishesNewLocalBundle(t *testing.T) { sourceRoot := t.TempDir() destinationRoot := t.TempDir() manifest := writeSourceBundle(t, sourceRoot, "", testBundleOptions{}) var stdout bytes.Buffer err := Run(context.Background(), RunOptions{ ConfigPath: writeLocalConfig(t, sourceRoot, destinationRoot), Stdout: &stdout, }) if err != nil { t.Fatalf("Run() error = %v", err) } testutil.AssertFile(t, filepath.Join(destinationRoot, "report.md"), "# Report\nSunny.\n") testutil.AssertFile(t, filepath.Join(destinationRoot, "summary.txt"), "Summary\n") if _, err := os.Stat(filepath.Join(destinationRoot, "manifest.json")); !os.IsNotExist(err) { t.Fatalf("destination manifest stat error = %v, want not exist", err) } destinationState := readStateFile(t, filepath.Join(destinationRoot, storage.StateFileName)) if destinationState.PipelineID != "reports" || destinationState.DestinationID != "archive" { t.Fatalf("state identity = %s/%s", destinationState.PipelineID, destinationState.DestinationID) } if destinationState.Source.Manifest.ID != manifest.ID { t.Fatalf("state source id = %q, want %q", destinationState.Source.Manifest.ID, manifest.ID) } if got, want := len(destinationState.Outputs), 2; got != want { t.Fatalf("state output count = %d, want %d", got, want) } if destinationState.Links != nil || destinationState.Outputs[0].URL != "" { t.Fatalf("state links = %#v output URL=%q, want absent", destinationState.Links, destinationState.Outputs[0].URL) } } func TestRunSharedRootDryRunWritesNoOutputsOrState(t *testing.T) { sourceRoot := t.TempDir() destinationRoot := t.TempDir() writeSourceBundle(t, sourceRoot, "", testBundleOptions{ Files: []testFile{{Path: "report.md", Data: "# Report\n"}}, }) configPath := writeSharedRootLocalConfig(t, sourceRoot, destinationRoot) cfg, err := config.LoadFile(configPath) if err != nil { t.Fatalf("load config: %v", err) } report, err := buildRunReportWithBackendFactory(context.Background(), cfg, RunOptions{DryRun: true}, newBackendFactoryWithEnvironment) if err != nil { t.Fatalf("dry-run buildRunReportWithBackendFactory() error = %v", err) } if got, want := report.Actions[0].Action, string(publish.ActionPublishNew); got != want { t.Fatalf("dry-run action = %q, want %q", got, want) } if _, statErr := os.Stat(filepath.Join(destinationRoot, "report.md")); !os.IsNotExist(statErr) { t.Fatalf("output stat error = %v, want absent", statErr) } if _, statErr := os.Stat(filepath.Join(destinationRoot, storage.StateFileName)); !os.IsNotExist(statErr) { t.Fatalf("state file stat error = %v, want absent", statErr) } } func TestRunPublishesTwoPipelinesIntoSharedRoot(t *testing.T) { firstSourceRoot := t.TempDir() secondSourceRoot := t.TempDir() destinationRoot := t.TempDir() writeSourceBundle(t, firstSourceRoot, "", testBundleOptions{ ID: "reports.first", Files: []testFile{{Path: "first.md", Data: "# First\n"}}, }) writeSourceBundle(t, secondSourceRoot, "", testBundleOptions{ ID: "reports.second", Files: []testFile{{Path: "second.md", Data: "# Second\n"}}, }) err := Run(context.Background(), RunOptions{ConfigPath: writeTwoPipelineSharedRootConfig(t, firstSourceRoot, secondSourceRoot, destinationRoot)}) if err != nil { t.Fatalf("Run() error = %v", err) } testutil.AssertFile(t, filepath.Join(destinationRoot, "first.md"), "# First\n") testutil.AssertFile(t, filepath.Join(destinationRoot, "second.md"), "# Second\n") destinationState := readSharedRootStateFile(t, filepath.Join(destinationRoot, storage.StateFileName)) if got, want := len(destinationState.Owners), 2; got != want { t.Fatalf("owner count = %d, want %d", got, want) } if _, ok := destinationState.Owner(state.CurrentOwnerScope("reports-first", "archive")); !ok { t.Fatal("reports-first/archive owner missing") } if _, ok := destinationState.Owner(state.CurrentOwnerScope("reports-second", "archive")); !ok { t.Fatal("reports-second/archive owner missing") } if got, want := strings.Join(destinationState.AllManagedOutputPaths(), ","), "first.md,second.md"; got != want { t.Fatalf("managed paths = %q, want %q", got, want) } } func TestRunPipelineWithLocalSourcePublishesConfiguredDestination(t *testing.T) { sourceRoot := t.TempDir() destinationRoot := t.TempDir() writeSourceBundle(t, sourceRoot, "", testBundleOptions{}) report, err := RunPipelineWithLocalSource(context.Background(), RunPipelineWithLocalSourceOptions{ ConfigPath: writeUploadPipelineConfig(t, destinationRoot), PipelineID: "reports", SourceRoot: sourceRoot, }) if err != nil { t.Fatalf("RunPipelineWithLocalSource() error = %v", err) } if got, want := report.Summary.Status, "ok"; got != want { t.Fatalf("report status = %q, want %q", got, want) } if got, want := len(report.Pipelines), 1; got != want { t.Fatalf("pipeline count = %d, want %d", got, want) } if got, want := report.Pipelines[0].SourceBackend, config.BackendLocal; got != want { t.Fatalf("source backend = %q, want %q", got, want) } if got, want := report.Pipelines[0].BundleCount, 1; got != want { t.Fatalf("bundle count = %d, want %d", got, want) } if got, want := len(report.Actions), 1; got != want { t.Fatalf("action count = %d, want %d", got, want) } if report.Actions[0].PipelineID != "reports" || report.Actions[0].DestinationID != "archive" { t.Fatalf("action = %#v, want reports/archive action", report.Actions[0]) } testutil.AssertFile(t, filepath.Join(destinationRoot, "report.md"), "# Report\nSunny.\n") testutil.AssertFile(t, filepath.Join(destinationRoot, "summary.txt"), "Summary\n") } func TestRunPipelineWithLocalSourceValidatesBeforeDestinationWrites(t *testing.T) { sourceRoot := t.TempDir() destinationRoot := t.TempDir() writeJSONManifest(t, sourceRoot, testutil.ValidManifest(testutil.BundleOptions{})) _, err := RunPipelineWithLocalSource(context.Background(), RunPipelineWithLocalSourceOptions{ ConfigPath: writeUploadPipelineConfig(t, destinationRoot), PipelineID: "reports", SourceRoot: sourceRoot, }) if err == nil { t.Fatal("RunPipelineWithLocalSource() error = nil, want validation error") } if !strings.Contains(err.Error(), "validate source bundle") { t.Fatalf("RunPipelineWithLocalSource() error = %v, want source validation context", err) } entries, readErr := os.ReadDir(destinationRoot) if readErr != nil { t.Fatalf("ReadDir() error = %v", readErr) } if len(entries) != 0 { t.Fatalf("destination entries = %d, want no writes", len(entries)) } } func TestRunPipelineWithLocalSourcePublishesToRegisteredDestinationBackends(t *testing.T) { sourceRoot := t.TempDir() writeSourceBundle(t, sourceRoot, "", testBundleOptions{}) s3Destination := fake.New() sshDestination := fake.New() cfg := config.Config{ Pipelines: []config.Pipeline{{ ID: "reports", Source: config.Backend{ Backend: config.BackendHTTPUpload, }, Destinations: []config.Destination{ { ID: "object-archive", Backend: config.BackendS3, Endpoint: "http://s3.test", Bucket: "destination-bucket", }, { ID: "ssh-archive", Backend: config.BackendSSH, Host: "ssh.test", Path: "/destination", }, }, }}, UploadTokens: []config.UploadToken{{ ID: "reporter", TokenEnv: "UPLOAD_TOKEN", AllowPipelines: []string{"reports"}, }}, } config.ApplyDefaults(&cfg) provider := fakeBackendFactoryProvider(t, map[string]storage.Backend{ "s3:destination-bucket": s3Destination, "ssh:/destination": sshDestination, }) report, err := runPipelineConfigWithLocalSourceAndBackendFactory(context.Background(), cfg, RunPipelineWithLocalSourceOptions{ PipelineID: "reports", SourceRoot: sourceRoot, }, provider) if err != nil { t.Fatalf("runPipelineConfigWithLocalSourceAndBackendFactory() error = %v", err) } if got, want := len(report.Actions), 2; got != want { t.Fatalf("action count = %d, want %d", got, want) } testutil.AssertFakeFile(t, s3Destination, "report.md", "# Report\nSunny.\n") testutil.AssertFakeFile(t, sshDestination, "summary.txt", "Summary\n") } func TestRunExplicitPreserveRelativePathMappingMatchesDefault(t *testing.T) { sourceRoot := t.TempDir() destinationRoot := t.TempDir() writeSourceBundle(t, sourceRoot, "daily/report", testBundleOptions{}) err := Run(context.Background(), RunOptions{ConfigPath: testutil.WriteLocalConfigWithPathMapping(t, sourceRoot, destinationRoot, config.PathMappingPreserveRelative)}) if err != nil { t.Fatalf("Run() error = %v", err) } testutil.AssertFile(t, filepath.Join(destinationRoot, "daily", "report", "report.md"), "# Report\nSunny.\n") if _, err := os.Stat(filepath.Join(destinationRoot, "report.md")); !os.IsNotExist(err) { t.Fatalf("root report.md stat error = %v, want not exist", err) } } func TestRunRecordsLinksForNestedBundlePath(t *testing.T) { sourceRoot := t.TempDir() destinationRoot := t.TempDir() writeSourceBundle(t, sourceRoot, "daily/brentwood", testBundleOptions{}) err := Run(context.Background(), RunOptions{ ConfigPath: testutil.WriteLocalConfigWithLinks(t, sourceRoot, destinationRoot, config.PathMappingPreserveRelative, "https://reports.example.com/archive", config.LinkPrimaryAuto, true, false, ""), }) if err != nil { t.Fatalf("Run() error = %v", err) } destinationState := readStateFile(t, filepath.Join(destinationRoot, "daily", "brentwood", storage.StateFileName)) if destinationState.Links == nil || destinationState.Links.PrimaryURL != "https://reports.example.com/archive/daily/brentwood/report.md" { t.Fatalf("state links = %#v, want source primary URL", destinationState.Links) } outputs := outputsByPath(destinationState.Outputs) if outputs["report.md"].URL != "https://reports.example.com/archive/daily/brentwood/report.md" { t.Fatalf("report URL = %q", outputs["report.md"].URL) } if outputs["summary.txt"].URL != "https://reports.example.com/archive/daily/brentwood/summary.txt" { t.Fatalf("summary URL = %q", outputs["summary.txt"].URL) } } func TestRunRecordsLinksForFixedIndexDestination(t *testing.T) { sourceRoot := t.TempDir() destinationRoot := t.TempDir() writeSourceBundle(t, sourceRoot, "older", testBundleOptions{ID: "reports.older", Created: testutil.DefaultCreated}) writeSourceBundle(t, sourceRoot, "newer", testBundleOptions{ID: "reports.newer", Created: testutil.DefaultCreated.Add(time.Hour)}) err := Run(context.Background(), RunOptions{ ConfigPath: testutil.WriteLocalConfigWithLinks(t, sourceRoot, destinationRoot, config.PathMappingFixed, "https://reports.example.com/latest", config.LinkPrimaryAuto, false, true, config.TransformModeIndex), }) if err != nil { t.Fatalf("Run() error = %v", err) } destinationState := readStateFile(t, filepath.Join(destinationRoot, storage.StateFileName)) if destinationState.Links == nil || destinationState.Links.PrimaryURL != "https://reports.example.com/latest/" { t.Fatalf("state links = %#v, want fixed index primary URL", destinationState.Links) } if got, want := len(destinationState.Outputs), 1; got != want { t.Fatalf("state output count = %d, want %d", got, want) } if destinationState.Outputs[0].Path != "index.html" || destinationState.Outputs[0].URL != "https://reports.example.com/latest/" { t.Fatalf("state output = %#v, want index URL", destinationState.Outputs[0]) } } func TestRunFixedPathPublishesNewestBundleAtDestinationRoot(t *testing.T) { sourceRoot := t.TempDir() destinationRoot := t.TempDir() writeSourceBundle(t, sourceRoot, "old", testBundleOptions{ ID: "reports.old", Created: testutil.DefaultCreated, Files: []testFile{ {Path: "report.md", Data: "# Report\nOld.\n"}, {Path: "summary.txt", Data: "Old summary\n"}, }, }) writeSourceBundle(t, sourceRoot, "new", testBundleOptions{ ID: "reports.new", Created: testutil.DefaultCreated.Add(time.Hour), Files: []testFile{ {Path: "report.md", Data: "# Report\nNew.\n"}, {Path: "summary.txt", Data: "New summary\n"}, }, }) err := Run(context.Background(), RunOptions{ConfigPath: testutil.WriteLocalConfigWithPathMapping(t, sourceRoot, destinationRoot, config.PathMappingFixed)}) if err != nil { t.Fatalf("Run() error = %v", err) } testutil.AssertFile(t, filepath.Join(destinationRoot, "report.md"), "# Report\nNew.\n") if _, err := os.Stat(filepath.Join(destinationRoot, "new", "report.md")); !os.IsNotExist(err) { t.Fatalf("nested new report stat error = %v, want not exist", err) } destinationState := readStateFile(t, filepath.Join(destinationRoot, storage.StateFileName)) if destinationState.Source.Manifest.ID != "reports.new" { t.Fatalf("state source id = %q, want reports.new", destinationState.Source.Manifest.ID) } } func TestRunFixedPathTieBreaksByBundlePath(t *testing.T) { sourceRoot := t.TempDir() destinationRoot := t.TempDir() writeSourceBundle(t, sourceRoot, "b", testBundleOptions{ ID: "reports.b", Created: testutil.DefaultCreated, Files: []testFile{ {Path: "report.md", Data: "# Report\nB.\n"}, {Path: "summary.txt", Data: "B summary\n"}, }, }) writeSourceBundle(t, sourceRoot, "a", testBundleOptions{ ID: "reports.a", Created: testutil.DefaultCreated, Files: []testFile{ {Path: "report.md", Data: "# Report\nA.\n"}, {Path: "summary.txt", Data: "A summary\n"}, }, }) err := Run(context.Background(), RunOptions{ConfigPath: testutil.WriteLocalConfigWithPathMapping(t, sourceRoot, destinationRoot, config.PathMappingFixed)}) if err != nil { t.Fatalf("Run() error = %v", err) } destinationState := readStateFile(t, filepath.Join(destinationRoot, storage.StateFileName)) if destinationState.Source.Manifest.ID != "reports.a" { t.Fatalf("state source id = %q, want reports.a", destinationState.Source.Manifest.ID) } } func TestRunFixedPathDryRunReportsSelection(t *testing.T) { sourceRoot := t.TempDir() destinationRoot := t.TempDir() writeSourceBundle(t, sourceRoot, "old", testBundleOptions{ID: "reports.old", Created: testutil.DefaultCreated}) writeSourceBundle(t, sourceRoot, "new", testBundleOptions{ID: "reports.new", Created: testutil.DefaultCreated.Add(time.Hour)}) var stdout bytes.Buffer err := Run(context.Background(), RunOptions{ ConfigPath: testutil.WriteLocalConfigWithPathMapping(t, sourceRoot, destinationRoot, config.PathMappingFixed), DryRun: true, Stdout: &stdout, }) if err != nil { t.Fatalf("Run() error = %v", err) } output := stdout.String() for _, want := range []string{ "Warning: pipeline=reports destination=archive path_mapping=fixed candidates=2 selected_bundle=new destination_bundle=.", "bundle=new destination=archive backend=local path_mapping=fixed target=. action=publish_new", "fixed_path=1", } { if !strings.Contains(output, want) { t.Fatalf("stdout = %q, want substring %q", output, want) } } if strings.Contains(output, "bundle=old destination=archive") { t.Fatalf("stdout = %q, older fixed candidate was planned", output) } if entries, err := os.ReadDir(destinationRoot); err != nil || len(entries) != 0 { t.Fatalf("destination entries = %v err=%v, want empty", entries, err) } } func TestRunFixedPathDryRunWarnsForReplacement(t *testing.T) { sourceRoot := t.TempDir() destinationRoot := t.TempDir() writeSourceBundle(t, sourceRoot, "old", testBundleOptions{ ID: "reports.old", Created: testutil.DefaultCreated, Files: []testFile{ {Path: "report.md", Data: "# Report\nOld.\n"}, {Path: "summary.txt", Data: "Old summary\n"}, }, }) configPath := testutil.WriteLocalConfigWithPathMapping(t, sourceRoot, destinationRoot, config.PathMappingFixed) if err := Run(context.Background(), RunOptions{ConfigPath: configPath}); err != nil { t.Fatalf("first Run() error = %v", err) } writeSourceBundle(t, sourceRoot, "new", testBundleOptions{ ID: "reports.new", Created: testutil.DefaultCreated.Add(time.Hour), Files: []testFile{ {Path: "report.md", Data: "# Report\nNew.\n"}, {Path: "summary.txt", Data: "New summary\n"}, }, }) var stdout bytes.Buffer err := Run(context.Background(), RunOptions{ ConfigPath: configPath, DryRun: true, Stdout: &stdout, }) if err != nil { t.Fatalf("Run() error = %v", err) } output := stdout.String() for _, want := range []string{ "Warning: pipeline=reports destination=archive path_mapping=fixed action=replace_takeover takeover_mode=same_pipeline replaces destination root for selected_bundle=new reason=\"destination source id differs from source\"", "bundle=new destination=archive backend=local path_mapping=fixed target=. action=replace_takeover takeover_mode=same_pipeline outputs=report.md,summary.txt reason=\"destination source id differs from source\"", "replace_takeover=1", } { if !strings.Contains(output, want) { t.Fatalf("stdout = %q, want substring %q", output, want) } } testutil.AssertFile(t, filepath.Join(destinationRoot, "report.md"), "# Report\nOld.\n") } func TestRunJSONIncludesTakeoverActionAndSummary(t *testing.T) { sourceRoot := t.TempDir() destinationRoot := t.TempDir() writeSourceBundle(t, sourceRoot, "old", testBundleOptions{ ID: "reports.old", Created: testutil.DefaultCreated, Files: []testFile{ {Path: "report.md", Data: "# Report\nOld.\n"}, {Path: "summary.txt", Data: "Old summary\n"}, }, }) configPath := testutil.WriteLocalConfigWithPathMapping(t, sourceRoot, destinationRoot, config.PathMappingFixed) if err := Run(context.Background(), RunOptions{ConfigPath: configPath}); err != nil { t.Fatalf("first Run() error = %v", err) } writeSourceBundle(t, sourceRoot, "new", testBundleOptions{ ID: "reports.new", Created: testutil.DefaultCreated.Add(time.Hour), Files: []testFile{ {Path: "report.md", Data: "# Report\nNew.\n"}, {Path: "summary.txt", Data: "New summary\n"}, }, }) var stdout bytes.Buffer err := Run(context.Background(), RunOptions{ ConfigPath: configPath, DryRun: true, Stdout: &stdout, OutputFormat: OutputFormatJSON, }) if err != nil { t.Fatalf("Run() error = %v", err) } result := decodeAppResult(t, stdout.String()) actions, ok := result["actions"].([]any) if !ok || len(actions) != 1 { t.Fatalf("actions = %#v, want one action", result["actions"]) } action, ok := actions[0].(map[string]any) if !ok { t.Fatalf("action = %#v, want object", actions[0]) } if action["action"] != "replace_takeover" || action["takeover_mode"] != "same_pipeline" || action["reason"] != "destination source id differs from source" { t.Fatalf("action = %#v, want takeover action metadata", action) } summary, ok := result["summary"].(map[string]any) if !ok { t.Fatalf("summary = %#v, want object", result["summary"]) } if summary["replace_takeover"] != float64(1) || summary["replace_older"] != float64(0) || summary["force_replace"] != float64(0) { t.Fatalf("summary = %#v, want takeover counter only", summary) } } func TestRunFixedPathReplacesOlderManagedState(t *testing.T) { sourceRoot := t.TempDir() destinationRoot := t.TempDir() writeSourceBundle(t, sourceRoot, "old", testBundleOptions{ ID: "reports.old", Created: testutil.DefaultCreated, Files: []testFile{ {Path: "report.md", Data: "# Report\nOld.\n"}, {Path: "summary.txt", Data: "Old summary\n"}, }, }) configPath := testutil.WriteLocalConfigWithPathMapping(t, sourceRoot, destinationRoot, config.PathMappingFixed) if err := Run(context.Background(), RunOptions{ConfigPath: configPath}); err != nil { t.Fatalf("first Run() error = %v", err) } testutil.AssertFile(t, filepath.Join(destinationRoot, "report.md"), "# Report\nOld.\n") writeSourceBundle(t, sourceRoot, "new", testBundleOptions{ ID: "reports.new", Created: testutil.DefaultCreated.Add(time.Hour), Files: []testFile{ {Path: "report.md", Data: "# Report\nNew.\n"}, {Path: "summary.txt", Data: "New summary\n"}, }, }) var stdout bytes.Buffer if err := Run(context.Background(), RunOptions{ConfigPath: configPath, Stdout: &stdout}); err != nil { t.Fatalf("second Run() error = %v", err) } if !strings.Contains(stdout.String(), "action=replace_takeover takeover_mode=same_pipeline") { t.Fatalf("stdout = %q, want same-pipeline takeover", stdout.String()) } testutil.AssertFile(t, filepath.Join(destinationRoot, "report.md"), "# Report\nNew.\n") destinationState := readStateFile(t, filepath.Join(destinationRoot, storage.StateFileName)) if destinationState.Source.Manifest.ID != "reports.new" { t.Fatalf("state source id = %q, want reports.new", destinationState.Source.Manifest.ID) } } func TestRunPreserveRelativeSameSourceTakeoverAllowsOwnerMismatch(t *testing.T) { sourceRoot := t.TempDir() destinationRoot := t.TempDir() sourceManifest := writeSourceBundle(t, sourceRoot, "daily/report", testBundleOptions{ ID: "reports.same", Files: []testFile{{Path: "report.md", Data: "# Report\nNew.\n"}}, }) testutil.WriteDestinationState(t, destinationRoot, "daily/report", sourceManifest, testutil.DestinationStateOptions{ PipelineID: "other", }) if err := os.WriteFile(filepath.Join(destinationRoot, "daily", "report", "report.md"), []byte("# Report\nOld.\n"), 0o600); err != nil { t.Fatalf("write old report: %v", err) } var stdout bytes.Buffer err := Run(context.Background(), RunOptions{ ConfigPath: writeSameSourcePreserveRelativeConfig(t, sourceRoot, destinationRoot), Stdout: &stdout, }) if err != nil { t.Fatalf("Run() error = %v", err) } if !strings.Contains(stdout.String(), "action=replace_takeover takeover_mode=same_source") { t.Fatalf("stdout = %q, want same-source takeover", stdout.String()) } testutil.AssertFile(t, filepath.Join(destinationRoot, "daily", "report", "report.md"), "# Report\nNew.\n") destinationState := readStateFile(t, filepath.Join(destinationRoot, "daily", "report", storage.StateFileName)) if destinationState.PipelineID != "reports" || destinationState.Source.Manifest.ID != "reports.same" { t.Fatalf("state owner/source = %s/%s source=%s, want reports/archive reports.same", destinationState.PipelineID, destinationState.DestinationID, destinationState.Source.Manifest.ID) } } func TestRunPreserveRelativeSameSourceRefusesDifferentSource(t *testing.T) { sourceRoot := t.TempDir() destinationRoot := t.TempDir() sourceManifest := writeSourceBundle(t, sourceRoot, "daily/report", testBundleOptions{ ID: "reports.same", Files: []testFile{{Path: "report.md", Data: "# Report\nNew.\n"}}, }) destinationManifest := sourceManifest destinationManifest.ID = "reports.other" testutil.WriteDestinationState(t, destinationRoot, "daily/report", destinationManifest, testutil.DestinationStateOptions{ PipelineID: "other", }) if err := os.WriteFile(filepath.Join(destinationRoot, "daily", "report", "report.md"), []byte("# Report\nOld.\n"), 0o600); err != nil { t.Fatalf("write old report: %v", err) } err := Run(context.Background(), RunOptions{ ConfigPath: writeSameSourcePreserveRelativeConfig(t, sourceRoot, destinationRoot), }) if err == nil || !strings.Contains(err.Error(), "fail_conflict") { t.Fatalf("Run() error = %v, want fail_conflict", err) } testutil.AssertFile(t, filepath.Join(destinationRoot, "daily", "report", "report.md"), "# Report\nOld.\n") destinationState := readStateFile(t, filepath.Join(destinationRoot, "daily", "report", storage.StateFileName)) if destinationState.PipelineID != "other" || destinationState.Source.Manifest.ID != "reports.other" { t.Fatalf("state owner/source = %s/%s source=%s, want unchanged other/archive reports.other", destinationState.PipelineID, destinationState.DestinationID, destinationState.Source.Manifest.ID) } } func TestRunFixedPathSkipsWhenDestinationStateIsNewer(t *testing.T) { sourceRoot := t.TempDir() destinationRoot := t.TempDir() newer := testutil.ValidManifest(testutil.BundleOptions{ ID: "reports.same", Created: testutil.DefaultCreated.Add(time.Hour), }) writeDestinationState(t, destinationRoot, "", newer) if err := os.WriteFile(filepath.Join(destinationRoot, "report.md"), []byte("# Report\nExisting.\n"), 0o600); err != nil { t.Fatalf("write existing report: %v", err) } writeSourceBundle(t, sourceRoot, "older", testBundleOptions{ ID: "reports.same", Created: testutil.DefaultCreated, }) var stdout bytes.Buffer err := Run(context.Background(), RunOptions{ ConfigPath: testutil.WriteLocalConfigWithPathMapping(t, sourceRoot, destinationRoot, config.PathMappingFixed), Stdout: &stdout, }) if err != nil { t.Fatalf("Run() error = %v", err) } if !strings.Contains(stdout.String(), "action=skip_destination_newer") { t.Fatalf("stdout = %q, want skip_destination_newer", stdout.String()) } testutil.AssertFile(t, filepath.Join(destinationRoot, "report.md"), "# Report\nExisting.\n") } func TestRunFixedPathFailsUnmanagedWithoutForce(t *testing.T) { sourceRoot := t.TempDir() destinationRoot := t.TempDir() writeSourceBundle(t, sourceRoot, "bundle", testBundleOptions{}) if err := os.WriteFile(filepath.Join(destinationRoot, "unmanaged.txt"), []byte("data"), 0o600); err != nil { t.Fatalf("write unmanaged file: %v", err) } err := Run(context.Background(), RunOptions{ConfigPath: testutil.WriteLocalConfigWithPathMapping(t, sourceRoot, destinationRoot, config.PathMappingFixed)}) if err == nil || !strings.Contains(err.Error(), "fail_unmanaged") { t.Fatalf("Run() error = %v, want unmanaged failure", err) } } func TestRunFixedPathForceReplacementStaysWithinDestinationRoot(t *testing.T) { sourceRoot := t.TempDir() parent := t.TempDir() destinationRoot := filepath.Join(parent, "latest") if err := os.MkdirAll(destinationRoot, 0o755); err != nil { t.Fatalf("mkdir destination: %v", err) } if err := os.WriteFile(filepath.Join(parent, "keep.txt"), []byte("keep"), 0o600); err != nil { t.Fatalf("write sibling: %v", err) } if err := os.WriteFile(filepath.Join(destinationRoot, "unmanaged.txt"), []byte("old"), 0o600); err != nil { t.Fatalf("write unmanaged: %v", err) } writeSourceBundle(t, sourceRoot, "bundle", testBundleOptions{}) err := Run(context.Background(), RunOptions{ ConfigPath: testutil.WriteLocalConfigWithPathMapping(t, sourceRoot, destinationRoot, config.PathMappingFixed), Force: true, }) if err != nil { t.Fatalf("Run() error = %v", err) } testutil.AssertFile(t, filepath.Join(destinationRoot, "report.md"), "# Report\nSunny.\n") testutil.AssertFile(t, filepath.Join(parent, "keep.txt"), "keep") if _, err := os.Stat(filepath.Join(destinationRoot, "unmanaged.txt")); !os.IsNotExist(err) { t.Fatalf("unmanaged stat error = %v, want removed", err) } } func TestRunFixedPathRemoteBackendsUseBackendRoots(t *testing.T) { localSourceRoot := t.TempDir() writeSourceBundle(t, localSourceRoot, "old", testBundleOptions{ ID: "reports.old", Created: testutil.DefaultCreated, Files: []testFile{ {Path: "report.md", Data: "# Report\nOld.\n"}, {Path: "summary.txt", Data: "Old summary\n"}, }, }) s3Destination := fake.New() sshDestination := fake.New() cfg := config.Config{Pipelines: []config.Pipeline{{ ID: "reports", Source: config.Backend{Backend: config.BackendLocal, Path: localSourceRoot}, Destinations: []config.Destination{ { ID: "object-latest", Backend: config.BackendS3, Endpoint: "http://s3.test", Bucket: "destination-bucket", PathMap: config.PathMapping{Mode: config.PathMappingFixed}, }, { ID: "ssh-latest", Backend: config.BackendSSH, Host: "ssh.test", Path: "/latest", PathMap: config.PathMapping{Mode: config.PathMappingFixed}, }, }, }}} config.ApplyDefaults(&cfg) provider := fakeBackendFactoryProvider(t, map[string]storage.Backend{ "s3:destination-bucket": s3Destination, "ssh:/latest": sshDestination, }) if err := runConfigWithBackendFactory(context.Background(), cfg, RunOptions{}, provider); err != nil { t.Fatalf("Run() error = %v", err) } testutil.AssertFakeFile(t, s3Destination, "report.md", "# Report\nOld.\n") testutil.AssertFakeFile(t, sshDestination, "summary.txt", "Old summary\n") writeSourceBundle(t, localSourceRoot, "new", testBundleOptions{ ID: "reports.new", Created: testutil.DefaultCreated.Add(time.Hour), Files: []testFile{ {Path: "report.md", Data: "# Report\nNew.\n"}, {Path: "summary.txt", Data: "New summary\n"}, }, }) var stdout bytes.Buffer if err := runConfigWithBackendFactory(context.Background(), cfg, RunOptions{Stdout: &stdout}, provider); err != nil { t.Fatalf("second Run() error = %v", err) } for _, want := range []string{ "destination=object-latest backend=s3 path_mapping=fixed target=. action=replace_takeover takeover_mode=same_pipeline", "destination=ssh-latest backend=ssh path_mapping=fixed target=. action=replace_takeover takeover_mode=same_pipeline", "replace_takeover=2", } { if !strings.Contains(stdout.String(), want) { t.Fatalf("stdout = %q, want substring %q", stdout.String(), want) } } testutil.AssertFakeFile(t, s3Destination, "report.md", "# Report\nNew.\n") testutil.AssertFakeFile(t, s3Destination, "summary.txt", "New summary\n") testutil.AssertFakeMissing(t, s3Destination, "new/report.md") testutil.AssertFakeFile(t, sshDestination, "report.md", "# Report\nNew.\n") testutil.AssertFakeFile(t, sshDestination, "summary.txt", "New summary\n") testutil.AssertFakeMissing(t, sshDestination, "new/report.md") } func TestRunNotifiesAfterPublication(t *testing.T) { sourceRoot := t.TempDir() destinationRoot := t.TempDir() writeSourceBundle(t, sourceRoot, "", testBundleOptions{}) notifier := &recordingNotifier{ check: func() { if _, err := os.Stat(filepath.Join(destinationRoot, storage.StateFileName)); err != nil { t.Fatalf("state stat during notify: %v", err) } }, } err := Run(context.Background(), RunOptions{ ConfigPath: writeLocalConfig(t, sourceRoot, destinationRoot), Notifier: notifier, }) if err != nil { t.Fatalf("Run() error = %v", err) } if got, want := len(notifier.events), 1; got != want { t.Fatalf("notification count = %d, want %d", got, want) } event := notifier.events[0] if event.PipelineID != "reports" || event.DestinationID != "archive" || event.BundleID == "" || event.Action != "publish_new" { t.Fatalf("notification event = %#v", event) } if got, want := len(event.Outputs), 2; got != want { t.Fatalf("notification output count = %d, want %d", got, want) } } func TestRunNotifiesGeneratedOutputMetadata(t *testing.T) { sourceRoot := t.TempDir() destinationRoot := t.TempDir() writeSourceBundle(t, sourceRoot, "", testBundleOptions{}) notifier := &recordingNotifier{} err := Run(context.Background(), RunOptions{ ConfigPath: testutil.WriteLocalConfigWithPublishPolicy(t, sourceRoot, destinationRoot, false, true), Notifier: notifier, }) if err != nil { t.Fatalf("Run() error = %v", err) } if got, want := len(notifier.events), 1; got != want { t.Fatalf("notification count = %d, want %d", got, want) } outputs := notifier.events[0].Outputs if got, want := len(outputs), 1; got != want { t.Fatalf("notification output count = %d, want %d", got, want) } output := outputs[0] if output.Path != "report.html" || output.Kind != state.OutputKindGenerated || output.SourcePath != "report.md" || output.Transform != "markdown_to_html" || output.SHA256 == "" || output.Size <= 0 { t.Fatalf("notification output = %#v", output) } } func TestRunNotifiesAfterReplacement(t *testing.T) { sourceRoot := t.TempDir() destinationRoot := t.TempDir() manifest := writeSourceBundle(t, sourceRoot, "", testBundleOptions{}) older := manifest older.Created = older.Created.Add(-time.Hour) writeDestinationState(t, destinationRoot, "", older) if err := os.WriteFile(filepath.Join(destinationRoot, "report.md"), []byte("old\n"), 0o600); err != nil { t.Fatalf("write old output: %v", err) } notifier := &recordingNotifier{} err := Run(context.Background(), RunOptions{ ConfigPath: writeLocalConfig(t, sourceRoot, destinationRoot), Notifier: notifier, }) if err != nil { t.Fatalf("Run() error = %v", err) } if got, want := len(notifier.events), 1; got != want { t.Fatalf("notification count = %d, want %d", got, want) } if notifier.events[0].Action != "replace_older" { t.Fatalf("notification action = %q, want replace_older", notifier.events[0].Action) } } func TestRunMergeReconciliationRetainsManagedOutput(t *testing.T) { sourceRoot := t.TempDir() destinationRoot := t.TempDir() manifest := testutil.WriteSourceBundle(t, sourceRoot, "", testutil.BundleOptions{ Files: []testutil.SourceFile{{Path: "report.md", Data: "# Report\nNew.\n"}}, }) older := manifest older.Created = older.Created.Add(-time.Hour) defaultManifest := testutil.ValidManifest(testutil.BundleOptions{}) older.Files = append([]bundle.ManifestFile(nil), defaultManifest.Files...) older.Digest = bundle.BundleDigest(older.Files) writeDestinationState(t, destinationRoot, "", older) if err := os.WriteFile(filepath.Join(destinationRoot, "report.md"), []byte("old\n"), 0o600); err != nil { t.Fatalf("write old report: %v", err) } if err := os.WriteFile(filepath.Join(destinationRoot, "summary.txt"), []byte("old summary\n"), 0o600); err != nil { t.Fatalf("write old summary: %v", err) } configPath := writeConfigFile(t, ` pipelines: - id: reports source: backend: local path: `+sourceRoot+` destinations: - id: archive backend: local path: `+destinationRoot+` reconciliation: mode: merge `) err := Run(context.Background(), RunOptions{ConfigPath: configPath}) if err != nil { t.Fatalf("Run() error = %v", err) } testutil.AssertFile(t, filepath.Join(destinationRoot, "report.md"), "# Report\nNew.\n") testutil.AssertFile(t, filepath.Join(destinationRoot, "summary.txt"), "old summary\n") destinationState := readStateFile(t, filepath.Join(destinationRoot, storage.StateFileName)) if got, want := destinationState.Reconciliation.Mode, config.ReconciliationModeMerge; got != want { t.Fatalf("reconciliation mode = %q, want %q", got, want) } if got, want := len(destinationState.Outputs), 2; got != want { t.Fatalf("state output count = %d, want %d", got, want) } } func TestRunJSONIncludesGeneratedOutputMetadata(t *testing.T) { sourceRoot := t.TempDir() destinationRoot := t.TempDir() writeSourceBundle(t, sourceRoot, "", testBundleOptions{}) var stdout bytes.Buffer err := Run(context.Background(), RunOptions{ ConfigPath: testutil.WriteLocalConfigWithLinks(t, sourceRoot, destinationRoot, config.PathMappingFixed, "https://reports.example.com/latest", config.LinkPrimaryAuto, false, true, config.TransformModeIndex), DryRun: true, Stdout: &stdout, OutputFormat: OutputFormatJSON, }) if err != nil { t.Fatalf("Run() error = %v", err) } result := decodeAppResult(t, stdout.String()) actions, ok := result["actions"].([]any) if !ok || len(actions) != 1 { t.Fatalf("actions = %#v, want one action", result["actions"]) } action, ok := actions[0].(map[string]any) if !ok { t.Fatalf("action = %#v, want object", actions[0]) } if action["primary_url"] != "https://reports.example.com/latest/" { t.Fatalf("action primary_url = %#v", action["primary_url"]) } outputs, ok := action["outputs"].([]any) if !ok || len(outputs) != 1 { t.Fatalf("outputs = %#v, want one output", action["outputs"]) } output, ok := outputs[0].(map[string]any) if !ok { t.Fatalf("output = %#v, want object", outputs[0]) } if output["path"] != "index.html" || output["kind"] != state.OutputKindGenerated || output["source_path"] != "report.md" || output["transform"] != "markdown_to_html" || output["url"] != "https://reports.example.com/latest/" { t.Fatalf("output = %#v, want generated index metadata", output) } if output["sha256"] == "" || output["size"] == nil { t.Fatalf("output = %#v, want digest and size", output) } } func TestBuildRunReportIncludesStructuredDryRunResults(t *testing.T) { sourceRoot := t.TempDir() destinationRoot := t.TempDir() writeSourceBundle(t, sourceRoot, "", testBundleOptions{}) configPath := testutil.WriteLocalConfigWithLinks(t, sourceRoot, destinationRoot, config.PathMappingFixed, "https://reports.example.com/latest", config.LinkPrimaryAuto, false, true, config.TransformModeIndex) cfg, err := config.LoadFile(configPath) if err != nil { t.Fatalf("load config: %v", err) } report, err := buildRunReportWithBackendFactory(context.Background(), cfg, RunOptions{DryRun: true}, newBackendFactoryWithEnvironment) if err != nil { t.Fatalf("buildRunReportWithBackendFactory() error = %v", err) } if !report.DryRun || report.Summary.Status != "ok" || !report.Summary.DryRun { t.Fatalf("report dry-run/status = dry_run:%t summary:%#v, want ok dry-run", report.DryRun, report.Summary) } if got, want := len(report.Pipelines), 1; got != want { t.Fatalf("pipeline count = %d, want %d", got, want) } pipeline := report.Pipelines[0] if pipeline.ID != "reports" || pipeline.SourceBackend != config.BackendLocal || pipeline.BundleCount != 1 || strings.Join(pipeline.Destinations, ",") != "archive" { t.Fatalf("pipeline summary = %#v, want reports/local bundle summary", pipeline) } if got, want := len(report.Warnings), 1; got != want { t.Fatalf("warning count = %d, want %d", got, want) } if !strings.Contains(report.Warnings[0].Message, "path_mapping=fixed candidates=1 selected_bundle=.") { t.Fatalf("warning = %#v, want fixed path selection", report.Warnings[0]) } if got, want := len(report.Actions), 1; got != want { t.Fatalf("action count = %d, want %d", got, want) } action := report.Actions[0] if action.PipelineID != "reports" || action.DestinationID != "archive" || action.Action != "publish_new" || action.PrimaryURL != "https://reports.example.com/latest/" { t.Fatalf("action = %#v, want publish_new with primary URL", action) } if action.PathMapping != config.PathMappingFixed || action.DestinationPath != "." { t.Fatalf("action path mapping = %q destination path = %q, want fixed root", action.PathMapping, action.DestinationPath) } if got, want := len(action.Outputs), 1; got != want { t.Fatalf("output count = %d, want %d", got, want) } output := action.Outputs[0] if output.Path != "index.html" || output.Kind != state.OutputKindGenerated || output.SourcePath != "report.md" || output.Transform != "markdown_to_html" || output.URL != "https://reports.example.com/latest/" { t.Fatalf("output = %#v, want generated index metadata", output) } if report.Summary.Planned != 1 || report.Summary.PublishNew != 1 || report.Summary.FixedPath != 1 || report.Summary.Failed != 0 { t.Fatalf("summary = %#v, want publish_new fixed path counters", report.Summary) } if len(report.OutputErrors) != 0 { t.Fatalf("output errors = %#v, want none", report.OutputErrors) } } func TestBuildRunReportIncludesPartialFailures(t *testing.T) { sourceRoot := t.TempDir() firstDestination := t.TempDir() secondDestination := t.TempDir() writeSourceBundle(t, sourceRoot, "", testBundleOptions{}) if err := os.WriteFile(filepath.Join(firstDestination, "unmanaged.txt"), []byte("data"), 0o600); err != nil { t.Fatalf("write unmanaged file: %v", err) } cfg, err := config.LoadFile(writeFanoutConfig(t, sourceRoot, firstDestination, secondDestination)) if err != nil { t.Fatalf("load config: %v", err) } report, err := buildRunReportWithBackendFactory(context.Background(), cfg, RunOptions{}, newBackendFactoryWithEnvironment) if err == nil || !IsPartialResultError(err) { t.Fatalf("buildRunReportWithBackendFactory() error = %v, want partial result error", err) } if report.Summary.Status != "failed" || report.Summary.Planned != 1 || report.Summary.PublishNew != 1 || report.Summary.Failed != 1 { t.Fatalf("summary = %#v, want one planned publish and one failure", report.Summary) } if got, want := len(report.Actions), 2; got != want { t.Fatalf("action count = %d, want %d", got, want) } if report.Actions[0].DestinationID != "archive-one" || report.Actions[0].Action != "error" || !strings.Contains(report.Actions[0].Reason, "fail_unmanaged") { t.Fatalf("first action = %#v, want archive-one error", report.Actions[0]) } if report.Actions[1].DestinationID != "archive-two" || report.Actions[1].Action != "publish_new" { t.Fatalf("second action = %#v, want archive-two publish_new", report.Actions[1]) } if got, want := len(report.OutputErrors), 1; got != want { t.Fatalf("output error count = %d, want %d", got, want) } outputError := report.OutputErrors[0] if outputError.PipelineID != "reports" || outputError.DestinationID != "archive-one" || outputError.Backend != config.BackendLocal || outputError.BundlePath != "." || !strings.Contains(outputError.Message, "fail_unmanaged") { t.Fatalf("output error = %#v, want archive-one unmanaged failure", outputError) } } func TestBuildRunReportAlignsDestinationOpenFailuresForSelectedBundles(t *testing.T) { sourceRoot := t.TempDir() writeSourceBundle(t, sourceRoot, "daily/one", testBundleOptions{ID: "reports.one"}) writeSourceBundle(t, sourceRoot, "daily/two", testBundleOptions{ID: "reports.two", Created: testutil.DefaultCreated.Add(time.Hour)}) cfg := config.Config{Pipelines: []config.Pipeline{{ ID: "reports", Source: config.Backend{Backend: config.BackendLocal, Path: sourceRoot}, Destinations: []config.Destination{{ ID: "object-archive", Backend: config.BackendS3, Endpoint: "http://s3.test", Bucket: "missing-destination", }}, }}} config.ApplyDefaults(&cfg) report, err := buildRunReportWithBackendFactory(context.Background(), cfg, RunOptions{}, fakeBackendFactoryProvider(t, nil)) if err == nil || !IsPartialResultError(err) { t.Fatalf("buildRunReportWithBackendFactory() error = %v, want partial result error", err) } if report.Summary.Status != "failed" || report.Summary.Planned != 0 || report.Summary.Failed != 2 { t.Fatalf("summary = %#v, want two destination open failures", report.Summary) } if got, want := len(report.Actions), 2; got != want { t.Fatalf("action count = %d, want %d", got, want) } if got, want := len(report.OutputErrors), 2; got != want { t.Fatalf("output error count = %d, want %d", got, want) } if got, want := len(report.Pipelines[0].events), 2; got != want { t.Fatalf("pipeline event count = %d, want %d", got, want) } for index, bundlePath := range []string{"daily/one", "daily/two"} { action := report.Actions[index] if action.PipelineID != "reports" || action.DestinationID != "object-archive" || action.Backend != config.BackendS3 || action.BundlePath != bundlePath || action.Action != "error" { t.Fatalf("action[%d] = %#v, want %s destination open error", index, action, bundlePath) } outputError := report.OutputErrors[index] if outputError.PipelineID != action.PipelineID || outputError.DestinationID != action.DestinationID || outputError.Backend != action.Backend || outputError.BundlePath != action.BundlePath { t.Fatalf("output error[%d] = %#v, action = %#v, want aligned identity", index, outputError, action) } } } func TestRunPipelineRunsOnlyRequestedPipeline(t *testing.T) { firstSource := t.TempDir() secondSource := t.TempDir() firstDestination := t.TempDir() secondDestination := t.TempDir() writeSourceBundle(t, firstSource, "", testBundleOptions{ID: "reports.one"}) writeSourceBundle(t, secondSource, "", testBundleOptions{ID: "reports.two"}) configPath := writeTwoPipelineConfig(t, firstSource, firstDestination, secondSource, secondDestination) notifier := &recordingNotifier{} report, err := RunPipeline(context.Background(), RunPipelineOptions{ ConfigPath: configPath, PipelineID: "reports-one", Notifier: notifier, }) if err != nil { t.Fatalf("RunPipeline() error = %v", err) } if got, want := len(report.Pipelines), 1; got != want { t.Fatalf("pipeline count = %d, want %d", got, want) } if report.Pipelines[0].ID != "reports-one" { t.Fatalf("pipeline id = %q, want reports-one", report.Pipelines[0].ID) } if got, want := len(report.Actions), 1; got != want { t.Fatalf("action count = %d, want %d", got, want) } if report.Actions[0].PipelineID != "reports-one" || report.Actions[0].Action != "publish_new" { t.Fatalf("action = %#v, want reports-one publish_new", report.Actions[0]) } if got, want := len(notifier.events), 1; got != want { t.Fatalf("notification count = %d, want %d", got, want) } if notifier.events[0].PipelineID != "reports-one" { t.Fatalf("notification pipeline = %q, want reports-one", notifier.events[0].PipelineID) } testutil.AssertFile(t, filepath.Join(firstDestination, "report.md"), "# Report\nSunny.\n") if entries, err := os.ReadDir(secondDestination); err != nil || len(entries) != 0 { t.Fatalf("second destination entries = %v err=%v, want empty", entries, err) } } func TestRunPipelineUnknownIDReturnsNotFound(t *testing.T) { sourceRoot := t.TempDir() destinationRoot := t.TempDir() writeSourceBundle(t, sourceRoot, "", testBundleOptions{}) _, err := RunPipeline(context.Background(), RunPipelineOptions{ ConfigPath: writeLocalConfig(t, sourceRoot, destinationRoot), PipelineID: "missing", }) if err == nil || !IsPipelineNotFound(err) { t.Fatalf("RunPipeline() error = %v, want pipeline not found", err) } if !strings.Contains(err.Error(), `pipeline "missing" not found`) { t.Fatalf("RunPipeline() error = %v, want pipeline id in message", err) } } func TestRunStillRunsAllConfiguredPipelines(t *testing.T) { firstSource := t.TempDir() secondSource := t.TempDir() firstDestination := t.TempDir() secondDestination := t.TempDir() writeSourceBundle(t, firstSource, "", testBundleOptions{ID: "reports.one"}) writeSourceBundle(t, secondSource, "", testBundleOptions{ID: "reports.two"}) err := Run(context.Background(), RunOptions{ ConfigPath: writeTwoPipelineConfig(t, firstSource, firstDestination, secondSource, secondDestination), }) if err != nil { t.Fatalf("Run() error = %v", err) } testutil.AssertFile(t, filepath.Join(firstDestination, "report.md"), "# Report\nSunny.\n") testutil.AssertFile(t, filepath.Join(secondDestination, "report.md"), "# Report\nSunny.\n") } func TestRunDoesNotNotifyForSkippedDestination(t *testing.T) { sourceRoot := t.TempDir() destinationRoot := t.TempDir() writeSourceBundle(t, sourceRoot, "", testBundleOptions{}) configPath := writeLocalConfig(t, sourceRoot, destinationRoot) if err := Run(context.Background(), RunOptions{ConfigPath: configPath}); err != nil { t.Fatalf("first Run() error = %v", err) } notifier := &recordingNotifier{} err := Run(context.Background(), RunOptions{ConfigPath: configPath, Notifier: notifier}) if err != nil { t.Fatalf("second Run() error = %v", err) } if len(notifier.events) != 0 { t.Fatalf("notifications = %#v, want none", notifier.events) } } func TestRunDoesNotNotifyDuringDryRun(t *testing.T) { sourceRoot := t.TempDir() destinationRoot := t.TempDir() writeSourceBundle(t, sourceRoot, "", testBundleOptions{}) notifier := &recordingNotifier{} err := Run(context.Background(), RunOptions{ ConfigPath: writeLocalConfig(t, sourceRoot, destinationRoot), DryRun: true, Notifier: notifier, }) if err != nil { t.Fatalf("Run() error = %v", err) } if len(notifier.events) != 0 { t.Fatalf("notifications = %#v, want none", notifier.events) } if entries, err := os.ReadDir(destinationRoot); err != nil || len(entries) != 0 { t.Fatalf("destination entries = %v err=%v, want empty", entries, err) } } func TestRunContinuesAfterDestinationFailure(t *testing.T) { sourceRoot := t.TempDir() firstDestination := t.TempDir() secondDestination := t.TempDir() writeSourceBundle(t, sourceRoot, "", testBundleOptions{}) if err := os.WriteFile(filepath.Join(firstDestination, "unmanaged.txt"), []byte("data"), 0o600); err != nil { t.Fatalf("write unmanaged file: %v", err) } var stdout bytes.Buffer err := Run(context.Background(), RunOptions{ ConfigPath: writeFanoutConfig(t, sourceRoot, firstDestination, secondDestination), Stdout: &stdout, }) if err == nil || !strings.Contains(err.Error(), "fail_unmanaged") { t.Fatalf("Run() error = %v, want unmanaged failure", err) } if !strings.Contains(err.Error(), "pipeline reports destination archive-one backend local bundle .") { t.Fatalf("Run() error = %v, want backend context", err) } output := stdout.String() for _, want := range []string{ "destination=archive-one backend=local action=error", "destination=archive-two backend=local action=publish_new", "Final status: failed planned=1 publish_new=1 replace_older=0 replace_takeover=0 force_replace=0 skipped=0 failed=1 dry_run=false", } { if !strings.Contains(output, want) { t.Fatalf("stdout = %q, want substring %q", output, want) } } testutil.AssertFile(t, filepath.Join(secondDestination, "report.md"), "# Report\nSunny.\n") } func TestRunPublishesHTMLOnly(t *testing.T) { sourceRoot := t.TempDir() destinationRoot := t.TempDir() writeSourceBundle(t, sourceRoot, "", testBundleOptions{}) err := Run(context.Background(), RunOptions{ConfigPath: testutil.WriteLocalConfigWithPublishPolicy(t, sourceRoot, destinationRoot, false, true)}) if err != nil { t.Fatalf("Run() error = %v", err) } testutil.AssertFileContains(t, filepath.Join(destinationRoot, "report.html"), "
Sunny.
") testutil.AssertFile(t, filepath.Join(destinationRoot, "summary.txt"), "Summary\n") destinationState := readStateFile(t, filepath.Join(destinationRoot, storage.StateFileName)) if got, want := len(destinationState.Outputs), 3; got != want { t.Fatalf("state output count = %d, want %d", got, want) } } func TestRunDoesNotMutateSourceBundle(t *testing.T) { sourceRoot := t.TempDir() destinationRoot := t.TempDir() writeSourceBundle(t, sourceRoot, "", testBundleOptions{}) sourcePath := filepath.Join(sourceRoot, "report.md") before, err := os.ReadFile(sourcePath) if err != nil { t.Fatalf("read source before: %v", err) } err = Run(context.Background(), RunOptions{ConfigPath: testutil.WriteLocalConfigWithPublishPolicy(t, sourceRoot, destinationRoot, true, true)}) if err != nil { t.Fatalf("Run() error = %v", err) } after, err := os.ReadFile(sourcePath) if err != nil { t.Fatalf("read source after: %v", err) } if string(after) != string(before) { t.Fatalf("source changed from %q to %q", before, after) } } func TestRunFailsOnOutputPathCollision(t *testing.T) { sourceRoot := t.TempDir() destinationRoot := t.TempDir() writeSourceBundle(t, sourceRoot, "", testBundleOptions{ExtraFiles: []testFile{{Path: "report.html", Data: "source html
\n"}}}) err := Run(context.Background(), RunOptions{ConfigPath: testutil.WriteLocalConfigWithPublishPolicy(t, sourceRoot, destinationRoot, true, true)}) if err == nil || !strings.Contains(err.Error(), "destination output path collision") { t.Fatalf("Run() error = %v, want collision", err) } if entries, err := os.ReadDir(destinationRoot); err != nil || len(entries) != 0 { t.Fatalf("destination entries = %v err=%v, want empty", entries, err) } } func TestRunFailsOnIndexOutputPathCollision(t *testing.T) { sourceRoot := t.TempDir() destinationRoot := t.TempDir() writeSourceBundle(t, sourceRoot, "", testBundleOptions{ ExtraFiles: []testFile{{Path: "index.html", Data: "source index
\n"}}, }) err := Run(context.Background(), RunOptions{ConfigPath: testutil.WriteLocalConfigWithMarkdownTransform(t, sourceRoot, destinationRoot, true, true, config.TransformModeIndex, "report.md")}) if err == nil || !strings.Contains(err.Error(), "destination output path collision") { t.Fatalf("Run() error = %v, want collision", err) } if entries, err := os.ReadDir(destinationRoot); err != nil || len(entries) != 0 { t.Fatalf("destination entries = %v err=%v, want empty", entries, err) } } func TestRunDryRunReportsGeneratedOutputs(t *testing.T) { sourceRoot := t.TempDir() destinationRoot := t.TempDir() writeSourceBundle(t, sourceRoot, "", testBundleOptions{}) var stdout bytes.Buffer err := Run(context.Background(), RunOptions{ ConfigPath: testutil.WriteLocalConfigWithPublishPolicy(t, sourceRoot, destinationRoot, false, true), DryRun: true, Stdout: &stdout, }) if err != nil { t.Fatalf("Run() error = %v", err) } if !strings.Contains(stdout.String(), "outputs=report.html") { t.Fatalf("stdout = %q, want generated output path", stdout.String()) } } func TestRunDryRunReportsIndexOutputWithoutWriting(t *testing.T) { sourceRoot := t.TempDir() destinationRoot := t.TempDir() writeSourceBundle(t, sourceRoot, "", testBundleOptions{}) var stdout bytes.Buffer err := Run(context.Background(), RunOptions{ ConfigPath: testutil.WriteLocalConfigWithMarkdownTransform(t, sourceRoot, destinationRoot, false, true, config.TransformModeIndex, ""), DryRun: true, Stdout: &stdout, }) if err != nil { t.Fatalf("Run() error = %v", err) } if !strings.Contains(stdout.String(), "outputs=index.html") { t.Fatalf("stdout = %q, want index output path", stdout.String()) } if entries, err := os.ReadDir(destinationRoot); err != nil || len(entries) != 0 { t.Fatalf("destination entries = %v err=%v, want empty", entries, err) } } func TestRunSourceOnlyDoesNotWriteIndexOutput(t *testing.T) { sourceRoot := t.TempDir() destinationRoot := t.TempDir() writeSourceBundle(t, sourceRoot, "", testBundleOptions{}) err := Run(context.Background(), RunOptions{ConfigPath: testutil.WriteLocalConfigWithMarkdownTransform(t, sourceRoot, destinationRoot, true, false, config.TransformModeIndex, "")}) if err != nil { t.Fatalf("Run() error = %v", err) } testutil.AssertFile(t, filepath.Join(destinationRoot, "report.md"), "# Report\nSunny.\n") if _, err := os.Stat(filepath.Join(destinationRoot, "index.html")); !os.IsNotExist(err) { t.Fatalf("index.html stat error = %v, want not exist", err) } } func TestRunReplacesHTMLIndexOutput(t *testing.T) { sourceRoot := t.TempDir() destinationRoot := t.TempDir() writeSourceBundle(t, sourceRoot, "", testBundleOptions{ Created: testutil.DefaultCreated, Files: []testFile{ {Path: "report.md", Data: "# Report\nOld.\n"}, {Path: "summary.txt", Data: "Summary\n"}, }, }) configPath := testutil.WriteLocalConfigWithMarkdownTransform(t, sourceRoot, destinationRoot, false, true, config.TransformModeIndex, "") if err := Run(context.Background(), RunOptions{ConfigPath: configPath}); err != nil { t.Fatalf("first Run() error = %v", err) } testutil.AssertFileContains(t, filepath.Join(destinationRoot, "index.html"), "Old.
") writeSourceBundle(t, sourceRoot, "", testBundleOptions{ Created: testutil.DefaultCreated.Add(time.Hour), Files: []testFile{ {Path: "report.md", Data: "# Report\nNew.\n"}, {Path: "summary.txt", Data: "Summary\n"}, }, }) if err := Run(context.Background(), RunOptions{ConfigPath: configPath}); err != nil { t.Fatalf("second Run() error = %v", err) } testutil.AssertFileContains(t, filepath.Join(destinationRoot, "index.html"), "New.
") } func TestRunSkipsWhenDestinationStateMatches(t *testing.T) { sourceRoot := t.TempDir() destinationRoot := t.TempDir() writeSourceBundle(t, sourceRoot, "", testBundleOptions{}) configPath := writeLocalConfig(t, sourceRoot, destinationRoot) if err := Run(context.Background(), RunOptions{ConfigPath: configPath}); err != nil { t.Fatalf("first Run() error = %v", err) } var stdout bytes.Buffer err := Run(context.Background(), RunOptions{ConfigPath: configPath, Stdout: &stdout}) if err != nil { t.Fatalf("second Run() error = %v", err) } if !strings.Contains(stdout.String(), "action=skip_same") { t.Fatalf("stdout = %q, want skip_same", stdout.String()) } } func TestRunReplacesOlderDestination(t *testing.T) { sourceRoot := t.TempDir() destinationRoot := t.TempDir() manifest := writeSourceBundle(t, sourceRoot, "", testBundleOptions{}) older := manifest older.Created = older.Created.Add(-time.Hour) writeDestinationState(t, destinationRoot, "", older) if err := os.WriteFile(filepath.Join(destinationRoot, "report.md"), []byte("old\n"), 0o600); err != nil { t.Fatalf("write old output: %v", err) } var stdout bytes.Buffer err := Run(context.Background(), RunOptions{ConfigPath: writeLocalConfig(t, sourceRoot, destinationRoot), Stdout: &stdout}) if err != nil { t.Fatalf("Run() error = %v", err) } if !strings.Contains(stdout.String(), "action=replace_older") { t.Fatalf("stdout = %q, want replace_older", stdout.String()) } testutil.AssertFile(t, filepath.Join(destinationRoot, "report.md"), "# Report\nSunny.\n") } func TestRunSkipsNewerDestination(t *testing.T) { sourceRoot := t.TempDir() destinationRoot := t.TempDir() manifest := writeSourceBundle(t, sourceRoot, "", testBundleOptions{}) newer := manifest newer.Created = newer.Created.Add(time.Hour) writeDestinationState(t, destinationRoot, "", newer) if err := os.WriteFile(filepath.Join(destinationRoot, "report.md"), []byte("newer\n"), 0o600); err != nil { t.Fatalf("write newer output: %v", err) } var stdout bytes.Buffer err := Run(context.Background(), RunOptions{ConfigPath: writeLocalConfig(t, sourceRoot, destinationRoot), Stdout: &stdout}) if err != nil { t.Fatalf("Run() error = %v", err) } if !strings.Contains(stdout.String(), "action=skip_destination_newer") { t.Fatalf("stdout = %q, want skip_destination_newer", stdout.String()) } testutil.AssertFile(t, filepath.Join(destinationRoot, "report.md"), "newer\n") } func TestRunTakeoverNeverFailsOnConflict(t *testing.T) { sourceRoot := t.TempDir() destinationRoot := t.TempDir() manifest := writeSourceBundle(t, sourceRoot, "", testBundleOptions{}) manifest.ID = "other.source" writeDestinationState(t, destinationRoot, "", manifest) configPath := writeConfigFile(t, ` pipelines: - id: reports source: backend: local path: `+sourceRoot+` destinations: - id: archive backend: local path: `+destinationRoot+` takeover: mode: never `) err := Run(context.Background(), RunOptions{ConfigPath: configPath}) if err == nil || !strings.Contains(err.Error(), "fail_conflict") { t.Fatalf("Run() error = %v, want fail_conflict", err) } } func TestRunFailsOnUnmanagedDestination(t *testing.T) { sourceRoot := t.TempDir() destinationRoot := t.TempDir() writeSourceBundle(t, sourceRoot, "", testBundleOptions{}) if err := os.WriteFile(filepath.Join(destinationRoot, "unmanaged.txt"), []byte("data"), 0o600); err != nil { t.Fatalf("write unmanaged file: %v", err) } err := Run(context.Background(), RunOptions{ConfigPath: writeLocalConfig(t, sourceRoot, destinationRoot)}) if err == nil || !strings.Contains(err.Error(), "fail_unmanaged") { t.Fatalf("Run() error = %v, want fail_unmanaged", err) } } func TestRunForceReplacesUnmanagedDestination(t *testing.T) { sourceRoot := t.TempDir() destinationRoot := t.TempDir() writeSourceBundle(t, sourceRoot, "", testBundleOptions{}) if err := os.WriteFile(filepath.Join(destinationRoot, "unmanaged.txt"), []byte("old"), 0o600); err != nil { t.Fatalf("write unmanaged file: %v", err) } var stdout bytes.Buffer err := Run(context.Background(), RunOptions{ ConfigPath: writeLocalConfig(t, sourceRoot, destinationRoot), Force: true, Stdout: &stdout, }) if err != nil { t.Fatalf("Run() error = %v", err) } if !strings.Contains(stdout.String(), "action=force_replace") { t.Fatalf("stdout = %q, want force_replace", stdout.String()) } if _, err := os.Stat(filepath.Join(destinationRoot, "unmanaged.txt")); !os.IsNotExist(err) { t.Fatalf("unmanaged file stat error = %v, want not exist", err) } testutil.AssertFile(t, filepath.Join(destinationRoot, "report.md"), "# Report\nSunny.\n") } func TestRunFansOutToLocalDestinations(t *testing.T) { sourceRoot := t.TempDir() firstDestination := t.TempDir() secondDestination := t.TempDir() writeSourceBundle(t, sourceRoot, "daily", testBundleOptions{}) err := Run(context.Background(), RunOptions{ConfigPath: writeFanoutConfig(t, sourceRoot, firstDestination, secondDestination)}) if err != nil { t.Fatalf("Run() error = %v", err) } testutil.AssertFile(t, filepath.Join(firstDestination, "daily", "report.md"), "# Report\nSunny.\n") testutil.AssertFile(t, filepath.Join(secondDestination, "daily", "summary.txt"), "Summary\n") } func TestRunFansOutWithDifferentPublishPolicies(t *testing.T) { sourceRoot := t.TempDir() archiveDestination := t.TempDir() htmlDestination := t.TempDir() writeSourceBundle(t, sourceRoot, "", testBundleOptions{}) err := Run(context.Background(), RunOptions{ConfigPath: testutil.WriteMixedPolicyFanoutLocalConfig(t, sourceRoot, archiveDestination, htmlDestination)}) if err != nil { t.Fatalf("Run() error = %v", err) } testutil.AssertFile(t, filepath.Join(archiveDestination, "report.md"), "# Report\nSunny.\n") testutil.AssertFile(t, filepath.Join(archiveDestination, "summary.txt"), "Summary\n") if _, err := os.Stat(filepath.Join(archiveDestination, "report.html")); !os.IsNotExist(err) { t.Fatalf("archive report.html stat error = %v, want not exist", err) } testutil.AssertFileContains(t, filepath.Join(htmlDestination, "report.html"), "