Add local source publication

2026-05-31 02:21:36 +00:00
parent aeee91940d
commit e296361042
17 changed files with 1019 additions and 84 deletions
--- a/internal/app/run_test.go
+++ b/internal/app/run_test.go
@@ -3,31 +3,25 @@ package app
 import (
 	"bytes"
 	"context"
+	"encoding/json"
 	"os"
 	"path/filepath"
 	"strings"
 	"testing"
+	"time"
+
+	"gitea.maximumdirect.net/eric/distributor/internal/bundle"
+	"gitea.maximumdirect.net/eric/distributor/internal/state"
 )

 func TestRunDryRunPrintsConfigSummary(t *testing.T) {
-	configPath := filepath.Join(t.TempDir(), "config.yml")
-	err := os.WriteFile(configPath, []byte(`
-pipelines:
-  - id: reports
-    source:
-      backend: local
-      path: /source
-    destinations:
-      - id: archive
-        backend: local
-        path: /archive
-`), 0o600)
-	if err != nil {
-		t.Fatalf("write config: %v", err)
-	}
+	sourceRoot := t.TempDir()
+	destinationRoot := t.TempDir()
+	writeSourceBundle(t, sourceRoot, "", testBundleOptions{})
+	configPath := writeLocalConfig(t, sourceRoot, destinationRoot)

 	var stdout bytes.Buffer
-	err = Run(context.Background(), RunOptions{
+	err := Run(context.Background(), RunOptions{
 		ConfigPath: configPath,
 		DryRun:     true,
 		Stdout:     &stdout,
@@ -39,8 +33,8 @@ pipelines:
 	output := stdout.String()
 	for _, want := range []string{
 		"Configured pipelines: 1",
-		"- reports: source=local destinations=1",
-		"archive: backend=local publish_source=true publish_html=false",
+		"- reports: source=local bundles=1 destinations=1",
+		"bundle=. destination=archive action=publish_new",
 	} {
 		if !strings.Contains(output, want) {
 			t.Fatalf("Run() output = %q, want substring %q", output, want)
@@ -48,9 +42,302 @@ pipelines:
 	}
 }

-func TestRunWithoutDryRunIsNotImplemented(t *testing.T) {
-	err := Run(context.Background(), RunOptions{})
-	if err == nil || !strings.Contains(err.Error(), "not implemented") {
-		t.Fatalf("Run() error = %v, want not implemented", err)
+func TestRunPublishesNewLocalBundle(t *testing.T) {
+	sourceRoot := t.TempDir()
+	destinationRoot := t.TempDir()
+	manifest := writeSourceBundle(t, sourceRoot, "", testBundleOptions{})
+
+	var stdout bytes.Buffer
+	err := Run(context.Background(), RunOptions{
+		ConfigPath: writeLocalConfig(t, sourceRoot, destinationRoot),
+		Stdout:     &stdout,
+	})
+	if err != nil {
+		t.Fatalf("Run() error = %v", err)
+	}
+	assertFile(t, filepath.Join(destinationRoot, "report.md"), "# Report\nSunny.\n")
+	assertFile(t, filepath.Join(destinationRoot, "summary.txt"), "Summary\n")
+	if _, err := os.Stat(filepath.Join(destinationRoot, "manifest.json")); !os.IsNotExist(err) {
+		t.Fatalf("destination manifest stat error = %v, want not exist", err)
+	}
+	destinationState := readStateFile(t, filepath.Join(destinationRoot, ".distributor.json"))
+	if destinationState.PipelineID != "reports" || destinationState.DestinationID != "archive" {
+		t.Fatalf("state identity = %s/%s", destinationState.PipelineID, destinationState.DestinationID)
+	}
+	if destinationState.Source.Manifest.ID != manifest.ID {
+		t.Fatalf("state source id = %q, want %q", destinationState.Source.Manifest.ID, manifest.ID)
+	}
+	if got, want := len(destinationState.Outputs), 2; got != want {
+		t.Fatalf("state output count = %d, want %d", got, want)
+	}
+}
+
+func TestRunSkipsWhenDestinationStateMatches(t *testing.T) {
+	sourceRoot := t.TempDir()
+	destinationRoot := t.TempDir()
+	writeSourceBundle(t, sourceRoot, "", testBundleOptions{})
+	configPath := writeLocalConfig(t, sourceRoot, destinationRoot)
+	if err := Run(context.Background(), RunOptions{ConfigPath: configPath}); err != nil {
+		t.Fatalf("first Run() error = %v", err)
+	}
+
+	var stdout bytes.Buffer
+	err := Run(context.Background(), RunOptions{ConfigPath: configPath, Stdout: &stdout})
+	if err != nil {
+		t.Fatalf("second Run() error = %v", err)
+	}
+	if !strings.Contains(stdout.String(), "action=skip_same") {
+		t.Fatalf("stdout = %q, want skip_same", stdout.String())
+	}
+}
+
+func TestRunReplacesOlderDestination(t *testing.T) {
+	sourceRoot := t.TempDir()
+	destinationRoot := t.TempDir()
+	manifest := writeSourceBundle(t, sourceRoot, "", testBundleOptions{})
+	older := manifest
+	older.Created = older.Created.Add(-time.Hour)
+	writeDestinationState(t, destinationRoot, "", older)
+	if err := os.WriteFile(filepath.Join(destinationRoot, "report.md"), []byte("old\n"), 0o600); err != nil {
+		t.Fatalf("write old output: %v", err)
+	}
+
+	var stdout bytes.Buffer
+	err := Run(context.Background(), RunOptions{ConfigPath: writeLocalConfig(t, sourceRoot, destinationRoot), Stdout: &stdout})
+	if err != nil {
+		t.Fatalf("Run() error = %v", err)
+	}
+	if !strings.Contains(stdout.String(), "action=replace_older") {
+		t.Fatalf("stdout = %q, want replace_older", stdout.String())
+	}
+	assertFile(t, filepath.Join(destinationRoot, "report.md"), "# Report\nSunny.\n")
+}
+
+func TestRunSkipsNewerDestination(t *testing.T) {
+	sourceRoot := t.TempDir()
+	destinationRoot := t.TempDir()
+	manifest := writeSourceBundle(t, sourceRoot, "", testBundleOptions{})
+	newer := manifest
+	newer.Created = newer.Created.Add(time.Hour)
+	writeDestinationState(t, destinationRoot, "", newer)
+	if err := os.WriteFile(filepath.Join(destinationRoot, "report.md"), []byte("newer\n"), 0o600); err != nil {
+		t.Fatalf("write newer output: %v", err)
+	}
+
+	var stdout bytes.Buffer
+	err := Run(context.Background(), RunOptions{ConfigPath: writeLocalConfig(t, sourceRoot, destinationRoot), Stdout: &stdout})
+	if err != nil {
+		t.Fatalf("Run() error = %v", err)
+	}
+	if !strings.Contains(stdout.String(), "action=skip_destination_newer") {
+		t.Fatalf("stdout = %q, want skip_destination_newer", stdout.String())
+	}
+	assertFile(t, filepath.Join(destinationRoot, "report.md"), "newer\n")
+}
+
+func TestRunFailsOnConflict(t *testing.T) {
+	sourceRoot := t.TempDir()
+	destinationRoot := t.TempDir()
+	manifest := writeSourceBundle(t, sourceRoot, "", testBundleOptions{})
+	manifest.ID = "other.source"
+	writeDestinationState(t, destinationRoot, "", manifest)
+
+	err := Run(context.Background(), RunOptions{ConfigPath: writeLocalConfig(t, sourceRoot, destinationRoot)})
+	if err == nil || !strings.Contains(err.Error(), "fail_conflict") {
+		t.Fatalf("Run() error = %v, want fail_conflict", err)
+	}
+}
+
+func TestRunFailsOnUnmanagedDestination(t *testing.T) {
+	sourceRoot := t.TempDir()
+	destinationRoot := t.TempDir()
+	writeSourceBundle(t, sourceRoot, "", testBundleOptions{})
+	if err := os.WriteFile(filepath.Join(destinationRoot, "unmanaged.txt"), []byte("data"), 0o600); err != nil {
+		t.Fatalf("write unmanaged file: %v", err)
+	}
+
+	err := Run(context.Background(), RunOptions{ConfigPath: writeLocalConfig(t, sourceRoot, destinationRoot)})
+	if err == nil || !strings.Contains(err.Error(), "fail_unmanaged") {
+		t.Fatalf("Run() error = %v, want fail_unmanaged", err)
+	}
+}
+
+func TestRunFansOutToLocalDestinations(t *testing.T) {
+	sourceRoot := t.TempDir()
+	firstDestination := t.TempDir()
+	secondDestination := t.TempDir()
+	writeSourceBundle(t, sourceRoot, "daily", testBundleOptions{})
+
+	err := Run(context.Background(), RunOptions{ConfigPath: writeFanoutConfig(t, sourceRoot, firstDestination, secondDestination)})
+	if err != nil {
+		t.Fatalf("Run() error = %v", err)
+	}
+	assertFile(t, filepath.Join(firstDestination, "daily", "report.md"), "# Report\nSunny.\n")
+	assertFile(t, filepath.Join(secondDestination, "daily", "summary.txt"), "Summary\n")
+}
+
+func TestRunDryRunDoesNotWrite(t *testing.T) {
+	sourceRoot := t.TempDir()
+	destinationRoot := t.TempDir()
+	writeSourceBundle(t, sourceRoot, "", testBundleOptions{})
+
+	err := Run(context.Background(), RunOptions{
+		ConfigPath: writeLocalConfig(t, sourceRoot, destinationRoot),
+		DryRun:     true,
+	})
+	if err != nil {
+		t.Fatalf("Run() error = %v", err)
+	}
+	if entries, err := os.ReadDir(destinationRoot); err != nil || len(entries) != 0 {
+		t.Fatalf("destination entries = %v err=%v, want empty", entries, err)
+	}
+}
+
+type testBundleOptions struct {
+	ID      string
+	Created time.Time
+}
+
+func writeSourceBundle(t *testing.T, root, relative string, opts testBundleOptions) bundle.Manifest {
+	t.Helper()
+	if opts.ID == "" {
+		opts.ID = "weather.daily.brentwood.2026-05-30"
+	}
+	if opts.Created.IsZero() {
+		opts.Created = time.Date(2026, 5, 30, 11, 10, 0, 0, time.UTC)
+	}
+	bundleRoot := filepath.Join(root, filepath.FromSlash(relative))
+	if err := os.MkdirAll(bundleRoot, 0o755); err != nil {
+		t.Fatalf("mkdir bundle: %v", err)
+	}
+	files := []struct {
+		path string
+		data string
+	}{
+		{path: "report.md", data: "# Report\nSunny.\n"},
+		{path: "summary.txt", data: "Summary\n"},
+	}
+	manifestFiles := make([]bundle.ManifestFile, 0, len(files))
+	for _, file := range files {
+		if err := os.WriteFile(filepath.Join(bundleRoot, filepath.FromSlash(file.path)), []byte(file.data), 0o600); err != nil {
+			t.Fatalf("write source file: %v", err)
+		}
+		manifestFiles = append(manifestFiles, bundle.ManifestFile{
+			Path:   file.path,
+			SHA256: bundle.FileDigest([]byte(file.data)),
+			Size:   int64(len(file.data)),
+		})
+	}
+	manifest := bundle.Manifest{
+		SchemaVersion: 1,
+		ID:            opts.ID,
+		Created:       opts.Created,
+		Files:         manifestFiles,
+	}
+	manifest.Digest = bundle.BundleDigest(manifest.Files)
+	data, err := json.MarshalIndent(manifest, "", "  ")
+	if err != nil {
+		t.Fatalf("marshal manifest: %v", err)
+	}
+	data = append(data, '\n')
+	if err := os.WriteFile(filepath.Join(bundleRoot, "manifest.json"), data, 0o600); err != nil {
+		t.Fatalf("write manifest: %v", err)
+	}
+	return manifest
+}
+
+func writeLocalConfig(t *testing.T, sourceRoot, destinationRoot string) string {
+	t.Helper()
+	return writeConfigFile(t, `
+pipelines:
+  - id: reports
+    source:
+      backend: local
+      path: `+sourceRoot+`
+    destinations:
+      - id: archive
+        backend: local
+        path: `+destinationRoot+`
+        publish:
+          source: true
+          html: false
+`)
+}
+
+func writeFanoutConfig(t *testing.T, sourceRoot, firstDestination, secondDestination string) string {
+	t.Helper()
+	return writeConfigFile(t, `
+pipelines:
+  - id: reports
+    source:
+      backend: local
+      path: `+sourceRoot+`
+    destinations:
+      - id: archive-one
+        backend: local
+        path: `+firstDestination+`
+      - id: archive-two
+        backend: local
+        path: `+secondDestination+`
+`)
+}
+
+func writeConfigFile(t *testing.T, body string) string {
+	t.Helper()
+	path := filepath.Join(t.TempDir(), "config.yml")
+	if err := os.WriteFile(path, []byte(strings.TrimSpace(body)+"\n"), 0o600); err != nil {
+		t.Fatalf("write config: %v", err)
+	}
+	return path
+}
+
+func writeDestinationState(t *testing.T, root, relative string, manifest bundle.Manifest) {
+	t.Helper()
+	bundleRoot := filepath.Join(root, filepath.FromSlash(relative))
+	if err := os.MkdirAll(bundleRoot, 0o755); err != nil {
+		t.Fatalf("mkdir destination: %v", err)
+	}
+	destinationState := state.DistributorState{
+		SchemaVersion: state.SchemaVersion,
+		PipelineID:    "reports",
+		DestinationID: "archive",
+		PublishedAt:   time.Date(2026, 5, 30, 11, 12, 0, 0, time.UTC),
+		Source:        state.SourceState{Manifest: manifest},
+		Outputs: []state.OutputFile{
+			{Path: "report.md", Kind: state.OutputKindSource, SourcePath: "report.md", SHA256: manifest.Files[0].SHA256, Size: manifest.Files[0].Size},
+			{Path: "summary.txt", Kind: state.OutputKindSource, SourcePath: "summary.txt", SHA256: manifest.Files[1].SHA256, Size: manifest.Files[1].Size},
+		},
+	}
+	data, err := json.MarshalIndent(destinationState, "", "  ")
+	if err != nil {
+		t.Fatalf("marshal state: %v", err)
+	}
+	data = append(data, '\n')
+	if err := os.WriteFile(filepath.Join(bundleRoot, ".distributor.json"), data, 0o600); err != nil {
+		t.Fatalf("write state: %v", err)
+	}
+}
+
+func readStateFile(t *testing.T, path string) state.DistributorState {
+	t.Helper()
+	data, err := os.ReadFile(path)
+	if err != nil {
+		t.Fatalf("read state: %v", err)
+	}
+	destinationState, err := state.Parse(data)
+	if err != nil {
+		t.Fatalf("parse state: %v", err)
+	}
+	return destinationState
+}
+
+func assertFile(t *testing.T, path, want string) {
+	t.Helper()
+	data, err := os.ReadFile(path)
+	if err != nil {
+		t.Fatalf("read file %s: %v", path, err)
+	}
+	if got := string(data); got != want {
+		t.Fatalf("%s = %q, want %q", path, got, want)
 	}
 }