Add local source publication

2026-05-31 02:21:36 +00:00
parent aeee91940d
commit e296361042
17 changed files with 1019 additions and 84 deletions
--- a/internal/publish/execute.go
+++ b/internal/publish/execute.go
@@ -0,0 +1,98 @@
+package publish
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"time"
+
+	"gitea.maximumdirect.net/eric/distributor/internal/state"
+	"gitea.maximumdirect.net/eric/distributor/internal/storage"
+)
+
+func Execute(ctx context.Context, req Request, plan Plan) error {
+	switch plan.Action {
+	case ActionSkipSame, ActionSkipDestinationNewer:
+		return nil
+	case ActionPublishNew, ActionReplaceOlder:
+	default:
+		return fmt.Errorf("cannot execute action %s: %s", plan.Action, plan.Reason)
+	}
+
+	if plan.Action == ActionReplaceOlder {
+		if plan.ExistingState == nil {
+			return fmt.Errorf("replace requires existing destination state")
+		}
+		if err := req.DestinationBackend.DeleteManagedBundle(ctx, req.DestinationBundlePath, existingManagedOutputPaths(*plan.ExistingState), storage.DeleteOptions{IgnoreMissing: true, PruneEmptyDirs: true}); err != nil {
+			return err
+		}
+		if err := ensureDestinationEmpty(ctx, req.DestinationBackend, req.DestinationBundlePath); err != nil {
+			return err
+		}
+	}
+
+	writtenOutputs := make([]Output, 0, len(plan.Outputs))
+	cleanup := func() {
+		_ = req.DestinationBackend.DeleteManagedBundle(ctx, req.DestinationBundlePath, managedOutputPaths(writtenOutputs), storage.DeleteOptions{IgnoreMissing: true, PruneEmptyDirs: true})
+	}
+	for _, output := range plan.Outputs {
+		sourcePath, err := storage.Join(req.SourceBundle.RootRelativePath, output.SourcePath)
+		if err != nil {
+			cleanup()
+			return err
+		}
+		destinationPath, err := storage.Join(req.DestinationBundlePath, output.DestinationPath)
+		if err != nil {
+			cleanup()
+			return err
+		}
+		data, err := req.SourceBackend.ReadFile(ctx, sourcePath)
+		if err != nil {
+			cleanup()
+			return err
+		}
+		if _, err := req.DestinationBackend.WriteFile(ctx, destinationPath, data, storage.WriteOptions{Overwrite: false, PreferAtomic: true}); err != nil {
+			cleanup()
+			return err
+		}
+		writtenOutputs = append(writtenOutputs, output)
+	}
+
+	destinationState := state.DistributorState{
+		SchemaVersion:      state.SchemaVersion,
+		DistributorVersion: req.DistributorVersion,
+		PipelineID:         req.PipelineID,
+		DestinationID:      req.DestinationID,
+		PublishedAt:        time.Now().UTC(),
+		Source:             state.SourceState{Manifest: req.SourceBundle.Manifest},
+		Outputs:            stateOutputs(plan.Outputs),
+	}
+	if err := state.Validate(destinationState); err != nil {
+		cleanup()
+		return err
+	}
+	data, err := json.MarshalIndent(destinationState, "", "  ")
+	if err != nil {
+		cleanup()
+		return err
+	}
+	data = append(data, '\n')
+	statePath, err := storage.StatePath(req.DestinationBundlePath)
+	if err != nil {
+		cleanup()
+		return err
+	}
+	if _, err := req.DestinationBackend.WriteFile(ctx, statePath, data, storage.WriteOptions{Overwrite: false, PreferAtomic: true}); err != nil {
+		cleanup()
+		return err
+	}
+	return nil
+}
+
+func existingManagedOutputPaths(destinationState state.DistributorState) []string {
+	paths := make([]string, 0, len(destinationState.Outputs))
+	for _, output := range destinationState.Outputs {
+		paths = append(paths, output.Path)
+	}
+	return paths
+}
--- a/internal/publish/execute_test.go
+++ b/internal/publish/execute_test.go
@@ -0,0 +1,91 @@
+package publish
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"testing"
+	"time"
+
+	"gitea.maximumdirect.net/eric/distributor/internal/bundle"
+	"gitea.maximumdirect.net/eric/distributor/internal/config"
+	"gitea.maximumdirect.net/eric/distributor/internal/storage"
+	"gitea.maximumdirect.net/eric/distributor/internal/storage/fake"
+)
+
+func TestExecuteCleansUpAfterWriteFailure(t *testing.T) {
+	sourceBackend := fake.New()
+	destinationBackend := &failingBackend{Backend: fake.New(), failPath: "summary.txt"}
+	sourceBundle := writeFakeSourceBundle(t, sourceBackend)
+	req := Request{
+		PipelineID:            "reports",
+		DestinationID:         "archive",
+		SourceBundle:          sourceBundle,
+		SourceBackend:         sourceBackend,
+		DestinationBackend:    destinationBackend,
+		DestinationBundlePath: "",
+		Publish:               config.PublishPolicy{Source: true},
+		Transfer:              config.TransferPolicy{OnDestinationSame: config.TransferActionSkip, OnDestinationOlder: config.TransferActionReplace, OnDestinationNewer: config.TransferActionSkip, OnConflict: config.TransferActionFail},
+		DistributorVersion:    "test",
+	}
+	plan, err := Build(context.Background(), req)
+	if err != nil {
+		t.Fatalf("Build() error = %v", err)
+	}
+	err = Execute(context.Background(), req, plan)
+	if err == nil {
+		t.Fatal("Execute() error = nil, want error")
+	}
+	found, err := destinationBackend.HasAny(context.Background(), "")
+	if err != nil {
+		t.Fatalf("HasAny() error = %v", err)
+	}
+	if found {
+		t.Fatal("destination has content after failed execution")
+	}
+}
+
+type failingBackend struct {
+	*fake.Backend
+	failPath string
+}
+
+func (b *failingBackend) WriteFile(ctx context.Context, path string, data []byte, opts storage.WriteOptions) (storage.Entry, error) {
+	if path == b.failPath {
+		return storage.Entry{}, fmt.Errorf("injected write failure")
+	}
+	return b.Backend.WriteFile(ctx, path, data, opts)
+}
+
+func (b *failingBackend) WriteFrom(ctx context.Context, path string, r io.Reader, opts storage.WriteOptions) (storage.Entry, error) {
+	if path == b.failPath {
+		return storage.Entry{}, fmt.Errorf("injected write failure")
+	}
+	return b.Backend.WriteFrom(ctx, path, r, opts)
+}
+
+func writeFakeSourceBundle(t *testing.T, backend *fake.Backend) bundle.Bundle {
+	t.Helper()
+	files := []struct {
+		path string
+		data string
+	}{
+		{path: "report.md", data: "# Report\nSunny.\n"},
+		{path: "summary.txt", data: "Summary\n"},
+	}
+	manifestFiles := make([]bundle.ManifestFile, 0, len(files))
+	for _, file := range files {
+		if _, err := backend.WriteFile(context.Background(), file.path, []byte(file.data), storage.WriteOptions{}); err != nil {
+			t.Fatalf("WriteFile() error = %v", err)
+		}
+		manifestFiles = append(manifestFiles, bundle.ManifestFile{Path: file.path, SHA256: bundle.FileDigest([]byte(file.data)), Size: int64(len(file.data))})
+	}
+	manifest := bundle.Manifest{
+		SchemaVersion: 1,
+		ID:            "weather.daily.brentwood.2026-05-30",
+		Created:       time.Date(2026, 5, 30, 11, 10, 0, 0, time.UTC),
+		Files:         manifestFiles,
+	}
+	manifest.Digest = bundle.BundleDigest(manifest.Files)
+	return bundle.Bundle{Manifest: manifest}
+}
--- a/internal/publish/output.go
+++ b/internal/publish/output.go
@@ -0,0 +1,54 @@
+package publish
+
+import (
+	"fmt"
+
+	"gitea.maximumdirect.net/eric/distributor/internal/state"
+	"gitea.maximumdirect.net/eric/distributor/internal/storage"
+)
+
+func PlanSourceOutputs(req Request) ([]Output, error) {
+	if !req.Publish.Source {
+		return nil, nil
+	}
+	outputs := make([]Output, 0, len(req.SourceBundle.Manifest.Files))
+	seen := make(map[string]struct{}, len(req.SourceBundle.Manifest.Files))
+	for _, file := range req.SourceBundle.Manifest.Files {
+		if _, exists := seen[file.Path]; exists {
+			return nil, fmt.Errorf("destination output path collision: %s", file.Path)
+		}
+		seen[file.Path] = struct{}{}
+		if err := storage.ValidatePath(file.Path); err != nil {
+			return nil, fmt.Errorf("destination output path %q: %w", file.Path, err)
+		}
+		outputs = append(outputs, Output{
+			SourcePath:      file.Path,
+			DestinationPath: file.Path,
+			SHA256:          file.SHA256,
+			Size:            file.Size,
+		})
+	}
+	return outputs, nil
+}
+
+func stateOutputs(outputs []Output) []state.OutputFile {
+	files := make([]state.OutputFile, 0, len(outputs))
+	for _, output := range outputs {
+		files = append(files, state.OutputFile{
+			Path:       output.DestinationPath,
+			Kind:       state.OutputKindSource,
+			SourcePath: output.SourcePath,
+			SHA256:     output.SHA256,
+			Size:       output.Size,
+		})
+	}
+	return files
+}
+
+func managedOutputPaths(outputs []Output) []string {
+	paths := make([]string, 0, len(outputs))
+	for _, output := range outputs {
+		paths = append(paths, output.DestinationPath)
+	}
+	return paths
+}
--- a/internal/publish/output_test.go
+++ b/internal/publish/output_test.go
@@ -0,0 +1,29 @@
+package publish
+
+import (
+	"testing"
+	"time"
+
+	"gitea.maximumdirect.net/eric/distributor/internal/bundle"
+	"gitea.maximumdirect.net/eric/distributor/internal/config"
+)
+
+func TestPlanSourceOutputsRejectsCollision(t *testing.T) {
+	_, err := PlanSourceOutputs(Request{
+		SourceBundle: bundle.Bundle{
+			Manifest: bundle.Manifest{
+				SchemaVersion: 1,
+				ID:            "bundle",
+				Created:       time.Date(2026, 5, 30, 11, 10, 0, 0, time.UTC),
+				Files: []bundle.ManifestFile{
+					{Path: "report.md", SHA256: "sha256:1111111111111111111111111111111111111111111111111111111111111111", Size: 1},
+					{Path: "report.md", SHA256: "sha256:2222222222222222222222222222222222222222222222222222222222222222", Size: 1},
+				},
+			},
+		},
+		Publish: config.PublishPolicy{Source: true},
+	})
+	if err == nil {
+		t.Fatal("PlanSourceOutputs() error = nil, want collision")
+	}
+}
--- a/internal/publish/plan.go
+++ b/internal/publish/plan.go
@@ -0,0 +1,141 @@
+package publish
+
+import (
+	"context"
+	"fmt"
+
+	"gitea.maximumdirect.net/eric/distributor/internal/bundle"
+	"gitea.maximumdirect.net/eric/distributor/internal/config"
+	"gitea.maximumdirect.net/eric/distributor/internal/state"
+	"gitea.maximumdirect.net/eric/distributor/internal/storage"
+)
+
+type Action string
+
+const (
+	ActionPublishNew           Action = "publish_new"
+	ActionReplaceOlder         Action = "replace_older"
+	ActionSkipSame             Action = "skip_same"
+	ActionSkipDestinationNewer Action = "skip_destination_newer"
+	ActionFailConflict         Action = "fail_conflict"
+	ActionFailUnmanaged        Action = "fail_unmanaged"
+)
+
+type Request struct {
+	PipelineID            string
+	DestinationID         string
+	SourceBundle          bundle.Bundle
+	SourceBackend         storage.Backend
+	DestinationBackend    storage.Backend
+	DestinationBundlePath string
+	Publish               config.PublishPolicy
+	Transfer              config.TransferPolicy
+	DistributorVersion    string
+}
+
+type Plan struct {
+	PipelineID            string
+	DestinationID         string
+	BundleID              string
+	BundlePath            string
+	DestinationBundlePath string
+	Action                Action
+	Reason                string
+	Outputs               []Output
+	ExistingState         *state.DistributorState
+}
+
+type Output struct {
+	SourcePath      string
+	DestinationPath string
+	SHA256          string
+	Size            int64
+}
+
+func Build(ctx context.Context, req Request) (Plan, error) {
+	if err := validateRequest(req); err != nil {
+		return Plan{}, err
+	}
+	outputs, err := PlanSourceOutputs(req)
+	if err != nil {
+		return Plan{}, err
+	}
+	status, err := inspectDestination(ctx, req.DestinationBackend, req.DestinationBundlePath)
+	if err != nil {
+		return Plan{}, err
+	}
+	comparison := state.Compare(req.SourceBundle.Manifest, req.PipelineID, req.DestinationID, status)
+	action, reason := actionForComparison(comparison, req.Transfer)
+	plan := Plan{
+		PipelineID:            req.PipelineID,
+		DestinationID:         req.DestinationID,
+		BundleID:              req.SourceBundle.Manifest.ID,
+		BundlePath:            req.SourceBundle.RootRelativePath,
+		DestinationBundlePath: req.DestinationBundlePath,
+		Action:                action,
+		Reason:                reason,
+		Outputs:               outputs,
+		ExistingState:         status.State,
+	}
+	if action == ActionFailConflict || action == ActionFailUnmanaged {
+		return plan, fmt.Errorf("%s: %s", action, reason)
+	}
+	return plan, nil
+}
+
+func validateRequest(req Request) error {
+	if req.PipelineID == "" {
+		return fmt.Errorf("pipeline id is required")
+	}
+	if req.DestinationID == "" {
+		return fmt.Errorf("destination id is required")
+	}
+	if req.SourceBackend == nil {
+		return fmt.Errorf("source backend is required")
+	}
+	if req.DestinationBackend == nil {
+		return fmt.Errorf("destination backend is required")
+	}
+	if req.Publish.HTML {
+		return fmt.Errorf("publish html is not implemented")
+	}
+	if !req.Publish.Source {
+		return fmt.Errorf("publish source must be enabled")
+	}
+	return nil
+}
+
+func actionForComparison(comparison state.Comparison, transfer config.TransferPolicy) (Action, string) {
+	switch comparison.Outcome {
+	case state.OutcomeDestinationAbsent:
+		return ActionPublishNew, comparison.Reason
+	case state.OutcomeDestinationUnmanaged:
+		return ActionFailUnmanaged, comparison.Reason
+	case state.OutcomeInvalidState, state.OutcomeIdentityMismatch, state.OutcomeSameCreatedConflict, state.OutcomeDifferentSourceConflict:
+		return ActionFailConflict, comparison.Reason
+	case state.OutcomeSameSource:
+		if transfer.OnDestinationSame == config.TransferActionFail {
+			return ActionFailConflict, "destination matches source and transfer policy requires failure"
+		}
+		return ActionSkipSame, comparison.Reason
+	case state.OutcomeDestinationOlder:
+		if transfer.OnDestinationOlder == config.TransferActionFail {
+			return ActionFailConflict, "destination is older and transfer policy requires failure"
+		}
+		return ActionReplaceOlder, comparison.Reason
+	case state.OutcomeDestinationNewer:
+		if transfer.OnDestinationNewer == config.TransferActionFail {
+			return ActionFailConflict, "destination is newer and transfer policy requires failure"
+		}
+		return ActionSkipDestinationNewer, comparison.Reason
+	default:
+		return ActionFailConflict, "unsupported comparison outcome"
+	}
+}
+
+func displayPath(path string) string {
+	if path == "" {
+		return "."
+	}
+	return path
+}
--- a/internal/publish/reconcile.go
+++ b/internal/publish/reconcile.go
@@ -0,0 +1,31 @@
+package publish
+
+import (
+	"context"
+
+	"gitea.maximumdirect.net/eric/distributor/internal/state"
+	"gitea.maximumdirect.net/eric/distributor/internal/storage"
+)
+
+func inspectDestination(ctx context.Context, backend storage.Backend, bundlePath string) (state.DestinationStatus, error) {
+	statePath, err := storage.StatePath(bundlePath)
+	if err != nil {
+		return state.DestinationStatus{}, err
+	}
+	data, err := backend.ReadFile(ctx, statePath)
+	if err == nil {
+		destinationState, parseErr := state.Parse(data)
+		if parseErr != nil {
+			return state.DestinationStatus{StateErr: parseErr}, nil
+		}
+		return state.DestinationStatus{State: &destinationState, HasContents: true}, nil
+	}
+	if !storage.IsNotFound(err) {
+		return state.DestinationStatus{}, err
+	}
+	hasContents, err := backend.HasAny(ctx, bundlePath)
+	if err != nil {
+		return state.DestinationStatus{}, err
+	}
+	return state.DestinationStatus{HasContents: hasContents}, nil
+}
--- a/internal/publish/safety.go
+++ b/internal/publish/safety.go
@@ -0,0 +1,19 @@
+package publish
+
+import (
+	"context"
+	"fmt"
+
+	"gitea.maximumdirect.net/eric/distributor/internal/storage"
+)
+
+func ensureDestinationEmpty(ctx context.Context, backend storage.Backend, bundlePath string) error {
+	hasAny, err := backend.HasAny(ctx, bundlePath)
+	if err != nil {
+		return err
+	}
+	if hasAny {
+		return fmt.Errorf("destination bundle path %q is not empty after managed cleanup", displayPath(bundlePath))
+	}
+	return nil
+}