Write catalog state during publish

2026-06-19 15:44:23 +00:00
parent 77cde40296
commit 52078e2195
5 changed files with 401 additions and 15 deletions
--- a/internal/publish/execute.go
+++ b/internal/publish/execute.go
@@ -4,6 +4,7 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"sort"
 	"time"

 	"gitea.maximumdirect.net/eric/distributor/internal/config"
@@ -16,7 +17,9 @@ func Execute(ctx context.Context, req Request, plan Plan) error {
 	switch plan.Action {
 	case ActionSkipSame, ActionSkipDestinationNewer:
 		return nil
-	case ActionPublishNew, ActionReplaceOlder, ActionReplaceConflict, ActionReplaceNewer, ActionReplaceTakeover, ActionForceReplace:
+	case ActionPublishNew, ActionUpsertAdditive, ActionReplaceCatalog:
+		return executeCatalog(ctx, req, plan)
+	case ActionReplaceOlder, ActionReplaceConflict, ActionReplaceNewer, ActionReplaceTakeover, ActionForceReplace:
 		if usesSharedRootState(req, plan) {
 			return executeSharedRoot(ctx, req, plan)
 		}
@@ -138,6 +141,152 @@ func Execute(ctx context.Context, req Request, plan Plan) error {
 	return nil
 }

+func executeCatalog(ctx context.Context, req Request, plan Plan) error {
+	if plan.Action == ActionReplaceCatalog {
+		if plan.ClearDestinationRoot {
+			if err := req.DestinationBackend.DeletePrefix(ctx, req.DestinationBundlePath, storage.DeleteOptions{IgnoreMissing: true, PruneEmptyDirs: true}); err != nil {
+				return err
+			}
+			if err := ensureDestinationEmpty(ctx, req.DestinationBackend, req.DestinationBundlePath); err != nil {
+				return err
+			}
+		} else if len(plan.CatalogOutputsToDelete) > 0 {
+			if err := req.DestinationBackend.DeleteManagedOutputs(ctx, req.DestinationBundlePath, catalogOutputPaths(plan.CatalogOutputsToDelete), storage.DeleteOptions{IgnoreMissing: true, PruneEmptyDirs: true}); err != nil {
+				return err
+			}
+		}
+	}
+
+	writtenOutputs := make([]Output, 0, len(plan.Outputs))
+	newOutputs := make([]Output, 0, len(plan.Outputs))
+	cleanup := func() {
+		outputs := writtenOutputs
+		if plan.Action == ActionUpsertAdditive {
+			outputs = newOutputs
+		}
+		_ = req.DestinationBackend.DeleteManagedOutputs(ctx, req.DestinationBundlePath, ManagedOutputPaths(outputs), storage.DeleteOptions{IgnoreMissing: true, PruneEmptyDirs: true})
+	}
+	for _, output := range plan.Outputs {
+		destinationPath, err := storage.Join(req.DestinationBundlePath, output.DestinationPath)
+		if err != nil {
+			cleanup()
+			return err
+		}
+		created, err := catalogWriteCreatesOutput(ctx, req.DestinationBackend, destinationPath)
+		if err != nil {
+			cleanup()
+			return err
+		}
+		data := output.Data
+		if output.Kind == state.OutputKindSource {
+			sourcePath, err := storage.Join(req.SourceBundle.RootRelativePath, output.SourcePath)
+			if err != nil {
+				cleanup()
+				return err
+			}
+			data, err = req.SourceBackend.ReadFile(ctx, sourcePath)
+			if err != nil {
+				cleanup()
+				return err
+			}
+		}
+		if _, err := req.DestinationBackend.WriteFile(ctx, destinationPath, data, storage.WriteOptions{Overwrite: catalogOutputOverwriteAllowed(plan, output), PreferAtomic: true}); err != nil {
+			cleanup()
+			return err
+		}
+		writtenOutputs = append(writtenOutputs, output)
+		if created {
+			newOutputs = append(newOutputs, output)
+		}
+	}
+
+	catalogState := catalogStateForPlan(req, plan)
+	if err := state.ValidateCatalog(catalogState); err != nil {
+		cleanup()
+		return err
+	}
+	data, err := json.MarshalIndent(catalogState, "", "  ")
+	if err != nil {
+		cleanup()
+		return err
+	}
+	data = append(data, '\n')
+	statePath, err := storage.StatePath(req.DestinationBundlePath)
+	if err != nil {
+		cleanup()
+		return err
+	}
+	if _, err := req.DestinationBackend.WriteFile(ctx, statePath, data, storage.WriteOptions{Overwrite: catalogStateWriteOverwrites(plan), PreferAtomic: true}); err != nil {
+		cleanup()
+		return err
+	}
+	return nil
+}
+
+func catalogWriteCreatesOutput(ctx context.Context, backend storage.Backend, destinationPath string) (bool, error) {
+	if _, err := backend.Stat(ctx, destinationPath); err == nil {
+		return false, nil
+	} else if storage.IsNotFound(err) {
+		return true, nil
+	} else {
+		return false, err
+	}
+}
+
+func catalogOutputOverwriteAllowed(plan Plan, output Output) bool {
+	if plan.ClearDestinationRoot {
+		return false
+	}
+	if plan.SupersededLegacy != nil {
+		return true
+	}
+	if plan.ExistingCatalog == nil {
+		return false
+	}
+	_, ok := state.FindCatalogOutputByPath(plan.ExistingCatalog.Outputs, output.DestinationPath)
+	return ok
+}
+
+func catalogStateForPlan(req Request, plan Plan) state.CatalogState {
+	now := requestTime(req)
+	createdAt := now
+	if plan.ExistingCatalog != nil {
+		createdAt = plan.ExistingCatalog.CreatedAt
+	}
+	outputs := make([]state.CatalogOutputFile, 0, len(plan.CatalogOutputsToRetain)+len(plan.CatalogOutputsToWrite))
+	outputs = append(outputs, plan.CatalogOutputsToRetain...)
+	outputs = append(outputs, plan.CatalogOutputsToWrite...)
+	sort.SliceStable(outputs, func(i, j int) bool {
+		if outputs[i].Path != outputs[j].Path {
+			return outputs[i].Path < outputs[j].Path
+		}
+		if outputs[i].PipelineID != outputs[j].PipelineID {
+			return outputs[i].PipelineID < outputs[j].PipelineID
+		}
+		return outputs[i].DestinationID < outputs[j].DestinationID
+	})
+	return state.CatalogState{
+		SchemaVersion:      state.CatalogSchemaVersion,
+		DistributorVersion: req.DistributorVersion,
+		CreatedAt:          createdAt,
+		UpdatedAt:          now,
+		State:              state.StatePolicy{Mode: state.StateModeCatalog},
+		Outputs:            outputs,
+	}
+}
+
+func catalogStateWriteOverwrites(plan Plan) bool {
+	return plan.ExistingCatalog != nil || plan.SupersededLegacy != nil
+}
+
+func catalogOutputPaths(outputs []state.CatalogOutputFile) []string {
+	paths := make([]string, 0, len(outputs))
+	for _, output := range outputs {
+		paths = append(paths, output.Path)
+	}
+	return paths
+}
+
 func executeSharedRoot(ctx context.Context, req Request, plan Plan) error {
 	plan.Reconciliation = normalizeReconciliation(plan.Reconciliation)
 	if plan.Action == ActionForceReplace {
--- a/internal/publish/execute_test.go
+++ b/internal/publish/execute_test.go
@@ -0,0 +1,168 @@
+package publish
+
+import (
+	"context"
+	"testing"
+
+	"gitea.maximumdirect.net/eric/distributor/internal/config"
+	"gitea.maximumdirect.net/eric/distributor/internal/state"
+	"gitea.maximumdirect.net/eric/distributor/internal/storage"
+	"gitea.maximumdirect.net/eric/distributor/internal/storage/fake"
+	"gitea.maximumdirect.net/eric/distributor/internal/testutil"
+)
+
+func TestExecuteAdditiveWritesOutputsAndCatalog(t *testing.T) {
+	_, destinationBackend, req := catalogPlanRequest(t, config.WorkflowAdditive)
+	existing := baseCatalog(req)
+	existing.Outputs = []state.CatalogOutputFile{
+		catalogOutput(req, "reports", "archive", "report.md", state.OutputKindSource, planCreatedAt),
+		catalogOutput(req, "reports", "web", "old.txt", state.OutputKindSource, planCreatedAt),
+	}
+	writeCatalogState(t, destinationBackend, "", existing)
+	testutil.WriteFakeFile(t, destinationBackend, "report.md", "old report")
+	testutil.WriteFakeFile(t, destinationBackend, "old.txt", "retained")
+
+	plan, err := Build(context.Background(), req)
+	if err != nil {
+		t.Fatalf("Build() error = %v", err)
+	}
+	if err := Execute(context.Background(), req, plan); err != nil {
+		t.Fatalf("Execute() error = %v", err)
+	}
+
+	testutil.AssertFakeFile(t, destinationBackend, "report.md", "# Report\nSunny.\n")
+	testutil.AssertFakeFile(t, destinationBackend, "summary.txt", "Summary\n")
+	testutil.AssertFakeFile(t, destinationBackend, "old.txt", "retained")
+	catalog := readCatalogState(t, destinationBackend, "")
+	if catalog.SchemaVersion != state.CatalogSchemaVersion || catalog.State.Mode != state.StateModeCatalog {
+		t.Fatalf("catalog identity = schema %d mode %s", catalog.SchemaVersion, catalog.State.Mode)
+	}
+	if len(catalog.Outputs) != 3 {
+		t.Fatalf("catalog outputs = %#v, want three outputs", catalog.Outputs)
+	}
+	report, ok := state.FindCatalogOutputByPath(catalog.Outputs, "report.md")
+	if !ok {
+		t.Fatalf("catalog outputs = %#v, want report.md", catalog.Outputs)
+	}
+	if !report.CreatedAt.Equal(planCreatedAt) || !report.UpdatedAt.Equal(planUpdatedAt) {
+		t.Fatalf("report times = %s/%s, want created preserved and updated now", report.CreatedAt, report.UpdatedAt)
+	}
+	if report.SourcePath != "" {
+		t.Fatalf("source catalog output source_path = %q, want empty", report.SourcePath)
+	}
+}
+
+func TestExecuteReplacementDeletesCurrentOwnerAndPreservesOtherOwners(t *testing.T) {
+	_, destinationBackend, req := catalogPlanRequest(t, config.WorkflowReplacement)
+	existing := baseCatalog(req)
+	existing.Outputs = []state.CatalogOutputFile{
+		catalogOutput(req, "reports", "archive", "report.md", state.OutputKindSource, planCreatedAt),
+		catalogOutput(req, "reports", "archive", "stale.txt", state.OutputKindSource, planCreatedAt),
+		catalogOutput(req, "reports", "web", "shared.txt", state.OutputKindSource, planCreatedAt),
+	}
+	writeCatalogState(t, destinationBackend, "", existing)
+	testutil.WriteFakeFile(t, destinationBackend, "report.md", "old report")
+	testutil.WriteFakeFile(t, destinationBackend, "stale.txt", "delete me")
+	testutil.WriteFakeFile(t, destinationBackend, "shared.txt", "keep me")
+
+	plan, err := Build(context.Background(), req)
+	if err != nil {
+		t.Fatalf("Build() error = %v", err)
+	}
+	if err := Execute(context.Background(), req, plan); err != nil {
+		t.Fatalf("Execute() error = %v", err)
+	}
+
+	testutil.AssertFakeFile(t, destinationBackend, "report.md", "# Report\nSunny.\n")
+	testutil.AssertFakeFile(t, destinationBackend, "summary.txt", "Summary\n")
+	testutil.AssertFakeMissing(t, destinationBackend, "stale.txt")
+	testutil.AssertFakeFile(t, destinationBackend, "shared.txt", "keep me")
+	catalog := readCatalogState(t, destinationBackend, "")
+	if _, ok := state.FindCatalogOutputByPath(catalog.Outputs, "stale.txt"); ok {
+		t.Fatalf("catalog outputs = %#v, want stale.txt removed", catalog.Outputs)
+	}
+	if _, ok := state.FindCatalogOutputByPath(catalog.Outputs, "shared.txt"); !ok {
+		t.Fatalf("catalog outputs = %#v, want shared.txt retained", catalog.Outputs)
+	}
+}
+
+func TestExecuteSupersededReplacementClearsDestinationRootOnly(t *testing.T) {
+	_, destinationBackend, req := catalogPlanRequest(t, config.WorkflowReplacement)
+	req.DestinationBundlePath = "bundle"
+	legacyState := testutil.DestinationState(req.SourceBundle.Manifest, testutil.DestinationStateOptions{})
+	writeJSONState(t, destinationBackend, req.DestinationBundlePath, legacyState)
+	testutil.WriteFakeFile(t, destinationBackend, "bundle/unplanned.txt", "remove")
+	testutil.WriteFakeFile(t, destinationBackend, "outside.txt", "keep")
+
+	plan, err := Build(context.Background(), req)
+	if err != nil {
+		t.Fatalf("Build() error = %v", err)
+	}
+	if err := Execute(context.Background(), req, plan); err != nil {
+		t.Fatalf("Execute() error = %v", err)
+	}
+
+	testutil.AssertFakeMissing(t, destinationBackend, "bundle/unplanned.txt")
+	testutil.AssertFakeFile(t, destinationBackend, "bundle/report.md", "# Report\nSunny.\n")
+	testutil.AssertFakeFile(t, destinationBackend, "outside.txt", "keep")
+	readCatalogState(t, destinationBackend, "bundle")
+}
+
+func TestExecuteSupersededAdditiveLeavesUnplannedFilesUnmanaged(t *testing.T) {
+	_, destinationBackend, req := catalogPlanRequest(t, config.WorkflowAdditive)
+	legacyState := testutil.DestinationState(req.SourceBundle.Manifest, testutil.DestinationStateOptions{})
+	writeJSONState(t, destinationBackend, "", legacyState)
+	testutil.WriteFakeFile(t, destinationBackend, "report.md", "legacy report")
+	testutil.WriteFakeFile(t, destinationBackend, "unplanned.txt", "leave me")
+
+	plan, err := Build(context.Background(), req)
+	if err != nil {
+		t.Fatalf("Build() error = %v", err)
+	}
+	if err := Execute(context.Background(), req, plan); err != nil {
+		t.Fatalf("Execute() error = %v", err)
+	}
+
+	testutil.AssertFakeFile(t, destinationBackend, "report.md", "# Report\nSunny.\n")
+	testutil.AssertFakeFile(t, destinationBackend, "unplanned.txt", "leave me")
+	catalog := readCatalogState(t, destinationBackend, "")
+	if _, ok := state.FindCatalogOutputByPath(catalog.Outputs, "unplanned.txt"); ok {
+		t.Fatalf("catalog outputs = %#v, want unplanned file omitted", catalog.Outputs)
+	}
+}
+
+func TestExecuteFailedWriteDoesNotWriteCatalogState(t *testing.T) {
+	_, destinationBackend, req := catalogPlanRequest(t, config.WorkflowAdditive)
+	plan, err := Build(context.Background(), req)
+	if err != nil {
+		t.Fatalf("Build() error = %v", err)
+	}
+	if err := destinationBackend.AddDirectory("report.md"); err != nil {
+		t.Fatalf("add conflicting directory: %v", err)
+	}
+
+	err = Execute(context.Background(), req, plan)
+	if err == nil {
+		t.Fatal("Execute() error = nil, want write failure")
+	}
+	if _, statErr := destinationBackend.Stat(context.Background(), storage.StateFileName); !storage.IsNotFound(statErr) {
+		t.Fatalf("state stat error = %v, want missing state", statErr)
+	}
+}
+
+func readCatalogState(t *testing.T, backend *fake.Backend, relative string) state.CatalogState {
+	t.Helper()
+	statePath, err := storage.StatePath(relative)
+	if err != nil {
+		t.Fatalf("state path: %v", err)
+	}
+	data, err := backend.ReadFile(context.Background(), statePath)
+	if err != nil {
+		t.Fatalf("read catalog state: %v", err)
+	}
+	catalog, err := state.ParseCatalog(data)
+	if err != nil {
+		t.Fatalf("parse catalog state: %v", err)
+	}
+	return catalog
+}