distributor/internal/publish/execute.go

package publish

import (
	"context"
	"encoding/json"
	"fmt"
	"sort"
	"time"

	"gitea.maximumdirect.net/eric/distributor/internal/config"
	"gitea.maximumdirect.net/eric/distributor/internal/state"
	"gitea.maximumdirect.net/eric/distributor/internal/storage"
)

func Execute(ctx context.Context, req Request, plan Plan) error {
	plan.Reconciliation = normalizeReconciliation(plan.Reconciliation)
	switch plan.Action {
	case ActionSkipSame, ActionSkipDestinationNewer:
		return nil
	case ActionPublishNew, ActionUpsertAdditive, ActionReplaceCatalog:
		return executeCatalog(ctx, req, plan)
	case ActionReplaceOlder, ActionReplaceConflict, ActionReplaceNewer, ActionReplaceTakeover, ActionForceReplace:
		if usesSharedRootState(req, plan) {
			return executeSharedRoot(ctx, req, plan)
		}
	default:
		return fmt.Errorf("cannot execute action %s: %s", plan.Action, plan.Reason)
	}

	if plan.Action == ActionReplaceOlder || plan.Action == ActionReplaceConflict || plan.Action == ActionReplaceNewer || plan.Action == ActionReplaceTakeover {
		if plan.ExistingState == nil {
			return fmt.Errorf("replace requires existing destination state")
		}
		if plan.Reconciliation.Mode == config.ReconciliationModeReplace || plan.Action == ActionReplaceConflict || plan.Action == ActionReplaceTakeover {
			if err := req.DestinationBackend.DeleteManagedBundle(ctx, req.DestinationBundlePath, state.ManagedOutputPaths(*plan.ExistingState), storage.DeleteOptions{IgnoreMissing: true, PruneEmptyDirs: true}); err != nil {
				return err
			}
			if err := ensureDestinationEmpty(ctx, req.DestinationBackend, req.DestinationBundlePath); err != nil {
				return err
			}
		}
	}
	if plan.Action == ActionForceReplace {
		if err := req.DestinationBackend.DeletePrefix(ctx, req.DestinationBundlePath, storage.DeleteOptions{IgnoreMissing: true, PruneEmptyDirs: true}); err != nil {
			return err
		}
		if err := ensureDestinationEmpty(ctx, req.DestinationBackend, req.DestinationBundlePath); err != nil {
			return err
		}
	}
	if usesMergeRetention(plan) {
		if err := ensureMergeOutputPaths(ctx, req.DestinationBackend, req.DestinationBundlePath, plan); err != nil {
			return err
		}
	}

	writtenOutputs := make([]Output, 0, len(plan.Outputs))
	newOutputs := make([]Output, 0, len(plan.Outputs))
	cleanup := func() {
		outputs := writtenOutputs
		if usesMergeRetention(plan) {
			outputs = newOutputs
		}
		_ = req.DestinationBackend.DeleteManagedBundle(ctx, req.DestinationBundlePath, ManagedOutputPaths(outputs), storage.DeleteOptions{IgnoreMissing: true, PruneEmptyDirs: true})
	}
	for _, output := range plan.Outputs {
		destinationPath, err := storage.Join(req.DestinationBundlePath, output.DestinationPath)
		if err != nil {
			cleanup()
			return err
		}
		data := output.Data
		if output.Kind == state.OutputKindSource {
			sourcePath, err := storage.Join(req.SourceBundle.RootRelativePath, output.SourcePath)
			if err != nil {
				cleanup()
				return err
			}
			data, err = req.SourceBackend.ReadFile(ctx, sourcePath)
			if err != nil {
				cleanup()
				return err
			}
		}
		managed := outputManagedByExistingState(output, plan.ExistingState)
		if _, err := req.DestinationBackend.WriteFile(ctx, destinationPath, data, storage.WriteOptions{Overwrite: managed && usesMergeRetention(plan), PreferAtomic: true}); err != nil {
			cleanup()
			return err
		}
		writtenOutputs = append(writtenOutputs, output)
		if !managed {
			newOutputs = append(newOutputs, output)
		}
	}

	now := time.Now().UTC()
	createdAt := now
	if plan.ExistingState != nil {
		createdAt = plan.ExistingState.CreatedAt
	}
	stateOutputs, err := stateOutputsForPlan(plan, now)
	if err != nil {
		cleanup()
		return err
	}
	destinationState := state.DistributorState{
		SchemaVersion:      state.SchemaVersion,
		DistributorVersion: req.DistributorVersion,
		PipelineID:         req.PipelineID,
		DestinationID:      req.DestinationID,
		PublishedAt:        now,
		CreatedAt:          createdAt,
		UpdatedAt:          now,
		State:              state.StatePolicy{Mode: state.StateModeSingleOwner},
		Reconciliation:     state.ReconciliationPolicy{Mode: plan.Reconciliation.Mode},
		Source:             state.SourceState{Manifest: req.SourceBundle.Manifest},
		Outputs:            stateOutputs,
	}
	if plan.PrimaryURL != "" {
		destinationState.Links = &state.LinkState{PrimaryURL: plan.PrimaryURL}
	}
	if err := state.Validate(destinationState); err != nil {
		cleanup()
		return err
	}
	data, err := json.MarshalIndent(destinationState, "", "  ")
	if err != nil {
		cleanup()
		return err
	}
	data = append(data, '\n')
	statePath, err := storage.StatePath(req.DestinationBundlePath)
	if err != nil {
		cleanup()
		return err
	}
	if _, err := req.DestinationBackend.WriteFile(ctx, statePath, data, storage.WriteOptions{Overwrite: plan.ExistingState != nil, PreferAtomic: true}); err != nil {
		cleanup()
		return err
	}
	return nil
}

func executeCatalog(ctx context.Context, req Request, plan Plan) error {
	if plan.Action == ActionReplaceCatalog {
		if plan.ClearDestinationRoot {
			if err := req.DestinationBackend.DeletePrefix(ctx, req.DestinationBundlePath, storage.DeleteOptions{IgnoreMissing: true, PruneEmptyDirs: true}); err != nil {
				return err
			}
			if err := ensureDestinationEmpty(ctx, req.DestinationBackend, req.DestinationBundlePath); err != nil {
				return err
			}
		} else if len(plan.CatalogOutputsToDelete) > 0 {
			if err := req.DestinationBackend.DeleteManagedOutputs(ctx, req.DestinationBundlePath, catalogOutputPaths(plan.CatalogOutputsToDelete), storage.DeleteOptions{IgnoreMissing: true, PruneEmptyDirs: true}); err != nil {
				return err
			}
		}
	}

	writtenOutputs := make([]Output, 0, len(plan.Outputs))
	newOutputs := make([]Output, 0, len(plan.Outputs))
	cleanup := func() {
		outputs := writtenOutputs
		if plan.Action == ActionUpsertAdditive {
			outputs = newOutputs
		}
		_ = req.DestinationBackend.DeleteManagedOutputs(ctx, req.DestinationBundlePath, ManagedOutputPaths(outputs), storage.DeleteOptions{IgnoreMissing: true, PruneEmptyDirs: true})
	}
	for _, output := range plan.Outputs {
		destinationPath, err := storage.Join(req.DestinationBundlePath, output.DestinationPath)
		if err != nil {
			cleanup()
			return err
		}
		created, err := catalogWriteCreatesOutput(ctx, req.DestinationBackend, destinationPath)
		if err != nil {
			cleanup()
			return err
		}
		data := output.Data
		if output.Kind == state.OutputKindSource {
			sourcePath, err := storage.Join(req.SourceBundle.RootRelativePath, output.SourcePath)
			if err != nil {
				cleanup()
				return err
			}
			data, err = req.SourceBackend.ReadFile(ctx, sourcePath)
			if err != nil {
				cleanup()
				return err
			}
		}
		if _, err := req.DestinationBackend.WriteFile(ctx, destinationPath, data, storage.WriteOptions{Overwrite: catalogOutputOverwriteAllowed(plan, output), PreferAtomic: true}); err != nil {
			cleanup()
			return err
		}
		writtenOutputs = append(writtenOutputs, output)
		if created {
			newOutputs = append(newOutputs, output)
		}
	}

	catalogState := catalogStateForPlan(req, plan)
	if err := state.ValidateCatalog(catalogState); err != nil {
		cleanup()
		return err
	}
	data, err := json.MarshalIndent(catalogState, "", "  ")
	if err != nil {
		cleanup()
		return err
	}
	data = append(data, '\n')
	statePath, err := storage.StatePath(req.DestinationBundlePath)
	if err != nil {
		cleanup()
		return err
	}
	if _, err := req.DestinationBackend.WriteFile(ctx, statePath, data, storage.WriteOptions{Overwrite: catalogStateWriteOverwrites(plan), PreferAtomic: true}); err != nil {
		cleanup()
		return err
	}
	return nil
}

func catalogWriteCreatesOutput(ctx context.Context, backend storage.Backend, destinationPath string) (bool, error) {
	if _, err := backend.Stat(ctx, destinationPath); err == nil {
		return false, nil
	} else if storage.IsNotFound(err) {
		return true, nil
	} else {
		return false, err
	}
}

func catalogOutputOverwriteAllowed(plan Plan, output Output) bool {
	if plan.ClearDestinationRoot {
		return false
	}
	if plan.SupersededLegacy != nil {
		return true
	}
	if plan.ExistingCatalog == nil {
		return false
	}
	_, ok := state.FindCatalogOutputByPath(plan.ExistingCatalog.Outputs, output.DestinationPath)
	return ok
}

func catalogStateForPlan(req Request, plan Plan) state.CatalogState {
	now := requestTime(req)
	createdAt := now
	if plan.ExistingCatalog != nil {
		createdAt = plan.ExistingCatalog.CreatedAt
	}
	outputs := make([]state.CatalogOutputFile, 0, len(plan.CatalogOutputsToRetain)+len(plan.CatalogOutputsToWrite))
	outputs = append(outputs, plan.CatalogOutputsToRetain...)
	outputs = append(outputs, plan.CatalogOutputsToWrite...)
	sort.SliceStable(outputs, func(i, j int) bool {
		if outputs[i].Path != outputs[j].Path {
			return outputs[i].Path < outputs[j].Path
		}
		if outputs[i].PipelineID != outputs[j].PipelineID {
			return outputs[i].PipelineID < outputs[j].PipelineID
		}
		return outputs[i].DestinationID < outputs[j].DestinationID
	})
	return state.CatalogState{
		SchemaVersion:      state.CatalogSchemaVersion,
		DistributorVersion: req.DistributorVersion,
		CreatedAt:          createdAt,
		UpdatedAt:          now,
		State:              state.StatePolicy{Mode: state.StateModeCatalog},
		Outputs:            outputs,
	}
}

func catalogStateWriteOverwrites(plan Plan) bool {
	return plan.ExistingCatalog != nil || plan.SupersededLegacy != nil
}

func catalogOutputPaths(outputs []state.CatalogOutputFile) []string {
	paths := make([]string, 0, len(outputs))
	for _, output := range outputs {
		paths = append(paths, output.Path)
	}
	return paths
}

func executeSharedRoot(ctx context.Context, req Request, plan Plan) error {
	plan.Reconciliation = normalizeReconciliation(plan.Reconciliation)
	if plan.Action == ActionForceReplace {
		if err := req.DestinationBackend.DeletePrefix(ctx, req.DestinationBundlePath, storage.DeleteOptions{IgnoreMissing: true, PruneEmptyDirs: true}); err != nil {
			return err
		}
		if err := ensureDestinationEmpty(ctx, req.DestinationBackend, req.DestinationBundlePath); err != nil {
			return err
		}
	}
	if plan.Action == ActionReplaceTakeover || plan.Action == ActionReplaceConflict || (isReconciliationReplacementAction(plan.Action) && plan.Reconciliation.Mode == config.ReconciliationModeReplace) {
		if err := req.DestinationBackend.DeleteManagedOutputs(ctx, req.DestinationBundlePath, sharedRootOutputPaths(plan.OwnerOutputsToDelete), storage.DeleteOptions{IgnoreMissing: true, PruneEmptyDirs: true}); err != nil {
			return err
		}
	}

	writtenOutputs := make([]Output, 0, len(plan.Outputs))
	newOutputs := make([]Output, 0, len(plan.Outputs))
	cleanup := func() {
		outputs := writtenOutputs
		if isReconciliationReplacementAction(plan.Action) && plan.Reconciliation.Mode == config.ReconciliationModeMerge {
			outputs = newOutputs
		}
		_ = req.DestinationBackend.DeleteManagedOutputs(ctx, req.DestinationBundlePath, ManagedOutputPaths(outputs), storage.DeleteOptions{IgnoreMissing: true, PruneEmptyDirs: true})
	}
	for _, output := range plan.Outputs {
		destinationPath, err := storage.Join(req.DestinationBundlePath, output.DestinationPath)
		if err != nil {
			cleanup()
			return err
		}
		data := output.Data
		if output.Kind == state.OutputKindSource {
			sourcePath, err := storage.Join(req.SourceBundle.RootRelativePath, output.SourcePath)
			if err != nil {
				cleanup()
				return err
			}
			data, err = req.SourceBackend.ReadFile(ctx, sourcePath)
			if err != nil {
				cleanup()
				return err
			}
		}
		managed := outputManagedBySharedRootPlan(output, plan)
		if _, err := req.DestinationBackend.WriteFile(ctx, destinationPath, data, storage.WriteOptions{Overwrite: managed, PreferAtomic: true}); err != nil {
			cleanup()
			return err
		}
		writtenOutputs = append(writtenOutputs, output)
		if !managed {
			newOutputs = append(newOutputs, output)
		}
	}

	now := time.Now().UTC()
	sharedRootState, err := sharedRootStateForPlan(req, plan, now)
	if err != nil {
		cleanup()
		return err
	}
	if err := state.ValidateSharedRoot(sharedRootState); err != nil {
		cleanup()
		return err
	}
	data, err := json.MarshalIndent(sharedRootState, "", "  ")
	if err != nil {
		cleanup()
		return err
	}
	data = append(data, '\n')
	statePath, err := storage.StatePath(req.DestinationBundlePath)
	if err != nil {
		cleanup()
		return err
	}
	if _, err := req.DestinationBackend.WriteFile(ctx, statePath, data, storage.WriteOptions{Overwrite: sharedRootStateWriteOverwrites(plan), PreferAtomic: true}); err != nil {
		cleanup()
		return err
	}
	return nil
}

func ensureMergeOutputPaths(ctx context.Context, backend storage.Backend, bundlePath string, plan Plan) error {
	for _, output := range plan.Outputs {
		if outputManagedByExistingState(output, plan.ExistingState) {
			continue
		}
		destinationPath, err := storage.Join(bundlePath, output.DestinationPath)
		if err != nil {
			return err
		}
		if _, err := backend.Stat(ctx, destinationPath); err == nil {
			return fmt.Errorf("merge output path %s exists but is not managed by destination state", storage.DisplayPath(output.DestinationPath))
		} else if !storage.IsNotFound(err) {
			return err
		}
	}
	return nil
}

func usesSharedRootState(req Request, plan Plan) bool {
	return plan.StateMode == config.StateModeSharedRoot
}

func outputManagedByExistingState(output Output, existing *state.DistributorState) bool {
	if existing == nil {
		return false
	}
	_, ok := state.FindOutputByPath(existing.Outputs, output.DestinationPath)
	return ok
}

func outputManagedBySharedRootPlan(output Output, plan Plan) bool {
	for _, existing := range currentOwnerSharedRootOutputs(plan) {
		if existing.Path == output.DestinationPath {
			return true
		}
	}
	for _, existing := range plan.TakenOverOwnerOutputs {
		if existing.Path == output.DestinationPath {
			return true
		}
	}
	return false
}

func stateOutputsForPlan(plan Plan, now time.Time) ([]state.OutputFile, error) {
	existingOutputs := []state.OutputFile(nil)
	if plan.ExistingState != nil {
		existingOutputs = plan.ExistingState.Outputs
	}
	planned := state.ProjectOutputs(StateOutputProjections(plan.Outputs), existingOutputs, now)
	if !usesMergeRetention(plan) || plan.ExistingState == nil {
		return planned, nil
	}
	return state.MergeOutputFiles(plan.ExistingState.Outputs, planned)
}

func usesMergeRetention(plan Plan) bool {
	return plan.Reconciliation.Mode == config.ReconciliationModeMerge && isReconciliationReplacementAction(plan.Action)
}

func sharedRootStateForPlan(req Request, plan Plan, now time.Time) (state.SharedRootState, error) {
	now = now.UTC()
	scope := plan.OwnerScope
	if scope.PipelineID == "" && scope.DestinationID == "" {
		scope = state.CurrentOwnerScope(req.PipelineID, req.DestinationID)
	}
	base := sharedRootBaseState(req, plan, now)
	base = removeTakenOverSharedRootOutputs(base, plan.TakenOverOwnerOutputs)
	owner := state.OwnerRecord{
		Scope:          scope,
		Reconciliation: state.ReconciliationPolicy{Mode: plan.Reconciliation.Mode},
		Source:         state.SourceState{Manifest: req.SourceBundle.Manifest},
	}
	if plan.PrimaryURL != "" {
		owner.Links = &state.LinkState{PrimaryURL: plan.PrimaryURL}
	}
	planned := state.ProjectSharedRootOutputs(StateOutputProjections(plan.Outputs), currentOwnerSharedRootOutputs(plan), scope, req.SourceBundle.Manifest, now)
	if isReconciliationReplacementAction(plan.Action) && plan.Reconciliation.Mode == config.ReconciliationModeMerge {
		return state.MergeOwnerOutputs(base, scope, owner, planned)
	}
	return state.ReplaceOwnerOutputs(base, scope, owner, planned)
}

func sharedRootBaseState(req Request, plan Plan, now time.Time) state.SharedRootState {
	if plan.Action == ActionForceReplace {
		return newSharedRootState(req, now)
	}
	if plan.ExistingSharedRoot != nil {
		base := *plan.ExistingSharedRoot
		base.Owners = append([]state.OwnerRecord(nil), plan.ExistingSharedRoot.Owners...)
		base.Outputs = append([]state.SharedRootOutputFile(nil), plan.ExistingSharedRoot.Outputs...)
		base.DistributorVersion = req.DistributorVersion
		base.UpdatedAt = now
		return base
	}
	if plan.ExistingState != nil {
		base := newSharedRootState(req, now)
		base.CreatedAt = plan.ExistingState.CreatedAt
		base.UpdatedAt = now
		return base
	}
	return newSharedRootState(req, now)
}

func removeTakenOverSharedRootOutputs(sharedRoot state.SharedRootState, takenOver []state.SharedRootOutputFile) state.SharedRootState {
	if len(takenOver) == 0 {
		return sharedRoot
	}
	paths := sharedRootOutputPathSet(takenOver)
	next := sharedRoot
	next.Outputs = make([]state.SharedRootOutputFile, 0, len(sharedRoot.Outputs))
	for _, output := range sharedRoot.Outputs {
		if _, remove := paths[output.Path]; remove {
			continue
		}
		next.Outputs = append(next.Outputs, output)
	}
	return next
}

func newSharedRootState(req Request, now time.Time) state.SharedRootState {
	return state.SharedRootState{
		SchemaVersion:      state.SharedRootSchemaVersion,
		DistributorVersion: req.DistributorVersion,
		CreatedAt:          now,
		UpdatedAt:          now,
		State:              state.StatePolicy{Mode: state.StateModeSharedRoot},
		Owners:             []state.OwnerRecord{},
		Outputs:            []state.SharedRootOutputFile{},
	}
}

func currentOwnerSharedRootOutputs(plan Plan) []state.SharedRootOutputFile {
	if plan.ExistingSharedRoot != nil {
		outputs := make([]state.SharedRootOutputFile, 0, len(plan.ExistingSharedRoot.Outputs))
		for _, output := range plan.ExistingSharedRoot.Outputs {
			if output.Owner == plan.OwnerScope {
				outputs = append(outputs, output)
			}
		}
		return outputs
	}
	if plan.ExistingState != nil {
		outputs := make([]state.SharedRootOutputFile, 0, len(plan.ExistingState.Outputs))
		for _, output := range plan.ExistingState.Outputs {
			outputs = append(outputs, state.SharedRootOutputFile{
				Path:          output.Path,
				Kind:          output.Kind,
				SourcePath:    output.SourcePath,
				Transform:     output.Transform,
				URL:           output.URL,
				SHA256:        output.SHA256,
				Size:          output.Size,
				Owner:         plan.OwnerScope,
				SourceID:      plan.ExistingState.Source.Manifest.ID,
				SourceDigest:  plan.ExistingState.Source.Manifest.Digest,
				SourceCreated: plan.ExistingState.Source.Manifest.Created,
				CreatedAt:     output.CreatedAt,
				UpdatedAt:     output.UpdatedAt,
			})
		}
		return outputs
	}
	return nil
}

func sharedRootOutputPaths(outputs []state.SharedRootOutputFile) []string {
	paths := make([]string, 0, len(outputs))
	for _, output := range outputs {
		paths = append(paths, output.Path)
	}
	return paths
}

func sharedRootStateWriteOverwrites(plan Plan) bool {
	return plan.ExistingSharedRoot != nil || plan.ExistingState != nil || plan.Action == ActionForceReplace
}