422 lines
15 KiB
Go
422 lines
15 KiB
Go
package publish
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
|
|
"gitea.maximumdirect.net/eric/distributor/internal/bundle"
|
|
"gitea.maximumdirect.net/eric/distributor/internal/config"
|
|
"gitea.maximumdirect.net/eric/distributor/internal/state"
|
|
"gitea.maximumdirect.net/eric/distributor/internal/storage"
|
|
"gitea.maximumdirect.net/eric/distributor/internal/transform"
|
|
)
|
|
|
|
type Action string
|
|
|
|
const (
|
|
ActionPublishNew Action = "publish_new"
|
|
ActionReplaceOlder Action = "replace_older"
|
|
ActionSkipSame Action = "skip_same"
|
|
ActionSkipDestinationNewer Action = "skip_destination_newer"
|
|
ActionFailConflict Action = "fail_conflict"
|
|
ActionFailUnmanaged Action = "fail_unmanaged"
|
|
ActionForceReplace Action = "force_replace"
|
|
)
|
|
|
|
type Request struct {
|
|
PipelineID string
|
|
DestinationID string
|
|
SourceBundle bundle.Bundle
|
|
SourceBackend storage.Backend
|
|
DestinationBackend storage.Backend
|
|
DestinationBundlePath string
|
|
PathMapping string
|
|
Publish config.PublishPolicy
|
|
Transform config.Transform
|
|
Links *config.Links
|
|
State config.StatePolicy
|
|
Reconciliation config.ReconciliationPolicy
|
|
Transformers TransformerResolver
|
|
Transfer config.TransferPolicy
|
|
DistributorVersion string
|
|
Force bool
|
|
}
|
|
|
|
type TransformerResolver interface {
|
|
Get(name string) (transform.Transformer, bool)
|
|
}
|
|
|
|
type Plan struct {
|
|
PipelineID string
|
|
DestinationID string
|
|
BundleID string
|
|
BundlePath string
|
|
DestinationBundlePath string
|
|
PathMapping string
|
|
Action Action
|
|
Reason string
|
|
Force bool
|
|
PrimaryURL string
|
|
StateMode string
|
|
OwnerScope state.OwnerScope
|
|
Reconciliation config.ReconciliationPolicy
|
|
Outputs []Output
|
|
ExistingState *state.DistributorState
|
|
ExistingSharedRoot *state.SharedRootState
|
|
OtherOwnerOutputs []state.SharedRootOutputFile
|
|
RetainedOwnerOutputs []state.SharedRootOutputFile
|
|
OwnerOutputsToDelete []state.SharedRootOutputFile
|
|
OwnerOutputsToWrite []Output
|
|
}
|
|
|
|
type Output struct {
|
|
SourcePath string
|
|
DestinationPath string
|
|
Kind string
|
|
Transform string
|
|
URL string
|
|
Data []byte
|
|
SHA256 string
|
|
Size int64
|
|
}
|
|
|
|
func Build(ctx context.Context, req Request) (Plan, error) {
|
|
if err := validateRequest(req); err != nil {
|
|
return Plan{}, err
|
|
}
|
|
outputs, err := PlanOutputs(ctx, req)
|
|
if err != nil {
|
|
return Plan{}, err
|
|
}
|
|
outputs, primaryURL, err := PlanLinks(req, outputs)
|
|
if err != nil {
|
|
return Plan{}, err
|
|
}
|
|
status, err := inspectDestination(ctx, req.DestinationBackend, req.DestinationBundlePath)
|
|
if err != nil {
|
|
return Plan{}, err
|
|
}
|
|
comparison := compareDestination(req, status)
|
|
action, reason := actionForComparison(comparison, req.Transfer, req.Force)
|
|
reconciliation := normalizeReconciliation(req.Reconciliation)
|
|
stateMode := normalizeState(req.State).Mode
|
|
plan := Plan{
|
|
PipelineID: req.PipelineID,
|
|
DestinationID: req.DestinationID,
|
|
BundleID: req.SourceBundle.Manifest.ID,
|
|
BundlePath: req.SourceBundle.RootRelativePath,
|
|
DestinationBundlePath: req.DestinationBundlePath,
|
|
PathMapping: req.PathMapping,
|
|
Action: action,
|
|
Reason: reason,
|
|
Force: action == ActionForceReplace,
|
|
PrimaryURL: primaryURL,
|
|
StateMode: stateMode,
|
|
OwnerScope: state.CurrentOwnerScope(req.PipelineID, req.DestinationID),
|
|
Reconciliation: reconciliation,
|
|
Outputs: outputs,
|
|
ExistingState: status.State,
|
|
ExistingSharedRoot: status.SharedRoot,
|
|
}
|
|
if stateMode == config.StateModeSharedRoot {
|
|
sharedDetails, err := planSharedRootOwner(ctx, req, status, action, reconciliation, outputs)
|
|
plan.OtherOwnerOutputs = sharedDetails.OtherOwnerOutputs
|
|
plan.RetainedOwnerOutputs = sharedDetails.RetainedOwnerOutputs
|
|
plan.OwnerOutputsToDelete = sharedDetails.OwnerOutputsToDelete
|
|
plan.OwnerOutputsToWrite = sharedDetails.OwnerOutputsToWrite
|
|
if err != nil {
|
|
plan.Action = sharedDetails.Action
|
|
plan.Reason = sharedDetails.Reason
|
|
return plan, err
|
|
}
|
|
}
|
|
if action == ActionFailConflict || action == ActionFailUnmanaged {
|
|
return plan, fmt.Errorf("%s: %s", action, reason)
|
|
}
|
|
return plan, nil
|
|
}
|
|
|
|
func validateRequest(req Request) error {
|
|
if req.PipelineID == "" {
|
|
return fmt.Errorf("pipeline id is required")
|
|
}
|
|
if req.DestinationID == "" {
|
|
return fmt.Errorf("destination id is required")
|
|
}
|
|
if req.SourceBackend == nil {
|
|
return fmt.Errorf("source backend is required")
|
|
}
|
|
if req.DestinationBackend == nil {
|
|
return fmt.Errorf("destination backend is required")
|
|
}
|
|
if err := config.ValidatePublishTransformPolicy(req.Publish, req.Transform); err != nil {
|
|
return fmt.Errorf("publish/transform policy: %w", err)
|
|
}
|
|
switch normalizeReconciliation(req.Reconciliation).Mode {
|
|
case config.ReconciliationModeReplace, config.ReconciliationModeMerge:
|
|
default:
|
|
return fmt.Errorf("reconciliation.mode must be %s or %s", config.ReconciliationModeReplace, config.ReconciliationModeMerge)
|
|
}
|
|
switch normalizeState(req.State).Mode {
|
|
case config.StateModeSingleOwner, config.StateModeSharedRoot:
|
|
default:
|
|
return fmt.Errorf("state.mode must be %s or %s", config.StateModeSingleOwner, config.StateModeSharedRoot)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func normalizeReconciliation(policy config.ReconciliationPolicy) config.ReconciliationPolicy {
|
|
if policy.Mode == "" {
|
|
policy.Mode = config.ReconciliationModeReplace
|
|
}
|
|
return policy
|
|
}
|
|
|
|
func normalizeState(policy config.StatePolicy) config.StatePolicy {
|
|
if policy.Mode == "" {
|
|
policy.Mode = config.StateModeSingleOwner
|
|
}
|
|
return policy
|
|
}
|
|
|
|
func compareDestination(req Request, status state.DestinationStatus) state.Comparison {
|
|
if normalizeState(req.State).Mode == config.StateModeSharedRoot {
|
|
scope := state.CurrentOwnerScope(req.PipelineID, req.DestinationID)
|
|
comparison := state.CompareSharedRootOwner(req.SourceBundle.Manifest, scope, status)
|
|
if req.PathMapping != config.PathMappingFixed || comparison.Outcome != state.OutcomeDifferentSourceConflict {
|
|
return comparison
|
|
}
|
|
destinationManifest, ok := sharedRootComparisonManifest(status, scope)
|
|
if !ok {
|
|
return comparison
|
|
}
|
|
if destinationManifest.Created.Before(req.SourceBundle.Manifest.Created) {
|
|
return state.Comparison{Outcome: state.OutcomeDestinationOlder, Reason: "fixed destination source is older than selected source"}
|
|
}
|
|
if destinationManifest.Created.After(req.SourceBundle.Manifest.Created) {
|
|
return state.Comparison{Outcome: state.OutcomeDestinationNewer, Reason: "fixed destination source is newer than selected source"}
|
|
}
|
|
return comparison
|
|
}
|
|
comparison := state.Compare(req.SourceBundle.Manifest, req.PipelineID, req.DestinationID, status)
|
|
if req.PathMapping != config.PathMappingFixed || comparison.Outcome != state.OutcomeDifferentSourceConflict || status.State == nil {
|
|
return comparison
|
|
}
|
|
destinationManifest := status.State.Source.Manifest
|
|
if destinationManifest.Created.Before(req.SourceBundle.Manifest.Created) {
|
|
return state.Comparison{Outcome: state.OutcomeDestinationOlder, Reason: "fixed destination source is older than selected source"}
|
|
}
|
|
if destinationManifest.Created.After(req.SourceBundle.Manifest.Created) {
|
|
return state.Comparison{Outcome: state.OutcomeDestinationNewer, Reason: "fixed destination source is newer than selected source"}
|
|
}
|
|
return comparison
|
|
}
|
|
|
|
func sharedRootComparisonManifest(status state.DestinationStatus, scope state.OwnerScope) (bundle.Manifest, bool) {
|
|
if status.SharedRoot != nil {
|
|
return status.SharedRoot.SourceManifest(scope)
|
|
}
|
|
if status.State != nil && status.State.PipelineID == scope.PipelineID && status.State.DestinationID == scope.DestinationID {
|
|
return status.State.Source.Manifest, true
|
|
}
|
|
return bundle.Manifest{}, false
|
|
}
|
|
|
|
type sharedRootPlanDetails struct {
|
|
Action Action
|
|
Reason string
|
|
OtherOwnerOutputs []state.SharedRootOutputFile
|
|
RetainedOwnerOutputs []state.SharedRootOutputFile
|
|
OwnerOutputsToDelete []state.SharedRootOutputFile
|
|
OwnerOutputsToWrite []Output
|
|
}
|
|
|
|
func planSharedRootOwner(ctx context.Context, req Request, status state.DestinationStatus, action Action, reconciliation config.ReconciliationPolicy, outputs []Output) (sharedRootPlanDetails, error) {
|
|
scope := state.CurrentOwnerScope(req.PipelineID, req.DestinationID)
|
|
details := sharedRootPlanDetails{Action: action}
|
|
if !isWriteAction(action) {
|
|
details.OtherOwnerOutputs = otherOwnerOutputs(status, scope)
|
|
return details, nil
|
|
}
|
|
|
|
plannedPaths := outputPaths(outputs)
|
|
if conflict, ok := sharedRootPathOwnershipConflict(status, scope, plannedPaths); ok {
|
|
reason := fmt.Sprintf("destination output path %s is owned by %s/%s", conflict.Path, conflict.Owner.PipelineID, conflict.Owner.DestinationID)
|
|
details.Action = ActionFailConflict
|
|
details.Reason = reason
|
|
return details, fmt.Errorf("%s: %s", ActionFailConflict, reason)
|
|
}
|
|
if err := rejectSharedRootUnmanagedCollisions(ctx, req.DestinationBackend, req.DestinationBundlePath, status, scope, plannedPaths); err != nil {
|
|
details.Action = ActionFailUnmanaged
|
|
details.Reason = err.Error()
|
|
return details, fmt.Errorf("%s: %s", ActionFailUnmanaged, err)
|
|
}
|
|
|
|
details.OtherOwnerOutputs = otherOwnerOutputs(status, scope)
|
|
ownerOutputs := currentOwnerOutputs(status, scope)
|
|
planned := make(map[string]struct{}, len(plannedPaths))
|
|
for _, path := range plannedPaths {
|
|
planned[path] = struct{}{}
|
|
}
|
|
for _, output := range ownerOutputs {
|
|
if _, exists := planned[output.Path]; exists {
|
|
continue
|
|
}
|
|
if action == ActionReplaceOlder && reconciliation.Mode == config.ReconciliationModeReplace {
|
|
details.OwnerOutputsToDelete = append(details.OwnerOutputsToDelete, output)
|
|
continue
|
|
}
|
|
if action == ActionReplaceOlder && reconciliation.Mode == config.ReconciliationModeMerge {
|
|
details.RetainedOwnerOutputs = append(details.RetainedOwnerOutputs, output)
|
|
}
|
|
}
|
|
details.OwnerOutputsToWrite = append([]Output(nil), outputs...)
|
|
return details, nil
|
|
}
|
|
|
|
func isWriteAction(action Action) bool {
|
|
switch action {
|
|
case ActionPublishNew, ActionReplaceOlder, ActionForceReplace:
|
|
return true
|
|
default:
|
|
return false
|
|
}
|
|
}
|
|
|
|
func outputPaths(outputs []Output) []string {
|
|
paths := make([]string, 0, len(outputs))
|
|
for _, output := range outputs {
|
|
paths = append(paths, output.DestinationPath)
|
|
}
|
|
return paths
|
|
}
|
|
|
|
func sharedRootPathOwnershipConflict(status state.DestinationStatus, scope state.OwnerScope, paths []string) (state.PathOwnershipConflict, bool) {
|
|
if status.SharedRoot != nil {
|
|
return status.SharedRoot.PathOwnershipConflict(scope, paths)
|
|
}
|
|
return state.PathOwnershipConflict{}, false
|
|
}
|
|
|
|
func rejectSharedRootUnmanagedCollisions(ctx context.Context, backend storage.Backend, bundlePath string, status state.DestinationStatus, scope state.OwnerScope, paths []string) error {
|
|
for _, path := range paths {
|
|
if pathManagedBySharedRootStatus(status, scope, path) {
|
|
continue
|
|
}
|
|
destinationPath, err := storage.Join(bundlePath, path)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if _, err := backend.Stat(ctx, destinationPath); err == nil {
|
|
return fmt.Errorf("destination output path %s exists but is not managed by destination state", storage.DisplayPath(path))
|
|
} else if !storage.IsNotFound(err) {
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func pathManagedBySharedRootStatus(status state.DestinationStatus, scope state.OwnerScope, path string) bool {
|
|
if status.SharedRoot != nil {
|
|
_, exists := status.SharedRoot.OutputOwner(path)
|
|
return exists
|
|
}
|
|
if status.State != nil && status.State.PipelineID == scope.PipelineID && status.State.DestinationID == scope.DestinationID {
|
|
_, exists := state.FindOutputByPath(status.State.Outputs, path)
|
|
return exists
|
|
}
|
|
return false
|
|
}
|
|
|
|
func otherOwnerOutputs(status state.DestinationStatus, scope state.OwnerScope) []state.SharedRootOutputFile {
|
|
if status.SharedRoot == nil {
|
|
return nil
|
|
}
|
|
outputs := make([]state.SharedRootOutputFile, 0, len(status.SharedRoot.Outputs))
|
|
for _, output := range status.SharedRoot.Outputs {
|
|
if output.Owner != scope {
|
|
outputs = append(outputs, output)
|
|
}
|
|
}
|
|
return outputs
|
|
}
|
|
|
|
func currentOwnerOutputs(status state.DestinationStatus, scope state.OwnerScope) []state.SharedRootOutputFile {
|
|
if status.SharedRoot != nil {
|
|
outputs := make([]state.SharedRootOutputFile, 0, len(status.SharedRoot.Outputs))
|
|
for _, output := range status.SharedRoot.Outputs {
|
|
if output.Owner == scope {
|
|
outputs = append(outputs, output)
|
|
}
|
|
}
|
|
return outputs
|
|
}
|
|
if status.State != nil && status.State.PipelineID == scope.PipelineID && status.State.DestinationID == scope.DestinationID {
|
|
outputs := make([]state.SharedRootOutputFile, 0, len(status.State.Outputs))
|
|
for _, output := range status.State.Outputs {
|
|
outputs = append(outputs, state.SharedRootOutputFile{
|
|
Path: output.Path,
|
|
Kind: output.Kind,
|
|
SourcePath: output.SourcePath,
|
|
Transform: output.Transform,
|
|
URL: output.URL,
|
|
SHA256: output.SHA256,
|
|
Size: output.Size,
|
|
Owner: scope,
|
|
SourceID: status.State.Source.Manifest.ID,
|
|
SourceDigest: status.State.Source.Manifest.Digest,
|
|
SourceCreated: status.State.Source.Manifest.Created,
|
|
CreatedAt: output.CreatedAt,
|
|
UpdatedAt: output.UpdatedAt,
|
|
})
|
|
}
|
|
return outputs
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func actionForComparison(comparison state.Comparison, transfer config.TransferPolicy, force bool) (Action, string) {
|
|
switch comparison.Outcome {
|
|
case state.OutcomeDestinationAbsent:
|
|
return ActionPublishNew, comparison.Reason
|
|
case state.OutcomeDestinationUnmanaged:
|
|
if force {
|
|
return ActionForceReplace, "forced replacement of unmanaged destination content"
|
|
}
|
|
return ActionFailUnmanaged, comparison.Reason
|
|
case state.OutcomeInvalidState:
|
|
return ActionFailConflict, comparison.Reason
|
|
case state.OutcomeIdentityMismatch, state.OutcomeSameCreatedConflict, state.OutcomeDifferentSourceConflict:
|
|
if transfer.OnConflict == config.TransferActionReplace {
|
|
if force {
|
|
return ActionForceReplace, "forced replacement of conflicting destination state: " + comparison.Reason
|
|
}
|
|
return ActionFailConflict, "destination conflict replacement requires --force"
|
|
}
|
|
return ActionFailConflict, comparison.Reason
|
|
case state.OutcomeSameSource:
|
|
if transfer.OnDestinationSame == config.TransferActionFail {
|
|
return ActionFailConflict, "destination matches source and transfer policy requires failure"
|
|
}
|
|
return ActionSkipSame, comparison.Reason
|
|
case state.OutcomeDestinationOlder:
|
|
if transfer.OnDestinationOlder == config.TransferActionFail {
|
|
return ActionFailConflict, "destination is older and transfer policy requires failure"
|
|
}
|
|
return ActionReplaceOlder, comparison.Reason
|
|
case state.OutcomeDestinationNewer:
|
|
if transfer.OnDestinationNewer == config.TransferActionReplace {
|
|
if force {
|
|
return ActionForceReplace, "forced replacement of newer destination state"
|
|
}
|
|
return ActionFailConflict, "destination is newer and replacement requires --force"
|
|
}
|
|
if transfer.OnDestinationNewer == config.TransferActionFail {
|
|
return ActionFailConflict, "destination is newer and transfer policy requires failure"
|
|
}
|
|
return ActionSkipDestinationNewer, comparison.Reason
|
|
default:
|
|
return ActionFailConflict, "unsupported comparison outcome"
|
|
}
|
|
}
|