package publish import ( "context" "fmt" "gitea.maximumdirect.net/eric/distributor/internal/bundle" "gitea.maximumdirect.net/eric/distributor/internal/config" "gitea.maximumdirect.net/eric/distributor/internal/state" "gitea.maximumdirect.net/eric/distributor/internal/storage" "gitea.maximumdirect.net/eric/distributor/internal/transform" ) type Action string const ( ActionPublishNew Action = "publish_new" ActionReplaceOlder Action = "replace_older" ActionSkipSame Action = "skip_same" ActionSkipDestinationNewer Action = "skip_destination_newer" ActionFailConflict Action = "fail_conflict" ActionFailUnmanaged Action = "fail_unmanaged" ActionForceReplace Action = "force_replace" ) type Request struct { PipelineID string DestinationID string SourceBundle bundle.Bundle SourceBackend storage.Backend DestinationBackend storage.Backend DestinationBundlePath string PathMapping string Publish config.PublishPolicy Transform config.Transform Links *config.Links State config.StatePolicy Reconciliation config.ReconciliationPolicy Transformers TransformerResolver Transfer config.TransferPolicy DistributorVersion string Force bool } type TransformerResolver interface { Get(name string) (transform.Transformer, bool) } type Plan struct { PipelineID string DestinationID string BundleID string BundlePath string DestinationBundlePath string PathMapping string Action Action Reason string Force bool PrimaryURL string StateMode string OwnerScope state.OwnerScope Reconciliation config.ReconciliationPolicy Outputs []Output ExistingState *state.DistributorState ExistingSharedRoot *state.SharedRootState OtherOwnerOutputs []state.SharedRootOutputFile RetainedOwnerOutputs []state.SharedRootOutputFile OwnerOutputsToDelete []state.SharedRootOutputFile OwnerOutputsToWrite []Output } type Output struct { SourcePath string DestinationPath string Kind string Transform string URL string Data []byte SHA256 string Size int64 } func Build(ctx context.Context, req Request) (Plan, error) { if err := validateRequest(req); err != nil { return Plan{}, err } outputs, err := PlanOutputs(ctx, req) if err != nil { return Plan{}, err } outputs, primaryURL, err := PlanLinks(req, outputs) if err != nil { return Plan{}, err } status, err := inspectDestination(ctx, req.DestinationBackend, req.DestinationBundlePath) if err != nil { return Plan{}, err } comparison := compareDestination(req, status) action, reason := actionForComparison(comparison, req.Transfer, req.Force) reconciliation := normalizeReconciliation(req.Reconciliation) stateMode := normalizeState(req.State).Mode plan := Plan{ PipelineID: req.PipelineID, DestinationID: req.DestinationID, BundleID: req.SourceBundle.Manifest.ID, BundlePath: req.SourceBundle.RootRelativePath, DestinationBundlePath: req.DestinationBundlePath, PathMapping: req.PathMapping, Action: action, Reason: reason, Force: action == ActionForceReplace, PrimaryURL: primaryURL, StateMode: stateMode, OwnerScope: state.CurrentOwnerScope(req.PipelineID, req.DestinationID), Reconciliation: reconciliation, Outputs: outputs, ExistingState: status.State, ExistingSharedRoot: status.SharedRoot, } if stateMode == config.StateModeSharedRoot { sharedDetails, err := planSharedRootOwner(ctx, req, status, action, reconciliation, outputs) plan.OtherOwnerOutputs = sharedDetails.OtherOwnerOutputs plan.RetainedOwnerOutputs = sharedDetails.RetainedOwnerOutputs plan.OwnerOutputsToDelete = sharedDetails.OwnerOutputsToDelete plan.OwnerOutputsToWrite = sharedDetails.OwnerOutputsToWrite if err != nil { plan.Action = sharedDetails.Action plan.Reason = sharedDetails.Reason return plan, err } } if action == ActionFailConflict || action == ActionFailUnmanaged { return plan, fmt.Errorf("%s: %s", action, reason) } return plan, nil } func validateRequest(req Request) error { if req.PipelineID == "" { return fmt.Errorf("pipeline id is required") } if req.DestinationID == "" { return fmt.Errorf("destination id is required") } if req.SourceBackend == nil { return fmt.Errorf("source backend is required") } if req.DestinationBackend == nil { return fmt.Errorf("destination backend is required") } if err := config.ValidatePublishTransformPolicy(req.Publish, req.Transform); err != nil { return fmt.Errorf("publish/transform policy: %w", err) } switch normalizeReconciliation(req.Reconciliation).Mode { case config.ReconciliationModeReplace, config.ReconciliationModeMerge: default: return fmt.Errorf("reconciliation.mode must be %s or %s", config.ReconciliationModeReplace, config.ReconciliationModeMerge) } switch normalizeState(req.State).Mode { case config.StateModeSingleOwner, config.StateModeSharedRoot: default: return fmt.Errorf("state.mode must be %s or %s", config.StateModeSingleOwner, config.StateModeSharedRoot) } return nil } func normalizeReconciliation(policy config.ReconciliationPolicy) config.ReconciliationPolicy { if policy.Mode == "" { policy.Mode = config.ReconciliationModeReplace } return policy } func normalizeState(policy config.StatePolicy) config.StatePolicy { if policy.Mode == "" { policy.Mode = config.StateModeSingleOwner } return policy } func compareDestination(req Request, status state.DestinationStatus) state.Comparison { if normalizeState(req.State).Mode == config.StateModeSharedRoot { scope := state.CurrentOwnerScope(req.PipelineID, req.DestinationID) comparison := state.CompareSharedRootOwner(req.SourceBundle.Manifest, scope, status) if req.PathMapping != config.PathMappingFixed || comparison.Outcome != state.OutcomeDifferentSourceConflict { return comparison } destinationManifest, ok := sharedRootComparisonManifest(status, scope) if !ok { return comparison } if destinationManifest.Created.Before(req.SourceBundle.Manifest.Created) { return state.Comparison{Outcome: state.OutcomeDestinationOlder, Reason: "fixed destination source is older than selected source"} } if destinationManifest.Created.After(req.SourceBundle.Manifest.Created) { return state.Comparison{Outcome: state.OutcomeDestinationNewer, Reason: "fixed destination source is newer than selected source"} } return comparison } comparison := state.Compare(req.SourceBundle.Manifest, req.PipelineID, req.DestinationID, status) if req.PathMapping != config.PathMappingFixed || comparison.Outcome != state.OutcomeDifferentSourceConflict || status.State == nil { return comparison } destinationManifest := status.State.Source.Manifest if destinationManifest.Created.Before(req.SourceBundle.Manifest.Created) { return state.Comparison{Outcome: state.OutcomeDestinationOlder, Reason: "fixed destination source is older than selected source"} } if destinationManifest.Created.After(req.SourceBundle.Manifest.Created) { return state.Comparison{Outcome: state.OutcomeDestinationNewer, Reason: "fixed destination source is newer than selected source"} } return comparison } func sharedRootComparisonManifest(status state.DestinationStatus, scope state.OwnerScope) (bundle.Manifest, bool) { if status.SharedRoot != nil { return status.SharedRoot.SourceManifest(scope) } if status.State != nil && status.State.PipelineID == scope.PipelineID && status.State.DestinationID == scope.DestinationID { return status.State.Source.Manifest, true } return bundle.Manifest{}, false } type sharedRootPlanDetails struct { Action Action Reason string OtherOwnerOutputs []state.SharedRootOutputFile RetainedOwnerOutputs []state.SharedRootOutputFile OwnerOutputsToDelete []state.SharedRootOutputFile OwnerOutputsToWrite []Output } func planSharedRootOwner(ctx context.Context, req Request, status state.DestinationStatus, action Action, reconciliation config.ReconciliationPolicy, outputs []Output) (sharedRootPlanDetails, error) { scope := state.CurrentOwnerScope(req.PipelineID, req.DestinationID) details := sharedRootPlanDetails{Action: action} if !isWriteAction(action) { details.OtherOwnerOutputs = otherOwnerOutputs(status, scope) return details, nil } plannedPaths := outputPaths(outputs) if conflict, ok := sharedRootPathOwnershipConflict(status, scope, plannedPaths); ok { reason := fmt.Sprintf("destination output path %s is owned by %s/%s", conflict.Path, conflict.Owner.PipelineID, conflict.Owner.DestinationID) details.Action = ActionFailConflict details.Reason = reason return details, fmt.Errorf("%s: %s", ActionFailConflict, reason) } if err := rejectSharedRootUnmanagedCollisions(ctx, req.DestinationBackend, req.DestinationBundlePath, status, scope, plannedPaths); err != nil { details.Action = ActionFailUnmanaged details.Reason = err.Error() return details, fmt.Errorf("%s: %s", ActionFailUnmanaged, err) } details.OtherOwnerOutputs = otherOwnerOutputs(status, scope) ownerOutputs := currentOwnerOutputs(status, scope) planned := make(map[string]struct{}, len(plannedPaths)) for _, path := range plannedPaths { planned[path] = struct{}{} } for _, output := range ownerOutputs { if _, exists := planned[output.Path]; exists { continue } if action == ActionReplaceOlder && reconciliation.Mode == config.ReconciliationModeReplace { details.OwnerOutputsToDelete = append(details.OwnerOutputsToDelete, output) continue } if action == ActionReplaceOlder && reconciliation.Mode == config.ReconciliationModeMerge { details.RetainedOwnerOutputs = append(details.RetainedOwnerOutputs, output) } } details.OwnerOutputsToWrite = append([]Output(nil), outputs...) return details, nil } func isWriteAction(action Action) bool { switch action { case ActionPublishNew, ActionReplaceOlder, ActionForceReplace: return true default: return false } } func outputPaths(outputs []Output) []string { paths := make([]string, 0, len(outputs)) for _, output := range outputs { paths = append(paths, output.DestinationPath) } return paths } func sharedRootPathOwnershipConflict(status state.DestinationStatus, scope state.OwnerScope, paths []string) (state.PathOwnershipConflict, bool) { if status.SharedRoot != nil { return status.SharedRoot.PathOwnershipConflict(scope, paths) } return state.PathOwnershipConflict{}, false } func rejectSharedRootUnmanagedCollisions(ctx context.Context, backend storage.Backend, bundlePath string, status state.DestinationStatus, scope state.OwnerScope, paths []string) error { for _, path := range paths { if pathManagedBySharedRootStatus(status, scope, path) { continue } destinationPath, err := storage.Join(bundlePath, path) if err != nil { return err } if _, err := backend.Stat(ctx, destinationPath); err == nil { return fmt.Errorf("destination output path %s exists but is not managed by destination state", storage.DisplayPath(path)) } else if !storage.IsNotFound(err) { return err } } return nil } func pathManagedBySharedRootStatus(status state.DestinationStatus, scope state.OwnerScope, path string) bool { if status.SharedRoot != nil { _, exists := status.SharedRoot.OutputOwner(path) return exists } if status.State != nil && status.State.PipelineID == scope.PipelineID && status.State.DestinationID == scope.DestinationID { _, exists := state.FindOutputByPath(status.State.Outputs, path) return exists } return false } func otherOwnerOutputs(status state.DestinationStatus, scope state.OwnerScope) []state.SharedRootOutputFile { if status.SharedRoot == nil { return nil } outputs := make([]state.SharedRootOutputFile, 0, len(status.SharedRoot.Outputs)) for _, output := range status.SharedRoot.Outputs { if output.Owner != scope { outputs = append(outputs, output) } } return outputs } func currentOwnerOutputs(status state.DestinationStatus, scope state.OwnerScope) []state.SharedRootOutputFile { if status.SharedRoot != nil { outputs := make([]state.SharedRootOutputFile, 0, len(status.SharedRoot.Outputs)) for _, output := range status.SharedRoot.Outputs { if output.Owner == scope { outputs = append(outputs, output) } } return outputs } if status.State != nil && status.State.PipelineID == scope.PipelineID && status.State.DestinationID == scope.DestinationID { outputs := make([]state.SharedRootOutputFile, 0, len(status.State.Outputs)) for _, output := range status.State.Outputs { outputs = append(outputs, state.SharedRootOutputFile{ Path: output.Path, Kind: output.Kind, SourcePath: output.SourcePath, Transform: output.Transform, URL: output.URL, SHA256: output.SHA256, Size: output.Size, Owner: scope, SourceID: status.State.Source.Manifest.ID, SourceDigest: status.State.Source.Manifest.Digest, SourceCreated: status.State.Source.Manifest.Created, CreatedAt: output.CreatedAt, UpdatedAt: output.UpdatedAt, }) } return outputs } return nil } func actionForComparison(comparison state.Comparison, transfer config.TransferPolicy, force bool) (Action, string) { switch comparison.Outcome { case state.OutcomeDestinationAbsent: return ActionPublishNew, comparison.Reason case state.OutcomeDestinationUnmanaged: if force { return ActionForceReplace, "forced replacement of unmanaged destination content" } return ActionFailUnmanaged, comparison.Reason case state.OutcomeInvalidState: return ActionFailConflict, comparison.Reason case state.OutcomeIdentityMismatch, state.OutcomeSameCreatedConflict, state.OutcomeDifferentSourceConflict: if transfer.OnConflict == config.TransferActionReplace { if force { return ActionForceReplace, "forced replacement of conflicting destination state: " + comparison.Reason } return ActionFailConflict, "destination conflict replacement requires --force" } return ActionFailConflict, comparison.Reason case state.OutcomeSameSource: if transfer.OnDestinationSame == config.TransferActionFail { return ActionFailConflict, "destination matches source and transfer policy requires failure" } return ActionSkipSame, comparison.Reason case state.OutcomeDestinationOlder: if transfer.OnDestinationOlder == config.TransferActionFail { return ActionFailConflict, "destination is older and transfer policy requires failure" } return ActionReplaceOlder, comparison.Reason case state.OutcomeDestinationNewer: if transfer.OnDestinationNewer == config.TransferActionReplace { if force { return ActionForceReplace, "forced replacement of newer destination state" } return ActionFailConflict, "destination is newer and replacement requires --force" } if transfer.OnDestinationNewer == config.TransferActionFail { return ActionFailConflict, "destination is newer and transfer policy requires failure" } return ActionSkipDestinationNewer, comparison.Reason default: return ActionFailConflict, "unsupported comparison outcome" } }