eric/distributor
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 7 Wiki Activity

Implement shared-root takeover policy

Browse Source
This commit is contained in:
Eric Rakestraw
2026-06-18 15:27:50 +00:00
parent c02106987f
commit 11d1eabe2a
7 changed files with 386 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files

144
internal/publish/plan.go
View File

@@ -66,6 +66,7 @@ type Plan struct {
ExistingState *state.DistributorState
ExistingSharedRoot *state.SharedRootState
OtherOwnerOutputs []state.SharedRootOutputFile
TakenOverOwnerOutputs []state.SharedRootOutputFile
RetainedOwnerOutputs []state.SharedRootOutputFile
OwnerOutputsToDelete []state.SharedRootOutputFile
OwnerOutputsToWrite []Output
@@ -126,18 +127,21 @@ func Build(ctx context.Context, req Request) (Plan, error) {
}
if stateMode == config.StateModeSharedRoot {
sharedDetails, err := planSharedRootOwner(ctx, req, status, action, reconciliation, outputs)
plan.Action = sharedDetails.Action
if sharedDetails.Reason != "" {
plan.Reason = sharedDetails.Reason
}
plan.OtherOwnerOutputs = sharedDetails.OtherOwnerOutputs
plan.TakenOverOwnerOutputs = sharedDetails.TakenOverOwnerOutputs
plan.RetainedOwnerOutputs = sharedDetails.RetainedOwnerOutputs
plan.OwnerOutputsToDelete = sharedDetails.OwnerOutputsToDelete
plan.OwnerOutputsToWrite = sharedDetails.OwnerOutputsToWrite
if err != nil {
plan.Action = sharedDetails.Action
plan.Reason = sharedDetails.Reason
return plan, err
}
}
if action == ActionFailConflict || action == ActionFailUnmanaged {
return plan, fmt.Errorf("%s: %s", action, reason)
if plan.Action == ActionFailConflict || plan.Action == ActionFailUnmanaged {
return plan, fmt.Errorf("%s: %s", plan.Action, plan.Reason)
}
return plan, nil
}
@@ -239,12 +243,13 @@ func sharedRootComparisonManifest(status state.DestinationStatus, scope state.Ow
}
type sharedRootPlanDetails struct {
Action Action
Reason string
OtherOwnerOutputs []state.SharedRootOutputFile
RetainedOwnerOutputs []state.SharedRootOutputFile
OwnerOutputsToDelete []state.SharedRootOutputFile
OwnerOutputsToWrite []Output
Action Action
Reason string
OtherOwnerOutputs []state.SharedRootOutputFile
TakenOverOwnerOutputs []state.SharedRootOutputFile
RetainedOwnerOutputs []state.SharedRootOutputFile
OwnerOutputsToDelete []state.SharedRootOutputFile
OwnerOutputsToWrite []Output
}
func planSharedRootOwner(ctx context.Context, req Request, status state.DestinationStatus, action Action, reconciliation config.ReconciliationPolicy, outputs []Output) (sharedRootPlanDetails, error) {
@@ -260,11 +265,20 @@ func planSharedRootOwner(ctx context.Context, req Request, status state.Destinat
details.OwnerOutputsToWrite = append([]Output(nil), outputs...)
return details, nil
}
if conflict, ok := sharedRootPathOwnershipConflict(status, scope, plannedPaths); ok {
reason := fmt.Sprintf("destination output path %s is owned by %s/%s", conflict.Path, conflict.Owner.PipelineID, conflict.Owner.DestinationID)
details.Action = ActionFailConflict
details.Reason = reason
return details, fmt.Errorf("%s: %s", ActionFailConflict, reason)
conflicts := sharedRootPathOwnershipConflicts(status, scope, plannedPaths)
if len(conflicts) > 0 {
for _, conflict := range conflicts {
if sharedRootTakeoverAllowed(req, status, conflict) {
continue
}
reason := sharedRootOwnershipConflictReason(conflict)
details.Action = ActionFailConflict
details.Reason = reason
return details, fmt.Errorf("%s: %s", ActionFailConflict, reason)
}
details.Action = ActionReplaceTakeover
details.Reason = sharedRootOwnershipConflictReason(conflicts[0])
details.TakenOverOwnerOutputs = sharedRootConflictOutputs(status.SharedRoot, conflicts)
}
if err := rejectSharedRootUnmanagedCollisions(ctx, req.DestinationBackend, req.DestinationBundlePath, status, scope, plannedPaths); err != nil {
details.Action = ActionFailUnmanaged
@@ -272,7 +286,8 @@ func planSharedRootOwner(ctx context.Context, req Request, status state.Destinat
return details, fmt.Errorf("%s: %s", ActionFailUnmanaged, err)
}
details.OtherOwnerOutputs = otherOwnerOutputs(status, scope)
takenOverPaths := sharedRootOutputPathSet(details.TakenOverOwnerOutputs)
details.OtherOwnerOutputs = otherOwnerOutputsExcept(status, scope, takenOverPaths)
ownerOutputs := currentOwnerOutputs(status, scope)
planned := make(map[string]struct{}, len(plannedPaths))
for _, path := range plannedPaths {
@@ -282,11 +297,11 @@ func planSharedRootOwner(ctx context.Context, req Request, status state.Destinat
if _, exists := planned[output.Path]; exists {
continue
}
if action == ActionReplaceOlder && reconciliation.Mode == config.ReconciliationModeReplace {
if details.Action == ActionReplaceTakeover || (details.Action == ActionReplaceOlder && reconciliation.Mode == config.ReconciliationModeReplace) {
details.OwnerOutputsToDelete = append(details.OwnerOutputsToDelete, output)
continue
}
if action == ActionReplaceOlder && reconciliation.Mode == config.ReconciliationModeMerge {
if details.Action == ActionReplaceOlder && reconciliation.Mode == config.ReconciliationModeMerge {
details.RetainedOwnerOutputs = append(details.RetainedOwnerOutputs, output)
}
}
@@ -311,11 +326,76 @@ func outputPaths(outputs []Output) []string {
return paths
}
func sharedRootPathOwnershipConflict(status state.DestinationStatus, scope state.OwnerScope, paths []string) (state.PathOwnershipConflict, bool) {
if status.SharedRoot != nil {
return status.SharedRoot.PathOwnershipConflict(scope, paths)
func sharedRootOwnershipConflictReason(conflict state.PathOwnershipConflict) string {
return fmt.Sprintf("destination output path %s is owned by %s/%s", conflict.Path, conflict.Owner.PipelineID, conflict.Owner.DestinationID)
}
func sharedRootPathOwnershipConflicts(status state.DestinationStatus, scope state.OwnerScope, paths []string) []state.PathOwnershipConflict {
if status.SharedRoot == nil {
return nil
}
conflicts := make([]state.PathOwnershipConflict, 0)
seen := make(map[string]struct{}, len(paths))
for _, path := range paths {
if _, exists := seen[path]; exists {
continue
}
seen[path] = struct{}{}
owner, exists := status.SharedRoot.OutputOwner(path)
if !exists || owner == scope {
continue
}
conflicts = append(conflicts, state.PathOwnershipConflict{
Path: path,
Owner: owner,
CurrentOwner: scope,
Detail: state.ComparisonDetail{
Kind: state.ComparisonDetailSharedRootOutputOwner,
Path: path,
CurrentOwner: scope,
ConflictingOwner: owner,
},
})
}
return conflicts
}
func sharedRootConflictOutputs(sharedRoot *state.SharedRootState, conflicts []state.PathOwnershipConflict) []state.SharedRootOutputFile {
if sharedRoot == nil || len(conflicts) == 0 {
return nil
}
paths := make(map[string]struct{}, len(conflicts))
for _, conflict := range conflicts {
paths[conflict.Path] = struct{}{}
}
outputs := make([]state.SharedRootOutputFile, 0, len(conflicts))
for _, output := range sharedRoot.Outputs {
if _, exists := paths[output.Path]; exists {
outputs = append(outputs, output)
}
}
return outputs
}
func sharedRootTakeoverAllowed(req Request, status state.DestinationStatus, conflict state.PathOwnershipConflict) bool {
if status.SharedRoot == nil {
return false
}
takeover := normalizeTakeover(req.Takeover)
switch takeover.Mode {
case config.TakeoverModeSamePipeline:
return conflict.Owner.PipelineID == req.PipelineID
case config.TakeoverModeSameSource:
owner, ok := status.SharedRoot.Owner(conflict.Owner)
return ok && owner.Source.Manifest.ID == req.SourceBundle.Manifest.ID
case config.TakeoverModeAnyManaged:
_, ok := status.SharedRoot.Owner(conflict.Owner)
return ok
case config.TakeoverModeNever:
return false
default:
return false
}
return state.PathOwnershipConflict{}, false
}
func rejectSharedRootUnmanagedCollisions(ctx context.Context, backend storage.Backend, bundlePath string, status state.DestinationStatus, scope state.OwnerScope, paths []string) error {
@@ -349,18 +429,34 @@ func pathManagedBySharedRootStatus(status state.DestinationStatus, scope state.O
}
func otherOwnerOutputs(status state.DestinationStatus, scope state.OwnerScope) []state.SharedRootOutputFile {
return otherOwnerOutputsExcept(status, scope, nil)
}
func otherOwnerOutputsExcept(status state.DestinationStatus, scope state.OwnerScope, exclude map[string]struct{}) []state.SharedRootOutputFile {
if status.SharedRoot == nil {
return nil
}
outputs := make([]state.SharedRootOutputFile, 0, len(status.SharedRoot.Outputs))
for _, output := range status.SharedRoot.Outputs {
if output.Owner != scope {
outputs = append(outputs, output)
if output.Owner == scope {
continue
}
if _, skip := exclude[output.Path]; skip {
continue
}
outputs = append(outputs, output)
}
return outputs
}
func sharedRootOutputPathSet(outputs []state.SharedRootOutputFile) map[string]struct{} {
paths := make(map[string]struct{}, len(outputs))
for _, output := range outputs {
paths[output.Path] = struct{}{}
}
return paths
}
func currentOwnerOutputs(status state.DestinationStatus, scope state.OwnerScope) []state.SharedRootOutputFile {
if status.SharedRoot != nil {
outputs := make([]state.SharedRootOutputFile, 0, len(status.SharedRoot.Outputs))