379 lines
13 KiB
Go
379 lines
13 KiB
Go
package publish
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"time"
|
|
|
|
"gitea.maximumdirect.net/eric/distributor/internal/config"
|
|
"gitea.maximumdirect.net/eric/distributor/internal/state"
|
|
"gitea.maximumdirect.net/eric/distributor/internal/storage"
|
|
)
|
|
|
|
func Execute(ctx context.Context, req Request, plan Plan) error {
|
|
plan.Reconciliation = normalizeReconciliation(plan.Reconciliation)
|
|
switch plan.Action {
|
|
case ActionSkipSame, ActionSkipDestinationNewer:
|
|
return nil
|
|
case ActionPublishNew, ActionReplaceOlder, ActionReplaceTakeover, ActionForceReplace:
|
|
if usesSharedRootState(req, plan) {
|
|
return executeSharedRoot(ctx, req, plan)
|
|
}
|
|
default:
|
|
return fmt.Errorf("cannot execute action %s: %s", plan.Action, plan.Reason)
|
|
}
|
|
|
|
if plan.Action == ActionReplaceOlder || plan.Action == ActionReplaceTakeover {
|
|
if plan.ExistingState == nil {
|
|
return fmt.Errorf("replace requires existing destination state")
|
|
}
|
|
if plan.Reconciliation.Mode == config.ReconciliationModeReplace || plan.Action == ActionReplaceTakeover {
|
|
if err := req.DestinationBackend.DeleteManagedBundle(ctx, req.DestinationBundlePath, state.ManagedOutputPaths(*plan.ExistingState), storage.DeleteOptions{IgnoreMissing: true, PruneEmptyDirs: true}); err != nil {
|
|
return err
|
|
}
|
|
if err := ensureDestinationEmpty(ctx, req.DestinationBackend, req.DestinationBundlePath); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
}
|
|
if plan.Action == ActionForceReplace {
|
|
if err := req.DestinationBackend.DeletePrefix(ctx, req.DestinationBundlePath, storage.DeleteOptions{IgnoreMissing: true, PruneEmptyDirs: true}); err != nil {
|
|
return err
|
|
}
|
|
if err := ensureDestinationEmpty(ctx, req.DestinationBackend, req.DestinationBundlePath); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
if usesMergeRetention(plan) {
|
|
if err := ensureMergeOutputPaths(ctx, req.DestinationBackend, req.DestinationBundlePath, plan); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
writtenOutputs := make([]Output, 0, len(plan.Outputs))
|
|
newOutputs := make([]Output, 0, len(plan.Outputs))
|
|
cleanup := func() {
|
|
outputs := writtenOutputs
|
|
if usesMergeRetention(plan) {
|
|
outputs = newOutputs
|
|
}
|
|
_ = req.DestinationBackend.DeleteManagedBundle(ctx, req.DestinationBundlePath, ManagedOutputPaths(outputs), storage.DeleteOptions{IgnoreMissing: true, PruneEmptyDirs: true})
|
|
}
|
|
for _, output := range plan.Outputs {
|
|
destinationPath, err := storage.Join(req.DestinationBundlePath, output.DestinationPath)
|
|
if err != nil {
|
|
cleanup()
|
|
return err
|
|
}
|
|
data := output.Data
|
|
if output.Kind == state.OutputKindSource {
|
|
sourcePath, err := storage.Join(req.SourceBundle.RootRelativePath, output.SourcePath)
|
|
if err != nil {
|
|
cleanup()
|
|
return err
|
|
}
|
|
data, err = req.SourceBackend.ReadFile(ctx, sourcePath)
|
|
if err != nil {
|
|
cleanup()
|
|
return err
|
|
}
|
|
}
|
|
managed := outputManagedByExistingState(output, plan.ExistingState)
|
|
if _, err := req.DestinationBackend.WriteFile(ctx, destinationPath, data, storage.WriteOptions{Overwrite: managed && usesMergeRetention(plan), PreferAtomic: true}); err != nil {
|
|
cleanup()
|
|
return err
|
|
}
|
|
writtenOutputs = append(writtenOutputs, output)
|
|
if !managed {
|
|
newOutputs = append(newOutputs, output)
|
|
}
|
|
}
|
|
|
|
now := time.Now().UTC()
|
|
createdAt := now
|
|
if plan.ExistingState != nil {
|
|
createdAt = plan.ExistingState.CreatedAt
|
|
}
|
|
stateOutputs, err := stateOutputsForPlan(plan, now)
|
|
if err != nil {
|
|
cleanup()
|
|
return err
|
|
}
|
|
destinationState := state.DistributorState{
|
|
SchemaVersion: state.SchemaVersion,
|
|
DistributorVersion: req.DistributorVersion,
|
|
PipelineID: req.PipelineID,
|
|
DestinationID: req.DestinationID,
|
|
PublishedAt: now,
|
|
CreatedAt: createdAt,
|
|
UpdatedAt: now,
|
|
State: state.StatePolicy{Mode: state.StateModeSingleOwner},
|
|
Reconciliation: state.ReconciliationPolicy{Mode: plan.Reconciliation.Mode},
|
|
Source: state.SourceState{Manifest: req.SourceBundle.Manifest},
|
|
Outputs: stateOutputs,
|
|
}
|
|
if plan.PrimaryURL != "" {
|
|
destinationState.Links = &state.LinkState{PrimaryURL: plan.PrimaryURL}
|
|
}
|
|
if err := state.Validate(destinationState); err != nil {
|
|
cleanup()
|
|
return err
|
|
}
|
|
data, err := json.MarshalIndent(destinationState, "", " ")
|
|
if err != nil {
|
|
cleanup()
|
|
return err
|
|
}
|
|
data = append(data, '\n')
|
|
statePath, err := storage.StatePath(req.DestinationBundlePath)
|
|
if err != nil {
|
|
cleanup()
|
|
return err
|
|
}
|
|
if _, err := req.DestinationBackend.WriteFile(ctx, statePath, data, storage.WriteOptions{Overwrite: plan.ExistingState != nil, PreferAtomic: true}); err != nil {
|
|
cleanup()
|
|
return err
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func executeSharedRoot(ctx context.Context, req Request, plan Plan) error {
|
|
plan.Reconciliation = normalizeReconciliation(plan.Reconciliation)
|
|
if plan.Action == ActionForceReplace {
|
|
if err := req.DestinationBackend.DeletePrefix(ctx, req.DestinationBundlePath, storage.DeleteOptions{IgnoreMissing: true, PruneEmptyDirs: true}); err != nil {
|
|
return err
|
|
}
|
|
if err := ensureDestinationEmpty(ctx, req.DestinationBackend, req.DestinationBundlePath); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
if plan.Action == ActionReplaceOlder && plan.Reconciliation.Mode == config.ReconciliationModeReplace {
|
|
if err := req.DestinationBackend.DeleteManagedOutputs(ctx, req.DestinationBundlePath, sharedRootOutputPaths(plan.OwnerOutputsToDelete), storage.DeleteOptions{IgnoreMissing: true, PruneEmptyDirs: true}); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
writtenOutputs := make([]Output, 0, len(plan.Outputs))
|
|
newOutputs := make([]Output, 0, len(plan.Outputs))
|
|
cleanup := func() {
|
|
outputs := writtenOutputs
|
|
if plan.Reconciliation.Mode == config.ReconciliationModeMerge {
|
|
outputs = newOutputs
|
|
}
|
|
_ = req.DestinationBackend.DeleteManagedOutputs(ctx, req.DestinationBundlePath, ManagedOutputPaths(outputs), storage.DeleteOptions{IgnoreMissing: true, PruneEmptyDirs: true})
|
|
}
|
|
for _, output := range plan.Outputs {
|
|
destinationPath, err := storage.Join(req.DestinationBundlePath, output.DestinationPath)
|
|
if err != nil {
|
|
cleanup()
|
|
return err
|
|
}
|
|
data := output.Data
|
|
if output.Kind == state.OutputKindSource {
|
|
sourcePath, err := storage.Join(req.SourceBundle.RootRelativePath, output.SourcePath)
|
|
if err != nil {
|
|
cleanup()
|
|
return err
|
|
}
|
|
data, err = req.SourceBackend.ReadFile(ctx, sourcePath)
|
|
if err != nil {
|
|
cleanup()
|
|
return err
|
|
}
|
|
}
|
|
managed := outputManagedBySharedRootPlan(output, plan)
|
|
if _, err := req.DestinationBackend.WriteFile(ctx, destinationPath, data, storage.WriteOptions{Overwrite: managed, PreferAtomic: true}); err != nil {
|
|
cleanup()
|
|
return err
|
|
}
|
|
writtenOutputs = append(writtenOutputs, output)
|
|
if !managed {
|
|
newOutputs = append(newOutputs, output)
|
|
}
|
|
}
|
|
|
|
now := time.Now().UTC()
|
|
sharedRootState, err := sharedRootStateForPlan(req, plan, now)
|
|
if err != nil {
|
|
cleanup()
|
|
return err
|
|
}
|
|
if err := state.ValidateSharedRoot(sharedRootState); err != nil {
|
|
cleanup()
|
|
return err
|
|
}
|
|
data, err := json.MarshalIndent(sharedRootState, "", " ")
|
|
if err != nil {
|
|
cleanup()
|
|
return err
|
|
}
|
|
data = append(data, '\n')
|
|
statePath, err := storage.StatePath(req.DestinationBundlePath)
|
|
if err != nil {
|
|
cleanup()
|
|
return err
|
|
}
|
|
if _, err := req.DestinationBackend.WriteFile(ctx, statePath, data, storage.WriteOptions{Overwrite: sharedRootStateWriteOverwrites(plan), PreferAtomic: true}); err != nil {
|
|
cleanup()
|
|
return err
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func ensureMergeOutputPaths(ctx context.Context, backend storage.Backend, bundlePath string, plan Plan) error {
|
|
for _, output := range plan.Outputs {
|
|
if outputManagedByExistingState(output, plan.ExistingState) {
|
|
continue
|
|
}
|
|
destinationPath, err := storage.Join(bundlePath, output.DestinationPath)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if _, err := backend.Stat(ctx, destinationPath); err == nil {
|
|
return fmt.Errorf("merge output path %s exists but is not managed by destination state", storage.DisplayPath(output.DestinationPath))
|
|
} else if !storage.IsNotFound(err) {
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func usesSharedRootState(req Request, plan Plan) bool {
|
|
return normalizeState(req.State).Mode == config.StateModeSharedRoot || plan.StateMode == config.StateModeSharedRoot
|
|
}
|
|
|
|
func outputManagedByExistingState(output Output, existing *state.DistributorState) bool {
|
|
if existing == nil {
|
|
return false
|
|
}
|
|
_, ok := state.FindOutputByPath(existing.Outputs, output.DestinationPath)
|
|
return ok
|
|
}
|
|
|
|
func outputManagedBySharedRootPlan(output Output, plan Plan) bool {
|
|
for _, existing := range currentOwnerSharedRootOutputs(plan) {
|
|
if existing.Path == output.DestinationPath {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
func stateOutputsForPlan(plan Plan, now time.Time) ([]state.OutputFile, error) {
|
|
existingOutputs := []state.OutputFile(nil)
|
|
if plan.ExistingState != nil {
|
|
existingOutputs = plan.ExistingState.Outputs
|
|
}
|
|
planned := state.ProjectOutputs(StateOutputProjections(plan.Outputs), existingOutputs, now)
|
|
if !usesMergeRetention(plan) || plan.ExistingState == nil {
|
|
return planned, nil
|
|
}
|
|
return state.MergeOutputFiles(plan.ExistingState.Outputs, planned)
|
|
}
|
|
|
|
func usesMergeRetention(plan Plan) bool {
|
|
return plan.Reconciliation.Mode == config.ReconciliationModeMerge && plan.Action == ActionReplaceOlder
|
|
}
|
|
|
|
func sharedRootStateForPlan(req Request, plan Plan, now time.Time) (state.SharedRootState, error) {
|
|
now = now.UTC()
|
|
scope := plan.OwnerScope
|
|
if scope.PipelineID == "" && scope.DestinationID == "" {
|
|
scope = state.CurrentOwnerScope(req.PipelineID, req.DestinationID)
|
|
}
|
|
base := sharedRootBaseState(req, plan, now)
|
|
owner := state.OwnerRecord{
|
|
Scope: scope,
|
|
Reconciliation: state.ReconciliationPolicy{Mode: plan.Reconciliation.Mode},
|
|
Source: state.SourceState{Manifest: req.SourceBundle.Manifest},
|
|
}
|
|
if plan.PrimaryURL != "" {
|
|
owner.Links = &state.LinkState{PrimaryURL: plan.PrimaryURL}
|
|
}
|
|
planned := state.ProjectSharedRootOutputs(StateOutputProjections(plan.Outputs), currentOwnerSharedRootOutputs(plan), scope, req.SourceBundle.Manifest, now)
|
|
if plan.Action == ActionReplaceOlder && plan.Reconciliation.Mode == config.ReconciliationModeMerge {
|
|
return state.MergeOwnerOutputs(base, scope, owner, planned)
|
|
}
|
|
return state.ReplaceOwnerOutputs(base, scope, owner, planned)
|
|
}
|
|
|
|
func sharedRootBaseState(req Request, plan Plan, now time.Time) state.SharedRootState {
|
|
if plan.Action == ActionForceReplace {
|
|
return newSharedRootState(req, now)
|
|
}
|
|
if plan.ExistingSharedRoot != nil {
|
|
base := *plan.ExistingSharedRoot
|
|
base.Owners = append([]state.OwnerRecord(nil), plan.ExistingSharedRoot.Owners...)
|
|
base.Outputs = append([]state.SharedRootOutputFile(nil), plan.ExistingSharedRoot.Outputs...)
|
|
base.DistributorVersion = req.DistributorVersion
|
|
base.UpdatedAt = now
|
|
return base
|
|
}
|
|
if plan.ExistingState != nil {
|
|
base := newSharedRootState(req, now)
|
|
base.CreatedAt = plan.ExistingState.CreatedAt
|
|
base.UpdatedAt = now
|
|
return base
|
|
}
|
|
return newSharedRootState(req, now)
|
|
}
|
|
|
|
func newSharedRootState(req Request, now time.Time) state.SharedRootState {
|
|
return state.SharedRootState{
|
|
SchemaVersion: state.SharedRootSchemaVersion,
|
|
DistributorVersion: req.DistributorVersion,
|
|
CreatedAt: now,
|
|
UpdatedAt: now,
|
|
State: state.StatePolicy{Mode: state.StateModeSharedRoot},
|
|
Owners: []state.OwnerRecord{},
|
|
Outputs: []state.SharedRootOutputFile{},
|
|
}
|
|
}
|
|
|
|
func currentOwnerSharedRootOutputs(plan Plan) []state.SharedRootOutputFile {
|
|
if plan.ExistingSharedRoot != nil {
|
|
outputs := make([]state.SharedRootOutputFile, 0, len(plan.ExistingSharedRoot.Outputs))
|
|
for _, output := range plan.ExistingSharedRoot.Outputs {
|
|
if output.Owner == plan.OwnerScope {
|
|
outputs = append(outputs, output)
|
|
}
|
|
}
|
|
return outputs
|
|
}
|
|
if plan.ExistingState != nil {
|
|
outputs := make([]state.SharedRootOutputFile, 0, len(plan.ExistingState.Outputs))
|
|
for _, output := range plan.ExistingState.Outputs {
|
|
outputs = append(outputs, state.SharedRootOutputFile{
|
|
Path: output.Path,
|
|
Kind: output.Kind,
|
|
SourcePath: output.SourcePath,
|
|
Transform: output.Transform,
|
|
URL: output.URL,
|
|
SHA256: output.SHA256,
|
|
Size: output.Size,
|
|
Owner: plan.OwnerScope,
|
|
SourceID: plan.ExistingState.Source.Manifest.ID,
|
|
SourceDigest: plan.ExistingState.Source.Manifest.Digest,
|
|
SourceCreated: plan.ExistingState.Source.Manifest.Created,
|
|
CreatedAt: output.CreatedAt,
|
|
UpdatedAt: output.UpdatedAt,
|
|
})
|
|
}
|
|
return outputs
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func sharedRootOutputPaths(outputs []state.SharedRootOutputFile) []string {
|
|
paths := make([]string, 0, len(outputs))
|
|
for _, output := range outputs {
|
|
paths = append(paths, output.Path)
|
|
}
|
|
return paths
|
|
}
|
|
|
|
func sharedRootStateWriteOverwrites(plan Plan) bool {
|
|
return plan.ExistingSharedRoot != nil || plan.ExistingState != nil || plan.Action == ActionForceReplace
|
|
}
|