package publish import ( "context" "encoding/json" "fmt" "time" "gitea.maximumdirect.net/eric/distributor/internal/config" "gitea.maximumdirect.net/eric/distributor/internal/state" "gitea.maximumdirect.net/eric/distributor/internal/storage" ) func Execute(ctx context.Context, req Request, plan Plan) error { plan.Reconciliation = normalizeReconciliation(plan.Reconciliation) switch plan.Action { case ActionSkipSame, ActionSkipDestinationNewer: return nil case ActionPublishNew, ActionReplaceOlder, ActionReplaceConflict, ActionReplaceNewer, ActionReplaceTakeover, ActionForceReplace: if usesSharedRootState(req, plan) { return executeSharedRoot(ctx, req, plan) } default: return fmt.Errorf("cannot execute action %s: %s", plan.Action, plan.Reason) } if plan.Action == ActionReplaceOlder || plan.Action == ActionReplaceConflict || plan.Action == ActionReplaceNewer || plan.Action == ActionReplaceTakeover { if plan.ExistingState == nil { return fmt.Errorf("replace requires existing destination state") } if plan.Reconciliation.Mode == config.ReconciliationModeReplace || plan.Action == ActionReplaceConflict || plan.Action == ActionReplaceTakeover { if err := req.DestinationBackend.DeleteManagedBundle(ctx, req.DestinationBundlePath, state.ManagedOutputPaths(*plan.ExistingState), storage.DeleteOptions{IgnoreMissing: true, PruneEmptyDirs: true}); err != nil { return err } if err := ensureDestinationEmpty(ctx, req.DestinationBackend, req.DestinationBundlePath); err != nil { return err } } } if plan.Action == ActionForceReplace { if err := req.DestinationBackend.DeletePrefix(ctx, req.DestinationBundlePath, storage.DeleteOptions{IgnoreMissing: true, PruneEmptyDirs: true}); err != nil { return err } if err := ensureDestinationEmpty(ctx, req.DestinationBackend, req.DestinationBundlePath); err != nil { return err } } if usesMergeRetention(plan) { if err := ensureMergeOutputPaths(ctx, req.DestinationBackend, req.DestinationBundlePath, plan); err != nil { return err } } writtenOutputs := make([]Output, 0, len(plan.Outputs)) newOutputs := make([]Output, 0, len(plan.Outputs)) cleanup := func() { outputs := writtenOutputs if usesMergeRetention(plan) { outputs = newOutputs } _ = req.DestinationBackend.DeleteManagedBundle(ctx, req.DestinationBundlePath, ManagedOutputPaths(outputs), storage.DeleteOptions{IgnoreMissing: true, PruneEmptyDirs: true}) } for _, output := range plan.Outputs { destinationPath, err := storage.Join(req.DestinationBundlePath, output.DestinationPath) if err != nil { cleanup() return err } data := output.Data if output.Kind == state.OutputKindSource { sourcePath, err := storage.Join(req.SourceBundle.RootRelativePath, output.SourcePath) if err != nil { cleanup() return err } data, err = req.SourceBackend.ReadFile(ctx, sourcePath) if err != nil { cleanup() return err } } managed := outputManagedByExistingState(output, plan.ExistingState) if _, err := req.DestinationBackend.WriteFile(ctx, destinationPath, data, storage.WriteOptions{Overwrite: managed && usesMergeRetention(plan), PreferAtomic: true}); err != nil { cleanup() return err } writtenOutputs = append(writtenOutputs, output) if !managed { newOutputs = append(newOutputs, output) } } now := time.Now().UTC() createdAt := now if plan.ExistingState != nil { createdAt = plan.ExistingState.CreatedAt } stateOutputs, err := stateOutputsForPlan(plan, now) if err != nil { cleanup() return err } destinationState := state.DistributorState{ SchemaVersion: state.SchemaVersion, DistributorVersion: req.DistributorVersion, PipelineID: req.PipelineID, DestinationID: req.DestinationID, PublishedAt: now, CreatedAt: createdAt, UpdatedAt: now, State: state.StatePolicy{Mode: state.StateModeSingleOwner}, Reconciliation: state.ReconciliationPolicy{Mode: plan.Reconciliation.Mode}, Source: state.SourceState{Manifest: req.SourceBundle.Manifest}, Outputs: stateOutputs, } if plan.PrimaryURL != "" { destinationState.Links = &state.LinkState{PrimaryURL: plan.PrimaryURL} } if err := state.Validate(destinationState); err != nil { cleanup() return err } data, err := json.MarshalIndent(destinationState, "", " ") if err != nil { cleanup() return err } data = append(data, '\n') statePath, err := storage.StatePath(req.DestinationBundlePath) if err != nil { cleanup() return err } if _, err := req.DestinationBackend.WriteFile(ctx, statePath, data, storage.WriteOptions{Overwrite: plan.ExistingState != nil, PreferAtomic: true}); err != nil { cleanup() return err } return nil } func executeSharedRoot(ctx context.Context, req Request, plan Plan) error { plan.Reconciliation = normalizeReconciliation(plan.Reconciliation) if plan.Action == ActionForceReplace { if err := req.DestinationBackend.DeletePrefix(ctx, req.DestinationBundlePath, storage.DeleteOptions{IgnoreMissing: true, PruneEmptyDirs: true}); err != nil { return err } if err := ensureDestinationEmpty(ctx, req.DestinationBackend, req.DestinationBundlePath); err != nil { return err } } if plan.Action == ActionReplaceTakeover || plan.Action == ActionReplaceConflict || (isReconciliationReplacementAction(plan.Action) && plan.Reconciliation.Mode == config.ReconciliationModeReplace) { if err := req.DestinationBackend.DeleteManagedOutputs(ctx, req.DestinationBundlePath, sharedRootOutputPaths(plan.OwnerOutputsToDelete), storage.DeleteOptions{IgnoreMissing: true, PruneEmptyDirs: true}); err != nil { return err } } writtenOutputs := make([]Output, 0, len(plan.Outputs)) newOutputs := make([]Output, 0, len(plan.Outputs)) cleanup := func() { outputs := writtenOutputs if isReconciliationReplacementAction(plan.Action) && plan.Reconciliation.Mode == config.ReconciliationModeMerge { outputs = newOutputs } _ = req.DestinationBackend.DeleteManagedOutputs(ctx, req.DestinationBundlePath, ManagedOutputPaths(outputs), storage.DeleteOptions{IgnoreMissing: true, PruneEmptyDirs: true}) } for _, output := range plan.Outputs { destinationPath, err := storage.Join(req.DestinationBundlePath, output.DestinationPath) if err != nil { cleanup() return err } data := output.Data if output.Kind == state.OutputKindSource { sourcePath, err := storage.Join(req.SourceBundle.RootRelativePath, output.SourcePath) if err != nil { cleanup() return err } data, err = req.SourceBackend.ReadFile(ctx, sourcePath) if err != nil { cleanup() return err } } managed := outputManagedBySharedRootPlan(output, plan) if _, err := req.DestinationBackend.WriteFile(ctx, destinationPath, data, storage.WriteOptions{Overwrite: managed, PreferAtomic: true}); err != nil { cleanup() return err } writtenOutputs = append(writtenOutputs, output) if !managed { newOutputs = append(newOutputs, output) } } now := time.Now().UTC() sharedRootState, err := sharedRootStateForPlan(req, plan, now) if err != nil { cleanup() return err } if err := state.ValidateSharedRoot(sharedRootState); err != nil { cleanup() return err } data, err := json.MarshalIndent(sharedRootState, "", " ") if err != nil { cleanup() return err } data = append(data, '\n') statePath, err := storage.StatePath(req.DestinationBundlePath) if err != nil { cleanup() return err } if _, err := req.DestinationBackend.WriteFile(ctx, statePath, data, storage.WriteOptions{Overwrite: sharedRootStateWriteOverwrites(plan), PreferAtomic: true}); err != nil { cleanup() return err } return nil } func ensureMergeOutputPaths(ctx context.Context, backend storage.Backend, bundlePath string, plan Plan) error { for _, output := range plan.Outputs { if outputManagedByExistingState(output, plan.ExistingState) { continue } destinationPath, err := storage.Join(bundlePath, output.DestinationPath) if err != nil { return err } if _, err := backend.Stat(ctx, destinationPath); err == nil { return fmt.Errorf("merge output path %s exists but is not managed by destination state", storage.DisplayPath(output.DestinationPath)) } else if !storage.IsNotFound(err) { return err } } return nil } func usesSharedRootState(req Request, plan Plan) bool { return plan.StateMode == config.StateModeSharedRoot } func outputManagedByExistingState(output Output, existing *state.DistributorState) bool { if existing == nil { return false } _, ok := state.FindOutputByPath(existing.Outputs, output.DestinationPath) return ok } func outputManagedBySharedRootPlan(output Output, plan Plan) bool { for _, existing := range currentOwnerSharedRootOutputs(plan) { if existing.Path == output.DestinationPath { return true } } for _, existing := range plan.TakenOverOwnerOutputs { if existing.Path == output.DestinationPath { return true } } return false } func stateOutputsForPlan(plan Plan, now time.Time) ([]state.OutputFile, error) { existingOutputs := []state.OutputFile(nil) if plan.ExistingState != nil { existingOutputs = plan.ExistingState.Outputs } planned := state.ProjectOutputs(StateOutputProjections(plan.Outputs), existingOutputs, now) if !usesMergeRetention(plan) || plan.ExistingState == nil { return planned, nil } return state.MergeOutputFiles(plan.ExistingState.Outputs, planned) } func usesMergeRetention(plan Plan) bool { return plan.Reconciliation.Mode == config.ReconciliationModeMerge && isReconciliationReplacementAction(plan.Action) } func sharedRootStateForPlan(req Request, plan Plan, now time.Time) (state.SharedRootState, error) { now = now.UTC() scope := plan.OwnerScope if scope.PipelineID == "" && scope.DestinationID == "" { scope = state.CurrentOwnerScope(req.PipelineID, req.DestinationID) } base := sharedRootBaseState(req, plan, now) base = removeTakenOverSharedRootOutputs(base, plan.TakenOverOwnerOutputs) owner := state.OwnerRecord{ Scope: scope, Reconciliation: state.ReconciliationPolicy{Mode: plan.Reconciliation.Mode}, Source: state.SourceState{Manifest: req.SourceBundle.Manifest}, } if plan.PrimaryURL != "" { owner.Links = &state.LinkState{PrimaryURL: plan.PrimaryURL} } planned := state.ProjectSharedRootOutputs(StateOutputProjections(plan.Outputs), currentOwnerSharedRootOutputs(plan), scope, req.SourceBundle.Manifest, now) if isReconciliationReplacementAction(plan.Action) && plan.Reconciliation.Mode == config.ReconciliationModeMerge { return state.MergeOwnerOutputs(base, scope, owner, planned) } return state.ReplaceOwnerOutputs(base, scope, owner, planned) } func sharedRootBaseState(req Request, plan Plan, now time.Time) state.SharedRootState { if plan.Action == ActionForceReplace { return newSharedRootState(req, now) } if plan.ExistingSharedRoot != nil { base := *plan.ExistingSharedRoot base.Owners = append([]state.OwnerRecord(nil), plan.ExistingSharedRoot.Owners...) base.Outputs = append([]state.SharedRootOutputFile(nil), plan.ExistingSharedRoot.Outputs...) base.DistributorVersion = req.DistributorVersion base.UpdatedAt = now return base } if plan.ExistingState != nil { base := newSharedRootState(req, now) base.CreatedAt = plan.ExistingState.CreatedAt base.UpdatedAt = now return base } return newSharedRootState(req, now) } func removeTakenOverSharedRootOutputs(sharedRoot state.SharedRootState, takenOver []state.SharedRootOutputFile) state.SharedRootState { if len(takenOver) == 0 { return sharedRoot } paths := sharedRootOutputPathSet(takenOver) next := sharedRoot next.Outputs = make([]state.SharedRootOutputFile, 0, len(sharedRoot.Outputs)) for _, output := range sharedRoot.Outputs { if _, remove := paths[output.Path]; remove { continue } next.Outputs = append(next.Outputs, output) } return next } func newSharedRootState(req Request, now time.Time) state.SharedRootState { return state.SharedRootState{ SchemaVersion: state.SharedRootSchemaVersion, DistributorVersion: req.DistributorVersion, CreatedAt: now, UpdatedAt: now, State: state.StatePolicy{Mode: state.StateModeSharedRoot}, Owners: []state.OwnerRecord{}, Outputs: []state.SharedRootOutputFile{}, } } func currentOwnerSharedRootOutputs(plan Plan) []state.SharedRootOutputFile { if plan.ExistingSharedRoot != nil { outputs := make([]state.SharedRootOutputFile, 0, len(plan.ExistingSharedRoot.Outputs)) for _, output := range plan.ExistingSharedRoot.Outputs { if output.Owner == plan.OwnerScope { outputs = append(outputs, output) } } return outputs } if plan.ExistingState != nil { outputs := make([]state.SharedRootOutputFile, 0, len(plan.ExistingState.Outputs)) for _, output := range plan.ExistingState.Outputs { outputs = append(outputs, state.SharedRootOutputFile{ Path: output.Path, Kind: output.Kind, SourcePath: output.SourcePath, Transform: output.Transform, URL: output.URL, SHA256: output.SHA256, Size: output.Size, Owner: plan.OwnerScope, SourceID: plan.ExistingState.Source.Manifest.ID, SourceDigest: plan.ExistingState.Source.Manifest.Digest, SourceCreated: plan.ExistingState.Source.Manifest.Created, CreatedAt: output.CreatedAt, UpdatedAt: output.UpdatedAt, }) } return outputs } return nil } func sharedRootOutputPaths(outputs []state.SharedRootOutputFile) []string { paths := make([]string, 0, len(outputs)) for _, output := range outputs { paths = append(paths, output.Path) } return paths } func sharedRootStateWriteOverwrites(plan Plan) bool { return plan.ExistingSharedRoot != nil || plan.ExistingState != nil || plan.Action == ActionForceReplace }