Split run orchestration helpers
This commit is contained in:
@@ -2,11 +2,8 @@ package app
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"errors"
|
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"sort"
|
|
||||||
"strings"
|
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/distributor/internal/bundle"
|
"gitea.maximumdirect.net/eric/distributor/internal/bundle"
|
||||||
"gitea.maximumdirect.net/eric/distributor/internal/config"
|
"gitea.maximumdirect.net/eric/distributor/internal/config"
|
||||||
@@ -260,433 +257,3 @@ func closeBackend(backend storage.Backend) {
|
|||||||
}
|
}
|
||||||
_ = closeable.Close()
|
_ = closeable.Close()
|
||||||
}
|
}
|
||||||
|
|
||||||
func writePlanLine(w io.Writer, backend string, plan publish.Plan, planErr error) {
|
|
||||||
if w == nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if planErr != nil {
|
|
||||||
destinationID := plan.DestinationID
|
|
||||||
if destinationID == "" {
|
|
||||||
destinationID = "unknown"
|
|
||||||
}
|
|
||||||
fmt.Fprintf(w, " - bundle=%s destination=%s backend=%s%s action=error reason=%q\n", storage.DisplayPath(plan.BundlePath), destinationID, backend, pathMappingSummary(plan), planErr.Error())
|
|
||||||
return
|
|
||||||
}
|
|
||||||
fmt.Fprintf(w, " - bundle=%s destination=%s backend=%s%s action=%s outputs=%s reason=%q\n", storage.DisplayPath(plan.BundlePath), plan.DestinationID, backend, pathMappingSummary(plan), plan.Action, outputSummary(plan.Outputs), plan.Reason)
|
|
||||||
}
|
|
||||||
|
|
||||||
func pathMappingSummary(plan publish.Plan) string {
|
|
||||||
if plan.PathMapping != config.PathMappingFixed {
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
return fmt.Sprintf(" path_mapping=fixed target=%s", storage.DisplayPath(plan.DestinationBundlePath))
|
|
||||||
}
|
|
||||||
|
|
||||||
func writeErrorLine(w io.Writer, bundlePath, destinationID, backend string, err error) {
|
|
||||||
if w == nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
fmt.Fprintf(w, " - bundle=%s destination=%s backend=%s action=error reason=%q\n", storage.DisplayPath(bundlePath), destinationID, backend, err.Error())
|
|
||||||
}
|
|
||||||
|
|
||||||
func outputSummary(outputs []publish.Output) string {
|
|
||||||
if len(outputs) == 0 {
|
|
||||||
return "none"
|
|
||||||
}
|
|
||||||
paths := make([]string, 0, len(outputs))
|
|
||||||
for _, output := range outputs {
|
|
||||||
paths = append(paths, output.DestinationPath)
|
|
||||||
}
|
|
||||||
return strings.Join(paths, ",")
|
|
||||||
}
|
|
||||||
|
|
||||||
type destinationBundleSelection struct {
|
|
||||||
SourceBundle bundle.Bundle
|
|
||||||
DestinationBundlePath string
|
|
||||||
}
|
|
||||||
|
|
||||||
func selectDestinationBundles(destination config.Destination, bundles []bundle.Bundle) []destinationBundleSelection {
|
|
||||||
if !isFixedPathDestination(destination) {
|
|
||||||
selections := make([]destinationBundleSelection, 0, len(bundles))
|
|
||||||
for _, sourceBundle := range bundles {
|
|
||||||
selections = append(selections, destinationBundleSelection{
|
|
||||||
SourceBundle: sourceBundle,
|
|
||||||
DestinationBundlePath: sourceBundle.RootRelativePath,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
return selections
|
|
||||||
}
|
|
||||||
if len(bundles) == 0 {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
sourceBundle := newestBundle(bundles)
|
|
||||||
return []destinationBundleSelection{{
|
|
||||||
SourceBundle: sourceBundle,
|
|
||||||
DestinationBundlePath: "",
|
|
||||||
}}
|
|
||||||
}
|
|
||||||
|
|
||||||
func newestBundle(bundles []bundle.Bundle) bundle.Bundle {
|
|
||||||
if len(bundles) == 0 {
|
|
||||||
return bundle.Bundle{}
|
|
||||||
}
|
|
||||||
sorted := append([]bundle.Bundle(nil), bundles...)
|
|
||||||
sort.Slice(sorted, func(i, j int) bool {
|
|
||||||
if sorted[i].Manifest.Created.Equal(sorted[j].Manifest.Created) {
|
|
||||||
return sorted[i].RootRelativePath < sorted[j].RootRelativePath
|
|
||||||
}
|
|
||||||
return sorted[i].Manifest.Created.After(sorted[j].Manifest.Created)
|
|
||||||
})
|
|
||||||
return sorted[0]
|
|
||||||
}
|
|
||||||
|
|
||||||
func isFixedPathDestination(destination config.Destination) bool {
|
|
||||||
return destination.PathMap.Mode == config.PathMappingFixed
|
|
||||||
}
|
|
||||||
|
|
||||||
func fixedPathSelectionWarning(pipelineID, destinationID string, selections []destinationBundleSelection, candidateCount int) OutputWarning {
|
|
||||||
selected := "none"
|
|
||||||
if len(selections) > 0 {
|
|
||||||
selected = storage.DisplayPath(selections[0].SourceBundle.RootRelativePath)
|
|
||||||
}
|
|
||||||
return OutputWarning{Message: fmt.Sprintf("pipeline=%s destination=%s path_mapping=fixed candidates=%d selected_bundle=%s destination_bundle=.", pipelineID, destinationID, candidateCount, selected)}
|
|
||||||
}
|
|
||||||
|
|
||||||
func isDestructiveFixedPathAction(action publish.Action) bool {
|
|
||||||
return action == publish.ActionReplaceOlder || action == publish.ActionForceReplace
|
|
||||||
}
|
|
||||||
|
|
||||||
func fixedPathReplacementWarning(plan publish.Plan) OutputWarning {
|
|
||||||
return OutputWarning{Message: fmt.Sprintf("pipeline=%s destination=%s path_mapping=fixed action=%s replaces destination root for selected_bundle=%s", plan.PipelineID, plan.DestinationID, plan.Action, storage.DisplayPath(plan.BundlePath))}
|
|
||||||
}
|
|
||||||
|
|
||||||
func destinationIDs(destinations []config.Destination) []string {
|
|
||||||
ids := make([]string, 0, len(destinations))
|
|
||||||
for _, destination := range destinations {
|
|
||||||
ids = append(ids, destination.ID)
|
|
||||||
}
|
|
||||||
return ids
|
|
||||||
}
|
|
||||||
|
|
||||||
func destinationSummary(destinations []config.Destination) string {
|
|
||||||
if len(destinations) == 0 {
|
|
||||||
return "none"
|
|
||||||
}
|
|
||||||
ids := make([]string, 0, len(destinations))
|
|
||||||
for _, destination := range destinations {
|
|
||||||
ids = append(ids, destination.ID)
|
|
||||||
}
|
|
||||||
return strings.Join(ids, ",")
|
|
||||||
}
|
|
||||||
|
|
||||||
func writeSecretConflictWarnings(w io.Writer, conflicts []config.SecretConflict) error {
|
|
||||||
return writeWarnings(w, secretConflictWarnings(conflicts))
|
|
||||||
}
|
|
||||||
|
|
||||||
func secretConflictWarnings(conflicts []config.SecretConflict) []OutputWarning {
|
|
||||||
warnings := make([]OutputWarning, 0, len(conflicts))
|
|
||||||
for _, conflict := range conflicts {
|
|
||||||
warnings = append(warnings, OutputWarning{
|
|
||||||
Message: fmt.Sprintf("secret %s ignored because the real environment already has that variable", conflict.Name),
|
|
||||||
})
|
|
||||||
}
|
|
||||||
return warnings
|
|
||||||
}
|
|
||||||
|
|
||||||
func writeSSHWarnings(w io.Writer, pipeline config.Pipeline) error {
|
|
||||||
return writeWarnings(w, sshWarnings(pipeline))
|
|
||||||
}
|
|
||||||
|
|
||||||
func sshWarnings(pipeline config.Pipeline) []OutputWarning {
|
|
||||||
var warnings []OutputWarning
|
|
||||||
if pipeline.Source.Backend == config.BackendSSH && pipeline.Source.SSH.HostKeyPolicy == config.HostKeyPolicyOff {
|
|
||||||
warnings = append(warnings, OutputWarning{
|
|
||||||
Message: fmt.Sprintf("pipeline=%s source host_key_policy=off disables SSH host key checking", pipeline.ID),
|
|
||||||
})
|
|
||||||
}
|
|
||||||
for _, destination := range pipeline.Destinations {
|
|
||||||
if destination.Backend == config.BackendSSH && destination.SSH.HostKeyPolicy == config.HostKeyPolicyOff {
|
|
||||||
warnings = append(warnings, OutputWarning{
|
|
||||||
Message: fmt.Sprintf("pipeline=%s destination=%s host_key_policy=off disables SSH host key checking", pipeline.ID, destination.ID),
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return warnings
|
|
||||||
}
|
|
||||||
|
|
||||||
func writeWarnings(w io.Writer, warnings []OutputWarning) error {
|
|
||||||
if w == nil {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
for _, warning := range warnings {
|
|
||||||
if _, err := fmt.Fprintf(w, "Warning: %s\n", warning.Message); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func shouldNotify(action publish.Action) bool {
|
|
||||||
return action == publish.ActionPublishNew || action == publish.ActionReplaceOlder || action == publish.ActionForceReplace
|
|
||||||
}
|
|
||||||
|
|
||||||
func notifyEvent(plan publish.Plan) notify.Event {
|
|
||||||
outputs := make([]notify.Output, 0, len(plan.Outputs))
|
|
||||||
for _, output := range plan.Outputs {
|
|
||||||
stateOutput := output.StateOutputFile()
|
|
||||||
outputs = append(outputs, notify.Output{
|
|
||||||
Path: stateOutput.Path,
|
|
||||||
Kind: stateOutput.Kind,
|
|
||||||
SourcePath: stateOutput.SourcePath,
|
|
||||||
Transform: stateOutput.Transform,
|
|
||||||
SHA256: stateOutput.SHA256,
|
|
||||||
Size: stateOutput.Size,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
return notify.Event{
|
|
||||||
PipelineID: plan.PipelineID,
|
|
||||||
DestinationID: plan.DestinationID,
|
|
||||||
BundleID: plan.BundleID,
|
|
||||||
BundlePath: plan.BundlePath,
|
|
||||||
Action: string(plan.Action),
|
|
||||||
Outputs: outputs,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
type runResult struct {
|
|
||||||
DryRun bool `json:"dry_run"`
|
|
||||||
Pipelines []runPipelineResult `json:"pipelines"`
|
|
||||||
Actions []runActionResult `json:"actions"`
|
|
||||||
Summary runSummaryResult `json:"summary"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type runPipelineResult struct {
|
|
||||||
ID string `json:"id"`
|
|
||||||
SourceBackend string `json:"source_backend"`
|
|
||||||
BundleCount int `json:"bundle_count"`
|
|
||||||
Destinations []string `json:"destinations"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type runActionResult struct {
|
|
||||||
PipelineID string `json:"pipeline_id,omitempty"`
|
|
||||||
DestinationID string `json:"destination_id"`
|
|
||||||
Backend string `json:"backend"`
|
|
||||||
BundleID string `json:"bundle_id,omitempty"`
|
|
||||||
BundlePath string `json:"bundle_path"`
|
|
||||||
DestinationPath string `json:"destination_path"`
|
|
||||||
PathMapping string `json:"path_mapping,omitempty"`
|
|
||||||
Action string `json:"action"`
|
|
||||||
PrimaryURL string `json:"primary_url,omitempty"`
|
|
||||||
Reason string `json:"reason,omitempty"`
|
|
||||||
Outputs []runOutputResult `json:"outputs"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type runOutputResult struct {
|
|
||||||
Path string `json:"path"`
|
|
||||||
Kind string `json:"kind"`
|
|
||||||
SourcePath string `json:"source_path,omitempty"`
|
|
||||||
Transform string `json:"transform,omitempty"`
|
|
||||||
URL string `json:"url,omitempty"`
|
|
||||||
SHA256 string `json:"sha256"`
|
|
||||||
Size int64 `json:"size"`
|
|
||||||
}
|
|
||||||
|
|
||||||
func runActionFromPlan(backend string, plan publish.Plan, planErr error) runActionResult {
|
|
||||||
if planErr != nil {
|
|
||||||
destinationID := plan.DestinationID
|
|
||||||
if destinationID == "" {
|
|
||||||
destinationID = "unknown"
|
|
||||||
}
|
|
||||||
return runActionResult{
|
|
||||||
PipelineID: plan.PipelineID,
|
|
||||||
DestinationID: destinationID,
|
|
||||||
Backend: backend,
|
|
||||||
BundleID: plan.BundleID,
|
|
||||||
BundlePath: storage.DisplayPath(plan.BundlePath),
|
|
||||||
DestinationPath: storage.DisplayPath(plan.DestinationBundlePath),
|
|
||||||
PathMapping: plan.PathMapping,
|
|
||||||
Action: "error",
|
|
||||||
PrimaryURL: plan.PrimaryURL,
|
|
||||||
Reason: planErr.Error(),
|
|
||||||
Outputs: []runOutputResult{},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return runActionResult{
|
|
||||||
PipelineID: plan.PipelineID,
|
|
||||||
DestinationID: plan.DestinationID,
|
|
||||||
Backend: backend,
|
|
||||||
BundleID: plan.BundleID,
|
|
||||||
BundlePath: storage.DisplayPath(plan.BundlePath),
|
|
||||||
DestinationPath: storage.DisplayPath(plan.DestinationBundlePath),
|
|
||||||
PathMapping: plan.PathMapping,
|
|
||||||
Action: string(plan.Action),
|
|
||||||
PrimaryURL: plan.PrimaryURL,
|
|
||||||
Reason: plan.Reason,
|
|
||||||
Outputs: runOutputsFromPlan(plan.Outputs),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func errorAction(pipelineID, destinationID, backend, bundlePath string, err error) runActionResult {
|
|
||||||
return runActionResult{
|
|
||||||
PipelineID: pipelineID,
|
|
||||||
DestinationID: destinationID,
|
|
||||||
Backend: backend,
|
|
||||||
BundlePath: storage.DisplayPath(bundlePath),
|
|
||||||
DestinationPath: storage.DisplayPath(bundlePath),
|
|
||||||
Action: "error",
|
|
||||||
Reason: err.Error(),
|
|
||||||
Outputs: []runOutputResult{},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func runOutputsFromPlan(outputs []publish.Output) []runOutputResult {
|
|
||||||
results := make([]runOutputResult, 0, len(outputs))
|
|
||||||
for _, output := range outputs {
|
|
||||||
stateOutput := output.StateOutputFile()
|
|
||||||
results = append(results, runOutputResult{
|
|
||||||
Path: stateOutput.Path,
|
|
||||||
Kind: stateOutput.Kind,
|
|
||||||
SourcePath: stateOutput.SourcePath,
|
|
||||||
Transform: stateOutput.Transform,
|
|
||||||
URL: stateOutput.URL,
|
|
||||||
SHA256: stateOutput.SHA256,
|
|
||||||
Size: stateOutput.Size,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
return results
|
|
||||||
}
|
|
||||||
|
|
||||||
type runSummary struct {
|
|
||||||
dryRun bool
|
|
||||||
planned int
|
|
||||||
publishNew int
|
|
||||||
replaceOlder int
|
|
||||||
forceReplace int
|
|
||||||
skipped int
|
|
||||||
failures int
|
|
||||||
fixedPath int
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *runSummary) recordPlan(action publish.Action) {
|
|
||||||
s.planned++
|
|
||||||
switch action {
|
|
||||||
case publish.ActionPublishNew:
|
|
||||||
s.publishNew++
|
|
||||||
case publish.ActionReplaceOlder:
|
|
||||||
s.replaceOlder++
|
|
||||||
case publish.ActionForceReplace:
|
|
||||||
s.forceReplace++
|
|
||||||
case publish.ActionSkipSame, publish.ActionSkipDestinationNewer:
|
|
||||||
s.skipped++
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *runSummary) recordFailure() {
|
|
||||||
s.failures++
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *runSummary) recordFixedPath() {
|
|
||||||
s.fixedPath++
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s runSummary) Line() string {
|
|
||||||
status := "ok"
|
|
||||||
if s.failures > 0 {
|
|
||||||
status = "failed"
|
|
||||||
}
|
|
||||||
return fmt.Sprintf("Final status: %s planned=%d publish_new=%d replace_older=%d force_replace=%d skipped=%d failed=%d dry_run=%t fixed_path=%d", status, s.planned, s.publishNew, s.replaceOlder, s.forceReplace, s.skipped, s.failures, s.dryRun, s.fixedPath)
|
|
||||||
}
|
|
||||||
|
|
||||||
type runSummaryResult struct {
|
|
||||||
Status string `json:"status"`
|
|
||||||
Planned int `json:"planned"`
|
|
||||||
PublishNew int `json:"publish_new"`
|
|
||||||
ReplaceOlder int `json:"replace_older"`
|
|
||||||
ForceReplace int `json:"force_replace"`
|
|
||||||
Skipped int `json:"skipped"`
|
|
||||||
Failed int `json:"failed"`
|
|
||||||
DryRun bool `json:"dry_run"`
|
|
||||||
FixedPath int `json:"fixed_path"`
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s runSummary) Result() runSummaryResult {
|
|
||||||
status := "ok"
|
|
||||||
if s.failures > 0 {
|
|
||||||
status = "failed"
|
|
||||||
}
|
|
||||||
return runSummaryResult{
|
|
||||||
Status: status,
|
|
||||||
Planned: s.planned,
|
|
||||||
PublishNew: s.publishNew,
|
|
||||||
ReplaceOlder: s.replaceOlder,
|
|
||||||
ForceReplace: s.forceReplace,
|
|
||||||
Skipped: s.skipped,
|
|
||||||
Failed: s.failures,
|
|
||||||
DryRun: s.dryRun,
|
|
||||||
FixedPath: s.fixedPath,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
type runFailure struct {
|
|
||||||
pipelineID string
|
|
||||||
destinationID string
|
|
||||||
backend string
|
|
||||||
bundlePath string
|
|
||||||
err error
|
|
||||||
}
|
|
||||||
|
|
||||||
type runFailures struct {
|
|
||||||
items []runFailure
|
|
||||||
}
|
|
||||||
|
|
||||||
func (f *runFailures) add(pipelineID, destinationID, backend, bundlePath string, err error) {
|
|
||||||
f.items = append(f.items, runFailure{
|
|
||||||
pipelineID: pipelineID,
|
|
||||||
destinationID: destinationID,
|
|
||||||
backend: backend,
|
|
||||||
bundlePath: bundlePath,
|
|
||||||
err: err,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func (f runFailures) Error() string {
|
|
||||||
if len(f.items) == 0 {
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
parts := make([]string, 0, len(f.items))
|
|
||||||
for _, item := range f.items {
|
|
||||||
parts = append(parts, fmt.Sprintf("pipeline %s destination %s backend %s bundle %s: %v", item.pipelineID, item.destinationID, item.backend, item.bundlePath, item.err))
|
|
||||||
}
|
|
||||||
return "run failed: " + strings.Join(parts, "; ")
|
|
||||||
}
|
|
||||||
|
|
||||||
func (f runFailures) outputErrors() []OutputError {
|
|
||||||
if len(f.items) == 0 {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
errors := make([]OutputError, 0, len(f.items))
|
|
||||||
for _, item := range f.items {
|
|
||||||
errors = append(errors, OutputError{
|
|
||||||
PipelineID: item.pipelineID,
|
|
||||||
DestinationID: item.destinationID,
|
|
||||||
Backend: item.backend,
|
|
||||||
BundlePath: item.bundlePath,
|
|
||||||
Message: item.err.Error(),
|
|
||||||
})
|
|
||||||
}
|
|
||||||
return errors
|
|
||||||
}
|
|
||||||
|
|
||||||
func IsPartialResultError(err error) bool {
|
|
||||||
var failures runFailures
|
|
||||||
return errors.As(err, &failures)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (f runFailures) Unwrap() error {
|
|
||||||
errs := make([]error, 0, len(f.items))
|
|
||||||
for _, item := range f.items {
|
|
||||||
errs = append(errs, item.err)
|
|
||||||
}
|
|
||||||
return errors.Join(errs...)
|
|
||||||
}
|
|
||||||
|
|||||||
70
internal/app/run_failures.go
Normal file
70
internal/app/run_failures.go
Normal file
@@ -0,0 +1,70 @@
|
|||||||
|
package app
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
type runFailure struct {
|
||||||
|
pipelineID string
|
||||||
|
destinationID string
|
||||||
|
backend string
|
||||||
|
bundlePath string
|
||||||
|
err error
|
||||||
|
}
|
||||||
|
|
||||||
|
type runFailures struct {
|
||||||
|
items []runFailure
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *runFailures) add(pipelineID, destinationID, backend, bundlePath string, err error) {
|
||||||
|
f.items = append(f.items, runFailure{
|
||||||
|
pipelineID: pipelineID,
|
||||||
|
destinationID: destinationID,
|
||||||
|
backend: backend,
|
||||||
|
bundlePath: bundlePath,
|
||||||
|
err: err,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f runFailures) Error() string {
|
||||||
|
if len(f.items) == 0 {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
parts := make([]string, 0, len(f.items))
|
||||||
|
for _, item := range f.items {
|
||||||
|
parts = append(parts, fmt.Sprintf("pipeline %s destination %s backend %s bundle %s: %v", item.pipelineID, item.destinationID, item.backend, item.bundlePath, item.err))
|
||||||
|
}
|
||||||
|
return "run failed: " + strings.Join(parts, "; ")
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f runFailures) outputErrors() []OutputError {
|
||||||
|
if len(f.items) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
errors := make([]OutputError, 0, len(f.items))
|
||||||
|
for _, item := range f.items {
|
||||||
|
errors = append(errors, OutputError{
|
||||||
|
PipelineID: item.pipelineID,
|
||||||
|
DestinationID: item.destinationID,
|
||||||
|
Backend: item.backend,
|
||||||
|
BundlePath: item.bundlePath,
|
||||||
|
Message: item.err.Error(),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
return errors
|
||||||
|
}
|
||||||
|
|
||||||
|
func IsPartialResultError(err error) bool {
|
||||||
|
var failures runFailures
|
||||||
|
return errors.As(err, &failures)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f runFailures) Unwrap() error {
|
||||||
|
errs := make([]error, 0, len(f.items))
|
||||||
|
for _, item := range f.items {
|
||||||
|
errs = append(errs, item.err)
|
||||||
|
}
|
||||||
|
return errors.Join(errs...)
|
||||||
|
}
|
||||||
33
internal/app/run_notify.go
Normal file
33
internal/app/run_notify.go
Normal file
@@ -0,0 +1,33 @@
|
|||||||
|
package app
|
||||||
|
|
||||||
|
import (
|
||||||
|
"gitea.maximumdirect.net/eric/distributor/internal/notify"
|
||||||
|
"gitea.maximumdirect.net/eric/distributor/internal/publish"
|
||||||
|
)
|
||||||
|
|
||||||
|
func shouldNotify(action publish.Action) bool {
|
||||||
|
return action == publish.ActionPublishNew || action == publish.ActionReplaceOlder || action == publish.ActionForceReplace
|
||||||
|
}
|
||||||
|
|
||||||
|
func notifyEvent(plan publish.Plan) notify.Event {
|
||||||
|
outputs := make([]notify.Output, 0, len(plan.Outputs))
|
||||||
|
for _, output := range plan.Outputs {
|
||||||
|
stateOutput := output.StateOutputFile()
|
||||||
|
outputs = append(outputs, notify.Output{
|
||||||
|
Path: stateOutput.Path,
|
||||||
|
Kind: stateOutput.Kind,
|
||||||
|
SourcePath: stateOutput.SourcePath,
|
||||||
|
Transform: stateOutput.Transform,
|
||||||
|
SHA256: stateOutput.SHA256,
|
||||||
|
Size: stateOutput.Size,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
return notify.Event{
|
||||||
|
PipelineID: plan.PipelineID,
|
||||||
|
DestinationID: plan.DestinationID,
|
||||||
|
BundleID: plan.BundleID,
|
||||||
|
BundlePath: plan.BundlePath,
|
||||||
|
Action: string(plan.Action),
|
||||||
|
Outputs: outputs,
|
||||||
|
}
|
||||||
|
}
|
||||||
154
internal/app/run_output.go
Normal file
154
internal/app/run_output.go
Normal file
@@ -0,0 +1,154 @@
|
|||||||
|
package app
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/distributor/internal/config"
|
||||||
|
"gitea.maximumdirect.net/eric/distributor/internal/publish"
|
||||||
|
"gitea.maximumdirect.net/eric/distributor/internal/storage"
|
||||||
|
)
|
||||||
|
|
||||||
|
func writePlanLine(w io.Writer, backend string, plan publish.Plan, planErr error) {
|
||||||
|
if w == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if planErr != nil {
|
||||||
|
destinationID := plan.DestinationID
|
||||||
|
if destinationID == "" {
|
||||||
|
destinationID = "unknown"
|
||||||
|
}
|
||||||
|
fmt.Fprintf(w, " - bundle=%s destination=%s backend=%s%s action=error reason=%q\n", storage.DisplayPath(plan.BundlePath), destinationID, backend, pathMappingSummary(plan), planErr.Error())
|
||||||
|
return
|
||||||
|
}
|
||||||
|
fmt.Fprintf(w, " - bundle=%s destination=%s backend=%s%s action=%s outputs=%s reason=%q\n", storage.DisplayPath(plan.BundlePath), plan.DestinationID, backend, pathMappingSummary(plan), plan.Action, outputSummary(plan.Outputs), plan.Reason)
|
||||||
|
}
|
||||||
|
|
||||||
|
func pathMappingSummary(plan publish.Plan) string {
|
||||||
|
if plan.PathMapping != config.PathMappingFixed {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return fmt.Sprintf(" path_mapping=fixed target=%s", storage.DisplayPath(plan.DestinationBundlePath))
|
||||||
|
}
|
||||||
|
|
||||||
|
func writeErrorLine(w io.Writer, bundlePath, destinationID, backend string, err error) {
|
||||||
|
if w == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
fmt.Fprintf(w, " - bundle=%s destination=%s backend=%s action=error reason=%q\n", storage.DisplayPath(bundlePath), destinationID, backend, err.Error())
|
||||||
|
}
|
||||||
|
|
||||||
|
func outputSummary(outputs []publish.Output) string {
|
||||||
|
if len(outputs) == 0 {
|
||||||
|
return "none"
|
||||||
|
}
|
||||||
|
paths := make([]string, 0, len(outputs))
|
||||||
|
for _, output := range outputs {
|
||||||
|
paths = append(paths, output.DestinationPath)
|
||||||
|
}
|
||||||
|
return strings.Join(paths, ",")
|
||||||
|
}
|
||||||
|
|
||||||
|
type runResult struct {
|
||||||
|
DryRun bool `json:"dry_run"`
|
||||||
|
Pipelines []runPipelineResult `json:"pipelines"`
|
||||||
|
Actions []runActionResult `json:"actions"`
|
||||||
|
Summary runSummaryResult `json:"summary"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type runPipelineResult struct {
|
||||||
|
ID string `json:"id"`
|
||||||
|
SourceBackend string `json:"source_backend"`
|
||||||
|
BundleCount int `json:"bundle_count"`
|
||||||
|
Destinations []string `json:"destinations"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type runActionResult struct {
|
||||||
|
PipelineID string `json:"pipeline_id,omitempty"`
|
||||||
|
DestinationID string `json:"destination_id"`
|
||||||
|
Backend string `json:"backend"`
|
||||||
|
BundleID string `json:"bundle_id,omitempty"`
|
||||||
|
BundlePath string `json:"bundle_path"`
|
||||||
|
DestinationPath string `json:"destination_path"`
|
||||||
|
PathMapping string `json:"path_mapping,omitempty"`
|
||||||
|
Action string `json:"action"`
|
||||||
|
PrimaryURL string `json:"primary_url,omitempty"`
|
||||||
|
Reason string `json:"reason,omitempty"`
|
||||||
|
Outputs []runOutputResult `json:"outputs"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type runOutputResult struct {
|
||||||
|
Path string `json:"path"`
|
||||||
|
Kind string `json:"kind"`
|
||||||
|
SourcePath string `json:"source_path,omitempty"`
|
||||||
|
Transform string `json:"transform,omitempty"`
|
||||||
|
URL string `json:"url,omitempty"`
|
||||||
|
SHA256 string `json:"sha256"`
|
||||||
|
Size int64 `json:"size"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func runActionFromPlan(backend string, plan publish.Plan, planErr error) runActionResult {
|
||||||
|
if planErr != nil {
|
||||||
|
destinationID := plan.DestinationID
|
||||||
|
if destinationID == "" {
|
||||||
|
destinationID = "unknown"
|
||||||
|
}
|
||||||
|
return runActionResult{
|
||||||
|
PipelineID: plan.PipelineID,
|
||||||
|
DestinationID: destinationID,
|
||||||
|
Backend: backend,
|
||||||
|
BundleID: plan.BundleID,
|
||||||
|
BundlePath: storage.DisplayPath(plan.BundlePath),
|
||||||
|
DestinationPath: storage.DisplayPath(plan.DestinationBundlePath),
|
||||||
|
PathMapping: plan.PathMapping,
|
||||||
|
Action: "error",
|
||||||
|
PrimaryURL: plan.PrimaryURL,
|
||||||
|
Reason: planErr.Error(),
|
||||||
|
Outputs: []runOutputResult{},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return runActionResult{
|
||||||
|
PipelineID: plan.PipelineID,
|
||||||
|
DestinationID: plan.DestinationID,
|
||||||
|
Backend: backend,
|
||||||
|
BundleID: plan.BundleID,
|
||||||
|
BundlePath: storage.DisplayPath(plan.BundlePath),
|
||||||
|
DestinationPath: storage.DisplayPath(plan.DestinationBundlePath),
|
||||||
|
PathMapping: plan.PathMapping,
|
||||||
|
Action: string(plan.Action),
|
||||||
|
PrimaryURL: plan.PrimaryURL,
|
||||||
|
Reason: plan.Reason,
|
||||||
|
Outputs: runOutputsFromPlan(plan.Outputs),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func errorAction(pipelineID, destinationID, backend, bundlePath string, err error) runActionResult {
|
||||||
|
return runActionResult{
|
||||||
|
PipelineID: pipelineID,
|
||||||
|
DestinationID: destinationID,
|
||||||
|
Backend: backend,
|
||||||
|
BundlePath: storage.DisplayPath(bundlePath),
|
||||||
|
DestinationPath: storage.DisplayPath(bundlePath),
|
||||||
|
Action: "error",
|
||||||
|
Reason: err.Error(),
|
||||||
|
Outputs: []runOutputResult{},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func runOutputsFromPlan(outputs []publish.Output) []runOutputResult {
|
||||||
|
results := make([]runOutputResult, 0, len(outputs))
|
||||||
|
for _, output := range outputs {
|
||||||
|
stateOutput := output.StateOutputFile()
|
||||||
|
results = append(results, runOutputResult{
|
||||||
|
Path: stateOutput.Path,
|
||||||
|
Kind: stateOutput.Kind,
|
||||||
|
SourcePath: stateOutput.SourcePath,
|
||||||
|
Transform: stateOutput.Transform,
|
||||||
|
URL: stateOutput.URL,
|
||||||
|
SHA256: stateOutput.SHA256,
|
||||||
|
Size: stateOutput.Size,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
return results
|
||||||
|
}
|
||||||
91
internal/app/run_selection.go
Normal file
91
internal/app/run_selection.go
Normal file
@@ -0,0 +1,91 @@
|
|||||||
|
package app
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/distributor/internal/bundle"
|
||||||
|
"gitea.maximumdirect.net/eric/distributor/internal/config"
|
||||||
|
"gitea.maximumdirect.net/eric/distributor/internal/publish"
|
||||||
|
"gitea.maximumdirect.net/eric/distributor/internal/storage"
|
||||||
|
)
|
||||||
|
|
||||||
|
type destinationBundleSelection struct {
|
||||||
|
SourceBundle bundle.Bundle
|
||||||
|
DestinationBundlePath string
|
||||||
|
}
|
||||||
|
|
||||||
|
func selectDestinationBundles(destination config.Destination, bundles []bundle.Bundle) []destinationBundleSelection {
|
||||||
|
if !isFixedPathDestination(destination) {
|
||||||
|
selections := make([]destinationBundleSelection, 0, len(bundles))
|
||||||
|
for _, sourceBundle := range bundles {
|
||||||
|
selections = append(selections, destinationBundleSelection{
|
||||||
|
SourceBundle: sourceBundle,
|
||||||
|
DestinationBundlePath: sourceBundle.RootRelativePath,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
return selections
|
||||||
|
}
|
||||||
|
if len(bundles) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
sourceBundle := newestBundle(bundles)
|
||||||
|
return []destinationBundleSelection{{
|
||||||
|
SourceBundle: sourceBundle,
|
||||||
|
DestinationBundlePath: "",
|
||||||
|
}}
|
||||||
|
}
|
||||||
|
|
||||||
|
func newestBundle(bundles []bundle.Bundle) bundle.Bundle {
|
||||||
|
if len(bundles) == 0 {
|
||||||
|
return bundle.Bundle{}
|
||||||
|
}
|
||||||
|
sorted := append([]bundle.Bundle(nil), bundles...)
|
||||||
|
sort.Slice(sorted, func(i, j int) bool {
|
||||||
|
if sorted[i].Manifest.Created.Equal(sorted[j].Manifest.Created) {
|
||||||
|
return sorted[i].RootRelativePath < sorted[j].RootRelativePath
|
||||||
|
}
|
||||||
|
return sorted[i].Manifest.Created.After(sorted[j].Manifest.Created)
|
||||||
|
})
|
||||||
|
return sorted[0]
|
||||||
|
}
|
||||||
|
|
||||||
|
func isFixedPathDestination(destination config.Destination) bool {
|
||||||
|
return destination.PathMap.Mode == config.PathMappingFixed
|
||||||
|
}
|
||||||
|
|
||||||
|
func fixedPathSelectionWarning(pipelineID, destinationID string, selections []destinationBundleSelection, candidateCount int) OutputWarning {
|
||||||
|
selected := "none"
|
||||||
|
if len(selections) > 0 {
|
||||||
|
selected = storage.DisplayPath(selections[0].SourceBundle.RootRelativePath)
|
||||||
|
}
|
||||||
|
return OutputWarning{Message: fmt.Sprintf("pipeline=%s destination=%s path_mapping=fixed candidates=%d selected_bundle=%s destination_bundle=.", pipelineID, destinationID, candidateCount, selected)}
|
||||||
|
}
|
||||||
|
|
||||||
|
func isDestructiveFixedPathAction(action publish.Action) bool {
|
||||||
|
return action == publish.ActionReplaceOlder || action == publish.ActionForceReplace
|
||||||
|
}
|
||||||
|
|
||||||
|
func fixedPathReplacementWarning(plan publish.Plan) OutputWarning {
|
||||||
|
return OutputWarning{Message: fmt.Sprintf("pipeline=%s destination=%s path_mapping=fixed action=%s replaces destination root for selected_bundle=%s", plan.PipelineID, plan.DestinationID, plan.Action, storage.DisplayPath(plan.BundlePath))}
|
||||||
|
}
|
||||||
|
|
||||||
|
func destinationIDs(destinations []config.Destination) []string {
|
||||||
|
ids := make([]string, 0, len(destinations))
|
||||||
|
for _, destination := range destinations {
|
||||||
|
ids = append(ids, destination.ID)
|
||||||
|
}
|
||||||
|
return ids
|
||||||
|
}
|
||||||
|
|
||||||
|
func destinationSummary(destinations []config.Destination) string {
|
||||||
|
if len(destinations) == 0 {
|
||||||
|
return "none"
|
||||||
|
}
|
||||||
|
ids := make([]string, 0, len(destinations))
|
||||||
|
for _, destination := range destinations {
|
||||||
|
ids = append(ids, destination.ID)
|
||||||
|
}
|
||||||
|
return strings.Join(ids, ",")
|
||||||
|
}
|
||||||
78
internal/app/run_summary.go
Normal file
78
internal/app/run_summary.go
Normal file
@@ -0,0 +1,78 @@
|
|||||||
|
package app
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/distributor/internal/publish"
|
||||||
|
)
|
||||||
|
|
||||||
|
type runSummary struct {
|
||||||
|
dryRun bool
|
||||||
|
planned int
|
||||||
|
publishNew int
|
||||||
|
replaceOlder int
|
||||||
|
forceReplace int
|
||||||
|
skipped int
|
||||||
|
failures int
|
||||||
|
fixedPath int
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *runSummary) recordPlan(action publish.Action) {
|
||||||
|
s.planned++
|
||||||
|
switch action {
|
||||||
|
case publish.ActionPublishNew:
|
||||||
|
s.publishNew++
|
||||||
|
case publish.ActionReplaceOlder:
|
||||||
|
s.replaceOlder++
|
||||||
|
case publish.ActionForceReplace:
|
||||||
|
s.forceReplace++
|
||||||
|
case publish.ActionSkipSame, publish.ActionSkipDestinationNewer:
|
||||||
|
s.skipped++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *runSummary) recordFailure() {
|
||||||
|
s.failures++
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *runSummary) recordFixedPath() {
|
||||||
|
s.fixedPath++
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s runSummary) Line() string {
|
||||||
|
status := "ok"
|
||||||
|
if s.failures > 0 {
|
||||||
|
status = "failed"
|
||||||
|
}
|
||||||
|
return fmt.Sprintf("Final status: %s planned=%d publish_new=%d replace_older=%d force_replace=%d skipped=%d failed=%d dry_run=%t fixed_path=%d", status, s.planned, s.publishNew, s.replaceOlder, s.forceReplace, s.skipped, s.failures, s.dryRun, s.fixedPath)
|
||||||
|
}
|
||||||
|
|
||||||
|
type runSummaryResult struct {
|
||||||
|
Status string `json:"status"`
|
||||||
|
Planned int `json:"planned"`
|
||||||
|
PublishNew int `json:"publish_new"`
|
||||||
|
ReplaceOlder int `json:"replace_older"`
|
||||||
|
ForceReplace int `json:"force_replace"`
|
||||||
|
Skipped int `json:"skipped"`
|
||||||
|
Failed int `json:"failed"`
|
||||||
|
DryRun bool `json:"dry_run"`
|
||||||
|
FixedPath int `json:"fixed_path"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s runSummary) Result() runSummaryResult {
|
||||||
|
status := "ok"
|
||||||
|
if s.failures > 0 {
|
||||||
|
status = "failed"
|
||||||
|
}
|
||||||
|
return runSummaryResult{
|
||||||
|
Status: status,
|
||||||
|
Planned: s.planned,
|
||||||
|
PublishNew: s.publishNew,
|
||||||
|
ReplaceOlder: s.replaceOlder,
|
||||||
|
ForceReplace: s.forceReplace,
|
||||||
|
Skipped: s.skipped,
|
||||||
|
Failed: s.failures,
|
||||||
|
DryRun: s.dryRun,
|
||||||
|
FixedPath: s.fixedPath,
|
||||||
|
}
|
||||||
|
}
|
||||||
55
internal/app/run_warnings.go
Normal file
55
internal/app/run_warnings.go
Normal file
@@ -0,0 +1,55 @@
|
|||||||
|
package app
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/distributor/internal/config"
|
||||||
|
)
|
||||||
|
|
||||||
|
func writeSecretConflictWarnings(w io.Writer, conflicts []config.SecretConflict) error {
|
||||||
|
return writeWarnings(w, secretConflictWarnings(conflicts))
|
||||||
|
}
|
||||||
|
|
||||||
|
func secretConflictWarnings(conflicts []config.SecretConflict) []OutputWarning {
|
||||||
|
warnings := make([]OutputWarning, 0, len(conflicts))
|
||||||
|
for _, conflict := range conflicts {
|
||||||
|
warnings = append(warnings, OutputWarning{
|
||||||
|
Message: fmt.Sprintf("secret %s ignored because the real environment already has that variable", conflict.Name),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
return warnings
|
||||||
|
}
|
||||||
|
|
||||||
|
func writeSSHWarnings(w io.Writer, pipeline config.Pipeline) error {
|
||||||
|
return writeWarnings(w, sshWarnings(pipeline))
|
||||||
|
}
|
||||||
|
|
||||||
|
func sshWarnings(pipeline config.Pipeline) []OutputWarning {
|
||||||
|
var warnings []OutputWarning
|
||||||
|
if pipeline.Source.Backend == config.BackendSSH && pipeline.Source.SSH.HostKeyPolicy == config.HostKeyPolicyOff {
|
||||||
|
warnings = append(warnings, OutputWarning{
|
||||||
|
Message: fmt.Sprintf("pipeline=%s source host_key_policy=off disables SSH host key checking", pipeline.ID),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
for _, destination := range pipeline.Destinations {
|
||||||
|
if destination.Backend == config.BackendSSH && destination.SSH.HostKeyPolicy == config.HostKeyPolicyOff {
|
||||||
|
warnings = append(warnings, OutputWarning{
|
||||||
|
Message: fmt.Sprintf("pipeline=%s destination=%s host_key_policy=off disables SSH host key checking", pipeline.ID, destination.ID),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return warnings
|
||||||
|
}
|
||||||
|
|
||||||
|
func writeWarnings(w io.Writer, warnings []OutputWarning) error {
|
||||||
|
if w == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
for _, warning := range warnings {
|
||||||
|
if _, err := fmt.Fprintf(w, "Warning: %s\n", warning.Message); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user