347 lines
12 KiB
Go
347 lines
12 KiB
Go
package app
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"io"
|
|
|
|
"gitea.maximumdirect.net/eric/distributor/internal/bundle"
|
|
"gitea.maximumdirect.net/eric/distributor/internal/config"
|
|
"gitea.maximumdirect.net/eric/distributor/internal/notify"
|
|
"gitea.maximumdirect.net/eric/distributor/internal/publish"
|
|
"gitea.maximumdirect.net/eric/distributor/internal/storage"
|
|
)
|
|
|
|
type RunOptions struct {
|
|
ConfigPath string
|
|
DryRun bool
|
|
Force bool
|
|
Stdout io.Writer
|
|
OutputFormat OutputFormat
|
|
Notifier notify.Notifier
|
|
}
|
|
|
|
type RunPipelineOptions struct {
|
|
ConfigPath string
|
|
PipelineID string
|
|
DryRun bool
|
|
Force bool
|
|
Notifier notify.Notifier
|
|
}
|
|
|
|
type RunPipelineWithLocalSourceOptions struct {
|
|
ConfigPath string
|
|
PipelineID string
|
|
SourceRoot string
|
|
DryRun bool
|
|
Force bool
|
|
Notifier notify.Notifier
|
|
}
|
|
|
|
func Run(ctx context.Context, options RunOptions) error {
|
|
if err := ValidateOutputFormat(options.OutputFormat); err != nil {
|
|
return err
|
|
}
|
|
if err := ctx.Err(); err != nil {
|
|
return err
|
|
}
|
|
|
|
configPath := options.ConfigPath
|
|
if configPath == "" {
|
|
configPath = config.DefaultConfigPath
|
|
}
|
|
cfg, err := config.LoadFile(configPath)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return runConfig(ctx, cfg, options)
|
|
}
|
|
|
|
func RunPipeline(ctx context.Context, options RunPipelineOptions) (RunReport, error) {
|
|
if err := ctx.Err(); err != nil {
|
|
return RunReport{}, err
|
|
}
|
|
|
|
configPath := options.ConfigPath
|
|
if configPath == "" {
|
|
configPath = config.DefaultConfigPath
|
|
}
|
|
cfg, err := config.LoadFile(configPath)
|
|
if err != nil {
|
|
return RunReport{}, err
|
|
}
|
|
return runPipelineConfig(ctx, cfg, options)
|
|
}
|
|
|
|
func RunPipelineWithLocalSource(ctx context.Context, options RunPipelineWithLocalSourceOptions) (RunReport, error) {
|
|
if err := ctx.Err(); err != nil {
|
|
return RunReport{}, err
|
|
}
|
|
if options.SourceRoot == "" {
|
|
return RunReport{}, fmt.Errorf("source root is required")
|
|
}
|
|
|
|
configPath := options.ConfigPath
|
|
if configPath == "" {
|
|
configPath = config.DefaultConfigPath
|
|
}
|
|
cfg, err := config.LoadFile(configPath)
|
|
if err != nil {
|
|
return RunReport{}, err
|
|
}
|
|
return runPipelineConfigWithLocalSource(ctx, cfg, options)
|
|
}
|
|
|
|
func runConfig(ctx context.Context, cfg config.Config, options RunOptions) error {
|
|
return runConfigWithBackendFactory(ctx, cfg, options, newBackendFactoryWithEnvironment)
|
|
}
|
|
|
|
type backendFactoryProvider func(config.Environment) *backendFactory
|
|
|
|
func runPipelineConfig(ctx context.Context, cfg config.Config, options RunPipelineOptions) (RunReport, error) {
|
|
return runPipelineConfigWithBackendFactory(ctx, cfg, options, newBackendFactoryWithEnvironment)
|
|
}
|
|
|
|
func runPipelineConfigWithLocalSource(ctx context.Context, cfg config.Config, options RunPipelineWithLocalSourceOptions) (RunReport, error) {
|
|
return runPipelineConfigWithLocalSourceAndBackendFactory(ctx, cfg, options, newBackendFactoryWithEnvironment)
|
|
}
|
|
|
|
func runPipelineConfigWithBackendFactory(ctx context.Context, cfg config.Config, options RunPipelineOptions, provider backendFactoryProvider) (RunReport, error) {
|
|
pipeline, ok := findPipeline(cfg, options.PipelineID)
|
|
if !ok {
|
|
return RunReport{}, PipelineNotFoundError{ID: options.PipelineID}
|
|
}
|
|
return buildRunReportWithBackendFactory(ctx, config.Config{
|
|
Server: cfg.Server,
|
|
Secrets: cfg.Secrets,
|
|
Pipelines: []config.Pipeline{pipeline},
|
|
}, RunOptions{
|
|
DryRun: options.DryRun,
|
|
Force: options.Force,
|
|
Notifier: options.Notifier,
|
|
}, provider)
|
|
}
|
|
|
|
func runPipelineConfigWithLocalSourceAndBackendFactory(ctx context.Context, cfg config.Config, options RunPipelineWithLocalSourceOptions, provider backendFactoryProvider) (RunReport, error) {
|
|
pipeline, ok := findPipeline(cfg, options.PipelineID)
|
|
if !ok {
|
|
return RunReport{}, PipelineNotFoundError{ID: options.PipelineID}
|
|
}
|
|
return buildRunReport(ctx, config.Config{
|
|
Server: cfg.Server,
|
|
Secrets: cfg.Secrets,
|
|
Pipelines: []config.Pipeline{pipeline},
|
|
}, RunOptions{
|
|
DryRun: options.DryRun,
|
|
Force: options.Force,
|
|
Notifier: options.Notifier,
|
|
}, provider, &localSourceRoot{
|
|
pipelineID: options.PipelineID,
|
|
root: options.SourceRoot,
|
|
})
|
|
}
|
|
|
|
func runConfigWithBackendFactory(ctx context.Context, cfg config.Config, options RunOptions, provider backendFactoryProvider) error {
|
|
report, err := buildRunReportWithBackendFactory(ctx, cfg, options, provider)
|
|
if err != nil && !IsPartialResultError(err) {
|
|
return err
|
|
}
|
|
if outputErr := WriteRunReport(options.Stdout, options.OutputFormat, report); outputErr != nil {
|
|
return outputErr
|
|
}
|
|
return err
|
|
}
|
|
|
|
func buildRunReportWithBackendFactory(ctx context.Context, cfg config.Config, options RunOptions, provider backendFactoryProvider) (RunReport, error) {
|
|
return buildRunReport(ctx, cfg, options, provider, nil)
|
|
}
|
|
|
|
type localSourceRoot struct {
|
|
pipelineID string
|
|
root string
|
|
}
|
|
|
|
func buildRunReport(ctx context.Context, cfg config.Config, options RunOptions, provider backendFactoryProvider, sourceRoot *localSourceRoot) (RunReport, error) {
|
|
notifier := options.Notifier
|
|
if notifier == nil {
|
|
notifier = notify.Noop{}
|
|
}
|
|
summary := runSummary{dryRun: options.DryRun}
|
|
report := RunReport{
|
|
DryRun: options.DryRun,
|
|
Pipelines: []RunPipelineSummary{},
|
|
Actions: []RunActionRecord{},
|
|
}
|
|
var failures runFailures
|
|
secretLoad, err := config.LoadSecretEnvironment(cfg.Secrets.Directory, nil)
|
|
if err != nil {
|
|
return report, err
|
|
}
|
|
secretWarnings := secretConflictWarnings(secretLoad.Conflicts)
|
|
report.PreambleWarnings = append(report.PreambleWarnings, secretWarnings...)
|
|
report.addWarnings(secretWarnings)
|
|
backends := provider(secretLoad.Environment)
|
|
backends.readOnlyKnownHosts = options.DryRun
|
|
transforms := newTransformRegistry()
|
|
for _, pipeline := range cfg.Pipelines {
|
|
pipelineWarnings := sshWarnings(pipeline)
|
|
report.addWarnings(pipelineWarnings)
|
|
sourceBackend, bundles, sourceBackendName, err := openPipelineSource(ctx, backends, pipeline, sourceRoot)
|
|
if err != nil {
|
|
return report, err
|
|
}
|
|
report.Pipelines = append(report.Pipelines, RunPipelineSummary{
|
|
ID: pipeline.ID,
|
|
SourceBackend: sourceBackendName,
|
|
BundleCount: len(bundles),
|
|
Destinations: destinationIDs(pipeline.Destinations),
|
|
Warnings: pipelineWarnings,
|
|
})
|
|
pipelineIndex := len(report.Pipelines) - 1
|
|
for _, destination := range pipeline.Destinations {
|
|
selections := selectDestinationBundles(destination, bundles)
|
|
if isFixedPathDestination(destination) {
|
|
summary.recordFixedPath()
|
|
if options.DryRun {
|
|
warning := fixedPathSelectionWarning(pipeline.ID, destination.ID, selections, len(bundles))
|
|
report.addWarning(warning)
|
|
report.Pipelines[pipelineIndex].events = append(report.Pipelines[pipelineIndex].events, warningEvent(warning))
|
|
}
|
|
}
|
|
if len(selections) == 0 {
|
|
continue
|
|
}
|
|
destinationBackend, err := backends.openDestination(ctx, destination)
|
|
if err != nil {
|
|
for _, selection := range selections {
|
|
failures.add(pipeline.ID, destination.ID, destination.Backend, storage.DisplayPath(selection.SourceBundle.RootRelativePath), err)
|
|
summary.recordFailure()
|
|
report.Actions = append(report.Actions, errorAction(pipeline.ID, destination.ID, destination.Backend, selection.SourceBundle.RootRelativePath, err))
|
|
report.Pipelines[pipelineIndex].events = append(report.Pipelines[pipelineIndex].events, actionEvent(len(report.Actions)-1))
|
|
}
|
|
continue
|
|
}
|
|
closeDestination := true
|
|
deferCloseDestination := func() {
|
|
if closeDestination {
|
|
closeBackend(destinationBackend)
|
|
closeDestination = false
|
|
}
|
|
}
|
|
for _, selection := range selections {
|
|
sourceBundle := selection.SourceBundle
|
|
req := publish.Request{
|
|
PipelineID: pipeline.ID,
|
|
DestinationID: destination.ID,
|
|
SourceBundle: sourceBundle,
|
|
SourceBackend: sourceBackend,
|
|
DestinationBackend: destinationBackend,
|
|
DestinationBundlePath: selection.DestinationBundlePath,
|
|
PathMapping: destination.PathMap.Mode,
|
|
Publish: *destination.Publish,
|
|
Transform: destination.Transform,
|
|
Links: destination.Links,
|
|
Transformers: transforms,
|
|
Transfer: destination.Transfer,
|
|
DistributorVersion: Version,
|
|
Force: options.Force,
|
|
}
|
|
plan, err := publish.Build(ctx, req)
|
|
if err != nil {
|
|
if plan.PipelineID == "" {
|
|
plan.PipelineID = pipeline.ID
|
|
}
|
|
if plan.DestinationID == "" {
|
|
plan.DestinationID = destination.ID
|
|
}
|
|
if plan.BundleID == "" {
|
|
plan.BundleID = sourceBundle.Manifest.ID
|
|
}
|
|
if plan.BundlePath == "" {
|
|
plan.BundlePath = sourceBundle.RootRelativePath
|
|
}
|
|
if plan.DestinationBundlePath == "" {
|
|
plan.DestinationBundlePath = selection.DestinationBundlePath
|
|
}
|
|
}
|
|
if isFixedPathDestination(destination) {
|
|
plan.PathMapping = config.PathMappingFixed
|
|
if options.DryRun && isDestructiveFixedPathAction(plan.Action) {
|
|
warning := fixedPathReplacementWarning(plan)
|
|
report.addWarning(warning)
|
|
report.Pipelines[pipelineIndex].events = append(report.Pipelines[pipelineIndex].events, warningEvent(warning))
|
|
}
|
|
}
|
|
report.Actions = append(report.Actions, runActionFromPlan(destination.Backend, plan, err))
|
|
report.Pipelines[pipelineIndex].events = append(report.Pipelines[pipelineIndex].events, actionEvent(len(report.Actions)-1))
|
|
if err != nil {
|
|
failures.add(pipeline.ID, destination.ID, destination.Backend, storage.DisplayPath(sourceBundle.RootRelativePath), err)
|
|
summary.recordFailure()
|
|
continue
|
|
}
|
|
summary.recordPlan(plan.Action)
|
|
if !options.DryRun {
|
|
if err := publish.Execute(ctx, req, plan); err != nil {
|
|
failures.add(pipeline.ID, destination.ID, destination.Backend, storage.DisplayPath(sourceBundle.RootRelativePath), err)
|
|
summary.recordFailure()
|
|
continue
|
|
}
|
|
if shouldNotify(plan.Action) {
|
|
if err := notifier.Notify(ctx, notifyEvent(plan)); err != nil {
|
|
failures.add(pipeline.ID, destination.ID, destination.Backend, storage.DisplayPath(sourceBundle.RootRelativePath), err)
|
|
summary.recordFailure()
|
|
continue
|
|
}
|
|
}
|
|
}
|
|
}
|
|
deferCloseDestination()
|
|
}
|
|
closeBackend(sourceBackend)
|
|
}
|
|
report.Summary = summary.Result()
|
|
report.OutputErrors = failures.outputErrors()
|
|
if len(failures.items) > 0 {
|
|
return report, failures
|
|
}
|
|
return report, nil
|
|
}
|
|
|
|
func openPipelineSource(ctx context.Context, backends *backendFactory, pipeline config.Pipeline, sourceRoot *localSourceRoot) (storage.Backend, []bundle.Bundle, string, error) {
|
|
if sourceRoot != nil && sourceRoot.pipelineID == pipeline.ID {
|
|
sourceBackend, err := backends.openLocalPath(ctx, sourceRoot.root)
|
|
if err != nil {
|
|
return nil, nil, config.BackendLocal, fmt.Errorf("pipeline %s source backend %s: %w", pipeline.ID, config.BackendLocal, err)
|
|
}
|
|
sourceBundle, err := bundle.Validate(ctx, sourceBackend, "")
|
|
if err != nil {
|
|
closeBackend(sourceBackend)
|
|
return nil, nil, config.BackendLocal, fmt.Errorf("pipeline %s source backend %s validate source bundle: %w", pipeline.ID, config.BackendLocal, err)
|
|
}
|
|
return sourceBackend, []bundle.Bundle{sourceBundle}, config.BackendLocal, nil
|
|
}
|
|
|
|
sourceBackend, err := backends.openSource(ctx, pipeline.Source)
|
|
if err != nil {
|
|
return nil, nil, pipeline.Source.Backend, fmt.Errorf("pipeline %s source backend %s: %w", pipeline.ID, pipeline.Source.Backend, err)
|
|
}
|
|
bundles, err := bundle.Discover(ctx, sourceBackend, "")
|
|
if err != nil {
|
|
closeBackend(sourceBackend)
|
|
return nil, nil, pipeline.Source.Backend, fmt.Errorf("pipeline %s source backend %s discover source bundles: %w", pipeline.ID, pipeline.Source.Backend, err)
|
|
}
|
|
return sourceBackend, bundles, pipeline.Source.Backend, nil
|
|
}
|
|
|
|
type closeableBackend interface {
|
|
Close() error
|
|
}
|
|
|
|
func closeBackend(backend storage.Backend) {
|
|
closeable, ok := backend.(closeableBackend)
|
|
if !ok {
|
|
return
|
|
}
|
|
_ = closeable.Close()
|
|
}
|