package app import ( "context" "fmt" "io" "gitea.maximumdirect.net/eric/distributor/internal/bundle" "gitea.maximumdirect.net/eric/distributor/internal/config" "gitea.maximumdirect.net/eric/distributor/internal/notify" "gitea.maximumdirect.net/eric/distributor/internal/publish" "gitea.maximumdirect.net/eric/distributor/internal/storage" ) type RunOptions struct { ConfigPath string DryRun bool Force bool Stdout io.Writer OutputFormat OutputFormat Notifier notify.Notifier } type RunPipelineOptions struct { ConfigPath string PipelineID string DryRun bool Force bool Notifier notify.Notifier } type RunPipelineWithLocalSourceOptions struct { ConfigPath string PipelineID string SourceRoot string DryRun bool Force bool Notifier notify.Notifier } func Run(ctx context.Context, options RunOptions) error { if err := ValidateOutputFormat(options.OutputFormat); err != nil { return err } if err := ctx.Err(); err != nil { return err } configPath := options.ConfigPath if configPath == "" { configPath = config.DefaultConfigPath } cfg, err := config.LoadFile(configPath) if err != nil { return err } return runConfig(ctx, cfg, options) } func RunPipeline(ctx context.Context, options RunPipelineOptions) (RunReport, error) { if err := ctx.Err(); err != nil { return RunReport{}, err } configPath := options.ConfigPath if configPath == "" { configPath = config.DefaultConfigPath } cfg, err := config.LoadFile(configPath) if err != nil { return RunReport{}, err } return runPipelineConfig(ctx, cfg, options) } func RunPipelineWithLocalSource(ctx context.Context, options RunPipelineWithLocalSourceOptions) (RunReport, error) { if err := ctx.Err(); err != nil { return RunReport{}, err } if options.SourceRoot == "" { return RunReport{}, fmt.Errorf("source root is required") } configPath := options.ConfigPath if configPath == "" { configPath = config.DefaultConfigPath } cfg, err := config.LoadFile(configPath) if err != nil { return RunReport{}, err } return runPipelineConfigWithLocalSource(ctx, cfg, options) } func runConfig(ctx context.Context, cfg config.Config, options RunOptions) error { return runConfigWithBackendFactory(ctx, cfg, options, newBackendFactoryWithEnvironment) } type backendFactoryProvider func(config.Environment) *backendFactory func runPipelineConfig(ctx context.Context, cfg config.Config, options RunPipelineOptions) (RunReport, error) { return runPipelineConfigWithBackendFactory(ctx, cfg, options, newBackendFactoryWithEnvironment) } func runPipelineConfigWithLocalSource(ctx context.Context, cfg config.Config, options RunPipelineWithLocalSourceOptions) (RunReport, error) { return runPipelineConfigWithLocalSourceAndBackendFactory(ctx, cfg, options, newBackendFactoryWithEnvironment) } func runPipelineConfigWithBackendFactory(ctx context.Context, cfg config.Config, options RunPipelineOptions, provider backendFactoryProvider) (RunReport, error) { pipeline, ok := findPipeline(cfg, options.PipelineID) if !ok { return RunReport{}, PipelineNotFoundError{ID: options.PipelineID} } return buildRunReportWithBackendFactory(ctx, config.Config{ Server: cfg.Server, Secrets: cfg.Secrets, Pipelines: []config.Pipeline{pipeline}, }, RunOptions{ DryRun: options.DryRun, Force: options.Force, Notifier: options.Notifier, }, provider) } func runPipelineConfigWithLocalSourceAndBackendFactory(ctx context.Context, cfg config.Config, options RunPipelineWithLocalSourceOptions, provider backendFactoryProvider) (RunReport, error) { pipeline, ok := findPipeline(cfg, options.PipelineID) if !ok { return RunReport{}, PipelineNotFoundError{ID: options.PipelineID} } return buildRunReport(ctx, config.Config{ Server: cfg.Server, Secrets: cfg.Secrets, Pipelines: []config.Pipeline{pipeline}, }, RunOptions{ DryRun: options.DryRun, Force: options.Force, Notifier: options.Notifier, }, provider, &localSourceRoot{ pipelineID: options.PipelineID, root: options.SourceRoot, }) } func runConfigWithBackendFactory(ctx context.Context, cfg config.Config, options RunOptions, provider backendFactoryProvider) error { report, err := buildRunReportWithBackendFactory(ctx, cfg, options, provider) if err != nil && !IsPartialResultError(err) { return err } if outputErr := WriteRunReport(options.Stdout, options.OutputFormat, report); outputErr != nil { return outputErr } return err } func buildRunReportWithBackendFactory(ctx context.Context, cfg config.Config, options RunOptions, provider backendFactoryProvider) (RunReport, error) { return buildRunReport(ctx, cfg, options, provider, nil) } type localSourceRoot struct { pipelineID string root string } func buildRunReport(ctx context.Context, cfg config.Config, options RunOptions, provider backendFactoryProvider, sourceRoot *localSourceRoot) (RunReport, error) { notifier := options.Notifier if notifier == nil { notifier = notify.Noop{} } summary := runSummary{dryRun: options.DryRun} report := RunReport{ DryRun: options.DryRun, Pipelines: []RunPipelineSummary{}, Actions: []RunActionRecord{}, } var failures runFailures secretLoad, err := config.LoadSecretEnvironment(cfg.Secrets.Directory, nil) if err != nil { return report, err } secretWarnings := secretConflictWarnings(secretLoad.Conflicts) report.PreambleWarnings = append(report.PreambleWarnings, secretWarnings...) report.addWarnings(secretWarnings) backends := provider(secretLoad.Environment) backends.readOnlyKnownHosts = options.DryRun transforms := newTransformRegistry() for _, pipeline := range cfg.Pipelines { pipelineWarnings := sshWarnings(pipeline) report.addWarnings(pipelineWarnings) sourceBackend, bundles, sourceBackendName, err := openPipelineSource(ctx, backends, pipeline, sourceRoot) if err != nil { return report, err } report.Pipelines = append(report.Pipelines, RunPipelineSummary{ ID: pipeline.ID, SourceBackend: sourceBackendName, BundleCount: len(bundles), Destinations: destinationIDs(pipeline.Destinations), Warnings: pipelineWarnings, }) pipelineIndex := len(report.Pipelines) - 1 for _, destination := range pipeline.Destinations { selections := selectDestinationBundles(destination, bundles) if isFixedPathDestination(destination) { summary.recordFixedPath() if options.DryRun { warning := fixedPathSelectionWarning(pipeline.ID, destination.ID, selections, len(bundles)) report.addWarning(warning) report.Pipelines[pipelineIndex].events = append(report.Pipelines[pipelineIndex].events, warningEvent(warning)) } } if len(selections) == 0 { continue } destinationBackend, err := backends.openDestination(ctx, destination) if err != nil { for _, selection := range selections { failures.add(pipeline.ID, destination.ID, destination.Backend, storage.DisplayPath(selection.SourceBundle.RootRelativePath), err) summary.recordFailure() report.Actions = append(report.Actions, errorAction(pipeline.ID, destination.ID, destination.Backend, selection.SourceBundle.RootRelativePath, err)) report.Pipelines[pipelineIndex].events = append(report.Pipelines[pipelineIndex].events, actionEvent(len(report.Actions)-1)) } continue } closeDestination := true deferCloseDestination := func() { if closeDestination { closeBackend(destinationBackend) closeDestination = false } } for _, selection := range selections { sourceBundle := selection.SourceBundle req := publish.Request{ PipelineID: pipeline.ID, DestinationID: destination.ID, SourceBundle: sourceBundle, SourceBackend: sourceBackend, DestinationBackend: destinationBackend, DestinationBundlePath: selection.DestinationBundlePath, PathMapping: destination.PathMap.Mode, Publish: *destination.Publish, Transform: destination.Transform, Links: destination.Links, Transformers: transforms, Transfer: destination.Transfer, DistributorVersion: Version, Force: options.Force, } plan, err := publish.Build(ctx, req) if err != nil { if plan.PipelineID == "" { plan.PipelineID = pipeline.ID } if plan.DestinationID == "" { plan.DestinationID = destination.ID } if plan.BundleID == "" { plan.BundleID = sourceBundle.Manifest.ID } if plan.BundlePath == "" { plan.BundlePath = sourceBundle.RootRelativePath } if plan.DestinationBundlePath == "" { plan.DestinationBundlePath = selection.DestinationBundlePath } } if isFixedPathDestination(destination) { plan.PathMapping = config.PathMappingFixed if options.DryRun && isDestructiveFixedPathAction(plan.Action) { warning := fixedPathReplacementWarning(plan) report.addWarning(warning) report.Pipelines[pipelineIndex].events = append(report.Pipelines[pipelineIndex].events, warningEvent(warning)) } } report.Actions = append(report.Actions, runActionFromPlan(destination.Backend, plan, err)) report.Pipelines[pipelineIndex].events = append(report.Pipelines[pipelineIndex].events, actionEvent(len(report.Actions)-1)) if err != nil { failures.add(pipeline.ID, destination.ID, destination.Backend, storage.DisplayPath(sourceBundle.RootRelativePath), err) summary.recordFailure() continue } summary.recordPlan(plan.Action) if !options.DryRun { if err := publish.Execute(ctx, req, plan); err != nil { failures.add(pipeline.ID, destination.ID, destination.Backend, storage.DisplayPath(sourceBundle.RootRelativePath), err) summary.recordFailure() continue } if shouldNotify(plan.Action) { if err := notifier.Notify(ctx, notifyEvent(plan)); err != nil { failures.add(pipeline.ID, destination.ID, destination.Backend, storage.DisplayPath(sourceBundle.RootRelativePath), err) summary.recordFailure() continue } } } } deferCloseDestination() } closeBackend(sourceBackend) } report.Summary = summary.Result() report.OutputErrors = failures.outputErrors() if len(failures.items) > 0 { return report, failures } return report, nil } func openPipelineSource(ctx context.Context, backends *backendFactory, pipeline config.Pipeline, sourceRoot *localSourceRoot) (storage.Backend, []bundle.Bundle, string, error) { if sourceRoot != nil && sourceRoot.pipelineID == pipeline.ID { sourceBackend, err := backends.openLocalPath(ctx, sourceRoot.root) if err != nil { return nil, nil, config.BackendLocal, fmt.Errorf("pipeline %s source backend %s: %w", pipeline.ID, config.BackendLocal, err) } sourceBundle, err := bundle.Validate(ctx, sourceBackend, "") if err != nil { closeBackend(sourceBackend) return nil, nil, config.BackendLocal, fmt.Errorf("pipeline %s source backend %s validate source bundle: %w", pipeline.ID, config.BackendLocal, err) } return sourceBackend, []bundle.Bundle{sourceBundle}, config.BackendLocal, nil } sourceBackend, err := backends.openSource(ctx, pipeline.Source) if err != nil { return nil, nil, pipeline.Source.Backend, fmt.Errorf("pipeline %s source backend %s: %w", pipeline.ID, pipeline.Source.Backend, err) } bundles, err := bundle.Discover(ctx, sourceBackend, "") if err != nil { closeBackend(sourceBackend) return nil, nil, pipeline.Source.Backend, fmt.Errorf("pipeline %s source backend %s discover source bundles: %w", pipeline.ID, pipeline.Source.Backend, err) } return sourceBackend, bundles, pipeline.Source.Backend, nil } type closeableBackend interface { Close() error } func closeBackend(backend storage.Backend) { closeable, ok := backend.(closeableBackend) if !ok { return } _ = closeable.Close() }