eric/distributor
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 7 Wiki Activity

Add local source pipeline execution

Browse Source
This commit is contained in:
Eric Rakestraw
2026-06-03 15:10:07 +00:00
parent 65dd22f974
commit f9436a7423
3 changed files with 250 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

98
internal/app/run.go
View File

@@ -29,6 +29,15 @@ type RunPipelineOptions struct {
Notifier notify.Notifier
}
type RunPipelineWithLocalSourceOptions struct {
ConfigPath string
PipelineID string
SourceRoot string
DryRun bool
Force bool
Notifier notify.Notifier
}
func Run(ctx context.Context, options RunOptions) error {
if err := ValidateOutputFormat(options.OutputFormat); err != nil {
return err
@@ -64,6 +73,25 @@ func RunPipeline(ctx context.Context, options RunPipelineOptions) (RunReport, er
return runPipelineConfig(ctx, cfg, options)
}
func RunPipelineWithLocalSource(ctx context.Context, options RunPipelineWithLocalSourceOptions) (RunReport, error) {
if err := ctx.Err(); err != nil {
return RunReport{}, err
}
if options.SourceRoot == "" {
return RunReport{}, fmt.Errorf("source root is required")
}
configPath := options.ConfigPath
if configPath == "" {
configPath = config.DefaultConfigPath
}
cfg, err := config.LoadFile(configPath)
if err != nil {
return RunReport{}, err
}
return runPipelineConfigWithLocalSource(ctx, cfg, options)
}
func runConfig(ctx context.Context, cfg config.Config, options RunOptions) error {
return runConfigWithBackendFactory(ctx, cfg, options, newBackendFactoryWithEnvironment)
}
@@ -74,12 +102,17 @@ func runPipelineConfig(ctx context.Context, cfg config.Config, options RunPipeli
return runPipelineConfigWithBackendFactory(ctx, cfg, options, newBackendFactoryWithEnvironment)
}
func runPipelineConfigWithLocalSource(ctx context.Context, cfg config.Config, options RunPipelineWithLocalSourceOptions) (RunReport, error) {
return runPipelineConfigWithLocalSourceAndBackendFactory(ctx, cfg, options, newBackendFactoryWithEnvironment)
}
func runPipelineConfigWithBackendFactory(ctx context.Context, cfg config.Config, options RunPipelineOptions, provider backendFactoryProvider) (RunReport, error) {
pipeline, ok := findPipeline(cfg, options.PipelineID)
if !ok {
return RunReport{}, PipelineNotFoundError{ID: options.PipelineID}
}
return buildRunReportWithBackendFactory(ctx, config.Config{
Server: cfg.Server,
Secrets: cfg.Secrets,
Pipelines: []config.Pipeline{pipeline},
}, RunOptions{
@@ -89,6 +122,25 @@ func runPipelineConfigWithBackendFactory(ctx context.Context, cfg config.Config,
}, provider)
}
func runPipelineConfigWithLocalSourceAndBackendFactory(ctx context.Context, cfg config.Config, options RunPipelineWithLocalSourceOptions, provider backendFactoryProvider) (RunReport, error) {
pipeline, ok := findPipeline(cfg, options.PipelineID)
if !ok {
return RunReport{}, PipelineNotFoundError{ID: options.PipelineID}
}
return buildRunReport(ctx, config.Config{
Server: cfg.Server,
Secrets: cfg.Secrets,
Pipelines: []config.Pipeline{pipeline},
}, RunOptions{
DryRun: options.DryRun,
Force: options.Force,
Notifier: options.Notifier,
}, provider, &localSourceRoot{
pipelineID: options.PipelineID,
root: options.SourceRoot,
})
}
func runConfigWithBackendFactory(ctx context.Context, cfg config.Config, options RunOptions, provider backendFactoryProvider) error {
report, err := buildRunReportWithBackendFactory(ctx, cfg, options, provider)
if err != nil && !IsPartialResultError(err) {
@@ -101,6 +153,15 @@ func runConfigWithBackendFactory(ctx context.Context, cfg config.Config, options
}
func buildRunReportWithBackendFactory(ctx context.Context, cfg config.Config, options RunOptions, provider backendFactoryProvider) (RunReport, error) {
return buildRunReport(ctx, cfg, options, provider, nil)
}
type localSourceRoot struct {
pipelineID string
root string
}
func buildRunReport(ctx context.Context, cfg config.Config, options RunOptions, provider backendFactoryProvider, sourceRoot *localSourceRoot) (RunReport, error) {
notifier := options.Notifier
if notifier == nil {
notifier = notify.Noop{}
@@ -125,18 +186,13 @@ func buildRunReportWithBackendFactory(ctx context.Context, cfg config.Config, op
for _, pipeline := range cfg.Pipelines {
pipelineWarnings := sshWarnings(pipeline)
report.addWarnings(pipelineWarnings)
sourceBackend, err := backends.openSource(ctx, pipeline.Source)
sourceBackend, bundles, sourceBackendName, err := openPipelineSource(ctx, backends, pipeline, sourceRoot)
if err != nil {
return report, fmt.Errorf("pipeline %s source backend %s: %w", pipeline.ID, pipeline.Source.Backend, err)
}
bundles, err := bundle.Discover(ctx, sourceBackend, "")
if err != nil {
closeBackend(sourceBackend)
return report, fmt.Errorf("pipeline %s source backend %s discover source bundles: %w", pipeline.ID, pipeline.Source.Backend, err)
return report, err
}
report.Pipelines = append(report.Pipelines, RunPipelineSummary{
ID: pipeline.ID,
SourceBackend: pipeline.Source.Backend,
SourceBackend: sourceBackendName,
BundleCount: len(bundles),
Destinations: destinationIDs(pipeline.Destinations),
Warnings: pipelineWarnings,
@@ -251,6 +307,32 @@ func buildRunReportWithBackendFactory(ctx context.Context, cfg config.Config, op
return report, nil
}
func openPipelineSource(ctx context.Context, backends *backendFactory, pipeline config.Pipeline, sourceRoot *localSourceRoot) (storage.Backend, []bundle.Bundle, string, error) {
if sourceRoot != nil && sourceRoot.pipelineID == pipeline.ID {
sourceBackend, err := backends.openLocalPath(ctx, sourceRoot.root)
if err != nil {
return nil, nil, config.BackendLocal, fmt.Errorf("pipeline %s source backend %s: %w", pipeline.ID, config.BackendLocal, err)
}
sourceBundle, err := bundle.Validate(ctx, sourceBackend, "")
if err != nil {
closeBackend(sourceBackend)
return nil, nil, config.BackendLocal, fmt.Errorf("pipeline %s source backend %s validate source bundle: %w", pipeline.ID, config.BackendLocal, err)
}
return sourceBackend, []bundle.Bundle{sourceBundle}, config.BackendLocal, nil
}
sourceBackend, err := backends.openSource(ctx, pipeline.Source)
if err != nil {
return nil, nil, pipeline.Source.Backend, fmt.Errorf("pipeline %s source backend %s: %w", pipeline.ID, pipeline.Source.Backend, err)
}
bundles, err := bundle.Discover(ctx, sourceBackend, "")
if err != nil {
closeBackend(sourceBackend)
return nil, nil, pipeline.Source.Backend, fmt.Errorf("pipeline %s source backend %s discover source bundles: %w", pipeline.ID, pipeline.Source.Backend, err)
}
return sourceBackend, bundles, pipeline.Source.Backend, nil
}
type closeableBackend interface {
Close() error
}