eric/distributor
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 7 Wiki Activity

Harden cross-backend run diagnostics

Browse Source
This commit is contained in:
Eric Rakestraw
2026-05-31 17:17:49 +00:00
parent 14fa9c8000
commit 7a174ce5f1
6 changed files with 262 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files

40
internal/app/run.go
View File

@@ -38,6 +38,12 @@ func Run(ctx context.Context, options RunOptions) error {
}
func runConfig(ctx context.Context, cfg config.Config, options RunOptions) error {
return runConfigWithBackendFactory(ctx, cfg, options, newBackendFactoryWithEnvironment)
}
type backendFactoryProvider func(config.Environment) *backendFactory
func runConfigWithBackendFactory(ctx context.Context, cfg config.Config, options RunOptions, provider backendFactoryProvider) error {
notifier := options.Notifier
if notifier == nil {
notifier = notify.Noop{}
@@ -53,7 +59,7 @@ func runConfig(ctx context.Context, cfg config.Config, options RunOptions) error
return err
}
}
backends := newBackendFactoryWithEnvironment(secretLoad.Environment)
backends := provider(secretLoad.Environment)
transforms := newTransformRegistry()
if options.Stdout != nil {
if _, err := fmt.Fprintf(options.Stdout, "Configured pipelines: %d\n", len(cfg.Pipelines)); err != nil {
@@ -68,12 +74,12 @@ func runConfig(ctx context.Context, cfg config.Config, options RunOptions) error
}
sourceBackend, err := backends.openSource(ctx, pipeline.Source)
if err != nil {
return fmt.Errorf("pipeline %s: %w", pipeline.ID, err)
return fmt.Errorf("pipeline %s source backend %s: %w", pipeline.ID, pipeline.Source.Backend, err)
}
bundles, err := bundle.Discover(ctx, sourceBackend, "")
if err != nil {
closeBackend(sourceBackend)
return fmt.Errorf("pipeline %s discover source bundles: %w", pipeline.ID, err)
return fmt.Errorf("pipeline %s source backend %s discover source bundles: %w", pipeline.ID, pipeline.Source.Backend, err)
}
if options.Stdout != nil {
if _, err := fmt.Fprintf(options.Stdout, "- pipeline=%s source=%s bundles=%d destinations=%s\n", pipeline.ID, pipeline.Source.Backend, len(bundles), destinationSummary(pipeline.Destinations)); err != nil {
@@ -85,10 +91,10 @@ func runConfig(ctx context.Context, cfg config.Config, options RunOptions) error
for _, destination := range pipeline.Destinations {
destinationBackend, err := backends.openDestination(ctx, destination)
if err != nil {
failures.add(pipeline.ID, destination.ID, storage.DisplayPath(sourceBundle.RootRelativePath), err)
failures.add(pipeline.ID, destination.ID, destination.Backend, storage.DisplayPath(sourceBundle.RootRelativePath), err)
summary.recordFailure()
if options.Stdout != nil {
writeErrorLine(options.Stdout, sourceBundle.RootRelativePath, destination.ID, err)
writeErrorLine(options.Stdout, sourceBundle.RootRelativePath, destination.ID, destination.Backend, err)
}
continue
}
@@ -117,11 +123,11 @@ func runConfig(ctx context.Context, cfg config.Config, options RunOptions) error
plan = publish.Plan{DestinationID: destination.ID, BundlePath: sourceBundle.RootRelativePath}
}
if options.Stdout != nil {
writePlanLine(options.Stdout, plan, err)
writePlanLine(options.Stdout, destination.Backend, plan, err)
}
if err != nil {
deferCloseDestination()
failures.add(pipeline.ID, destination.ID, storage.DisplayPath(sourceBundle.RootRelativePath), err)
failures.add(pipeline.ID, destination.ID, destination.Backend, storage.DisplayPath(sourceBundle.RootRelativePath), err)
summary.recordFailure()
continue
}
@@ -129,14 +135,14 @@ func runConfig(ctx context.Context, cfg config.Config, options RunOptions) error
if !options.DryRun {
if err := publish.Execute(ctx, req, plan); err != nil {
deferCloseDestination()
failures.add(pipeline.ID, destination.ID, storage.DisplayPath(sourceBundle.RootRelativePath), err)
failures.add(pipeline.ID, destination.ID, destination.Backend, storage.DisplayPath(sourceBundle.RootRelativePath), err)
summary.recordFailure()
continue
}
if shouldNotify(plan.Action) {
if err := notifier.Notify(ctx, notifyEvent(plan)); err != nil {
deferCloseDestination()
failures.add(pipeline.ID, destination.ID, storage.DisplayPath(sourceBundle.RootRelativePath), err)
failures.add(pipeline.ID, destination.ID, destination.Backend, storage.DisplayPath(sourceBundle.RootRelativePath), err)
summary.recordFailure()
continue
}
@@ -170,7 +176,7 @@ func closeBackend(backend storage.Backend) {
_ = closeable.Close()
}
func writePlanLine(w io.Writer, plan publish.Plan, planErr error) {
func writePlanLine(w io.Writer, backend string, plan publish.Plan, planErr error) {
if w == nil {
return
}
@@ -179,17 +185,17 @@ func writePlanLine(w io.Writer, plan publish.Plan, planErr error) {
if destinationID == "" {
destinationID = "unknown"
}
fmt.Fprintf(w, " - bundle=%s destination=%s action=error reason=%q\n", storage.DisplayPath(plan.BundlePath), destinationID, planErr.Error())
fmt.Fprintf(w, " - bundle=%s destination=%s backend=%s action=error reason=%q\n", storage.DisplayPath(plan.BundlePath), destinationID, backend, planErr.Error())
return
}
fmt.Fprintf(w, " - bundle=%s destination=%s action=%s outputs=%s reason=%q\n", storage.DisplayPath(plan.BundlePath), plan.DestinationID, plan.Action, outputSummary(plan.Outputs), plan.Reason)
fmt.Fprintf(w, " - bundle=%s destination=%s backend=%s action=%s outputs=%s reason=%q\n", storage.DisplayPath(plan.BundlePath), plan.DestinationID, backend, plan.Action, outputSummary(plan.Outputs), plan.Reason)
}
func writeErrorLine(w io.Writer, bundlePath, destinationID string, err error) {
func writeErrorLine(w io.Writer, bundlePath, destinationID, backend string, err error) {
if w == nil {
return
}
fmt.Fprintf(w, " - bundle=%s destination=%s action=error reason=%q\n", storage.DisplayPath(bundlePath), destinationID, err.Error())
fmt.Fprintf(w, " - bundle=%s destination=%s backend=%s action=error reason=%q\n", storage.DisplayPath(bundlePath), destinationID, backend, err.Error())
}
func outputSummary(outputs []publish.Output) string {
@@ -301,6 +307,7 @@ func (s runSummary) Line() string {
type runFailure struct {
pipelineID string
destinationID string
backend string
bundlePath string
err error
}
@@ -309,10 +316,11 @@ type runFailures struct {
items []runFailure
}
func (f *runFailures) add(pipelineID, destinationID, bundlePath string, err error) {
func (f *runFailures) add(pipelineID, destinationID, backend, bundlePath string, err error) {
f.items = append(f.items, runFailure{
pipelineID: pipelineID,
destinationID: destinationID,
backend: backend,
bundlePath: bundlePath,
err: err,
})
@@ -324,7 +332,7 @@ func (f runFailures) Error() string {
}
parts := make([]string, 0, len(f.items))
for _, item := range f.items {
parts = append(parts, fmt.Sprintf("pipeline %s destination %s bundle %s: %v", item.pipelineID, item.destinationID, item.bundlePath, item.err))
parts = append(parts, fmt.Sprintf("pipeline %s destination %s backend %s bundle %s: %v", item.pipelineID, item.destinationID, item.backend, item.bundlePath, item.err))
}
return "run failed: " + strings.Join(parts, "; ")
}