eric/narratio
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 15 Wiki Activity

Terminalize handled invocation failures

Browse Source
This commit is contained in:
Eric Rakestraw
2026-08-10 19:34:46 +00:00
parent a1ceb457e9
commit ee747243fe
6 changed files with 341 additions and 74 deletions
Download Patch File Download Diff File Expand all files Collapse all files

238
internal/app/runner.go
View File

@@ -26,6 +26,7 @@ type RunOptions struct {
Force bool
SelectedArtifacts []string
Env *Env
RunManifestStore manifest.RunStore
}
type RunSummary struct {
@@ -79,53 +80,6 @@ func executeStages(ctx context.Context, cfg *config.Config, stages []stage.Stage
if env.Logger == nil {
env.Logger = logging.NewLogger(os.Stderr, slog.LevelInfo)
}
if _, err := loadSecretsFromConfig(env.Config, env.Logger); err != nil {
return nil, fmt.Errorf("load secrets from files: %w", err)
}
if env.WhisperX == nil {
client, err := buildDefaultWhisperXClient(env.Config)
if err != nil {
return nil, fmt.Errorf("initialize whisperx client: %w", err)
}
env.WhisperX = client
}
if env.Seriatim == nil {
runner, err := buildDefaultSeriatimRunner(env.Config)
if err != nil {
return nil, fmt.Errorf("initialize seriatim runner: %w", err)
}
env.Seriatim = runner
}
if env.Audita == nil {
runner, err := buildDefaultAuditaRunner(env.Config)
if err != nil {
return nil, fmt.Errorf("initialize audita runner: %w", err)
}
env.Audita = runner
}
if env.Notarius == nil && needsNotariusForRun(env.Config, stages) {
env.Notarius = notarius.NewSubprocessRunner()
}
if env.Scriptorium == nil {
env.Scriptorium = scriptorium.NewSubprocessRunner()
}
if env.ObjectStore == nil && needsObjectStoreForRun(env.Config, stages) {
objectStore, err := newCommandObjectStore(ctx, env.Config, nil)
if err != nil {
return nil, err
}
env.ObjectStore = objectStore
}
if needsRemoteLocksForRun(env.Config, stages) {
locks, err := loadEffectiveLocks(ctx, env.Config, env.ObjectStore)
if err != nil {
return nil, fmt.Errorf("load remote publish locks: %w", err)
}
applyEffectiveLocks(env.Config, locks.All)
}
if env.Notifier == nil {
env.Notifier = &notify.NoopSender{}
}
artifactStore := env.ArtifactStore
paths, err := artifactStore.EnsureLayoutFor(identity.Campaign, identity.SessionID)
@@ -174,7 +128,10 @@ func executeStages(ctx context.Context, cfg *config.Config, stages []stage.Stage
identity.SessionID,
identity.RunID,
)
runManifestStore := &manifest.LocalStore{}
runManifestStore := opts.RunManifestStore
if runManifestStore == nil {
runManifestStore = &manifest.LocalStore{}
}
runManifest, err := runManifestStore.CreateRun(
ctx,
identity.SessionID,
@@ -188,7 +145,74 @@ func executeStages(ctx context.Context, cfg *config.Config, stages []stage.Stage
}
identity.applyToRunManifest(runManifest, manifestPath)
if err := runManifestStore.SaveRun(ctx, runManifestPath, runManifest); err != nil {
return nil, fmt.Errorf("save initial run manifest %q: %w", runManifestPath, err)
return nil, persistTerminalFailure(
ctx, env.ManifestStore, manifestPath, m, runManifestStore, runManifestPath, runManifest,
fmt.Errorf("save initial run manifest %q: %w", runManifestPath, err),
)
}
if _, err := loadSecretsFromConfig(env.Config, env.Logger); err != nil {
return nil, persistTerminalFailure(
ctx, env.ManifestStore, manifestPath, m, runManifestStore, runManifestPath, runManifest,
fmt.Errorf("load secrets from files: %w", err),
)
}
if env.WhisperX == nil {
client, err := buildDefaultWhisperXClient(env.Config)
if err != nil {
return nil, persistTerminalFailure(
ctx, env.ManifestStore, manifestPath, m, runManifestStore, runManifestPath, runManifest,
fmt.Errorf("initialize whisperx client: %w", err),
)
}
env.WhisperX = client
}
if env.Seriatim == nil {
runner, err := buildDefaultSeriatimRunner(env.Config)
if err != nil {
return nil, persistTerminalFailure(
ctx, env.ManifestStore, manifestPath, m, runManifestStore, runManifestPath, runManifest,
fmt.Errorf("initialize seriatim runner: %w", err),
)
}
env.Seriatim = runner
}
if env.Audita == nil {
runner, err := buildDefaultAuditaRunner(env.Config)
if err != nil {
return nil, persistTerminalFailure(
ctx, env.ManifestStore, manifestPath, m, runManifestStore, runManifestPath, runManifest,
fmt.Errorf("initialize audita runner: %w", err),
)
}
env.Audita = runner
}
if env.Notarius == nil && needsNotariusForRun(env.Config, stages) {
env.Notarius = notarius.NewSubprocessRunner()
}
if env.Scriptorium == nil {
env.Scriptorium = scriptorium.NewSubprocessRunner()
}
if env.ObjectStore == nil && needsObjectStoreForRun(env.Config, stages) {
objectStore, err := newCommandObjectStore(ctx, env.Config, nil)
if err != nil {
return nil, persistTerminalFailure(
ctx, env.ManifestStore, manifestPath, m, runManifestStore, runManifestPath, runManifest, err,
)
}
env.ObjectStore = objectStore
}
if needsRemoteLocksForRun(env.Config, stages) {
locks, err := loadEffectiveLocks(ctx, env.Config, env.ObjectStore)
if err != nil {
return nil, persistTerminalFailure(
ctx, env.ManifestStore, manifestPath, m, runManifestStore, runManifestPath, runManifest,
fmt.Errorf("load remote publish locks: %w", err),
)
}
applyEffectiveLocks(env.Config, locks.All)
}
if env.Notifier == nil {
env.Notifier = &notify.NoopSender{}
}
stageEnv := env
@@ -207,7 +231,10 @@ func executeStages(ctx context.Context, cfg *config.Config, stages []stage.Stage
if validator, ok := s.(stage.ResumeValidator); ok {
validation, err := validator.ValidateResume(ctx, stageEnv, m)
if err != nil {
return nil, fmt.Errorf("validate resume for stage %q: %w", s.Name(), err)
return nil, persistTerminalFailure(
ctx, env.ManifestStore, manifestPath, m, runManifestStore, runManifestPath, runManifest,
fmt.Errorf("validate resume for stage %q: %w", s.Name(), err),
)
}
validation = validation.Normalized()
if !validation.Resumable {
@@ -217,7 +244,10 @@ func executeStages(ctx context.Context, cfg *config.Config, stages []stage.Stage
m, s.Name(), staleAt, staleReasonNotResumable,
)
if err := env.ManifestStore.Save(ctx, manifestPath, m); err != nil {
return nil, fmt.Errorf("save manifest after resume validation for stage %q: %w", s.Name(), err)
return nil, persistTerminalFailure(
ctx, env.ManifestStore, manifestPath, m, runManifestStore, runManifestPath, runManifest,
fmt.Errorf("save manifest after resume validation for stage %q: %w", s.Name(), err),
)
}
env.Logger.Info("stage result is not resumable", "stage", s.Name(), "reason", validation.Reason)
d.Action = stageActionRun
@@ -231,7 +261,10 @@ func executeStages(ctx context.Context, cfg *config.Config, stages []stage.Stage
runManifest.SetStageAction(s.Name(), manifest.RunStageActionSkip, skipAt)
runManifest.MarkStageSkipped(s.Name(), skipAt, "already_succeeded")
if err := runManifestStore.SaveRun(ctx, runManifestPath, runManifest); err != nil {
return nil, fmt.Errorf("save run manifest after skip %q: %w", s.Name(), err)
return nil, persistTerminalFailure(
ctx, env.ManifestStore, manifestPath, m, runManifestStore, runManifestPath, runManifest,
fmt.Errorf("save run manifest after skip %q: %w", s.Name(), err),
)
}
env.Logger.Info("skipping stage", "stage", s.Name(), "reason", "already_succeeded", "force", opts.Force)
continue
@@ -243,7 +276,11 @@ func executeStages(ctx context.Context, cfg *config.Config, stages []stage.Stage
runManifest.SetStageAction(s.Name(), manifest.RunStageActionRun, now)
runManifest.MarkStageRunning(s.Name(), now)
if err := runManifestStore.SaveRun(ctx, runManifestPath, runManifest); err != nil {
return nil, fmt.Errorf("save run manifest before stage %q: %w", s.Name(), err)
operationErr := fmt.Errorf("save run manifest before stage %q: %w", s.Name(), err)
runManifest.MarkStageFailed(s.Name(), nowUTC(), operationErr.Error())
return nil, persistTerminalFailure(
ctx, env.ManifestStore, manifestPath, m, runManifestStore, runManifestPath, runManifest, operationErr,
)
}
m.MarkStageRunning(s.Name(), now)
if opts.Force {
@@ -251,7 +288,12 @@ func executeStages(ctx context.Context, cfg *config.Config, stages []stage.Stage
}
env.Logger.Info("starting stage", "stage", s.Name())
if err := env.ManifestStore.Save(ctx, manifestPath, m); err != nil {
return nil, fmt.Errorf("save manifest before stage %q: %w", s.Name(), err)
operationErr := fmt.Errorf("save manifest before stage %q: %w", s.Name(), err)
m.MarkStageFailed(s.Name(), nowUTC(), operationErr.Error())
invalidateDownstreamSucceededStagesWithReason(m, s.Name(), nowUTC(), staleReasonFailure)
return nil, persistTerminalFailure(
ctx, env.ManifestStore, manifestPath, m, runManifestStore, runManifestPath, runManifest, operationErr,
)
}
env.Logger.Debug("manifest saved", "stage", s.Name(), "transition", "running", "path", manifestPath)
@@ -263,16 +305,13 @@ func executeStages(ctx context.Context, cfg *config.Config, stages []stage.Stage
failedAt := nowUTC()
m.MarkStageFailed(s.Name(), failedAt, err.Error())
invalidateDownstreamSucceededStagesWithReason(m, s.Name(), failedAt, staleReasonFailure)
if saveErr := env.ManifestStore.Save(ctx, manifestPath, m); saveErr != nil {
return nil, fmt.Errorf("stage %q failed (%v) and manifest save failed (%v)", s.Name(), err, saveErr)
}
runManifest.MarkStageFailed(s.Name(), failedAt, err.Error())
identity.applyToRunManifest(runManifest, manifestPath)
if saveErr := runManifestStore.SaveRun(ctx, runManifestPath, runManifest); saveErr != nil {
return nil, fmt.Errorf("stage %q failed (%v) and run-manifest save failed (%v)", s.Name(), err, saveErr)
}
env.Logger.Info("stage failed", "stage", s.Name(), "error", err)
return nil, fmt.Errorf("stage %q failed: %w", s.Name(), err)
return nil, persistTerminalFailure(
ctx, env.ManifestStore, manifestPath, m, runManifestStore, runManifestPath, runManifest,
fmt.Errorf("stage %q failed: %w", s.Name(), err),
)
}
if result != nil && result.Disposition == stage.StageDispositionSkipped {
skipped = append(skipped, s.Name())
@@ -283,13 +322,19 @@ func executeStages(ctx context.Context, cfg *config.Config, stages []stage.Stage
invalidateDownstreamSucceededStagesWithReason(m, s.Name(), skippedAt, staleReasonSelfSkip)
}
if err := env.ManifestStore.Save(ctx, manifestPath, m); err != nil {
return nil, fmt.Errorf("save manifest after self-skip %q: %w", s.Name(), err)
return nil, persistTerminalFailure(
ctx, env.ManifestStore, manifestPath, m, runManifestStore, runManifestPath, runManifest,
fmt.Errorf("save manifest after self-skip %q: %w", s.Name(), err),
)
}
runManifest.MarkStageSkipped(s.Name(), skippedAt, result.SkipReason)
applyStageResultToRunManifest(runManifest, s.Name(), result)
identity.applyToRunManifest(runManifest, manifestPath)
if err := runManifestStore.SaveRun(ctx, runManifestPath, runManifest); err != nil {
return nil, fmt.Errorf("save run manifest after self-skip %q: %w", s.Name(), err)
return nil, persistTerminalFailure(
ctx, env.ManifestStore, manifestPath, m, runManifestStore, runManifestPath, runManifest,
fmt.Errorf("save run manifest after self-skip %q: %w", s.Name(), err),
)
}
env.Logger.Debug("manifest saved", "stage", s.Name(), "transition", "skipped", "path", manifestPath)
env.Logger.Info("stage skipped", "stage", s.Name(), "reason", result.SkipReason)
@@ -305,32 +350,38 @@ func executeStages(ctx context.Context, cfg *config.Config, stages []stage.Stage
}
if err := env.ManifestStore.Save(ctx, manifestPath, m); err != nil {
return nil, fmt.Errorf("save manifest after stage %q: %w", s.Name(), err)
return nil, persistTerminalFailure(
ctx, env.ManifestStore, manifestPath, m, runManifestStore, runManifestPath, runManifest,
fmt.Errorf("save manifest after stage %q: %w", s.Name(), err),
)
}
runManifest.MarkStageSucceeded(s.Name(), succeededAt, outputs)
applyStageResultToRunManifest(runManifest, s.Name(), result)
identity.applyToRunManifest(runManifest, manifestPath)
if err := runManifestStore.SaveRun(ctx, runManifestPath, runManifest); err != nil {
return nil, fmt.Errorf("save run manifest after stage %q: %w", s.Name(), err)
return nil, persistTerminalFailure(
ctx, env.ManifestStore, manifestPath, m, runManifestStore, runManifestPath, runManifest,
fmt.Errorf("save run manifest after stage %q: %w", s.Name(), err),
)
}
env.Logger.Debug("manifest saved", "stage", s.Name(), "transition", "succeeded", "path", manifestPath)
env.Logger.Info("stage succeeded", "stage", s.Name())
}
if err := runPostPublishCleanup(ctx, env, manifestPath, m, executed); err != nil {
failedAt := nowUTC()
runManifest.MarkFailed(failedAt, err.Error())
identity.applyToRunManifest(runManifest, manifestPath)
if saveErr := runManifestStore.SaveRun(ctx, runManifestPath, runManifest); saveErr != nil {
return nil, fmt.Errorf("post-publish cleanup failed (%v) and run-manifest save failed (%v)", err, saveErr)
}
return nil, fmt.Errorf("post-publish cleanup: %w", err)
return nil, persistTerminalFailure(
ctx, env.ManifestStore, manifestPath, m, runManifestStore, runManifestPath, runManifest,
fmt.Errorf("post-publish cleanup: %w", err),
)
}
completedAt := nowUTC()
runManifest.MarkSucceeded(completedAt)
identity.applyToRunManifest(runManifest, manifestPath)
if err := runManifestStore.SaveRun(ctx, runManifestPath, runManifest); err != nil {
return nil, fmt.Errorf("save final run manifest %q: %w", runManifestPath, err)
return nil, persistTerminalFailure(
ctx, env.ManifestStore, manifestPath, m, runManifestStore, runManifestPath, runManifest,
fmt.Errorf("save final run manifest %q: %w", runManifestPath, err),
)
}
return &RunSummary{
@@ -344,6 +395,47 @@ func executeStages(ctx context.Context, cfg *config.Config, stages []stage.Stage
}, nil
}
func persistTerminalFailure(
ctx context.Context,
sessionStore manifest.Store,
sessionManifestPath string,
sessionManifest *manifest.Manifest,
runStore manifest.RunStore,
runManifestPath string,
runManifest *manifest.RunManifest,
operationErr error,
) error {
if operationErr == nil {
return nil
}
at := nowUTC()
if sessionManifest != nil {
sessionManifest.RecordFailure(at, operationErr.Error())
}
// A running run record is durable before execution. At termination, session
// state controls resume while the run record only diagnoses this invocation,
// so session persistence comes first and any disagreement remains visible.
sessionErr := sessionStore.Save(ctx, sessionManifestPath, sessionManifest)
if runManifest != nil {
runManifest.MarkFailed(at, operationErr.Error())
}
runErr := runStore.SaveRun(ctx, runManifestPath, runManifest)
return errors.Join(
operationErr,
wrapTerminalPersistenceError("save terminal session manifest", sessionErr),
wrapTerminalPersistenceError("save terminal run manifest", runErr),
)
}
func wrapTerminalPersistenceError(operation string, err error) error {
if err == nil {
return nil
}
return fmt.Errorf("%s: %w", operation, err)
}
func buildDefaultWhisperXClient(cfg *config.Config) (whisperx.Client, error) {
if cfg == nil || cfg.Pipeline == nil {
return &whisperx.NoopClient{}, nil