Terminalize handled invocation failures

2026-08-10 19:34:46 +00:00
parent a1ceb457e9
commit ee747243fe
6 changed files with 341 additions and 74 deletions
--- a/internal/app/runner.go
+++ b/internal/app/runner.go
@@ -26,6 +26,7 @@ type RunOptions struct {
 	Force             bool
 	SelectedArtifacts []string
 	Env               *Env
+	RunManifestStore  manifest.RunStore
 }

 type RunSummary struct {
@@ -79,53 +80,6 @@ func executeStages(ctx context.Context, cfg *config.Config, stages []stage.Stage
 	if env.Logger == nil {
 		env.Logger = logging.NewLogger(os.Stderr, slog.LevelInfo)
 	}
-	if _, err := loadSecretsFromConfig(env.Config, env.Logger); err != nil {
-		return nil, fmt.Errorf("load secrets from files: %w", err)
-	}
-	if env.WhisperX == nil {
-		client, err := buildDefaultWhisperXClient(env.Config)
-		if err != nil {
-			return nil, fmt.Errorf("initialize whisperx client: %w", err)
-		}
-		env.WhisperX = client
-	}
-	if env.Seriatim == nil {
-		runner, err := buildDefaultSeriatimRunner(env.Config)
-		if err != nil {
-			return nil, fmt.Errorf("initialize seriatim runner: %w", err)
-		}
-		env.Seriatim = runner
-	}
-	if env.Audita == nil {
-		runner, err := buildDefaultAuditaRunner(env.Config)
-		if err != nil {
-			return nil, fmt.Errorf("initialize audita runner: %w", err)
-		}
-		env.Audita = runner
-	}
-	if env.Notarius == nil && needsNotariusForRun(env.Config, stages) {
-		env.Notarius = notarius.NewSubprocessRunner()
-	}
-	if env.Scriptorium == nil {
-		env.Scriptorium = scriptorium.NewSubprocessRunner()
-	}
-	if env.ObjectStore == nil && needsObjectStoreForRun(env.Config, stages) {
-		objectStore, err := newCommandObjectStore(ctx, env.Config, nil)
-		if err != nil {
-			return nil, err
-		}
-		env.ObjectStore = objectStore
-	}
-	if needsRemoteLocksForRun(env.Config, stages) {
-		locks, err := loadEffectiveLocks(ctx, env.Config, env.ObjectStore)
-		if err != nil {
-			return nil, fmt.Errorf("load remote publish locks: %w", err)
-		}
-		applyEffectiveLocks(env.Config, locks.All)
-	}
-	if env.Notifier == nil {
-		env.Notifier = &notify.NoopSender{}
-	}

 	artifactStore := env.ArtifactStore
 	paths, err := artifactStore.EnsureLayoutFor(identity.Campaign, identity.SessionID)
@@ -174,7 +128,10 @@ func executeStages(ctx context.Context, cfg *config.Config, stages []stage.Stage
 		identity.SessionID,
 		identity.RunID,
 	)
-	runManifestStore := &manifest.LocalStore{}
+	runManifestStore := opts.RunManifestStore
+	if runManifestStore == nil {
+		runManifestStore = &manifest.LocalStore{}
+	}
 	runManifest, err := runManifestStore.CreateRun(
 		ctx,
 		identity.SessionID,
@@ -188,7 +145,74 @@ func executeStages(ctx context.Context, cfg *config.Config, stages []stage.Stage
 	}
 	identity.applyToRunManifest(runManifest, manifestPath)
 	if err := runManifestStore.SaveRun(ctx, runManifestPath, runManifest); err != nil {
-		return nil, fmt.Errorf("save initial run manifest %q: %w", runManifestPath, err)
+		return nil, persistTerminalFailure(
+			ctx, env.ManifestStore, manifestPath, m, runManifestStore, runManifestPath, runManifest,
+			fmt.Errorf("save initial run manifest %q: %w", runManifestPath, err),
+		)
+	}
+	if _, err := loadSecretsFromConfig(env.Config, env.Logger); err != nil {
+		return nil, persistTerminalFailure(
+			ctx, env.ManifestStore, manifestPath, m, runManifestStore, runManifestPath, runManifest,
+			fmt.Errorf("load secrets from files: %w", err),
+		)
+	}
+	if env.WhisperX == nil {
+		client, err := buildDefaultWhisperXClient(env.Config)
+		if err != nil {
+			return nil, persistTerminalFailure(
+				ctx, env.ManifestStore, manifestPath, m, runManifestStore, runManifestPath, runManifest,
+				fmt.Errorf("initialize whisperx client: %w", err),
+			)
+		}
+		env.WhisperX = client
+	}
+	if env.Seriatim == nil {
+		runner, err := buildDefaultSeriatimRunner(env.Config)
+		if err != nil {
+			return nil, persistTerminalFailure(
+				ctx, env.ManifestStore, manifestPath, m, runManifestStore, runManifestPath, runManifest,
+				fmt.Errorf("initialize seriatim runner: %w", err),
+			)
+		}
+		env.Seriatim = runner
+	}
+	if env.Audita == nil {
+		runner, err := buildDefaultAuditaRunner(env.Config)
+		if err != nil {
+			return nil, persistTerminalFailure(
+				ctx, env.ManifestStore, manifestPath, m, runManifestStore, runManifestPath, runManifest,
+				fmt.Errorf("initialize audita runner: %w", err),
+			)
+		}
+		env.Audita = runner
+	}
+	if env.Notarius == nil && needsNotariusForRun(env.Config, stages) {
+		env.Notarius = notarius.NewSubprocessRunner()
+	}
+	if env.Scriptorium == nil {
+		env.Scriptorium = scriptorium.NewSubprocessRunner()
+	}
+	if env.ObjectStore == nil && needsObjectStoreForRun(env.Config, stages) {
+		objectStore, err := newCommandObjectStore(ctx, env.Config, nil)
+		if err != nil {
+			return nil, persistTerminalFailure(
+				ctx, env.ManifestStore, manifestPath, m, runManifestStore, runManifestPath, runManifest, err,
+			)
+		}
+		env.ObjectStore = objectStore
+	}
+	if needsRemoteLocksForRun(env.Config, stages) {
+		locks, err := loadEffectiveLocks(ctx, env.Config, env.ObjectStore)
+		if err != nil {
+			return nil, persistTerminalFailure(
+				ctx, env.ManifestStore, manifestPath, m, runManifestStore, runManifestPath, runManifest,
+				fmt.Errorf("load remote publish locks: %w", err),
+			)
+		}
+		applyEffectiveLocks(env.Config, locks.All)
+	}
+	if env.Notifier == nil {
+		env.Notifier = &notify.NoopSender{}
 	}

 	stageEnv := env
@@ -207,7 +231,10 @@ func executeStages(ctx context.Context, cfg *config.Config, stages []stage.Stage
 			if validator, ok := s.(stage.ResumeValidator); ok {
 				validation, err := validator.ValidateResume(ctx, stageEnv, m)
 				if err != nil {
-					return nil, fmt.Errorf("validate resume for stage %q: %w", s.Name(), err)
+					return nil, persistTerminalFailure(
+						ctx, env.ManifestStore, manifestPath, m, runManifestStore, runManifestPath, runManifest,
+						fmt.Errorf("validate resume for stage %q: %w", s.Name(), err),
+					)
 				}
 				validation = validation.Normalized()
 				if !validation.Resumable {
@@ -217,7 +244,10 @@ func executeStages(ctx context.Context, cfg *config.Config, stages []stage.Stage
 						m, s.Name(), staleAt, staleReasonNotResumable,
 					)
 					if err := env.ManifestStore.Save(ctx, manifestPath, m); err != nil {
-						return nil, fmt.Errorf("save manifest after resume validation for stage %q: %w", s.Name(), err)
+						return nil, persistTerminalFailure(
+							ctx, env.ManifestStore, manifestPath, m, runManifestStore, runManifestPath, runManifest,
+							fmt.Errorf("save manifest after resume validation for stage %q: %w", s.Name(), err),
+						)
 					}
 					env.Logger.Info("stage result is not resumable", "stage", s.Name(), "reason", validation.Reason)
 					d.Action = stageActionRun
@@ -231,7 +261,10 @@ func executeStages(ctx context.Context, cfg *config.Config, stages []stage.Stage
 			runManifest.SetStageAction(s.Name(), manifest.RunStageActionSkip, skipAt)
 			runManifest.MarkStageSkipped(s.Name(), skipAt, "already_succeeded")
 			if err := runManifestStore.SaveRun(ctx, runManifestPath, runManifest); err != nil {
-				return nil, fmt.Errorf("save run manifest after skip %q: %w", s.Name(), err)
+				return nil, persistTerminalFailure(
+					ctx, env.ManifestStore, manifestPath, m, runManifestStore, runManifestPath, runManifest,
+					fmt.Errorf("save run manifest after skip %q: %w", s.Name(), err),
+				)
 			}
 			env.Logger.Info("skipping stage", "stage", s.Name(), "reason", "already_succeeded", "force", opts.Force)
 			continue
@@ -243,7 +276,11 @@ func executeStages(ctx context.Context, cfg *config.Config, stages []stage.Stage
 		runManifest.SetStageAction(s.Name(), manifest.RunStageActionRun, now)
 		runManifest.MarkStageRunning(s.Name(), now)
 		if err := runManifestStore.SaveRun(ctx, runManifestPath, runManifest); err != nil {
-			return nil, fmt.Errorf("save run manifest before stage %q: %w", s.Name(), err)
+			operationErr := fmt.Errorf("save run manifest before stage %q: %w", s.Name(), err)
+			runManifest.MarkStageFailed(s.Name(), nowUTC(), operationErr.Error())
+			return nil, persistTerminalFailure(
+				ctx, env.ManifestStore, manifestPath, m, runManifestStore, runManifestPath, runManifest, operationErr,
+			)
 		}
 		m.MarkStageRunning(s.Name(), now)
 		if opts.Force {
@@ -251,7 +288,12 @@ func executeStages(ctx context.Context, cfg *config.Config, stages []stage.Stage
 		}
 		env.Logger.Info("starting stage", "stage", s.Name())
 		if err := env.ManifestStore.Save(ctx, manifestPath, m); err != nil {
-			return nil, fmt.Errorf("save manifest before stage %q: %w", s.Name(), err)
+			operationErr := fmt.Errorf("save manifest before stage %q: %w", s.Name(), err)
+			m.MarkStageFailed(s.Name(), nowUTC(), operationErr.Error())
+			invalidateDownstreamSucceededStagesWithReason(m, s.Name(), nowUTC(), staleReasonFailure)
+			return nil, persistTerminalFailure(
+				ctx, env.ManifestStore, manifestPath, m, runManifestStore, runManifestPath, runManifest, operationErr,
+			)
 		}
 		env.Logger.Debug("manifest saved", "stage", s.Name(), "transition", "running", "path", manifestPath)

@@ -263,16 +305,13 @@ func executeStages(ctx context.Context, cfg *config.Config, stages []stage.Stage
 			failedAt := nowUTC()
 			m.MarkStageFailed(s.Name(), failedAt, err.Error())
 			invalidateDownstreamSucceededStagesWithReason(m, s.Name(), failedAt, staleReasonFailure)
-			if saveErr := env.ManifestStore.Save(ctx, manifestPath, m); saveErr != nil {
-				return nil, fmt.Errorf("stage %q failed (%v) and manifest save failed (%v)", s.Name(), err, saveErr)
-			}
 			runManifest.MarkStageFailed(s.Name(), failedAt, err.Error())
 			identity.applyToRunManifest(runManifest, manifestPath)
-			if saveErr := runManifestStore.SaveRun(ctx, runManifestPath, runManifest); saveErr != nil {
-				return nil, fmt.Errorf("stage %q failed (%v) and run-manifest save failed (%v)", s.Name(), err, saveErr)
-			}
 			env.Logger.Info("stage failed", "stage", s.Name(), "error", err)
-			return nil, fmt.Errorf("stage %q failed: %w", s.Name(), err)
+			return nil, persistTerminalFailure(
+				ctx, env.ManifestStore, manifestPath, m, runManifestStore, runManifestPath, runManifest,
+				fmt.Errorf("stage %q failed: %w", s.Name(), err),
+			)
 		}
 		if result != nil && result.Disposition == stage.StageDispositionSkipped {
 			skipped = append(skipped, s.Name())
@@ -283,13 +322,19 @@ func executeStages(ctx context.Context, cfg *config.Config, stages []stage.Stage
 				invalidateDownstreamSucceededStagesWithReason(m, s.Name(), skippedAt, staleReasonSelfSkip)
 			}
 			if err := env.ManifestStore.Save(ctx, manifestPath, m); err != nil {
-				return nil, fmt.Errorf("save manifest after self-skip %q: %w", s.Name(), err)
+				return nil, persistTerminalFailure(
+					ctx, env.ManifestStore, manifestPath, m, runManifestStore, runManifestPath, runManifest,
+					fmt.Errorf("save manifest after self-skip %q: %w", s.Name(), err),
+				)
 			}
 			runManifest.MarkStageSkipped(s.Name(), skippedAt, result.SkipReason)
 			applyStageResultToRunManifest(runManifest, s.Name(), result)
 			identity.applyToRunManifest(runManifest, manifestPath)
 			if err := runManifestStore.SaveRun(ctx, runManifestPath, runManifest); err != nil {
-				return nil, fmt.Errorf("save run manifest after self-skip %q: %w", s.Name(), err)
+				return nil, persistTerminalFailure(
+					ctx, env.ManifestStore, manifestPath, m, runManifestStore, runManifestPath, runManifest,
+					fmt.Errorf("save run manifest after self-skip %q: %w", s.Name(), err),
+				)
 			}
 			env.Logger.Debug("manifest saved", "stage", s.Name(), "transition", "skipped", "path", manifestPath)
 			env.Logger.Info("stage skipped", "stage", s.Name(), "reason", result.SkipReason)
@@ -305,32 +350,38 @@ func executeStages(ctx context.Context, cfg *config.Config, stages []stage.Stage
 		}

 		if err := env.ManifestStore.Save(ctx, manifestPath, m); err != nil {
-			return nil, fmt.Errorf("save manifest after stage %q: %w", s.Name(), err)
+			return nil, persistTerminalFailure(
+				ctx, env.ManifestStore, manifestPath, m, runManifestStore, runManifestPath, runManifest,
+				fmt.Errorf("save manifest after stage %q: %w", s.Name(), err),
+			)
 		}
 		runManifest.MarkStageSucceeded(s.Name(), succeededAt, outputs)
 		applyStageResultToRunManifest(runManifest, s.Name(), result)
 		identity.applyToRunManifest(runManifest, manifestPath)
 		if err := runManifestStore.SaveRun(ctx, runManifestPath, runManifest); err != nil {
-			return nil, fmt.Errorf("save run manifest after stage %q: %w", s.Name(), err)
+			return nil, persistTerminalFailure(
+				ctx, env.ManifestStore, manifestPath, m, runManifestStore, runManifestPath, runManifest,
+				fmt.Errorf("save run manifest after stage %q: %w", s.Name(), err),
+			)
 		}
 		env.Logger.Debug("manifest saved", "stage", s.Name(), "transition", "succeeded", "path", manifestPath)
 		env.Logger.Info("stage succeeded", "stage", s.Name())
 	}

 	if err := runPostPublishCleanup(ctx, env, manifestPath, m, executed); err != nil {
-		failedAt := nowUTC()
-		runManifest.MarkFailed(failedAt, err.Error())
-		identity.applyToRunManifest(runManifest, manifestPath)
-		if saveErr := runManifestStore.SaveRun(ctx, runManifestPath, runManifest); saveErr != nil {
-			return nil, fmt.Errorf("post-publish cleanup failed (%v) and run-manifest save failed (%v)", err, saveErr)
-		}
-		return nil, fmt.Errorf("post-publish cleanup: %w", err)
+		return nil, persistTerminalFailure(
+			ctx, env.ManifestStore, manifestPath, m, runManifestStore, runManifestPath, runManifest,
+			fmt.Errorf("post-publish cleanup: %w", err),
+		)
 	}
 	completedAt := nowUTC()
 	runManifest.MarkSucceeded(completedAt)
 	identity.applyToRunManifest(runManifest, manifestPath)
 	if err := runManifestStore.SaveRun(ctx, runManifestPath, runManifest); err != nil {
-		return nil, fmt.Errorf("save final run manifest %q: %w", runManifestPath, err)
+		return nil, persistTerminalFailure(
+			ctx, env.ManifestStore, manifestPath, m, runManifestStore, runManifestPath, runManifest,
+			fmt.Errorf("save final run manifest %q: %w", runManifestPath, err),
+		)
 	}

 	return &RunSummary{
@@ -344,6 +395,47 @@ func executeStages(ctx context.Context, cfg *config.Config, stages []stage.Stage
 	}, nil
 }

+func persistTerminalFailure(
+	ctx context.Context,
+	sessionStore manifest.Store,
+	sessionManifestPath string,
+	sessionManifest *manifest.Manifest,
+	runStore manifest.RunStore,
+	runManifestPath string,
+	runManifest *manifest.RunManifest,
+	operationErr error,
+) error {
+	if operationErr == nil {
+		return nil
+	}
+
+	at := nowUTC()
+	if sessionManifest != nil {
+		sessionManifest.RecordFailure(at, operationErr.Error())
+	}
+
+	// A running run record is durable before execution. At termination, session
+	// state controls resume while the run record only diagnoses this invocation,
+	// so session persistence comes first and any disagreement remains visible.
+	sessionErr := sessionStore.Save(ctx, sessionManifestPath, sessionManifest)
+	if runManifest != nil {
+		runManifest.MarkFailed(at, operationErr.Error())
+	}
+	runErr := runStore.SaveRun(ctx, runManifestPath, runManifest)
+	return errors.Join(
+		operationErr,
+		wrapTerminalPersistenceError("save terminal session manifest", sessionErr),
+		wrapTerminalPersistenceError("save terminal run manifest", runErr),
+	)
+}
+
+func wrapTerminalPersistenceError(operation string, err error) error {
+	if err == nil {
+		return nil
+	}
+	return fmt.Errorf("%s: %w", operation, err)
+}
+
 func buildDefaultWhisperXClient(cfg *config.Config) (whisperx.Client, error) {
 	if cfg == nil || cfg.Pipeline == nil {
 		return &whisperx.NoopClient{}, nil
--- a/internal/app/runner_terminal_failure_test.go
+++ b/internal/app/runner_terminal_failure_test.go
@@ -0,0 +1,154 @@
+package app
+
+import (
+	"context"
+	"errors"
+	"strings"
+	"testing"
+	"time"
+
+	"gitea.maximumdirect.net/eric/narratio/internal/manifest"
+	"gitea.maximumdirect.net/eric/narratio/internal/stage"
+)
+
+type terminalSessionStore struct {
+	events  []string
+	saveErr error
+	saved   *manifest.Manifest
+}
+
+func (s *terminalSessionStore) Create(_ context.Context, sessionID string) (*manifest.Manifest, error) {
+	return manifest.New(sessionID, time.Now().UTC()), nil
+}
+
+func (s *terminalSessionStore) Load(context.Context, string) (*manifest.Manifest, error) {
+	return nil, errors.New("unexpected manifest load")
+}
+
+func (s *terminalSessionStore) Save(_ context.Context, _ string, m *manifest.Manifest) error {
+	s.events = append(s.events, "session")
+	s.saved = m
+	return s.saveErr
+}
+
+type terminalRunStore struct {
+	events  []string
+	saveErr error
+	saved   *manifest.RunManifest
+}
+
+func (s *terminalRunStore) CreateRun(_ context.Context, sessionID, campaign, runID string, force bool, requestedStages []string) (*manifest.RunManifest, error) {
+	return manifest.NewRun(sessionID, campaign, runID, force, requestedStages, time.Now().UTC()), nil
+}
+
+func (s *terminalRunStore) SaveRun(_ context.Context, _ string, m *manifest.RunManifest) error {
+	s.events = append(s.events, "run")
+	s.saved = m
+	return s.saveErr
+}
+
+func TestPersistTerminalFailureKeepsSessionAuthorityBeforeRunAudit(t *testing.T) {
+	cause := errors.New("stage failed")
+	sessionFailure := errors.New("session unavailable")
+	runFailure := errors.New("run unavailable")
+
+	for _, tc := range []struct {
+		name            string
+		sessionErr      error
+		runErr          error
+		cause           error
+		alreadyTerminal bool
+	}{
+		{name: "both ledgers saved", cause: cause},
+		{name: "session save fails", sessionErr: sessionFailure, cause: cause},
+		{name: "run save fails", runErr: runFailure, cause: cause},
+		{name: "both saves fail", sessionErr: sessionFailure, runErr: runFailure, cause: cause},
+		{name: "cancellation", cause: context.Canceled},
+		{name: "already terminal run", cause: cause, alreadyTerminal: true},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			sessionStore := &terminalSessionStore{saveErr: tc.sessionErr}
+			runStore := &terminalRunStore{saveErr: tc.runErr}
+			sessionManifest := manifest.New("2026-05-03", time.Now().UTC())
+			runManifest := manifest.NewRun("2026-05-03", "sample-campaign", "20260503T010203Z-a1b2c3d4", false, nil, time.Now().UTC())
+			if tc.alreadyTerminal {
+				sessionManifest.RecordFailure(time.Now().UTC(), "earlier failure")
+				runManifest.MarkFailed(time.Now().UTC(), "earlier failure")
+			}
+
+			err := persistTerminalFailure(
+				context.Background(), sessionStore, "session.json", sessionManifest,
+				runStore, "run.json", runManifest, tc.cause,
+			)
+			if !errors.Is(err, tc.cause) {
+				t.Fatalf("error = %v, want original cause %v", err, tc.cause)
+			}
+			if tc.sessionErr != nil && !errors.Is(err, tc.sessionErr) {
+				t.Fatalf("error = %v, want session failure %v", err, tc.sessionErr)
+			}
+			if tc.runErr != nil && !errors.Is(err, tc.runErr) {
+				t.Fatalf("error = %v, want run failure %v", err, tc.runErr)
+			}
+			if got := append(sessionStore.events, runStore.events...); len(got) != 2 || got[0] != "session" || got[1] != "run" {
+				t.Fatalf("save order = %v, want [session run]", got)
+			}
+			if sessionStore.saved == nil || sessionStore.saved.LastError == nil || sessionStore.saved.LastError.Message != tc.cause.Error() {
+				t.Fatalf("session terminal error = %#v, want %q", sessionStore.saved, tc.cause)
+			}
+			if runStore.saved == nil || runStore.saved.Status != manifest.RunManifestStatusFailed || runStore.saved.LastError == nil || runStore.saved.LastError.Message != tc.cause.Error() {
+				t.Fatalf("run terminal record = %#v, want failed with %q", runStore.saved, tc.cause)
+			}
+		})
+	}
+}
+
+func TestExecuteStagesTerminalizesCancelledStage(t *testing.T) {
+	cfg := testConfig(t)
+	runStore := &terminalRunStore{}
+
+	_, err := executeStages(context.Background(), cfg, []stage.Stage{
+		failingStage{name: "prepare", err: context.Canceled},
+	}, RunOptions{RunManifestStore: runStore})
+	if !errors.Is(err, context.Canceled) {
+		t.Fatalf("executeStages() error = %v, want context cancellation", err)
+	}
+	if runStore.saved == nil || runStore.saved.Status != manifest.RunManifestStatusFailed {
+		t.Fatalf("run terminal record = %#v, want failed", runStore.saved)
+	}
+	store := &manifest.LocalStore{}
+	sessionManifest, loadErr := store.Load(context.Background(), manifestPathFor(cfg))
+	if loadErr != nil {
+		t.Fatalf("load session manifest: %v", loadErr)
+	}
+	if sessionManifest.Stages["prepare"] == nil || sessionManifest.Stages["prepare"].Status != manifest.StatusFailed || sessionManifest.LastError == nil {
+		t.Fatalf("session terminal record = %#v, want failed prepare stage and last error", sessionManifest)
+	}
+}
+
+func TestExecuteStagesResumeValidationFailureTerminalizesRunRecord(t *testing.T) {
+	cfg := testConfig(t)
+	store := &manifest.LocalStore{}
+	seed := manifest.New(cfg.Session.SessionID, time.Now().UTC())
+	seed.MarkStageSucceeded("checked", time.Now().UTC(), nil)
+	if err := store.Save(context.Background(), manifestPathFor(cfg), seed); err != nil {
+		t.Fatalf("save session manifest: %v", err)
+	}
+	runStore := &terminalRunStore{}
+
+	_, err := executeStages(context.Background(), cfg, []stage.Stage{
+		resumeCheckingStage{name: "checked", validateErr: errors.New("inspection unavailable"), runs: new(int)},
+	}, RunOptions{RunManifestStore: runStore})
+	if err == nil || !strings.Contains(err.Error(), "inspection unavailable") || runStore.saved == nil {
+		t.Fatalf("executeStages() error = %v, want resume validation failure and terminal run", err)
+	}
+	if runStore.saved.Status != manifest.RunManifestStatusFailed {
+		t.Fatalf("run manifest status = %q, want failed", runStore.saved.Status)
+	}
+	persisted, loadErr := store.Load(context.Background(), manifestPathFor(cfg))
+	if loadErr != nil {
+		t.Fatalf("load session manifest: %v", loadErr)
+	}
+	if persisted.Stages["checked"].Status != manifest.StatusSucceeded || persisted.LastError == nil {
+		t.Fatalf("session manifest = %#v, want preserved success with terminal error", persisted)
+	}
+}
--- a/internal/manifest/manifest.go
+++ b/internal/manifest/manifest.go
@@ -121,6 +121,15 @@ func (m *Manifest) MarkStageFailed(name string, at time.Time, message string) {
 	m.UpdatedAt = at
 }

+// RecordFailure records an invocation failure without changing session progress.
+func (m *Manifest) RecordFailure(at time.Time, message string) {
+	if m == nil {
+		return
+	}
+	m.LastError = &ErrorRecord{Message: strings.TrimSpace(message), At: timePtr(at)}
+	m.UpdatedAt = at
+}
+
 // MarkStageSkipped marks a stage as skipped and records the skip reason.
 func (m *Manifest) MarkStageSkipped(name string, at time.Time, reason string) {
 	s := m.ensureStage(name, at)
--- a/internal/manifest/store.go
+++ b/internal/manifest/store.go
@@ -21,6 +21,12 @@ type Store interface {
 	Save(ctx context.Context, path string, m *Manifest) error
 }

+// RunStore persists invocation-scoped run manifests.
+type RunStore interface {
+	CreateRun(ctx context.Context, sessionID, campaign, runID string, force bool, requestedStages []string) (*RunManifest, error)
+	SaveRun(ctx context.Context, path string, m *RunManifest) error
+}
+
 // LocalStore stores manifests as JSON on the local filesystem.
 type LocalStore struct{}