Terminalize handled invocation failures

internal/app/runner_terminal_failure_test.go

@@ -0,0 +1,154 @@
+package app
+
+import (
+	"context"
+	"errors"
+	"strings"
+	"testing"
+	"time"
+
+	"gitea.maximumdirect.net/eric/narratio/internal/manifest"
+	"gitea.maximumdirect.net/eric/narratio/internal/stage"
+)
+
+type terminalSessionStore struct {
+	events  []string
+	saveErr error
+	saved   *manifest.Manifest
+}
+
+func (s *terminalSessionStore) Create(_ context.Context, sessionID string) (*manifest.Manifest, error) {
+	return manifest.New(sessionID, time.Now().UTC()), nil
+}
+
+func (s *terminalSessionStore) Load(context.Context, string) (*manifest.Manifest, error) {
+	return nil, errors.New("unexpected manifest load")
+}
+
+func (s *terminalSessionStore) Save(_ context.Context, _ string, m *manifest.Manifest) error {
+	s.events = append(s.events, "session")
+	s.saved = m
+	return s.saveErr
+}
+
+type terminalRunStore struct {
+	events  []string
+	saveErr error
+	saved   *manifest.RunManifest
+}
+
+func (s *terminalRunStore) CreateRun(_ context.Context, sessionID, campaign, runID string, force bool, requestedStages []string) (*manifest.RunManifest, error) {
+	return manifest.NewRun(sessionID, campaign, runID, force, requestedStages, time.Now().UTC()), nil
+}
+
+func (s *terminalRunStore) SaveRun(_ context.Context, _ string, m *manifest.RunManifest) error {
+	s.events = append(s.events, "run")
+	s.saved = m
+	return s.saveErr
+}
+
+func TestPersistTerminalFailureKeepsSessionAuthorityBeforeRunAudit(t *testing.T) {
+	cause := errors.New("stage failed")
+	sessionFailure := errors.New("session unavailable")
+	runFailure := errors.New("run unavailable")
+
+	for _, tc := range []struct {
+		name            string
+		sessionErr      error
+		runErr          error
+		cause           error
+		alreadyTerminal bool
+	}{
+		{name: "both ledgers saved", cause: cause},
+		{name: "session save fails", sessionErr: sessionFailure, cause: cause},
+		{name: "run save fails", runErr: runFailure, cause: cause},
+		{name: "both saves fail", sessionErr: sessionFailure, runErr: runFailure, cause: cause},
+		{name: "cancellation", cause: context.Canceled},
+		{name: "already terminal run", cause: cause, alreadyTerminal: true},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			sessionStore := &terminalSessionStore{saveErr: tc.sessionErr}
+			runStore := &terminalRunStore{saveErr: tc.runErr}
+			sessionManifest := manifest.New("2026-05-03", time.Now().UTC())
+			runManifest := manifest.NewRun("2026-05-03", "sample-campaign", "20260503T010203Z-a1b2c3d4", false, nil, time.Now().UTC())
+			if tc.alreadyTerminal {
+				sessionManifest.RecordFailure(time.Now().UTC(), "earlier failure")
+				runManifest.MarkFailed(time.Now().UTC(), "earlier failure")
+			}
+
+			err := persistTerminalFailure(
+				context.Background(), sessionStore, "session.json", sessionManifest,
+				runStore, "run.json", runManifest, tc.cause,
+			)
+			if !errors.Is(err, tc.cause) {
+				t.Fatalf("error = %v, want original cause %v", err, tc.cause)
+			}
+			if tc.sessionErr != nil && !errors.Is(err, tc.sessionErr) {
+				t.Fatalf("error = %v, want session failure %v", err, tc.sessionErr)
+			}
+			if tc.runErr != nil && !errors.Is(err, tc.runErr) {
+				t.Fatalf("error = %v, want run failure %v", err, tc.runErr)
+			}
+			if got := append(sessionStore.events, runStore.events...); len(got) != 2 || got[0] != "session" || got[1] != "run" {
+				t.Fatalf("save order = %v, want [session run]", got)
+			}
+			if sessionStore.saved == nil || sessionStore.saved.LastError == nil || sessionStore.saved.LastError.Message != tc.cause.Error() {
+				t.Fatalf("session terminal error = %#v, want %q", sessionStore.saved, tc.cause)
+			}
+			if runStore.saved == nil || runStore.saved.Status != manifest.RunManifestStatusFailed || runStore.saved.LastError == nil || runStore.saved.LastError.Message != tc.cause.Error() {
+				t.Fatalf("run terminal record = %#v, want failed with %q", runStore.saved, tc.cause)
+			}
+		})
+	}
+}
+
+func TestExecuteStagesTerminalizesCancelledStage(t *testing.T) {
+	cfg := testConfig(t)
+	runStore := &terminalRunStore{}
+
+	_, err := executeStages(context.Background(), cfg, []stage.Stage{
+		failingStage{name: "prepare", err: context.Canceled},
+	}, RunOptions{RunManifestStore: runStore})
+	if !errors.Is(err, context.Canceled) {
+		t.Fatalf("executeStages() error = %v, want context cancellation", err)
+	}
+	if runStore.saved == nil || runStore.saved.Status != manifest.RunManifestStatusFailed {
+		t.Fatalf("run terminal record = %#v, want failed", runStore.saved)
+	}
+	store := &manifest.LocalStore{}
+	sessionManifest, loadErr := store.Load(context.Background(), manifestPathFor(cfg))
+	if loadErr != nil {
+		t.Fatalf("load session manifest: %v", loadErr)
+	}
+	if sessionManifest.Stages["prepare"] == nil || sessionManifest.Stages["prepare"].Status != manifest.StatusFailed || sessionManifest.LastError == nil {
+		t.Fatalf("session terminal record = %#v, want failed prepare stage and last error", sessionManifest)
+	}
+}
+
+func TestExecuteStagesResumeValidationFailureTerminalizesRunRecord(t *testing.T) {
+	cfg := testConfig(t)
+	store := &manifest.LocalStore{}
+	seed := manifest.New(cfg.Session.SessionID, time.Now().UTC())
+	seed.MarkStageSucceeded("checked", time.Now().UTC(), nil)
+	if err := store.Save(context.Background(), manifestPathFor(cfg), seed); err != nil {
+		t.Fatalf("save session manifest: %v", err)
+	}
+	runStore := &terminalRunStore{}
+
+	_, err := executeStages(context.Background(), cfg, []stage.Stage{
+		resumeCheckingStage{name: "checked", validateErr: errors.New("inspection unavailable"), runs: new(int)},
+	}, RunOptions{RunManifestStore: runStore})
+	if err == nil || !strings.Contains(err.Error(), "inspection unavailable") || runStore.saved == nil {
+		t.Fatalf("executeStages() error = %v, want resume validation failure and terminal run", err)
+	}
+	if runStore.saved.Status != manifest.RunManifestStatusFailed {
+		t.Fatalf("run manifest status = %q, want failed", runStore.saved.Status)
+	}
+	persisted, loadErr := store.Load(context.Background(), manifestPathFor(cfg))
+	if loadErr != nil {
+		t.Fatalf("load session manifest: %v", loadErr)
+	}
+	if persisted.Stages["checked"].Status != manifest.StatusSucceeded || persisted.LastError == nil {
+		t.Fatalf("session manifest = %#v, want preserved success with terminal error", persisted)
+	}
+}