Integrate transcription checkpoints across lifecycle

2026-09-22 00:00:51 +00:00
parent 2ae63dbc2b
commit d872edca94
7 changed files with 581 additions and 2 deletions
--- a/internal/app/restore_execution_test.go
+++ b/internal/app/restore_execution_test.go
@@ -82,6 +82,28 @@ func TestExecuteRestoreRoundTripsPublishedExtractionAndManifestMetadata(t *testi
 	remoteManifest.RunID = "20260519T010203Z-a1b2c3d4"
 	producerRoot := "/prior/workspace/work/sample-campaign/2026-05-03"
 	remoteManifest.LocalWorkDir = filepath.Join(producerRoot, "runs", remoteManifest.RunID)
+	rawRelative := "transcripts/raw/alice.json"
+	rawOutput := manifest.ArtifactRecord{
+		Kind: "transcript_raw", LocalPath: filepath.Join(producerRoot, filepath.FromSlash(rawRelative)),
+		ProducerRunID: remoteManifest.RunID, Checksum: strings.Repeat("b", 64),
+	}
+	remoteManifest.MarkStageSucceeded("transcribe", time.Now().UTC(), []manifest.ArtifactRecord{rawOutput})
+	remoteManifest.Stages["transcribe"].TranscribeStateVersion = manifest.TranscribeStateContractVersion
+	remoteManifest.Stages["transcribe"].TranscribeSpeakers = map[string]manifest.TranscribeSpeakerRecord{
+		"alice": {
+			FingerprintVersion: manifest.TranscribeFingerprintContractVersion,
+			Fingerprint:        strings.Repeat("c", 64),
+			AudioPath:          "audio/alice.flac",
+			AudioSHA256:        strings.Repeat("a", 64),
+			AudioSize:          12,
+			Output: manifest.ArtifactRecord{
+				Kind: "transcript_raw", LocalPath: rawRelative,
+				ProducerRunID: remoteManifest.RunID, Checksum: strings.Repeat("b", 64),
+			},
+			OutputSize:    15,
+			ProducerRunID: remoteManifest.RunID,
+		},
+	}
 	remoteManifest.Stages["extract"] = &manifest.StageRecord{
 		Name: "extract", Status: manifest.StatusSucceeded,
 		Outputs: []manifest.ArtifactRecord{{
@@ -121,6 +143,13 @@ func TestExecuteRestoreRoundTripsPublishedExtractionAndManifestMetadata(t *testi
 	if want := filepath.Join(sessionRoot, "artifacts", "encounters.json"); lane.LocalPath != want {
 		t.Fatalf("rebased extraction path = %q, want %q", lane.LocalPath, want)
 	}
+	speaker := restored.Stages["transcribe"].TranscribeSpeakers["alice"]
+	if speaker.AudioPath != "audio/alice.flac" || speaker.Output.LocalPath != rawRelative {
+		t.Fatalf("restored speaker paths = %#v, want session-relative paths", speaker)
+	}
+	if _, err := os.Stat(filepath.Join(sessionRoot, filepath.FromSlash(rawRelative))); !os.IsNotExist(err) {
+		t.Fatalf("unpublished raw transcript should not be restored; stat err=%v", err)
+	}
 }

 func TestExecuteRestoreIncludeAudioRestoresAudio(t *testing.T) {
--- a/internal/app/runner_test_helpers_test.go
+++ b/internal/app/runner_test_helpers_test.go
@@ -6,6 +6,7 @@ import (
 	"encoding/hex"
 	"os"
 	"path/filepath"
+	"slices"
 	"strings"
 	"testing"

@@ -126,6 +127,14 @@ func seedCurrentTranscribeEvidence(
 	record := m.Stages["transcribe"]
 	record.TranscribeStateVersion = manifest.TranscribeStateContractVersion
 	record.TranscribeSpeakers = records
+	if len(record.Outputs) == 0 {
+		for _, speakerRecord := range records {
+			record.Outputs = append(record.Outputs, speakerRecord.Output)
+		}
+		slices.SortFunc(record.Outputs, func(a, b manifest.ArtifactRecord) int {
+			return strings.Compare(a.LocalPath, b.LocalPath)
+		})
+	}
 }

 func loadConfigForSemanticEvidence(t *testing.T, pipelinePath, campaignPath, sessionPath string) *config.Config {
--- a/internal/app/transcribe_checkpoint_recovery_test.go
+++ b/internal/app/transcribe_checkpoint_recovery_test.go
@@ -0,0 +1,338 @@
+package app
+
+import (
+	"context"
+	"errors"
+	"os"
+	"path/filepath"
+	"slices"
+	"testing"
+	"time"
+
+	"gitea.maximumdirect.net/eric/narratio/internal/adapters/whisperx"
+	"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
+	"gitea.maximumdirect.net/eric/narratio/internal/config"
+	"gitea.maximumdirect.net/eric/narratio/internal/manifest"
+	"gitea.maximumdirect.net/eric/narratio/internal/stage"
+)
+
+func TestExecuteStagesRetriesOnlyUncommittedTranscribeSpeakers(t *testing.T) {
+	cfg, baseStore := setupTranscribeRecoveryFixture(t, []string{"a", "b", "c", "d"})
+	concurrency := 1
+	cfg.Pipeline.WhisperX.Concurrency = &concurrency
+	cause := errors.New("speaker c failed")
+	failing := &whisperx.FakeClient{TranscribeFn: func(
+		_ context.Context,
+		req whisperx.TranscribeRequest,
+	) (whisperx.TranscribeResult, error) {
+		if req.SpeakerID == "c" {
+			return whisperx.TranscribeResult{}, cause
+		}
+		if req.SpeakerID == "d" {
+			t.Fatalf("speaker d was dispatched after c failed")
+		}
+		return writeTranscribeRecoveryResult(req)
+	}}
+	transcribe := mustSelectStage(t, "transcribe")
+
+	if _, err := executeStages(context.Background(), cfg, []stage.Stage{transcribe}, RunOptions{
+		Env: &Env{ManifestStore: baseStore, WhisperX: failing},
+	}); !errors.Is(err, cause) {
+		t.Fatalf("failed executeStages() error = %v, want %v", err, cause)
+	}
+	failed := loadTranscribeRecoveryManifest(t, baseStore, cfg)
+	if failed.Stages["transcribe"].Status != manifest.StatusFailed {
+		t.Fatalf("transcribe status = %q, want failed", failed.Stages["transcribe"].Status)
+	}
+	if got := transcribeSpeakerKeys(failed); !slices.Equal(got, []string{"a", "b"}) {
+		t.Fatalf("checkpointed speakers = %#v, want a and b", got)
+	}
+	if failed.Stages["prepare"].Status != manifest.StatusSucceeded {
+		t.Fatalf("prepare status = %q, want unchanged success", failed.Stages["prepare"].Status)
+	}
+	if failed.Stages["merge"].Status != manifest.StatusStale {
+		t.Fatalf("merge status = %q, want stale after transcribe failure", failed.Stages["merge"].Status)
+	}
+
+	retry := &whisperx.FakeClient{}
+	if _, err := executeStages(context.Background(), cfg, []stage.Stage{transcribe}, RunOptions{
+		Env: &Env{ManifestStore: baseStore, WhisperX: retry},
+	}); err != nil {
+		t.Fatalf("retry executeStages() error = %v", err)
+	}
+	if got := requestedTranscribeSpeakers(retry); !slices.Equal(got, []string{"c", "d"}) {
+		t.Fatalf("retry speakers = %#v, want c and d", got)
+	}
+	completed := loadTranscribeRecoveryManifest(t, baseStore, cfg)
+	if completed.Stages["transcribe"].Status != manifest.StatusSucceeded {
+		t.Fatalf("transcribe status = %q, want succeeded", completed.Stages["transcribe"].Status)
+	}
+	if got := transcribeSpeakerKeys(completed); !slices.Equal(got, []string{"a", "b", "c", "d"}) {
+		t.Fatalf("completed speakers = %#v", got)
+	}
+}
+
+func TestExecuteStagesRebuildsTranscribeAggregateAfterFinalSaveFailure(t *testing.T) {
+	cfg, baseStore := setupTranscribeRecoveryFixture(t, []string{"alice", "bob"})
+	store := &transcribeAggregateSaveFailingStore{
+		Store: baseStore,
+		Err:   errors.New("aggregate save failed"),
+	}
+	fake := &whisperx.FakeClient{}
+	transcribe := mustSelectStage(t, "transcribe")
+
+	if _, err := executeStages(context.Background(), cfg, []stage.Stage{transcribe}, RunOptions{
+		Env: &Env{ManifestStore: store, WhisperX: fake},
+	}); !errors.Is(err, store.Err) {
+		t.Fatalf("executeStages() error = %v, want aggregate save failure", err)
+	}
+	failed := loadTranscribeRecoveryManifest(t, baseStore, cfg)
+	if failed.Stages["transcribe"].Status != manifest.StatusFailed {
+		t.Fatalf("transcribe status = %q, want failed", failed.Stages["transcribe"].Status)
+	}
+	if got := transcribeSpeakerKeys(failed); !slices.Equal(got, []string{"alice", "bob"}) {
+		t.Fatalf("checkpointed speakers = %#v, want both speakers", got)
+	}
+
+	retry := &whisperx.FakeClient{}
+	if _, err := executeStages(context.Background(), cfg, []stage.Stage{transcribe}, RunOptions{
+		Env: &Env{ManifestStore: baseStore, WhisperX: retry},
+	}); err != nil {
+		t.Fatalf("retry executeStages() error = %v", err)
+	}
+	if got := len(retry.RequestsSnapshot()); got != 0 {
+		t.Fatalf("retry adapter requests = %d, want aggregate-only rebuild", got)
+	}
+	completed := loadTranscribeRecoveryManifest(t, baseStore, cfg)
+	if completed.Stages["transcribe"].Status != manifest.StatusSucceeded || len(completed.Stages["transcribe"].Outputs) != 2 {
+		t.Fatalf("rebuilt aggregate = %#v", completed.Stages["transcribe"])
+	}
+}
+
+func TestExecuteStagesResumesInterruptedRunningTranscribeState(t *testing.T) {
+	cfg, baseStore := setupTranscribeRecoveryFixture(t, []string{"alice", "bob"})
+	concurrency := 1
+	cfg.Pipeline.WhisperX.Concurrency = &concurrency
+	stageErr := errors.New("speaker b failed")
+	terminalErr := errors.New("terminal save interrupted")
+	store := &interruptAfterTranscribeCheckpointStore{Store: baseStore, Err: terminalErr}
+	fake := &whisperx.FakeClient{TranscribeFn: func(
+		_ context.Context,
+		req whisperx.TranscribeRequest,
+	) (whisperx.TranscribeResult, error) {
+		if req.SpeakerID == "bob" {
+			return whisperx.TranscribeResult{}, stageErr
+		}
+		return writeTranscribeRecoveryResult(req)
+	}}
+	transcribe := mustSelectStage(t, "transcribe")
+
+	_, err := executeStages(context.Background(), cfg, []stage.Stage{transcribe}, RunOptions{
+		Env: &Env{ManifestStore: store, WhisperX: fake},
+	})
+	if !errors.Is(err, stageErr) || !errors.Is(err, terminalErr) {
+		t.Fatalf("executeStages() error = %v, want stage and terminal failures", err)
+	}
+	interrupted := loadTranscribeRecoveryManifest(t, baseStore, cfg)
+	if interrupted.Stages["transcribe"].Status != manifest.StatusRunning {
+		t.Fatalf("transcribe status = %q, want durable running state", interrupted.Stages["transcribe"].Status)
+	}
+	if got := transcribeSpeakerKeys(interrupted); !slices.Equal(got, []string{"alice"}) {
+		t.Fatalf("checkpointed speakers = %#v, want alice", got)
+	}
+
+	retry := &whisperx.FakeClient{}
+	if _, err := executeStages(context.Background(), cfg, []stage.Stage{transcribe}, RunOptions{
+		Env: &Env{ManifestStore: baseStore, WhisperX: retry},
+	}); err != nil {
+		t.Fatalf("retry executeStages() error = %v", err)
+	}
+	if got := requestedTranscribeSpeakers(retry); !slices.Equal(got, []string{"bob"}) {
+		t.Fatalf("retry speakers = %#v, want bob", got)
+	}
+}
+
+func TestExecuteStagesPersistsForcedTranscribeResetAcrossRetry(t *testing.T) {
+	cfg, store := setupTranscribeRecoveryFixture(t, []string{"alice", "bob"})
+	concurrency := 1
+	cfg.Pipeline.WhisperX.Concurrency = &concurrency
+	transcribe := mustSelectStage(t, "transcribe")
+	if _, err := executeStages(context.Background(), cfg, []stage.Stage{transcribe}, RunOptions{
+		Env: &Env{ManifestStore: store, WhisperX: &whisperx.FakeClient{}},
+	}); err != nil {
+		t.Fatalf("baseline executeStages() error = %v", err)
+	}
+
+	cause := errors.New("forced bob failed")
+	forced := &whisperx.FakeClient{TranscribeFn: func(
+		_ context.Context,
+		req whisperx.TranscribeRequest,
+	) (whisperx.TranscribeResult, error) {
+		if req.SpeakerID == "bob" {
+			return whisperx.TranscribeResult{}, cause
+		}
+		return writeTranscribeRecoveryResult(req)
+	}}
+	if _, err := executeStages(context.Background(), cfg, []stage.Stage{transcribe}, RunOptions{
+		Force: true,
+		Env:   &Env{ManifestStore: store, WhisperX: forced},
+	}); !errors.Is(err, cause) {
+		t.Fatalf("forced executeStages() error = %v, want %v", err, cause)
+	}
+	failed := loadTranscribeRecoveryManifest(t, store, cfg)
+	if got := transcribeSpeakerKeys(failed); !slices.Equal(got, []string{"alice"}) {
+		t.Fatalf("speakers after forced failure = %#v, want only replacement alice", got)
+	}
+
+	retry := &whisperx.FakeClient{}
+	if _, err := executeStages(context.Background(), cfg, []stage.Stage{transcribe}, RunOptions{
+		Env: &Env{ManifestStore: store, WhisperX: retry},
+	}); err != nil {
+		t.Fatalf("retry executeStages() error = %v", err)
+	}
+	if got := requestedTranscribeSpeakers(retry); !slices.Equal(got, []string{"bob"}) {
+		t.Fatalf("retry speakers = %#v, want bob", got)
+	}
+}
+
+func TestExecuteStagesUpgradesLegacyTranscribeSuccessOnce(t *testing.T) {
+	cfg, store := setupTranscribeRecoveryFixture(t, []string{"alice"})
+	m := loadTranscribeRecoveryManifest(t, store, cfg)
+	m.MarkStageSucceeded("transcribe", time.Now().UTC(), []manifest.ArtifactRecord{{
+		Kind: "transcript_raw", LocalPath: "transcripts/raw/alice.json",
+	}})
+	if err := store.Save(context.Background(), manifestPathFor(cfg), m); err != nil {
+		t.Fatalf("save legacy transcribe success: %v", err)
+	}
+	transcribe := mustSelectStage(t, "transcribe")
+	upgrade := &whisperx.FakeClient{}
+	if _, err := executeStages(context.Background(), cfg, []stage.Stage{transcribe}, RunOptions{
+		Env: &Env{ManifestStore: store, WhisperX: upgrade},
+	}); err != nil {
+		t.Fatalf("upgrade executeStages() error = %v", err)
+	}
+	if got := requestedTranscribeSpeakers(upgrade); !slices.Equal(got, []string{"alice"}) {
+		t.Fatalf("upgrade speakers = %#v, want alice", got)
+	}
+	upgraded := loadTranscribeRecoveryManifest(t, store, cfg)
+	if upgraded.Stages["transcribe"].TranscribeStateVersion != manifest.TranscribeStateContractVersion {
+		t.Fatalf("transcribe state version = %d, want current", upgraded.Stages["transcribe"].TranscribeStateVersion)
+	}
+
+	reused := &whisperx.FakeClient{}
+	summary, err := executeStages(context.Background(), cfg, []stage.Stage{transcribe}, RunOptions{
+		Env: &Env{ManifestStore: store, WhisperX: reused},
+	})
+	if err != nil {
+		t.Fatalf("reuse executeStages() error = %v", err)
+	}
+	if len(summary.Skipped) != 1 || len(reused.RequestsSnapshot()) != 0 {
+		t.Fatalf("reuse summary = %#v, requests = %#v, want skipped without HTTP", summary, reused.RequestsSnapshot())
+	}
+}
+
+type transcribeAggregateSaveFailingStore struct {
+	manifest.Store
+	Err    error
+	Failed bool
+}
+
+func (store *transcribeAggregateSaveFailingStore) Save(
+	ctx context.Context,
+	path string,
+	m *manifest.Manifest,
+) error {
+	record := m.Stages["transcribe"]
+	if !store.Failed && record != nil && record.Status == manifest.StatusSucceeded &&
+		record.TranscribeStateVersion == manifest.TranscribeStateContractVersion {
+		store.Failed = true
+		return store.Err
+	}
+	return store.Store.Save(ctx, path, m)
+}
+
+type interruptAfterTranscribeCheckpointStore struct {
+	manifest.Store
+	Err         error
+	Interrupted bool
+}
+
+func (store *interruptAfterTranscribeCheckpointStore) Save(
+	ctx context.Context,
+	path string,
+	m *manifest.Manifest,
+) error {
+	if store.Interrupted {
+		return store.Err
+	}
+	record := m.Stages["transcribe"]
+	err := store.Store.Save(ctx, path, m)
+	if err == nil && record != nil && record.Status == manifest.StatusRunning && len(record.TranscribeSpeakers) == 1 {
+		store.Interrupted = true
+	}
+	return err
+}
+
+func setupTranscribeRecoveryFixture(t *testing.T, speakers []string) (*config.Config, *manifest.LocalStore) {
+	t.Helper()
+	cfg := testConfig(t)
+	store := &manifest.LocalStore{}
+	artifactStore := artifacts.NewLocalStore(cfg.Pipeline.Workspace.Root)
+	paths, err := artifactStore.EnsureLayoutFor(cfg.Session.Campaign, cfg.Session.SessionID)
+	if err != nil {
+		t.Fatalf("prepare session layout: %v", err)
+	}
+	m := manifest.New(cfg.Session.SessionID, time.Now().UTC())
+	m.Campaign = cfg.Session.Campaign
+	m.Inputs = make([]manifest.InputRecord, 0, len(speakers))
+	for _, speaker := range speakers {
+		relative := filepath.ToSlash(filepath.Join("audio", speaker+".flac"))
+		if err := os.WriteFile(filepath.Join(paths.AudioDir, speaker+".flac"), []byte("audio-"+speaker), 0o644); err != nil {
+			t.Fatalf("write audio fixture: %v", err)
+		}
+		m.Inputs = append(m.Inputs, manifest.InputRecord{Kind: "audio", Path: relative})
+	}
+	m.MarkStageSucceeded("prepare", time.Now().UTC(), nil)
+	m.MarkStageSucceeded("merge", time.Now().UTC(), nil)
+	if err := store.Save(context.Background(), paths.ManifestPath, m); err != nil {
+		t.Fatalf("save recovery fixture: %v", err)
+	}
+	return cfg, store
+}
+
+func writeTranscribeRecoveryResult(req whisperx.TranscribeRequest) (whisperx.TranscribeResult, error) {
+	if err := os.MkdirAll(filepath.Dir(req.OutputRawTranscriptPath), 0o755); err != nil {
+		return whisperx.TranscribeResult{}, err
+	}
+	if err := os.WriteFile(req.OutputRawTranscriptPath, []byte(`{"segments":[]}`), 0o644); err != nil {
+		return whisperx.TranscribeResult{}, err
+	}
+	return whisperx.TranscribeResult{OutputRawTranscriptPath: req.OutputRawTranscriptPath}, nil
+}
+
+func loadTranscribeRecoveryManifest(t *testing.T, store manifest.Store, cfg *config.Config) *manifest.Manifest {
+	t.Helper()
+	m, err := store.Load(context.Background(), manifestPathFor(cfg))
+	if err != nil {
+		t.Fatalf("load transcribe recovery manifest: %v", err)
+	}
+	return m
+}
+
+func transcribeSpeakerKeys(m *manifest.Manifest) []string {
+	keys := make([]string, 0, len(m.Stages["transcribe"].TranscribeSpeakers))
+	for key := range m.Stages["transcribe"].TranscribeSpeakers {
+		keys = append(keys, key)
+	}
+	slices.Sort(keys)
+	return keys
+}
+
+func requestedTranscribeSpeakers(fake *whisperx.FakeClient) []string {
+	requests := fake.RequestsSnapshot()
+	speakers := make([]string, 0, len(requests))
+	for _, request := range requests {
+		speakers = append(speakers, request.SpeakerID)
+	}
+	return speakers
+}