package app import ( "context" "errors" "os" "path/filepath" "slices" "testing" "time" "gitea.maximumdirect.net/eric/narratio/internal/adapters/whisperx" "gitea.maximumdirect.net/eric/narratio/internal/artifacts" "gitea.maximumdirect.net/eric/narratio/internal/config" "gitea.maximumdirect.net/eric/narratio/internal/manifest" "gitea.maximumdirect.net/eric/narratio/internal/stage" ) func TestExecuteStagesRetriesOnlyUncommittedTranscribeSpeakers(t *testing.T) { cfg, baseStore := setupTranscribeRecoveryFixture(t, []string{"a", "b", "c", "d"}) concurrency := 1 cfg.Pipeline.WhisperX.Concurrency = &concurrency cause := errors.New("speaker c failed") failing := &whisperx.FakeClient{TranscribeFn: func( _ context.Context, req whisperx.TranscribeRequest, ) (whisperx.TranscribeResult, error) { if req.SpeakerID == "c" { return whisperx.TranscribeResult{}, cause } if req.SpeakerID == "d" { t.Fatalf("speaker d was dispatched after c failed") } return writeTranscribeRecoveryResult(req) }} transcribe := mustSelectStage(t, "transcribe") if _, err := executeStages(context.Background(), cfg, []stage.Stage{transcribe}, RunOptions{ Env: &Env{ManifestStore: baseStore, WhisperX: failing}, }); !errors.Is(err, cause) { t.Fatalf("failed executeStages() error = %v, want %v", err, cause) } failed := loadTranscribeRecoveryManifest(t, baseStore, cfg) if failed.Stages["transcribe"].Status != manifest.StatusFailed { t.Fatalf("transcribe status = %q, want failed", failed.Stages["transcribe"].Status) } if got := transcribeSpeakerKeys(failed); !slices.Equal(got, []string{"a", "b"}) { t.Fatalf("checkpointed speakers = %#v, want a and b", got) } if failed.Stages["prepare"].Status != manifest.StatusSucceeded { t.Fatalf("prepare status = %q, want unchanged success", failed.Stages["prepare"].Status) } if failed.Stages["merge"].Status != manifest.StatusStale { t.Fatalf("merge status = %q, want stale after transcribe failure", failed.Stages["merge"].Status) } retry := &whisperx.FakeClient{} if _, err := executeStages(context.Background(), cfg, []stage.Stage{transcribe}, RunOptions{ Env: &Env{ManifestStore: baseStore, WhisperX: retry}, }); err != nil { t.Fatalf("retry executeStages() error = %v", err) } if got := requestedTranscribeSpeakers(retry); !slices.Equal(got, []string{"c", "d"}) { t.Fatalf("retry speakers = %#v, want c and d", got) } completed := loadTranscribeRecoveryManifest(t, baseStore, cfg) if completed.Stages["transcribe"].Status != manifest.StatusSucceeded { t.Fatalf("transcribe status = %q, want succeeded", completed.Stages["transcribe"].Status) } if got := transcribeSpeakerKeys(completed); !slices.Equal(got, []string{"a", "b", "c", "d"}) { t.Fatalf("completed speakers = %#v", got) } } func TestExecuteStagesRebuildsTranscribeAggregateAfterFinalSaveFailure(t *testing.T) { cfg, baseStore := setupTranscribeRecoveryFixture(t, []string{"alice", "bob"}) store := &transcribeAggregateSaveFailingStore{ Store: baseStore, Err: errors.New("aggregate save failed"), } fake := &whisperx.FakeClient{} transcribe := mustSelectStage(t, "transcribe") if _, err := executeStages(context.Background(), cfg, []stage.Stage{transcribe}, RunOptions{ Env: &Env{ManifestStore: store, WhisperX: fake}, }); !errors.Is(err, store.Err) { t.Fatalf("executeStages() error = %v, want aggregate save failure", err) } failed := loadTranscribeRecoveryManifest(t, baseStore, cfg) if failed.Stages["transcribe"].Status != manifest.StatusFailed { t.Fatalf("transcribe status = %q, want failed", failed.Stages["transcribe"].Status) } if got := transcribeSpeakerKeys(failed); !slices.Equal(got, []string{"alice", "bob"}) { t.Fatalf("checkpointed speakers = %#v, want both speakers", got) } retry := &whisperx.FakeClient{} if _, err := executeStages(context.Background(), cfg, []stage.Stage{transcribe}, RunOptions{ Env: &Env{ManifestStore: baseStore, WhisperX: retry}, }); err != nil { t.Fatalf("retry executeStages() error = %v", err) } if got := len(retry.RequestsSnapshot()); got != 0 { t.Fatalf("retry adapter requests = %d, want aggregate-only rebuild", got) } completed := loadTranscribeRecoveryManifest(t, baseStore, cfg) if completed.Stages["transcribe"].Status != manifest.StatusSucceeded || len(completed.Stages["transcribe"].Outputs) != 2 { t.Fatalf("rebuilt aggregate = %#v", completed.Stages["transcribe"]) } } func TestExecuteStagesResumesInterruptedRunningTranscribeState(t *testing.T) { cfg, baseStore := setupTranscribeRecoveryFixture(t, []string{"alice", "bob"}) concurrency := 1 cfg.Pipeline.WhisperX.Concurrency = &concurrency stageErr := errors.New("speaker b failed") terminalErr := errors.New("terminal save interrupted") store := &interruptAfterTranscribeCheckpointStore{Store: baseStore, Err: terminalErr} fake := &whisperx.FakeClient{TranscribeFn: func( _ context.Context, req whisperx.TranscribeRequest, ) (whisperx.TranscribeResult, error) { if req.SpeakerID == "bob" { return whisperx.TranscribeResult{}, stageErr } return writeTranscribeRecoveryResult(req) }} transcribe := mustSelectStage(t, "transcribe") _, err := executeStages(context.Background(), cfg, []stage.Stage{transcribe}, RunOptions{ Env: &Env{ManifestStore: store, WhisperX: fake}, }) if !errors.Is(err, stageErr) || !errors.Is(err, terminalErr) { t.Fatalf("executeStages() error = %v, want stage and terminal failures", err) } interrupted := loadTranscribeRecoveryManifest(t, baseStore, cfg) if interrupted.Stages["transcribe"].Status != manifest.StatusRunning { t.Fatalf("transcribe status = %q, want durable running state", interrupted.Stages["transcribe"].Status) } if got := transcribeSpeakerKeys(interrupted); !slices.Equal(got, []string{"alice"}) { t.Fatalf("checkpointed speakers = %#v, want alice", got) } retry := &whisperx.FakeClient{} if _, err := executeStages(context.Background(), cfg, []stage.Stage{transcribe}, RunOptions{ Env: &Env{ManifestStore: baseStore, WhisperX: retry}, }); err != nil { t.Fatalf("retry executeStages() error = %v", err) } if got := requestedTranscribeSpeakers(retry); !slices.Equal(got, []string{"bob"}) { t.Fatalf("retry speakers = %#v, want bob", got) } } func TestExecuteStagesPersistsForcedTranscribeResetAcrossRetry(t *testing.T) { cfg, store := setupTranscribeRecoveryFixture(t, []string{"alice", "bob"}) concurrency := 1 cfg.Pipeline.WhisperX.Concurrency = &concurrency transcribe := mustSelectStage(t, "transcribe") if _, err := executeStages(context.Background(), cfg, []stage.Stage{transcribe}, RunOptions{ Env: &Env{ManifestStore: store, WhisperX: &whisperx.FakeClient{}}, }); err != nil { t.Fatalf("baseline executeStages() error = %v", err) } cause := errors.New("forced bob failed") forced := &whisperx.FakeClient{TranscribeFn: func( _ context.Context, req whisperx.TranscribeRequest, ) (whisperx.TranscribeResult, error) { if req.SpeakerID == "bob" { return whisperx.TranscribeResult{}, cause } return writeTranscribeRecoveryResult(req) }} if _, err := executeStages(context.Background(), cfg, []stage.Stage{transcribe}, RunOptions{ Force: true, Env: &Env{ManifestStore: store, WhisperX: forced}, }); !errors.Is(err, cause) { t.Fatalf("forced executeStages() error = %v, want %v", err, cause) } failed := loadTranscribeRecoveryManifest(t, store, cfg) if got := transcribeSpeakerKeys(failed); !slices.Equal(got, []string{"alice"}) { t.Fatalf("speakers after forced failure = %#v, want only replacement alice", got) } retry := &whisperx.FakeClient{} if _, err := executeStages(context.Background(), cfg, []stage.Stage{transcribe}, RunOptions{ Env: &Env{ManifestStore: store, WhisperX: retry}, }); err != nil { t.Fatalf("retry executeStages() error = %v", err) } if got := requestedTranscribeSpeakers(retry); !slices.Equal(got, []string{"bob"}) { t.Fatalf("retry speakers = %#v, want bob", got) } } func TestExecuteStagesUpgradesLegacyTranscribeSuccessOnce(t *testing.T) { cfg, store := setupTranscribeRecoveryFixture(t, []string{"alice"}) m := loadTranscribeRecoveryManifest(t, store, cfg) m.MarkStageSucceeded("transcribe", time.Now().UTC(), []manifest.ArtifactRecord{{ Kind: "transcript_raw", LocalPath: "transcripts/raw/alice.json", }}) if err := store.Save(context.Background(), manifestPathFor(cfg), m); err != nil { t.Fatalf("save legacy transcribe success: %v", err) } transcribe := mustSelectStage(t, "transcribe") upgrade := &whisperx.FakeClient{} if _, err := executeStages(context.Background(), cfg, []stage.Stage{transcribe}, RunOptions{ Env: &Env{ManifestStore: store, WhisperX: upgrade}, }); err != nil { t.Fatalf("upgrade executeStages() error = %v", err) } if got := requestedTranscribeSpeakers(upgrade); !slices.Equal(got, []string{"alice"}) { t.Fatalf("upgrade speakers = %#v, want alice", got) } upgraded := loadTranscribeRecoveryManifest(t, store, cfg) if upgraded.Stages["transcribe"].TranscribeStateVersion != manifest.TranscribeStateContractVersion { t.Fatalf("transcribe state version = %d, want current", upgraded.Stages["transcribe"].TranscribeStateVersion) } reused := &whisperx.FakeClient{} summary, err := executeStages(context.Background(), cfg, []stage.Stage{transcribe}, RunOptions{ Env: &Env{ManifestStore: store, WhisperX: reused}, }) if err != nil { t.Fatalf("reuse executeStages() error = %v", err) } if len(summary.Skipped) != 1 || len(reused.RequestsSnapshot()) != 0 { t.Fatalf("reuse summary = %#v, requests = %#v, want skipped without HTTP", summary, reused.RequestsSnapshot()) } } type transcribeAggregateSaveFailingStore struct { manifest.Store Err error Failed bool } func (store *transcribeAggregateSaveFailingStore) Save( ctx context.Context, path string, m *manifest.Manifest, ) error { record := m.Stages["transcribe"] if !store.Failed && record != nil && record.Status == manifest.StatusSucceeded && record.TranscribeStateVersion == manifest.TranscribeStateContractVersion { store.Failed = true return store.Err } return store.Store.Save(ctx, path, m) } type interruptAfterTranscribeCheckpointStore struct { manifest.Store Err error Interrupted bool } func (store *interruptAfterTranscribeCheckpointStore) Save( ctx context.Context, path string, m *manifest.Manifest, ) error { if store.Interrupted { return store.Err } record := m.Stages["transcribe"] err := store.Store.Save(ctx, path, m) if err == nil && record != nil && record.Status == manifest.StatusRunning && len(record.TranscribeSpeakers) == 1 { store.Interrupted = true } return err } func setupTranscribeRecoveryFixture(t *testing.T, speakers []string) (*config.Config, *manifest.LocalStore) { t.Helper() cfg := testConfig(t) store := &manifest.LocalStore{} artifactStore := artifacts.NewLocalStore(cfg.Pipeline.Workspace.Root) paths, err := artifactStore.EnsureLayoutFor(cfg.Session.Campaign, cfg.Session.SessionID) if err != nil { t.Fatalf("prepare session layout: %v", err) } m := manifest.New(cfg.Session.SessionID, time.Now().UTC()) m.Campaign = cfg.Session.Campaign m.Inputs = make([]manifest.InputRecord, 0, len(speakers)) for _, speaker := range speakers { relative := filepath.ToSlash(filepath.Join("audio", speaker+".flac")) if err := os.WriteFile(filepath.Join(paths.AudioDir, speaker+".flac"), []byte("audio-"+speaker), 0o644); err != nil { t.Fatalf("write audio fixture: %v", err) } m.Inputs = append(m.Inputs, manifest.InputRecord{Kind: "audio", Path: relative}) } m.MarkStageSucceeded("prepare", time.Now().UTC(), nil) m.MarkStageSucceeded("merge", time.Now().UTC(), nil) if err := store.Save(context.Background(), paths.ManifestPath, m); err != nil { t.Fatalf("save recovery fixture: %v", err) } return cfg, store } func writeTranscribeRecoveryResult(req whisperx.TranscribeRequest) (whisperx.TranscribeResult, error) { if err := os.MkdirAll(filepath.Dir(req.OutputRawTranscriptPath), 0o755); err != nil { return whisperx.TranscribeResult{}, err } if err := os.WriteFile(req.OutputRawTranscriptPath, []byte(`{"segments":[]}`), 0o644); err != nil { return whisperx.TranscribeResult{}, err } return whisperx.TranscribeResult{OutputRawTranscriptPath: req.OutputRawTranscriptPath}, nil } func loadTranscribeRecoveryManifest(t *testing.T, store manifest.Store, cfg *config.Config) *manifest.Manifest { t.Helper() m, err := store.Load(context.Background(), manifestPathFor(cfg)) if err != nil { t.Fatalf("load transcribe recovery manifest: %v", err) } return m } func transcribeSpeakerKeys(m *manifest.Manifest) []string { keys := make([]string, 0, len(m.Stages["transcribe"].TranscribeSpeakers)) for key := range m.Stages["transcribe"].TranscribeSpeakers { keys = append(keys, key) } slices.Sort(keys) return keys } func requestedTranscribeSpeakers(fake *whisperx.FakeClient) []string { requests := fake.RequestsSnapshot() speakers := make([]string, 0, len(requests)) for _, request := range requests { speakers = append(speakers, request.SpeakerID) } return speakers }