eric/narratio
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 27 Wiki Activity

Integrate transcription checkpoints across lifecycle

Browse Source
This commit is contained in:
Eric Rakestraw
2026-09-22 00:00:51 +00:00
parent 2ae63dbc2b
commit d872edca94
7 changed files with 581 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

64
internal/stage/merge.go
View File

@@ -306,6 +306,12 @@ func normalizeMergeInputs(
}
func discoverRawTranscripts(m *manifest.Manifest, paths artifacts.SessionPaths) ([]string, error) {
if record, versioned, err := transcribeManifestState(m); err != nil {
return nil, err
} else if versioned {
return discoverVersionedRawTranscripts(record, paths)
}
fromManifest := make([]string, 0)
if m != nil && m.Stages != nil {
if tr := m.Stages["transcribe"]; tr != nil {
@@ -350,6 +356,64 @@ func discoverRawTranscripts(m *manifest.Manifest, paths artifacts.SessionPaths)
return dedupeAndSortPaths(out), nil
}
func discoverVersionedRawTranscripts(
record *manifest.StageRecord,
paths artifacts.SessionPaths,
) ([]string, error) {
if record.Status != manifest.StatusSucceeded {
return nil, fmt.Errorf("versioned transcribe state is not aggregate-successful (status %q)", record.Status)
}
if err := manifest.ValidateTranscribeSpeakerCollection(
record.TranscribeStateVersion,
record.TranscribeSpeakers,
); err != nil {
return nil, fmt.Errorf("validate versioned transcribe state: %w", err)
}
if len(record.TranscribeSpeakers) == 0 {
return nil, fmt.Errorf("versioned transcribe state has no completed speakers")
}
expected := make(map[string]string, len(record.TranscribeSpeakers))
for speaker, speakerRecord := range record.TranscribeSpeakers {
path := filepath.Clean(artifacts.ResolveSessionLocalPathForRead(paths, speakerRecord.Output.LocalPath))
if other, exists := expected[path]; exists {
return nil, fmt.Errorf("versioned transcribe speakers %q and %q declare the same output", other, speaker)
}
expected[path] = speaker
}
inputs := make([]string, 0, len(expected))
for _, output := range record.Outputs {
if output.Kind != "transcript_raw" {
continue
}
path := strings.TrimSpace(output.LocalPath)
if path == "" {
return nil, fmt.Errorf("versioned transcribe aggregate output has an empty local path")
}
path = filepath.Clean(artifacts.ResolveSessionLocalPathForRead(paths, path))
speaker, declared := expected[path]
if !declared {
return nil, fmt.Errorf("versioned transcribe aggregate output %q is not declared by current speaker state", output.LocalPath)
}
if speaker == "" {
return nil, fmt.Errorf("versioned transcribe aggregate output %q is duplicated", output.LocalPath)
}
expected[path] = ""
inputs = append(inputs, path)
}
missing := make([]string, 0)
for _, speaker := range expected {
if speaker != "" {
missing = append(missing, speaker)
}
}
if len(missing) > 0 {
sort.Strings(missing)
return nil, fmt.Errorf("versioned transcribe aggregate outputs are incomplete for speakers: %s", strings.Join(missing, ", "))
}
return dedupeAndSortPaths(inputs), nil
}
func dedupeAndSortPaths(paths []string) []string {
if len(paths) == 0 {
return nil

106
internal/stage/merge_test.go
View File

@@ -5,6 +5,7 @@ import (
"fmt"
"os"
"path/filepath"
"slices"
"strings"
"testing"
"time"
@@ -191,6 +192,70 @@ func TestMergeStageFallsBackToRawDirectoryWhenTranscribeOutputsMissing(t *testin
}
}
func TestMergeStageRejectsPartialVersionedTranscribeState(t *testing.T) {
env, m := setupMergeEnv(t)
paths := sessionPathsForEnv(env, m.SessionID)
writeFile(t, filepath.Join(paths.TranscriptsRawDir, "alice.json"), `{"segments":[]}`)
writeFile(t, filepath.Join(paths.TranscriptsRawDir, "uncommitted.json"), `{"segments":[]}`)
writeFile(t, filepath.Join(paths.InputsDir, "speakers.yml"), "match: []\n")
writeFile(t, filepath.Join(paths.InputsDir, "autocorrect.yml"), "rules: []\n")
setVersionedTranscribeMergeState(t, m, manifest.StatusFailed, map[string]string{
"alice": "transcripts/raw/alice.json",
})
fake := &seriatim.FakeRunner{}
env.Seriatim = fake
_, err := (mergeStage{}).Run(context.Background(), env, m)
if err == nil || !strings.Contains(err.Error(), "not aggregate-successful") {
t.Fatalf("merge.Run() error = %v, want incomplete aggregate rejection", err)
}
if len(fake.Requests) != 0 || len(fake.NormalizeRequests) != 0 {
t.Fatalf("adapter requests = %#v / %#v, want none", fake.Requests, fake.NormalizeRequests)
}
}
func TestMergeStageUsesOnlyCurrentVersionedAggregateOutputs(t *testing.T) {
env, m := setupMergeEnv(t)
paths := sessionPathsForEnv(env, m.SessionID)
alice := filepath.Join(paths.TranscriptsRawDir, "alice.json")
writeFile(t, alice, `{"segments":[]}`)
writeFile(t, filepath.Join(paths.TranscriptsRawDir, "removed.json"), `{"segments":[]}`)
writeFile(t, filepath.Join(paths.InputsDir, "speakers.yml"), "match: []\n")
writeFile(t, filepath.Join(paths.InputsDir, "autocorrect.yml"), "rules: []\n")
setVersionedTranscribeMergeState(t, m, manifest.StatusSucceeded, map[string]string{
"alice": "transcripts/raw/alice.json",
})
fake := &seriatim.FakeRunner{}
env.Seriatim = fake
if _, err := (mergeStage{}).Run(context.Background(), env, m); err != nil {
t.Fatalf("merge.Run() error = %v", err)
}
if len(fake.NormalizeRequests) != 1 || fake.NormalizeRequests[0].InputTranscriptPath != alice {
t.Fatalf("normalize requests = %#v, want only current alice", fake.NormalizeRequests)
}
}
func TestMergeStageRejectsIncompleteVersionedAggregateOutputs(t *testing.T) {
env, m := setupMergeEnv(t)
paths := sessionPathsForEnv(env, m.SessionID)
writeFile(t, filepath.Join(paths.TranscriptsRawDir, "alice.json"), `{"segments":[]}`)
writeFile(t, filepath.Join(paths.TranscriptsRawDir, "bob.json"), `{"segments":[]}`)
writeFile(t, filepath.Join(paths.InputsDir, "speakers.yml"), "match: []\n")
writeFile(t, filepath.Join(paths.InputsDir, "autocorrect.yml"), "rules: []\n")
setVersionedTranscribeMergeState(t, m, manifest.StatusSucceeded, map[string]string{
"alice": "transcripts/raw/alice.json",
"bob": "transcripts/raw/bob.json",
})
m.Stages["transcribe"].Outputs = m.Stages["transcribe"].Outputs[:1]
env.Seriatim = &seriatim.FakeRunner{}
_, err := (mergeStage{}).Run(context.Background(), env, m)
if err == nil || !strings.Contains(err.Error(), "incomplete for speakers: bob") {
t.Fatalf("merge.Run() error = %v, want incomplete aggregate output rejection", err)
}
}
func TestMergeStageResolvesWorkspaceQualifiedManifestOutputsWithoutDuplication(t *testing.T) {
env, m := setupMergeEnvWithRelativeWorkspaceRoot(t)
paths := sessionPathsForEnv(env, m.SessionID)
@@ -414,3 +479,44 @@ func setupMergeEnvWithWorkspace(t *testing.T, workspace string) (*Env, *manifest
ArtifactStore: store,
}, manifest.New("2026-05-03", time.Now().UTC())
}
func setVersionedTranscribeMergeState(
t *testing.T,
m *manifest.Manifest,
status manifest.StageStatus,
outputs map[string]string,
) {
t.Helper()
if m.RunID == "" {
m.RunID = "20260503T010203Z-a1b2c3d4"
}
aggregate := make([]manifest.ArtifactRecord, 0, len(outputs))
records := make(map[string]manifest.TranscribeSpeakerRecord, len(outputs))
for speaker, outputPath := range outputs {
output := manifest.ArtifactRecord{
Kind: "transcript_raw", LocalPath: outputPath,
ProducerRunID: m.RunID, Checksum: strings.Repeat("b", 64),
}
aggregate = append(aggregate, output)
records[speaker] = manifest.TranscribeSpeakerRecord{
FingerprintVersion: manifest.TranscribeFingerprintContractVersion,
Fingerprint: strings.Repeat("c", 64),
AudioPath: "audio/" + speaker + ".flac",
AudioSHA256: strings.Repeat("a", 64),
AudioSize: 1,
Output: output,
OutputSize: 1,
ProducerRunID: m.RunID,
}
}
slices.SortFunc(aggregate, func(a, b manifest.ArtifactRecord) int {
return strings.Compare(a.LocalPath, b.LocalPath)
})
m.MarkStageSucceeded("transcribe", time.Now().UTC(), aggregate)
record := m.Stages["transcribe"]
record.TranscribeStateVersion = manifest.TranscribeStateContractVersion
record.TranscribeSpeakers = records
if status != manifest.StatusSucceeded {
m.MarkStageFailed("transcribe", time.Now().UTC(), "incomplete")
}
}

22
internal/stage/transcribe_checkpoint_test.go
View File

@@ -271,6 +271,28 @@ func TestTranscribeRebuildsAggregateFromCurrentSpeakersWithoutRequests(t *testin
}
}
func TestTranscribeReuseDoesNotDependOnRunDirectory(t *testing.T) {
env, m := setupTranscribeEnv(t, []string{"alice.flac"})
env.WhisperX = &whisperx.FakeClient{}
if _, err := (prepareStage{}).Run(context.Background(), env, m); err != nil {
t.Fatalf("prepare.Run() error = %v", err)
}
if _, err := (transcribeStage{}).Run(context.Background(), env, m); err != nil {
t.Fatalf("Run() error = %v", err)
}
paths := sessionPathsForEnv(env, m.SessionID)
if err := os.RemoveAll(paths.RunsDir); err != nil {
t.Fatalf("remove run directory: %v", err)
}
if _, err := os.Stat(paths.TranscriptsRawDir + "/alice.json"); err != nil {
t.Fatalf("canonical transcript missing after run cleanup: %v", err)
}
validation, err := (transcribeStage{}).ValidateResume(context.Background(), env, m)
if err != nil || !validation.Resumable {
t.Fatalf("ValidateResume() = %#v, %v, want canonical reuse", validation, err)
}
}
type failingTranscribeManifestStore struct {
manifest.Store
Saves atomic.Int64