eric/narratio
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 27 Wiki Activity

Reconcile reusable speaker transcriptions

Browse Source
This commit is contained in:
Eric Rakestraw
2026-09-21 16:09:45 +00:00
parent 7b0bf99ed1
commit 26aa0cce29
4 changed files with 700 additions and 52 deletions
Download Patch File Download Diff File Expand all files Collapse all files

109
internal/stage/transcribe.go
View File

@@ -51,48 +51,26 @@ func (transcribeStage) Run(ctx context.Context, env *Env, m *manifest.Manifest)
if err != nil {
return nil, fmt.Errorf("transcribe: resolve run-stage layout: %w", err)
}
audioFiles, err := discoverPreparedAudio(m, paths.AudioDir)
jobs, err := buildTranscribeJobs(m, paths)
if err != nil {
return nil, fmt.Errorf("transcribe: resolve audio inputs: %w", err)
}
if len(audioFiles) == 0 {
if len(jobs) == 0 {
return nil, fmt.Errorf("transcribe: no prepared audio files found")
}
for index := range jobs {
jobs[index].runOutputPath, err = runLocalPathForCanonical(runLayout, paths, jobs[index].canonicalOutputPath)
if err != nil {
return nil, fmt.Errorf("transcribe: resolve run-local output path for %q: %w", jobs[index].speakerID, err)
}
}
concurrency := 1
if env.Config.Pipeline.WhisperX.Concurrency != nil && *env.Config.Pipeline.WhisperX.Concurrency > 0 {
concurrency = *env.Config.Pipeline.WhisperX.Concurrency
}
if concurrency > len(audioFiles) {
concurrency = len(audioFiles)
}
type job struct {
speakerID string
audioPath string
outPath string
}
jobs := make([]job, 0, len(audioFiles))
seenSpeaker := map[string]string{}
for _, audioPath := range audioFiles {
base := strings.TrimSuffix(filepath.Base(audioPath), filepath.Ext(audioPath))
if base == "" {
return nil, fmt.Errorf("transcribe: could not derive speaker id from %q", audioPath)
}
if prev, ok := seenSpeaker[base]; ok {
return nil, fmt.Errorf("transcribe: duplicate speaker/audio basename %q from %q and %q", base, prev, audioPath)
}
seenSpeaker[base] = audioPath
canonicalOut := filepath.Join(paths.TranscriptsRawDir, base+".json")
runOut, err := runLocalPathForCanonical(runLayout, paths, canonicalOut)
if err != nil {
return nil, fmt.Errorf("transcribe: resolve run-local output path for %q: %w", base, err)
}
jobs = append(jobs, job{
speakerID: base,
audioPath: audioPath,
outPath: runOut,
})
if concurrency > len(jobs) {
concurrency = len(jobs)
}
var (
@@ -111,24 +89,24 @@ func (transcribeStage) Run(ctx context.Context, env *Env, m *manifest.Manifest)
res, runErr := env.WhisperX.Transcribe(groupCtx, whisperx.TranscribeRequest{
SpeakerID: job.speakerID,
AudioPath: job.audioPath,
OutputRawTranscriptPath: job.outPath,
OutputRawTranscriptPath: job.runOutputPath,
})
if runErr != nil {
return fmt.Errorf("speaker %q (%s): %w", job.speakerID, filepath.Base(job.audioPath), runErr)
}
if _, err := authoritativeOutputPath(job.outPath, res.OutputRawTranscriptPath); err != nil {
if _, err := authoritativeOutputPath(job.runOutputPath, res.OutputRawTranscriptPath); err != nil {
return fmt.Errorf("speaker %q: %w", job.speakerID, err)
}
if err := validateTranscriptJSONFile(job.outPath); err != nil {
return fmt.Errorf("speaker %q output %q invalid: %w", job.speakerID, job.outPath, err)
if err := validateTranscriptJSONFile(job.runOutputPath); err != nil {
return fmt.Errorf("speaker %q output %q invalid: %w", job.speakerID, job.runOutputPath, err)
}
meta := map[string]any{
"attempts": res.Attempts,
"http_status": res.HTTPStatus,
"duration_ms": res.Duration.Milliseconds(),
"output_path": job.outPath,
"output_path": job.runOutputPath,
}
mu.Lock()
@@ -163,22 +141,21 @@ func (transcribeStage) Run(ctx context.Context, env *Env, m *manifest.Manifest)
outputPaths := make([]string, 0, len(speakers))
orderedPerFile := make(map[string]any, len(speakers))
for _, speaker := range speakers {
var runOut string
var selected transcribeJob
for _, job := range jobs {
if job.speakerID == speaker {
runOut = job.outPath
selected = job
break
}
}
runOutputPaths = append(runOutputPaths, runOut)
canonicalOut := filepath.Join(paths.TranscriptsRawDir, speaker+".json")
ref := artifacts.Ref{Kind: "transcript_raw", Category: "transcripts", SessionID: sessionID, AbsolutePath: runOut}
materialized, err := materializeRunLocalOutput(env.ArtifactStore, runOut, canonicalOut, ref)
runOutputPaths = append(runOutputPaths, selected.runOutputPath)
ref := artifacts.Ref{Kind: "transcript_raw", Category: "transcripts", SessionID: sessionID, AbsolutePath: selected.runOutputPath}
materialized, err := materializeRunLocalOutput(env.ArtifactStore, selected.runOutputPath, selected.canonicalOutputPath, ref)
if err != nil {
return nil, fmt.Errorf("transcribe: materialize %q output: %w", speaker, err)
}
outputs = append(outputs, materialized)
outputPaths = append(outputPaths, canonicalOut)
outputPaths = append(outputPaths, selected.canonicalOutputPath)
orderedPerFile[speaker] = perFile[speaker]
}
@@ -191,7 +168,7 @@ func (transcribeStage) Run(ctx context.Context, env *Env, m *manifest.Manifest)
Outputs: outputs,
Metadata: map[string]any{
"stage": "transcribe",
"audio_files_count": len(audioFiles),
"audio_files_count": len(jobs),
"language": env.Config.Pipeline.WhisperX.Language,
"concurrency": concurrency,
"retries": retries,
@@ -204,7 +181,39 @@ func (transcribeStage) Run(ctx context.Context, env *Env, m *manifest.Manifest)
}, nil
}
func discoverPreparedAudio(m *manifest.Manifest, audioDir string) ([]string, error) {
type transcribeJob struct {
speakerID string
audioPath string
canonicalOutputPath string
runOutputPath string
}
func buildTranscribeJobs(m *manifest.Manifest, paths artifacts.SessionPaths) ([]transcribeJob, error) {
audioFiles, err := discoverPreparedAudio(m, paths)
if err != nil {
return nil, err
}
jobs := make([]transcribeJob, 0, len(audioFiles))
seenSpeaker := map[string]string{}
for _, audioPath := range audioFiles {
base := strings.TrimSuffix(filepath.Base(audioPath), filepath.Ext(audioPath))
if base == "" {
return nil, fmt.Errorf("could not derive speaker id from %q", audioPath)
}
if previous, exists := seenSpeaker[base]; exists {
return nil, fmt.Errorf("duplicate speaker/audio basename %q from %q and %q", base, previous, audioPath)
}
seenSpeaker[base] = audioPath
jobs = append(jobs, transcribeJob{
speakerID: base,
audioPath: audioPath,
canonicalOutputPath: filepath.Join(paths.TranscriptsRawDir, base+".json"),
})
}
return jobs, nil
}
func discoverPreparedAudio(m *manifest.Manifest, paths artifacts.SessionPaths) ([]string, error) {
fromManifest := make([]string, 0)
if m != nil {
for _, in := range m.Inputs {
@@ -215,7 +224,7 @@ func discoverPreparedAudio(m *manifest.Manifest, audioDir string) ([]string, err
if p == "" {
continue
}
fromManifest = append(fromManifest, filepath.Clean(p))
fromManifest = append(fromManifest, artifacts.ResolveSessionLocalPathForRead(paths, p))
}
}
if len(fromManifest) > 0 {
@@ -226,9 +235,9 @@ func discoverPreparedAudio(m *manifest.Manifest, audioDir string) ([]string, err
return fromManifest, nil
}
entries, err := os.ReadDir(audioDir)
entries, err := os.ReadDir(paths.AudioDir)
if err != nil {
return nil, fmt.Errorf("read audio directory %q: %w", audioDir, err)
return nil, fmt.Errorf("read audio directory %q: %w", paths.AudioDir, err)
}
out := make([]string, 0)
@@ -236,7 +245,7 @@ func discoverPreparedAudio(m *manifest.Manifest, audioDir string) ([]string, err
if entry.IsDir() {
continue
}
p := filepath.Join(audioDir, entry.Name())
p := filepath.Join(paths.AudioDir, entry.Name())
if !isFlac(p) {
continue
}