eric/narratio
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 15 Wiki Activity

Download S3 audio during prepare

Browse Source
This commit is contained in:
Eric Rakestraw
2026-05-16 14:33:42 +00:00
parent 1e6db89dd4
commit 24cce49a70
10 changed files with 429 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files

182
internal/stage/prepare.go
View File

@@ -6,10 +6,12 @@ import (
"encoding/hex"
"fmt"
"os"
"path"
"path/filepath"
"sort"
"strings"
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
"gitea.maximumdirect.net/eric/narratio/internal/config"
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
@@ -33,7 +35,7 @@ func (prepareStage) Declares() IODecl {
}
}
func (prepareStage) Run(_ context.Context, env *Env, m *manifest.Manifest) (*StageResult, error) {
func (prepareStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*StageResult, error) {
if env == nil || env.Config == nil {
return nil, fmt.Errorf("prepare: stage environment config is required")
}
@@ -89,13 +91,12 @@ func (prepareStage) Run(_ context.Context, env *Env, m *manifest.Manifest) (*Sta
}
}
resolvedAudio, err := resolveAudioFiles(sessionDir, env.Config.Session.Inputs)
resolvedLocalAudio, useS3Audio, err := resolveAudioInputs(sessionDir, env.Config.Session.Inputs)
if err != nil {
return nil, fmt.Errorf("prepare: resolve audio inputs: %w", err)
}
copiedByDest := map[string]string{}
inputs := make([]manifest.InputRecord, 0, 5+len(resolvedAudio))
inputs := make([]manifest.InputRecord, 0, 5+len(resolvedLocalAudio))
registerInput := func(kind, path, checksum string) {
inputs = append(inputs, manifest.InputRecord{Kind: kind, Path: path, Checksum: checksum})
}
@@ -134,19 +135,14 @@ func (prepareStage) Run(_ context.Context, env *Env, m *manifest.Manifest) (*Sta
registerInput(cfgFile.kind, cfgFile.dst, checksum)
}
for _, src := range resolvedAudio {
base := filepath.Base(src)
if prev, exists := copiedByDest[base]; exists && prev != src {
return nil, fmt.Errorf("prepare: duplicate audio basename %q from %q and %q", base, prev, src)
if useS3Audio {
if err := materializeS3AudioInputs(ctx, env, m, sessionID, &inputs); err != nil {
return nil, fmt.Errorf("prepare: materialize s3 audio: %w", err)
}
copiedByDest[base] = src
dst := filepath.Join(paths.AudioDir, base)
checksum, err := copyFileIfChanged(env.ArtifactStore, src, dst)
if err != nil {
return nil, fmt.Errorf("prepare: materialize audio %q: %w", base, err)
} else {
if err := materializeLocalAudioInputs(env, paths, resolvedLocalAudio, registerInput); err != nil {
return nil, fmt.Errorf("prepare: %w", err)
}
registerInput("audio", dst, checksum)
}
sort.Slice(inputs, func(i, j int) bool {
@@ -162,7 +158,7 @@ func (prepareStage) Run(_ context.Context, env *Env, m *manifest.Manifest) (*Sta
"prepared": true,
"stage": "prepare",
"inputs_count": len(inputs),
"audio_files_resolved": len(resolvedAudio),
"audio_files_resolved": countAudioInputs(inputs),
},
}, nil
}
@@ -171,7 +167,23 @@ func renderResolvedPipeline(cfg *config.PipelineConfig) ([]byte, error) {
return yaml.Marshal(cfg)
}
func resolveAudioFiles(sessionDir string, inputs config.SessionInputsConfig) ([]string, error) {
func resolveAudioInputs(sessionDir string, inputs config.SessionInputsConfig) ([]string, bool, error) {
hasLocal := strings.TrimSpace(inputs.AudioDir) != "" || len(inputs.AudioFiles) > 0
if inputs.AudioS3 != nil {
if hasLocal {
return nil, false, fmt.Errorf("audio_dir/audio_files and audio_s3 are mutually exclusive")
}
return nil, true, nil
}
local, err := resolveLocalAudioFiles(sessionDir, inputs)
if err != nil {
return nil, false, err
}
return local, false, nil
}
func resolveLocalAudioFiles(sessionDir string, inputs config.SessionInputsConfig) ([]string, error) {
if len(inputs.AudioFiles) > 0 {
out := make([]string, 0, len(inputs.AudioFiles))
for _, p := range inputs.AudioFiles {
@@ -223,6 +235,142 @@ func resolveAudioFiles(sessionDir string, inputs config.SessionInputsConfig) ([]
return out, nil
}
func materializeLocalAudioInputs(env *Env, paths artifacts.SessionPaths, resolvedAudio []string, registerInput func(kind, path, checksum string)) error {
copiedByDest := map[string]string{}
for _, src := range resolvedAudio {
base := filepath.Base(src)
if prev, exists := copiedByDest[base]; exists && prev != src {
return fmt.Errorf("duplicate audio basename %q from %q and %q", base, prev, src)
}
copiedByDest[base] = src
dst := filepath.Join(paths.AudioDir, base)
checksum, err := copyFileIfChanged(env.ArtifactStore, src, dst)
if err != nil {
return fmt.Errorf("materialize audio %q: %w", base, err)
}
registerInput("audio", dst, checksum)
}
return nil
}
func materializeS3AudioInputs(ctx context.Context, env *Env, m *manifest.Manifest, sessionID string, inputs *[]manifest.InputRecord) error {
if env.ObjectStore == nil {
return fmt.Errorf("s3 audio input requires object store backend")
}
if env.Config == nil || env.Config.Pipeline == nil || env.Config.Session == nil || env.Config.Pipeline.Storage.S3 == nil || env.Config.Session.Inputs.AudioS3 == nil {
return fmt.Errorf("s3 audio input requires pipeline.storage.s3 and session.inputs.audio_s3 configuration")
}
campaign := strings.TrimSpace(env.Config.Session.Campaign)
if campaign == "" {
return fmt.Errorf("session campaign is required for s3 audio input")
}
runID := strings.TrimSpace(m.RunID)
if runID == "" {
return fmt.Errorf("run id is required for s3 audio input")
}
sessionPrefix := artifacts.S3SessionPrefix(env.Config.Pipeline.Storage.S3.RootPrefix, campaign, sessionID)
audioPrefix := artifacts.S3AudioPrefix(sessionPrefix, env.Config.Session.Inputs.AudioS3.Prefix)
objects, err := env.ObjectStore.List(ctx, audioPrefix)
if err != nil {
return fmt.Errorf("list s3 audio objects under %q: %w", audioPrefix, err)
}
audioObjects := make([]storage.ObjectInfo, 0, len(objects))
for _, obj := range objects {
key := strings.TrimSpace(obj.Key)
if key == "" || strings.HasSuffix(key, "/") {
continue
}
if !isFlac(key) {
continue
}
audioObjects = append(audioObjects, obj)
}
sort.Slice(audioObjects, func(i, j int) bool {
return audioObjects[i].Key < audioObjects[j].Key
})
if len(audioObjects) == 0 {
return fmt.Errorf("no .flac files found under s3 audio prefix %q", audioPrefix)
}
spoolAudioDir := strings.TrimSpace(m.LocalSpoolDir)
if spoolAudioDir == "" {
spoolAudioDir = artifacts.SessionSpoolAudioDir(env.Config.Pipeline.Spool.Root, campaign, sessionID, runID)
}
workAudioDir := filepath.Join(pathsWorkDirForManifest(env, m, sessionID), "audio")
if err := os.MkdirAll(spoolAudioDir, 0o755); err != nil {
return fmt.Errorf("create spool audio directory %q: %w", spoolAudioDir, err)
}
if err := os.MkdirAll(workAudioDir, 0o755); err != nil {
return fmt.Errorf("create work audio directory %q: %w", workAudioDir, err)
}
seenBase := map[string]string{}
for _, obj := range audioObjects {
base := path.Base(obj.Key)
if prev, exists := seenBase[base]; exists && prev != obj.Key {
return fmt.Errorf("duplicate s3 audio basename %q from %q and %q", base, prev, obj.Key)
}
seenBase[base] = obj.Key
spoolPath := filepath.Join(spoolAudioDir, base)
if err := env.ObjectStore.Download(ctx, obj.Key, spoolPath); err != nil {
return fmt.Errorf("download s3 audio object %q: %w", obj.Key, err)
}
workPath := filepath.Join(workAudioDir, base)
checksum, err := copyFileIfChanged(env.ArtifactStore, spoolPath, workPath)
if err != nil {
return fmt.Errorf("materialize downloaded audio %q: %w", base, err)
}
*inputs = append(*inputs, manifest.InputRecord{
Kind: "audio",
Path: workPath,
Checksum: checksum,
Source: "s3",
S3Bucket: strings.TrimSpace(env.Config.Pipeline.Storage.S3.Bucket),
S3Key: obj.Key,
S3Size: obj.Size,
S3ETag: obj.ETag,
SpoolPath: spoolPath,
})
}
return nil
}
func countAudioInputs(inputs []manifest.InputRecord) int {
count := 0
for _, in := range inputs {
if in.Kind == "audio" {
count++
}
}
return count
}
func pathsWorkDirForManifest(env *Env, m *manifest.Manifest, sessionID string) string {
if env == nil || env.Config == nil || env.Config.Pipeline == nil || env.Config.Session == nil {
return ""
}
if m != nil && strings.TrimSpace(m.LocalWorkDir) != "" {
return strings.TrimSpace(m.LocalWorkDir)
}
runID := ""
if m != nil {
runID = strings.TrimSpace(m.RunID)
}
if runID != "" {
return artifacts.SessionRunWorkDir(env.Config.Pipeline.Workspace.Root, env.Config.Session.Campaign, sessionID, runID)
}
return artifacts.SessionWorkDir(env.Config.Pipeline.Workspace.Root, sessionID)
}
func resolvePath(baseDir, p string) (string, error) {
trimmed := strings.TrimSpace(p)
if trimmed == "" {