Implement prepare stage input resolution
This commit is contained in:
307
internal/stage/prepare.go
Normal file
307
internal/stage/prepare.go
Normal file
@@ -0,0 +1,307 @@
|
||||
package stage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
type prepareStage struct{}
|
||||
|
||||
func (prepareStage) Name() string { return "prepare" }
|
||||
|
||||
func (prepareStage) Declares() IODecl {
|
||||
return IODecl{
|
||||
Inputs: []artifacts.Ref{
|
||||
{Kind: "config", Category: "inputs", RelativePath: "session.yml"},
|
||||
{Kind: "config", Category: "inputs", RelativePath: "pipeline.resolved.yml"},
|
||||
{Kind: "config", Category: "inputs", RelativePath: "speakers.yml"},
|
||||
{Kind: "config", Category: "inputs", RelativePath: "autocorrect.yml"},
|
||||
{Kind: "config", Category: "inputs", RelativePath: "glossary.yml"},
|
||||
{Kind: "audio", Category: "audio", RelativePath: "*.flac"},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func (prepareStage) Run(_ context.Context, env *Env, m *manifest.Manifest) (*StageResult, error) {
|
||||
if env == nil || env.Config == nil {
|
||||
return nil, fmt.Errorf("prepare: stage environment config is required")
|
||||
}
|
||||
if env.ArtifactStore == nil {
|
||||
return nil, fmt.Errorf("prepare: artifact store is required")
|
||||
}
|
||||
if env.Config.Session == nil || env.Config.Pipeline == nil {
|
||||
return nil, fmt.Errorf("prepare: resolved config must include pipeline and session")
|
||||
}
|
||||
|
||||
sessionID := strings.TrimSpace(m.SessionID)
|
||||
if sessionID == "" {
|
||||
sessionID = strings.TrimSpace(env.Config.Session.SessionID)
|
||||
}
|
||||
if sessionID == "" {
|
||||
return nil, fmt.Errorf("prepare: session id is required")
|
||||
}
|
||||
|
||||
paths, err := env.ArtifactStore.EnsureLayout(sessionID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("prepare: ensure workdir layout: %w", err)
|
||||
}
|
||||
|
||||
sessionSrc := env.Config.SessionPath
|
||||
if err := requireFile(sessionSrc, "session.yml"); err != nil {
|
||||
return nil, fmt.Errorf("prepare: %w", err)
|
||||
}
|
||||
sessionDir := filepath.Dir(sessionSrc)
|
||||
|
||||
speakersSrc, err := resolvePath(sessionDir, env.Config.Session.Inputs.SpeakersFile)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("prepare: speakers path: %w", err)
|
||||
}
|
||||
autocorrectSrc, err := resolvePath(sessionDir, env.Config.Session.Inputs.AutocorrectFile)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("prepare: autocorrect path: %w", err)
|
||||
}
|
||||
glossarySrc, err := resolvePath(sessionDir, env.Config.Session.Inputs.GlossaryFile)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("prepare: glossary path: %w", err)
|
||||
}
|
||||
|
||||
for _, required := range []struct {
|
||||
path string
|
||||
name string
|
||||
}{
|
||||
{path: speakersSrc, name: "speakers.yml"},
|
||||
{path: autocorrectSrc, name: "autocorrect.yml"},
|
||||
{path: glossarySrc, name: "glossary.yml"},
|
||||
} {
|
||||
if err := requireFile(required.path, required.name); err != nil {
|
||||
return nil, fmt.Errorf("prepare: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
resolvedAudio, err := resolveAudioFiles(sessionDir, env.Config.Session.Inputs)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("prepare: resolve audio inputs: %w", err)
|
||||
}
|
||||
|
||||
copiedByDest := map[string]string{}
|
||||
inputs := make([]manifest.InputRecord, 0, 5+len(resolvedAudio))
|
||||
registerInput := func(kind, path, checksum string) {
|
||||
inputs = append(inputs, manifest.InputRecord{Kind: kind, Path: path, Checksum: checksum})
|
||||
}
|
||||
|
||||
sessionDst := filepath.Join(paths.InputsDir, "session.yml")
|
||||
sessionChecksum, err := copyFileIfChanged(env.ArtifactStore, sessionSrc, sessionDst)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("prepare: materialize session.yml: %w", err)
|
||||
}
|
||||
registerInput("session_config", sessionDst, sessionChecksum)
|
||||
|
||||
pipelineResolvedBytes, err := renderResolvedPipeline(env.Config.Pipeline)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("prepare: render pipeline.resolved.yml: %w", err)
|
||||
}
|
||||
pipelineDst := filepath.Join(paths.InputsDir, "pipeline.resolved.yml")
|
||||
pipelineChecksum, err := writeBytesIfChanged(env.ArtifactStore, pipelineDst, pipelineResolvedBytes)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("prepare: materialize pipeline.resolved.yml: %w", err)
|
||||
}
|
||||
registerInput("pipeline_resolved", pipelineDst, pipelineChecksum)
|
||||
|
||||
for _, cfgFile := range []struct {
|
||||
kind string
|
||||
src string
|
||||
dst string
|
||||
}{
|
||||
{kind: "speakers", src: speakersSrc, dst: filepath.Join(paths.InputsDir, "speakers.yml")},
|
||||
{kind: "autocorrect", src: autocorrectSrc, dst: filepath.Join(paths.InputsDir, "autocorrect.yml")},
|
||||
{kind: "glossary", src: glossarySrc, dst: filepath.Join(paths.InputsDir, "glossary.yml")},
|
||||
} {
|
||||
checksum, err := copyFileIfChanged(env.ArtifactStore, cfgFile.src, cfgFile.dst)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("prepare: materialize %s: %w", cfgFile.kind, err)
|
||||
}
|
||||
registerInput(cfgFile.kind, cfgFile.dst, checksum)
|
||||
}
|
||||
|
||||
for _, src := range resolvedAudio {
|
||||
base := filepath.Base(src)
|
||||
if prev, exists := copiedByDest[base]; exists && prev != src {
|
||||
return nil, fmt.Errorf("prepare: duplicate audio basename %q from %q and %q", base, prev, src)
|
||||
}
|
||||
copiedByDest[base] = src
|
||||
|
||||
dst := filepath.Join(paths.AudioDir, base)
|
||||
checksum, err := copyFileIfChanged(env.ArtifactStore, src, dst)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("prepare: materialize audio %q: %w", base, err)
|
||||
}
|
||||
registerInput("audio", dst, checksum)
|
||||
}
|
||||
|
||||
sort.Slice(inputs, func(i, j int) bool {
|
||||
if inputs[i].Kind != inputs[j].Kind {
|
||||
return inputs[i].Kind < inputs[j].Kind
|
||||
}
|
||||
return inputs[i].Path < inputs[j].Path
|
||||
})
|
||||
m.Inputs = inputs
|
||||
|
||||
return &StageResult{
|
||||
Metadata: map[string]any{
|
||||
"prepared": true,
|
||||
"stage": "prepare",
|
||||
"inputs_count": len(inputs),
|
||||
"audio_files_resolved": len(resolvedAudio),
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
func renderResolvedPipeline(cfg *config.PipelineConfig) ([]byte, error) {
|
||||
return yaml.Marshal(cfg)
|
||||
}
|
||||
|
||||
func resolveAudioFiles(sessionDir string, inputs config.SessionInputsConfig) ([]string, error) {
|
||||
if len(inputs.AudioFiles) > 0 {
|
||||
out := make([]string, 0, len(inputs.AudioFiles))
|
||||
for _, p := range inputs.AudioFiles {
|
||||
resolved, err := resolvePath(sessionDir, p)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !isFlac(resolved) {
|
||||
return nil, fmt.Errorf("audio file %q must have .flac extension", resolved)
|
||||
}
|
||||
if err := requireFile(resolved, "audio file"); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, resolved)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
audioDir, err := resolvePath(sessionDir, inputs.AudioDir)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
entries, err := os.ReadDir(audioDir)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read audio directory %q: %w", audioDir, err)
|
||||
}
|
||||
|
||||
out := make([]string, 0)
|
||||
for _, entry := range entries {
|
||||
if entry.IsDir() {
|
||||
continue
|
||||
}
|
||||
name := entry.Name()
|
||||
full := filepath.Join(audioDir, name)
|
||||
if !isFlac(full) {
|
||||
continue
|
||||
}
|
||||
if err := requireFile(full, "audio file"); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, full)
|
||||
}
|
||||
|
||||
if len(out) == 0 {
|
||||
return nil, fmt.Errorf("no .flac files found in audio directory %q", audioDir)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func resolvePath(baseDir, p string) (string, error) {
|
||||
trimmed := strings.TrimSpace(p)
|
||||
if trimmed == "" {
|
||||
return "", fmt.Errorf("path is required")
|
||||
}
|
||||
if filepath.IsAbs(trimmed) {
|
||||
return filepath.Clean(trimmed), nil
|
||||
}
|
||||
return filepath.Clean(filepath.Join(baseDir, trimmed)), nil
|
||||
}
|
||||
|
||||
func requireFile(path string, label string) error {
|
||||
if strings.TrimSpace(path) == "" {
|
||||
return fmt.Errorf("%s path is required", label)
|
||||
}
|
||||
info, err := os.Stat(path)
|
||||
if err != nil {
|
||||
return fmt.Errorf("%s %q not found: %w", label, path, err)
|
||||
}
|
||||
if info.IsDir() {
|
||||
return fmt.Errorf("%s %q is a directory", label, path)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func isFlac(path string) bool {
|
||||
return strings.EqualFold(filepath.Ext(path), ".flac")
|
||||
}
|
||||
|
||||
func copyFileIfChanged(store artifacts.Store, src, dst string) (string, error) {
|
||||
srcChecksum, err := store.Checksum(src)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
exists, err := store.Exists(dst)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if exists {
|
||||
dstChecksum, err := store.Checksum(dst)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if dstChecksum == srcChecksum {
|
||||
return srcChecksum, nil
|
||||
}
|
||||
}
|
||||
|
||||
data, err := os.ReadFile(src)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if err := store.WriteFileAtomic(dst, data, 0o644); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return srcChecksum, nil
|
||||
}
|
||||
|
||||
func writeBytesIfChanged(store artifacts.Store, dst string, data []byte) (string, error) {
|
||||
digest := sha256.Sum256(data)
|
||||
targetChecksum := hex.EncodeToString(digest[:])
|
||||
|
||||
exists, err := store.Exists(dst)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if exists {
|
||||
dstChecksum, err := store.Checksum(dst)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if dstChecksum == targetChecksum {
|
||||
return targetChecksum, nil
|
||||
}
|
||||
}
|
||||
|
||||
if err := store.WriteFileAtomic(dst, data, 0o644); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return targetChecksum, nil
|
||||
}
|
||||
Reference in New Issue
Block a user