eric/narratio
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 15 Wiki Activity

Add archive storage path configuration

Browse Source
This commit is contained in:
Eric Rakestraw
2026-05-16 14:11:59 +00:00
parent 58c6ab2d54
commit 0454296c81
23 changed files with 950 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
internal/app/commands_test.go
View File

@@ -208,6 +208,7 @@ notification:
timeout: 10s
`
sessionYAML := `session_id: ` + sessionID + `
campaign: sample-campaign
inputs:
audio_dir: ./audio
speakers_file: ./speakers.yml
@@ -271,6 +272,7 @@ notification:
timeout: 10s
`
sessionYAML := `session_id: 2026-05-03
campaign: sample-campaign
inputs:
audio_dir: ./audio
speakers_file: ./speakers.yml
@@ -377,6 +379,8 @@ func writeValidConfigFiles(t *testing.T, workspaceRoot string, transcribeURL ...
root: ` + workspaceRoot + `
storage:
backend: s3
s3:
bucket: test-bucket
whisperx:
transcribe_url: ` + url + `
timeout: 2s
@@ -400,6 +404,7 @@ notification:
`
sessionYAML := `session_id: 2026-05-03
campaign: sample-campaign
inputs:
audio_dir: ./audio
speakers_file: ./speakers.yml

1
internal/app/plan_test.go
View File

@@ -113,6 +113,7 @@ notification:
timeout: 10s
`
sessionYAML := `session_id: 2026-05-03
campaign: sample-campaign
inputs:
audio_dir: ./audio
speakers_file: ./speakers.yml

61
internal/app/runner.go
View File

@@ -104,6 +104,15 @@ func executeStages(ctx context.Context, cfg *config.Config, stages []stage.Stage
if err != nil {
return nil, err
}
identityChanged, err := ensureManifestIdentity(cfg, m)
if err != nil {
return nil, fmt.Errorf("initialize manifest identity: %w", err)
}
if identityChanged {
if err := env.ManifestStore.Save(ctx, manifestPath, m); err != nil {
return nil, fmt.Errorf("save manifest identity %q: %w", manifestPath, err)
}
}
stageEnv := env
@@ -338,6 +347,58 @@ func applyStageResultToManifest(m *manifest.Manifest, stageName string, result *
}
}
func ensureManifestIdentity(cfg *config.Config, m *manifest.Manifest) (bool, error) {
if cfg == nil || cfg.Pipeline == nil || cfg.Session == nil || m == nil {
return false, nil
}
changed := false
campaign := strings.TrimSpace(cfg.Session.Campaign)
sessionID := strings.TrimSpace(cfg.Session.SessionID)
if sessionID == "" {
sessionID = strings.TrimSpace(m.SessionID)
}
if m.Campaign == "" && campaign != "" {
m.Campaign = campaign
changed = true
}
if m.RunID == "" {
runID, err := artifacts.NewRunID()
if err != nil {
return false, err
}
m.RunID = runID
changed = true
}
if m.LocalWorkDir == "" && campaign != "" && sessionID != "" && m.RunID != "" {
m.LocalWorkDir = artifacts.SessionRunWorkDir(cfg.Pipeline.Workspace.Root, campaign, sessionID, m.RunID)
changed = true
}
if m.LocalSpoolDir == "" && campaign != "" && sessionID != "" && m.RunID != "" && strings.TrimSpace(cfg.Pipeline.Spool.Root) != "" {
m.LocalSpoolDir = artifacts.SessionSpoolAudioDir(cfg.Pipeline.Spool.Root, campaign, sessionID, m.RunID)
changed = true
}
if cfg.Pipeline.Storage.S3 != nil {
if m.S3Bucket == "" && strings.TrimSpace(cfg.Pipeline.Storage.S3.Bucket) != "" {
m.S3Bucket = strings.TrimSpace(cfg.Pipeline.Storage.S3.Bucket)
changed = true
}
sessionPrefix := artifacts.S3SessionPrefix(cfg.Pipeline.Storage.S3.RootPrefix, campaign, sessionID)
if m.S3SessionPrefix == "" && sessionPrefix != "" {
m.S3SessionPrefix = sessionPrefix
changed = true
}
runPrefix := artifacts.S3RunPrefix(sessionPrefix, m.RunID)
if m.S3RunPrefix == "" && runPrefix != "" {
m.S3RunPrefix = runPrefix
changed = true
}
}
return changed, nil
}
func manifestPathFor(cfg *config.Config) string {
return filepath.Join(cfg.Pipeline.Workspace.Root, "work", cfg.Session.SessionID, "manifest.json")
}

3
internal/app/runner_test.go
View File

@@ -430,7 +430,7 @@ func testConfig(t *testing.T) *config.Config {
pipelinePath := filepath.Join(cfgDir, "pipeline.yml")
mustWriteFile(t, pipelinePath, "workspace:\n root: "+workspace+"\n")
mustWriteFile(t, sessionPath, "session_id: 2026-05-03\n")
mustWriteFile(t, sessionPath, "session_id: 2026-05-03\ncampaign: sample-campaign\n")
mustWriteFile(t, filepath.Join(cfgDir, "speakers.yml"), "alice: alice.flac\n")
mustWriteFile(t, filepath.Join(cfgDir, "autocorrect.yml"), "[]\n")
mustWriteFile(t, filepath.Join(cfgDir, "glossary.yml"), "[]\n")
@@ -442,6 +442,7 @@ func testConfig(t *testing.T) *config.Config {
SessionPath: sessionPath,
Session: &config.SessionConfig{
SessionID: "2026-05-03",
Campaign: "sample-campaign",
Inputs: config.SessionInputsConfig{
AudioDir: "./audio",
SpeakersFile: "./speakers.yml",

14
internal/artifacts/paths.go
View File

@@ -1,6 +1,8 @@
package artifacts
import "path/filepath"
import (
"path/filepath"
)
// SessionPaths contains canonical local paths for one session work directory.
type SessionPaths struct {
@@ -23,6 +25,16 @@ func SessionWorkDir(rootDir, sessionID string) string {
return filepath.Join(rootDir, "work", sessionID)
}
// SessionRunWorkDir returns the campaign/session/run scoped local work directory.
func SessionRunWorkDir(rootDir, campaign, sessionID, runID string) string {
return filepath.Join(rootDir, "work", campaign, sessionID, runID)
}
// SessionSpoolAudioDir returns the campaign/session/run scoped local spool audio path.
func SessionSpoolAudioDir(spoolRoot, campaign, sessionID, runID string) string {
return filepath.Join(spoolRoot, campaign, sessionID, runID, "audio")
}
func buildSessionPaths(workspaceRoot, sessionID string) SessionPaths {
root := SessionWorkDir(workspaceRoot, sessionID)
transcripts := filepath.Join(root, "transcripts")

24
internal/artifacts/paths_model_test.go Normal file
View File

@@ -0,0 +1,24 @@
package artifacts
import (
"path/filepath"
"testing"
)
func TestSessionRunWorkDir(t *testing.T) {
root := "/tmp/workspace"
got := SessionRunWorkDir(root, "forsaken", "2026-04-19", "20260515T031522Z-a1b2c3d4")
want := filepath.Join(root, "work", "forsaken", "2026-04-19", "20260515T031522Z-a1b2c3d4")
if got != want {
t.Fatalf("SessionRunWorkDir() = %q, want %q", got, want)
}
}
func TestSessionSpoolAudioDir(t *testing.T) {
root := "/var/spool/narratio"
got := SessionSpoolAudioDir(root, "forsaken", "2026-04-19", "20260515T031522Z-a1b2c3d4")
want := filepath.Join(root, "forsaken", "2026-04-19", "20260515T031522Z-a1b2c3d4", "audio")
if got != want {
t.Fatalf("SessionSpoolAudioDir() = %q, want %q", got, want)
}
}

30
internal/artifacts/run_id.go Normal file
View File

@@ -0,0 +1,30 @@
package artifacts
import (
"crypto/rand"
"encoding/hex"
"fmt"
"io"
"time"
)
// NewRunID returns a run ID in format: YYYYMMDDTHHMMSSZ-xxxxxxxx.
func NewRunID() (string, error) {
return NewRunIDWith(time.Now().UTC(), rand.Reader)
}
// NewRunIDWith returns a run ID in format: YYYYMMDDTHHMMSSZ-xxxxxxxx
// using an injected timestamp and randomness source.
func NewRunIDWith(now time.Time, random io.Reader) (string, error) {
if random == nil {
random = rand.Reader
}
var suffix [4]byte
if _, err := io.ReadFull(random, suffix[:]); err != nil {
return "", fmt.Errorf("generate run id random suffix: %w", err)
}
ts := now.UTC().Format("20060102T150405Z")
return ts + "-" + hex.EncodeToString(suffix[:]), nil
}

37
internal/artifacts/run_id_test.go Normal file
View File

@@ -0,0 +1,37 @@
package artifacts
import (
"bytes"
"regexp"
"strings"
"testing"
"time"
)
func TestNewRunIDWithFormat(t *testing.T) {
now := time.Date(2026, 5, 15, 3, 15, 22, 0, time.UTC)
random := bytes.NewReader([]byte{0xa1, 0xb2, 0xc3, 0xd4})
runID, err := NewRunIDWith(now, random)
if err != nil {
t.Fatalf("NewRunIDWith() error = %v", err)
}
if runID != "20260515T031522Z-a1b2c3d4" {
t.Fatalf("runID = %q, want %q", runID, "20260515T031522Z-a1b2c3d4")
}
}
func TestNewRunIDWithShape(t *testing.T) {
runID, err := NewRunIDWith(time.Now().UTC(), bytes.NewReader([]byte{0x01, 0x02, 0x03, 0x04}))
if err != nil {
t.Fatalf("NewRunIDWith() error = %v", err)
}
pattern := regexp.MustCompile(`^\d{8}T\d{6}Z-[0-9a-f]{8}$`)
if !pattern.MatchString(runID) {
t.Fatalf("runID = %q, want pattern %q", runID, pattern.String())
}
suffix := runID[len(runID)-8:]
if strings.ToLower(suffix) != suffix {
t.Fatalf("runID suffix = %q, want lowercase", suffix)
}
}

73
internal/artifacts/s3_keys.go Normal file
View File

@@ -0,0 +1,73 @@
package artifacts
import (
"path"
"strings"
)
// S3SessionPrefix builds the canonical S3 session prefix.
// Format: {root_prefix}/campaigns/{campaign}/sessions/{session_id}/
func S3SessionPrefix(rootPrefix, campaign, sessionID string) string {
prefix := path.Join(
cleanS3PathPart(rootPrefix),
"campaigns",
cleanS3PathPart(campaign),
"sessions",
cleanS3PathPart(sessionID),
)
return ensureS3TrailingSlash(prefix)
}
// S3RunPrefix builds the canonical S3 run prefix.
// Format: {session_prefix}/runs/{run_id}/
func S3RunPrefix(sessionPrefix, runID string) string {
prefix := path.Join(strings.TrimSuffix(cleanS3Key(sessionPrefix), "/"), "runs", cleanS3PathPart(runID))
return ensureS3TrailingSlash(prefix)
}
// S3AudioPrefix builds the session audio prefix from configured audio_s3.prefix.
// Format: {session_prefix}/{audio_s3.prefix}
func S3AudioPrefix(sessionPrefix, audioPrefix string) string {
key := path.Join(strings.TrimSuffix(cleanS3Key(sessionPrefix), "/"), cleanS3Key(audioPrefix))
return ensureS3TrailingSlash(key)
}
// S3CurrentManifestKey returns the current manifest pointer key.
// Format: {session_prefix}/current/manifest.json
func S3CurrentManifestKey(sessionPrefix string) string {
return path.Join(strings.TrimSuffix(cleanS3Key(sessionPrefix), "/"), "current", "manifest.json")
}
// S3CurrentRunPointerKey returns the current run pointer key.
// Format: {session_prefix}/current/run_id.txt
func S3CurrentRunPointerKey(sessionPrefix string) string {
return path.Join(strings.TrimSuffix(cleanS3Key(sessionPrefix), "/"), "current", "run_id.txt")
}
// S3PromotedArtifactKey returns the destination key for one promoted artifact.
// Format: {session_prefix}/{promotion.to}
func S3PromotedArtifactKey(sessionPrefix, to string) string {
return path.Join(strings.TrimSuffix(cleanS3Key(sessionPrefix), "/"), cleanS3Key(to))
}
// S3RunRelativeDestinationKey returns a run-scoped key for a workdir-relative path.
// Format: {run_prefix}/{relative_workdir_path}
func S3RunRelativeDestinationKey(runPrefix, relativeWorkdirPath string) string {
return path.Join(strings.TrimSuffix(cleanS3Key(runPrefix), "/"), cleanS3Key(relativeWorkdirPath))
}
func ensureS3TrailingSlash(v string) string {
key := cleanS3Key(v)
if key == "" {
return ""
}
return strings.TrimSuffix(key, "/") + "/"
}
func cleanS3PathPart(v string) string {
return strings.Trim(strings.ReplaceAll(strings.TrimSpace(v), "\\", "/"), "/")
}
func cleanS3Key(v string) string {
return strings.ReplaceAll(strings.TrimSpace(v), "\\", "/")
}

45
internal/artifacts/s3_keys_test.go Normal file
View File

@@ -0,0 +1,45 @@
package artifacts
import (
"strings"
"testing"
)
func TestS3KeyConstruction(t *testing.T) {
runID := "20260515T031522Z-a1b2c3d4"
sessionPrefix := S3SessionPrefix("dnd", "forsaken", "2026-04-19")
if sessionPrefix != "dnd/campaigns/forsaken/sessions/2026-04-19/" {
t.Fatalf("sessionPrefix = %q", sessionPrefix)
}
audioPrefix := S3AudioPrefix(sessionPrefix, "audio/")
if audioPrefix != "dnd/campaigns/forsaken/sessions/2026-04-19/audio/" {
t.Fatalf("audioPrefix = %q", audioPrefix)
}
runPrefix := S3RunPrefix(sessionPrefix, runID)
wantRunPrefix := "dnd/campaigns/forsaken/sessions/2026-04-19/runs/" + runID + "/"
if runPrefix != wantRunPrefix {
t.Fatalf("runPrefix = %q, want %q", runPrefix, wantRunPrefix)
}
runPointer := S3CurrentRunPointerKey(sessionPrefix)
if runPointer != "dnd/campaigns/forsaken/sessions/2026-04-19/current/run_id.txt" {
t.Fatalf("run pointer key = %q", runPointer)
}
manifestKey := S3CurrentManifestKey(sessionPrefix)
if manifestKey != "dnd/campaigns/forsaken/sessions/2026-04-19/current/manifest.json" {
t.Fatalf("manifest key = %q", manifestKey)
}
promoted := S3PromotedArtifactKey(sessionPrefix, "transcripts/trimmed.json")
if promoted != "dnd/campaigns/forsaken/sessions/2026-04-19/transcripts/trimmed.json" {
t.Fatalf("promoted key = %q", promoted)
}
runRelative := S3RunRelativeDestinationKey(runPrefix, `logs\whisperx.stdout.log`)
if !strings.HasSuffix(runRelative, "/logs/whisperx.stdout.log") {
t.Fatalf("runRelative key = %q, want normalized forward slashes", runRelative)
}
}

54
internal/config/config.go
View File

@@ -12,6 +12,8 @@ type Config struct {
type PipelineConfig struct {
Workspace WorkspaceConfig `yaml:"workspace"`
Storage StorageConfig `yaml:"storage"`
Spool SpoolConfig `yaml:"spool"`
Archive *ArchiveConfig `yaml:"archive"`
Secrets *SecretsConfig `yaml:"secrets"`
WhisperX WhisperXConfig `yaml:"whisperx"`
Seriatim SeriatimConfig `yaml:"seriatim"`
@@ -44,9 +46,39 @@ type SecretsConfig struct {
// StorageConfig configures storage backends and related parameters.
type StorageConfig struct {
Backend string `yaml:"backend"`
Bucket string `yaml:"bucket"`
Prefix string `yaml:"prefix"`
Backend string `yaml:"backend"`
Bucket string `yaml:"bucket"`
Prefix string `yaml:"prefix"`
S3 *StorageS3Config `yaml:"s3"`
}
// StorageS3Config configures S3 storage coordinates.
type StorageS3Config struct {
Bucket string `yaml:"bucket"`
RootPrefix string `yaml:"root_prefix"`
Region string `yaml:"region"`
Endpoint string `yaml:"endpoint"`
ForcePathStyle bool `yaml:"force_path_style"`
}
// SpoolConfig configures local spool storage for staged data.
type SpoolConfig struct {
Root string `yaml:"root"`
DeleteAudioAfterArchive bool `yaml:"delete_audio_after_archive"`
}
// ArchiveConfig configures archive behavior and artifact promotions.
type ArchiveConfig struct {
Enabled *bool `yaml:"enabled"`
UploadRun *bool `yaml:"upload_run"`
PromoteArtifacts []ArchivePromotionRule `yaml:"promote_artifacts"`
}
// ArchivePromotionRule configures one artifact promotion mapping.
type ArchivePromotionRule struct {
From string `yaml:"from"`
To string `yaml:"to"`
Required *bool `yaml:"required"`
}
// WhisperXConfig configures WhisperX adapter settings.
@@ -180,9 +212,15 @@ type ArtifactSettings struct {
// SessionInputsConfig contains per-session input references.
type SessionInputsConfig struct {
AudioDir string `yaml:"audio_dir"`
AudioFiles []string `yaml:"audio_files"`
SpeakersFile string `yaml:"speakers_file"`
AutocorrectFile string `yaml:"autocorrect_file"`
GlossaryFile string `yaml:"glossary_file"`
AudioDir string `yaml:"audio_dir"`
AudioFiles []string `yaml:"audio_files"`
AudioS3 *SessionAudioS3Input `yaml:"audio_s3"`
SpeakersFile string `yaml:"speakers_file"`
AutocorrectFile string `yaml:"autocorrect_file"`
GlossaryFile string `yaml:"glossary_file"`
}
// SessionAudioS3Input configures S3 session-audio input discovery.
type SessionAudioS3Input struct {
Prefix string `yaml:"prefix"`
}

51
internal/config/load.go
View File

@@ -81,6 +81,9 @@ func applyPipelineDefaults(cfg *PipelineConfig) {
if cfg == nil {
return
}
applyStorageDefaults(&cfg.Storage)
applySpoolDefaults(&cfg.Spool)
applyArchiveDefaults(&cfg.Archive)
applyWhisperXDefaults(&cfg.WhisperX)
applySeriatimDefaults(&cfg.Seriatim)
applyAuditaDefaults(&cfg.Audita)
@@ -92,6 +95,54 @@ func applyPipelineDefaults(cfg *PipelineConfig) {
applyScriptoriumDefaults(cfg.Scriptorium)
}
func applyStorageDefaults(cfg *StorageConfig) {
if cfg == nil {
return
}
if cfg.S3 == nil {
cfg.S3 = &StorageS3Config{}
}
if cfg.S3.RootPrefix == "" {
cfg.S3.RootPrefix = "dnd"
}
}
func applySpoolDefaults(cfg *SpoolConfig) {
if cfg == nil {
return
}
if cfg.Root == "" {
cfg.Root = "/var/spool/narratio"
}
}
func applyArchiveDefaults(cfg **ArchiveConfig) {
if cfg == nil {
return
}
if *cfg == nil {
*cfg = &ArchiveConfig{}
}
if (*cfg).Enabled == nil {
(*cfg).Enabled = boolPtr(true)
}
if (*cfg).UploadRun == nil {
(*cfg).UploadRun = boolPtr(true)
}
if len((*cfg).PromoteArtifacts) == 0 {
(*cfg).PromoteArtifacts = []ArchivePromotionRule{
{From: "transcripts/trimmed.json", To: "transcripts/trimmed.json", Required: boolPtr(true)},
{From: "artifacts/session_recap.md", To: "artifacts/session_recap.md", Required: boolPtr(true)},
}
}
for i := range (*cfg).PromoteArtifacts {
if (*cfg).PromoteArtifacts[i].Required == nil {
(*cfg).PromoteArtifacts[i].Required = boolPtr(true)
}
}
}
func applyWhisperXDefaults(cfg *WhisperXConfig) {
if cfg == nil {
return

9
internal/config/load_validate_test.go
View File

@@ -820,6 +820,7 @@ func TestValidateMissingAudioSource(t *testing.T) {
},
Session: &SessionConfig{
SessionID: "2026-05-03",
Campaign: "sample-campaign",
Inputs: SessionInputsConfig{
SpeakersFile: "speakers.yml",
AutocorrectFile: "autocorrect.yml",
@@ -832,7 +833,7 @@ func TestValidateMissingAudioSource(t *testing.T) {
if err == nil {
t.Fatal("expected validation error, got nil")
}
if !strings.Contains(err.Error(), "audio_dir or at least one audio_files") {
if !strings.Contains(err.Error(), "audio_dir, at least one audio_files entry, or audio_s3") {
t.Fatalf("error = %q, want audio source guidance", err.Error())
}
if !strings.Contains(err.Error(), "session config") {
@@ -861,6 +862,12 @@ func writeConfigFiles(t *testing.T, pipelineYAML, sessionYAML string) (string, s
}
pipelineYAML += "audita:\n binary: audita\n"
}
if !strings.Contains(sessionYAML, "\ncampaign:") && !strings.HasPrefix(sessionYAML, "campaign:") {
if !strings.HasSuffix(sessionYAML, "\n") {
sessionYAML += "\n"
}
sessionYAML += "campaign: sample-campaign\n"
}
dir := t.TempDir()
pipelinePath := filepath.Join(dir, "pipeline.yml")

235
internal/config/storage_archive_test.go Normal file
View File

@@ -0,0 +1,235 @@
package config
import (
"strings"
"testing"
)
func TestStorageS3DefaultsAndValidation(t *testing.T) {
pipelineYAML := testPipelineBaseYAML + `
storage:
backend: s3
s3:
bucket: my-dnd-archive
`
pipelinePath, sessionPath := writeConfigFiles(t, pipelineYAML, testSessionBaseYAML)
cfg, err := Load(pipelinePath, sessionPath)
if err != nil {
t.Fatalf("Load() error = %v", err)
}
if cfg.Pipeline.Storage.S3 == nil {
t.Fatal("storage.s3 should be initialized")
}
if cfg.Pipeline.Storage.S3.RootPrefix != "dnd" {
t.Fatalf("storage.s3.root_prefix = %q, want dnd", cfg.Pipeline.Storage.S3.RootPrefix)
}
if cfg.Pipeline.Storage.S3.ForcePathStyle {
t.Fatalf("storage.s3.force_path_style = true, want false default")
}
if err := Validate(cfg); err != nil {
t.Fatalf("Validate() error = %v", err)
}
}
func TestSpoolAndArchiveDefaults(t *testing.T) {
pipelinePath, sessionPath := writeConfigFiles(t, testPipelineBaseYAML, testSessionBaseYAML)
cfg, err := Load(pipelinePath, sessionPath)
if err != nil {
t.Fatalf("Load() error = %v", err)
}
if cfg.Pipeline.Spool.Root != "/var/spool/narratio" {
t.Fatalf("spool.root = %q, want /var/spool/narratio", cfg.Pipeline.Spool.Root)
}
if cfg.Pipeline.Spool.DeleteAudioAfterArchive {
t.Fatalf("spool.delete_audio_after_archive = true, want false")
}
if cfg.Pipeline.Archive == nil {
t.Fatal("archive should be initialized by defaults")
}
if cfg.Pipeline.Archive.Enabled == nil || !*cfg.Pipeline.Archive.Enabled {
t.Fatalf("archive.enabled = %#v, want true", cfg.Pipeline.Archive.Enabled)
}
if cfg.Pipeline.Archive.UploadRun == nil || !*cfg.Pipeline.Archive.UploadRun {
t.Fatalf("archive.upload_run = %#v, want true", cfg.Pipeline.Archive.UploadRun)
}
if len(cfg.Pipeline.Archive.PromoteArtifacts) != 2 {
t.Fatalf("archive.promote_artifacts len = %d, want 2 defaults", len(cfg.Pipeline.Archive.PromoteArtifacts))
}
for i, item := range cfg.Pipeline.Archive.PromoteArtifacts {
if item.Required == nil || !*item.Required {
t.Fatalf("archive.promote_artifacts[%d].required = %#v, want true", i, item.Required)
}
}
}
func TestArchivePromotionPathValidation(t *testing.T) {
tests := []struct {
name string
ruleYML string
wantErr string
}{
{
name: "absolute from path rejected",
ruleYML: `archive:
promote_artifacts:
- from: "/transcripts/trimmed.json"
to: "transcripts/trimmed.json"
`,
wantErr: "must be a relative path",
},
{
name: "traversal to path rejected",
ruleYML: `archive:
promote_artifacts:
- from: "transcripts/trimmed.json"
to: "../trimmed.json"
`,
wantErr: "must not contain path traversal",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
pipelineYAML := testPipelineBaseYAML + "\n" + tt.ruleYML
pipelinePath, sessionPath := writeConfigFiles(t, pipelineYAML, testSessionBaseYAML)
cfg, err := Load(pipelinePath, sessionPath)
if err != nil {
t.Fatalf("Load() error = %v", err)
}
err = Validate(cfg)
if err == nil || !strings.Contains(err.Error(), tt.wantErr) {
t.Fatalf("Validate() error = %v, want to contain %q", err, tt.wantErr)
}
})
}
}
func TestSessionAudioS3Validation(t *testing.T) {
tests := []struct {
name string
sessionYAML string
wantErr string
}{
{
name: "valid audio_s3 prefix",
sessionYAML: `session_id: 2026-05-03
campaign: forsaken
inputs:
audio_s3:
prefix: audio/
speakers_file: ./speakers.yml
autocorrect_file: ./autocorrect.yml
glossary_file: ./glossary.yml
`,
},
{
name: "invalid audio_s3 absolute prefix",
sessionYAML: `session_id: 2026-05-03
campaign: forsaken
inputs:
audio_s3:
prefix: /audio/
speakers_file: ./speakers.yml
autocorrect_file: ./autocorrect.yml
glossary_file: ./glossary.yml
`,
wantErr: "session.inputs.audio_s3.prefix must be a relative path",
},
{
name: "invalid audio_s3 traversal prefix",
sessionYAML: `session_id: 2026-05-03
campaign: forsaken
inputs:
audio_s3:
prefix: ../audio/
speakers_file: ./speakers.yml
autocorrect_file: ./autocorrect.yml
glossary_file: ./glossary.yml
`,
wantErr: "session.inputs.audio_s3.prefix must not contain path traversal",
},
{
name: "local and s3 audio conflict",
sessionYAML: `session_id: 2026-05-03
campaign: forsaken
inputs:
audio_dir: ./audio
audio_s3:
prefix: audio/
speakers_file: ./speakers.yml
autocorrect_file: ./autocorrect.yml
glossary_file: ./glossary.yml
`,
wantErr: "mutually exclusive",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
pipelineYAML := testPipelineBaseYAML + `
storage:
backend: s3
s3:
bucket: my-dnd-archive
`
pipelinePath, sessionPath := writeConfigFiles(t, pipelineYAML, tt.sessionYAML)
cfg, err := Load(pipelinePath, sessionPath)
if err != nil {
t.Fatalf("Load() error = %v", err)
}
err = Validate(cfg)
if tt.wantErr != "" {
if err == nil || !strings.Contains(err.Error(), tt.wantErr) {
t.Fatalf("Validate() error = %v, want to contain %q", err, tt.wantErr)
}
return
}
if err != nil {
t.Fatalf("Validate() error = %v", err)
}
})
}
}
func TestStorageS3BucketRequiredWhenS3DependentFeatureEnabled(t *testing.T) {
pipelineYAML := testPipelineBaseYAML + `
storage:
backend: s3
`
sessionYAML := `session_id: 2026-05-03
campaign: forsaken
inputs:
audio_s3:
prefix: audio/
speakers_file: ./speakers.yml
autocorrect_file: ./autocorrect.yml
glossary_file: ./glossary.yml
`
pipelinePath, sessionPath := writeConfigFiles(t, pipelineYAML, sessionYAML)
cfg, err := Load(pipelinePath, sessionPath)
if err != nil {
t.Fatalf("Load() error = %v", err)
}
err = Validate(cfg)
if err == nil || !strings.Contains(err.Error(), "pipeline.storage.s3.bucket is required") {
t.Fatalf("Validate() error = %v, want bucket requirement", err)
}
}
func TestLocalAudioConfigStillValid(t *testing.T) {
pipelinePath, sessionPath := writeConfigFiles(t, testPipelineBaseYAML, testSessionBaseYAML)
cfg, err := Load(pipelinePath, sessionPath)
if err != nil {
t.Fatalf("Load() error = %v", err)
}
if err := Validate(cfg); err != nil {
t.Fatalf("Validate() error = %v", err)
}
}

129
internal/config/validate.go
View File

@@ -3,6 +3,8 @@ package config
import (
"fmt"
"net/url"
"path/filepath"
"regexp"
"strings"
"time"
)
@@ -25,6 +27,9 @@ func Validate(cfg *Config) error {
if err := validateSession(cfg.Session); err != nil {
return fmt.Errorf("session config %q invalid: %w", shortName(cfg.SessionPath, "session.yml"), err)
}
if err := validateCrossConfig(cfg.Pipeline, cfg.Session); err != nil {
return fmt.Errorf("pipeline/session config invalid: %w", err)
}
return nil
}
@@ -36,6 +41,15 @@ func validatePipeline(cfg *PipelineConfig) error {
if err := validateSecrets(cfg.Secrets); err != nil {
return err
}
if err := validateStorage(cfg.Storage); err != nil {
return err
}
if err := validateSpool(cfg.Spool); err != nil {
return err
}
if err := validateArchive(cfg.Archive); err != nil {
return err
}
if err := validateWhisperX(cfg.WhisperX); err != nil {
return err
}
@@ -64,6 +78,48 @@ func validatePipeline(cfg *PipelineConfig) error {
return nil
}
func validateStorage(cfg StorageConfig) error {
if cfg.S3 == nil {
return nil
}
if strings.TrimSpace(cfg.S3.RootPrefix) == "" {
return fmt.Errorf("pipeline.storage.s3.root_prefix must be non-empty")
}
if err := validateRelativeSafePath("pipeline.storage.s3.root_prefix", cfg.S3.RootPrefix); err != nil {
return err
}
if cfg.S3.Endpoint != "" && strings.TrimSpace(cfg.S3.Endpoint) == "" {
return fmt.Errorf("pipeline.storage.s3.endpoint must be non-empty when provided")
}
return nil
}
func validateSpool(cfg SpoolConfig) error {
return nil
}
func validateArchive(cfg *ArchiveConfig) error {
if cfg == nil {
return nil
}
for i, item := range cfg.PromoteArtifacts {
prefix := fmt.Sprintf("pipeline.archive.promote_artifacts[%d]", i)
if strings.TrimSpace(item.From) == "" {
return fmt.Errorf("%s.from is required", prefix)
}
if strings.TrimSpace(item.To) == "" {
return fmt.Errorf("%s.to is required", prefix)
}
if err := validateRelativeSafePath(prefix+".from", item.From); err != nil {
return err
}
if err := validateRelativeSafePath(prefix+".to", item.To); err != nil {
return err
}
}
return nil
}
func validateSecrets(cfg *SecretsConfig) error {
if cfg == nil {
return nil
@@ -307,6 +363,9 @@ func validateSession(cfg *SessionConfig) error {
if strings.TrimSpace(cfg.SessionID) == "" {
return fmt.Errorf("session.session_id is required")
}
if strings.TrimSpace(cfg.Campaign) == "" {
return fmt.Errorf("session.campaign is required")
}
if strings.TrimSpace(cfg.Inputs.SpeakersFile) == "" {
return fmt.Errorf("session.inputs.speakers_file is required")
@@ -320,13 +379,79 @@ func validateSession(cfg *SessionConfig) error {
hasAudioDir := strings.TrimSpace(cfg.Inputs.AudioDir) != ""
hasAudioFiles := len(cfg.Inputs.AudioFiles) > 0
if !hasAudioDir && !hasAudioFiles {
return fmt.Errorf("session.inputs requires audio_dir or at least one audio_files entry")
hasAudioS3 := cfg.Inputs.AudioS3 != nil
if hasAudioS3 {
if strings.TrimSpace(cfg.Inputs.AudioS3.Prefix) == "" {
return fmt.Errorf("session.inputs.audio_s3.prefix is required when session.inputs.audio_s3 is configured")
}
if err := validateRelativeSafePath("session.inputs.audio_s3.prefix", cfg.Inputs.AudioS3.Prefix); err != nil {
return err
}
}
if hasAudioS3 && (hasAudioDir || hasAudioFiles) {
return fmt.Errorf("session.inputs.audio_dir/audio_files and session.inputs.audio_s3 are mutually exclusive")
}
if !hasAudioDir && !hasAudioFiles && !hasAudioS3 {
return fmt.Errorf("session.inputs requires audio_dir, at least one audio_files entry, or audio_s3")
}
return nil
}
func validateCrossConfig(pipeline *PipelineConfig, session *SessionConfig) error {
if pipeline == nil || session == nil {
return nil
}
if pipeline.Storage.S3 == nil {
return nil
}
audioS3Enabled := session.Inputs.AudioS3 != nil
archiveUploadEnabled := archiveUploadConfiguredForS3(pipeline)
if (audioS3Enabled || archiveUploadEnabled) && strings.TrimSpace(pipeline.Storage.S3.Bucket) == "" {
return fmt.Errorf("pipeline.storage.s3.bucket is required when S3 session audio or archive upload is enabled")
}
return nil
}
func archiveUploadConfiguredForS3(pipeline *PipelineConfig) bool {
if pipeline == nil || pipeline.Archive == nil {
return false
}
if !strings.EqualFold(strings.TrimSpace(pipeline.Storage.Backend), "s3") {
return false
}
enabled := true
if pipeline.Archive.Enabled != nil {
enabled = *pipeline.Archive.Enabled
}
upload := true
if pipeline.Archive.UploadRun != nil {
upload = *pipeline.Archive.UploadRun
}
return enabled && upload
}
var windowsAbsPathRE = regexp.MustCompile(`^[A-Za-z]:[\\/].*`)
func validateRelativeSafePath(fieldName, value string) error {
trimmed := strings.TrimSpace(value)
if trimmed == "" {
return fmt.Errorf("%s must be non-empty", fieldName)
}
if filepath.IsAbs(trimmed) || strings.HasPrefix(trimmed, "/") || strings.HasPrefix(trimmed, "\\") || windowsAbsPathRE.MatchString(trimmed) {
return fmt.Errorf("%s must be a relative path", fieldName)
}
normalized := strings.ReplaceAll(trimmed, "\\", "/")
for _, segment := range strings.Split(normalized, "/") {
if segment == ".." {
return fmt.Errorf("%s must not contain path traversal", fieldName)
}
}
return nil
}
func validateDuration(fieldName, value string) error {
trimmed := strings.TrimSpace(value)
if trimmed == "" {

7
internal/manifest/manifest.go
View File

@@ -45,6 +45,13 @@ type StageRecord struct {
// Manifest is the durable run-state record for a session execution.
type Manifest struct {
SessionID string `json:"session_id"`
Campaign string `json:"campaign,omitempty"`
RunID string `json:"run_id,omitempty"`
LocalWorkDir string `json:"local_workdir,omitempty"`
LocalSpoolDir string `json:"local_spool_dir,omitempty"`
S3Bucket string `json:"s3_bucket,omitempty"`
S3SessionPrefix string `json:"s3_session_prefix,omitempty"`
S3RunPrefix string `json:"s3_run_prefix,omitempty"`
PipelineVersion string `json:"pipeline_version,omitempty"`
CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"`

13
internal/manifest/store_test.go
View File

@@ -22,6 +22,13 @@ func TestLocalStoreCreateSaveLoadRoundTrip(t *testing.T) {
now := time.Date(2026, 5, 3, 12, 0, 0, 0, time.UTC)
m.MarkStageRunning("prepare", now)
m.MarkStageSucceeded("prepare", now.Add(2*time.Second), []ArtifactRecord{{Kind: "transcript", LocalPath: "transcripts/merged.json"}})
m.Campaign = "forsaken"
m.RunID = "20260515T031522Z-a1b2c3d4"
m.LocalWorkDir = "/var/lib/narratio/work/forsaken/2026-05-03/20260515T031522Z-a1b2c3d4"
m.LocalSpoolDir = "/var/spool/narratio/forsaken/2026-05-03/20260515T031522Z-a1b2c3d4/audio"
m.S3Bucket = "my-dnd-archive"
m.S3SessionPrefix = "dnd/campaigns/forsaken/sessions/2026-05-03/"
m.S3RunPrefix = "dnd/campaigns/forsaken/sessions/2026-05-03/runs/20260515T031522Z-a1b2c3d4/"
path := filepath.Join(t.TempDir(), "manifest.json")
if err := store.Save(ctx, path, m); err != nil {
@@ -36,6 +43,12 @@ func TestLocalStoreCreateSaveLoadRoundTrip(t *testing.T) {
if loaded.SessionID != "2026-05-03" {
t.Fatalf("SessionID = %q, want %q", loaded.SessionID, "2026-05-03")
}
if loaded.Campaign != "forsaken" {
t.Fatalf("Campaign = %q, want %q", loaded.Campaign, "forsaken")
}
if loaded.RunID != "20260515T031522Z-a1b2c3d4" {
t.Fatalf("RunID = %q, want run id", loaded.RunID)
}
stage, ok := loaded.Stages["prepare"]
if !ok {
t.Fatalf("stage prepare not found")