Upload successful run records to storage

This commit is contained in:
2026-05-16 14:43:29 +00:00
parent ed4dcf1ef7
commit d40c91acde
11 changed files with 716 additions and 34 deletions

View File

@@ -381,6 +381,9 @@ storage:
backend: s3
s3:
bucket: test-bucket
archive:
enabled: true
upload_run: false
whisperx:
transcribe_url: ` + url + `
timeout: 2s

View File

@@ -81,7 +81,7 @@ func executeStages(ctx context.Context, cfg *config.Config, stages []stage.Stage
if env.Storage == nil {
env.Storage = &storage.NoopBackend{}
}
if env.ObjectStore == nil && env.Config != nil && env.Config.Session != nil && env.Config.Session.Inputs.AudioS3 != nil {
if env.ObjectStore == nil && needsObjectStoreForRun(env.Config, stages) {
objectStore, err := storage.NewObjectStoreFromConfig(ctx, env.Config)
if err != nil {
return nil, fmt.Errorf("initialize object store backend: %w", err)
@@ -409,3 +409,34 @@ func ensureManifestIdentity(cfg *config.Config, m *manifest.Manifest) (bool, err
func manifestPathFor(cfg *config.Config) string {
return filepath.Join(cfg.Pipeline.Workspace.Root, "work", cfg.Session.SessionID, "manifest.json")
}
func needsObjectStoreForRun(cfg *config.Config, stages []stage.Stage) bool {
if cfg == nil || cfg.Pipeline == nil || cfg.Session == nil {
return false
}
stageRequested := func(name string) bool {
for _, s := range stages {
if s != nil && s.Name() == name {
return true
}
}
return false
}
if cfg.Session.Inputs.AudioS3 != nil && stageRequested("prepare") {
return true
}
if !stageRequested("archive") {
return false
}
if cfg.Pipeline.Archive == nil {
return false
}
if cfg.Pipeline.Archive.Enabled != nil && !*cfg.Pipeline.Archive.Enabled {
return false
}
if cfg.Pipeline.Archive.UploadRun != nil && !*cfg.Pipeline.Archive.UploadRun {
return false
}
return true
}

View File

@@ -144,6 +144,15 @@ func TestExecuteStagesPlaceholderSuccessUpdatesManifest(t *testing.T) {
}
continue
}
if name == "archive" {
if sr.Metadata == nil || sr.Metadata["stage"] != "archive" {
t.Fatalf("archive metadata missing stage=archive: %#v", sr.Metadata)
}
if sr.Metadata["skipped"] != true {
t.Fatalf("archive metadata missing skipped=true for test config without archive section: %#v", sr.Metadata)
}
continue
}
if sr.Metadata == nil || sr.Metadata["placeholder"] != true {
t.Fatalf("stage %q missing placeholder metadata", name)
}
@@ -315,7 +324,7 @@ func TestAdapterBackedStageFailureMarksManifestFailed(t *testing.T) {
{name: "merge", env: &Env{Seriatim: &seriatim.FakeRunner{Err: errors.New("merge fail")}}},
{name: "polish", env: &Env{Audita: &audita.FakeRunner{Err: errors.New("polish fail")}}},
{name: "analyze", env: &Env{Scriptorium: &scriptorium.FakeRunner{RunErr: errors.New("analyze fail")}}},
{name: "archive", env: &Env{Storage: &storage.FakeBackend{Err: errors.New("archive fail")}}},
{name: "archive", env: &Env{ObjectStore: &storage.FakeBackend{UploadErr: errors.New("archive fail")}}},
{name: "notify", env: &Env{Notifier: &notify.FakeSender{Err: errors.New("notify fail")}}},
}
@@ -400,6 +409,41 @@ func TestAdapterBackedStageFailureMarksManifestFailed(t *testing.T) {
},
}
}
if tc.name == "archive" {
cfg.Pipeline.Archive = &config.ArchiveConfig{
Enabled: boolPtr(true),
UploadRun: boolPtr(true),
}
cfg.Pipeline.Storage.S3 = &config.StorageS3Config{
Bucket: "my-dnd-archive",
RootPrefix: "dnd",
}
runID := "20260516T010203Z-0a1b2c3d"
runWorkDir := filepath.Join(cfg.Pipeline.Workspace.Root, "work", cfg.Session.Campaign, cfg.Session.SessionID, runID)
if err := os.MkdirAll(filepath.Join(runWorkDir, "inputs"), 0o755); err != nil {
t.Fatalf("mkdir archive inputs dir: %v", err)
}
if err := os.WriteFile(filepath.Join(runWorkDir, "inputs", "session.yml"), []byte("session_id: 2026-05-03\n"), 0o644); err != nil {
t.Fatalf("write archive fixture session.yml: %v", err)
}
if err := os.WriteFile(filepath.Join(runWorkDir, "manifest.json"), []byte("{}\n"), 0o644); err != nil {
t.Fatalf("write archive fixture manifest.json: %v", err)
}
seed := manifest.New(cfg.Session.SessionID, time.Now().UTC())
seed.Campaign = cfg.Session.Campaign
seed.RunID = runID
seed.LocalWorkDir = runWorkDir
seed.S3Bucket = "my-dnd-archive"
seed.S3SessionPrefix = "dnd/campaigns/" + cfg.Session.Campaign + "/sessions/" + cfg.Session.SessionID + "/"
seed.S3RunPrefix = seed.S3SessionPrefix + "runs/" + runID + "/"
for _, name := range []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim", "analyze"} {
seed.MarkStageSucceeded(name, time.Now().UTC(), nil)
}
if err := tc.env.ManifestStore.Save(context.Background(), manifestPathFor(cfg), seed); err != nil {
t.Fatalf("seed archive manifest: %v", err)
}
}
_, runErr := executeStages(context.Background(), cfg, []stage.Stage{selected}, RunOptions{Env: tc.env})
if runErr == nil {
@@ -462,3 +506,8 @@ func mustWriteFile(t *testing.T, path, contents string) {
t.Fatalf("WriteFile(%q): %v", path, err)
}
}
func boolPtr(v bool) *bool {
p := v
return &p
}