Upload successful run records to storage
This commit is contained in:
@@ -381,6 +381,9 @@ storage:
|
||||
backend: s3
|
||||
s3:
|
||||
bucket: test-bucket
|
||||
archive:
|
||||
enabled: true
|
||||
upload_run: false
|
||||
whisperx:
|
||||
transcribe_url: ` + url + `
|
||||
timeout: 2s
|
||||
|
||||
@@ -81,7 +81,7 @@ func executeStages(ctx context.Context, cfg *config.Config, stages []stage.Stage
|
||||
if env.Storage == nil {
|
||||
env.Storage = &storage.NoopBackend{}
|
||||
}
|
||||
if env.ObjectStore == nil && env.Config != nil && env.Config.Session != nil && env.Config.Session.Inputs.AudioS3 != nil {
|
||||
if env.ObjectStore == nil && needsObjectStoreForRun(env.Config, stages) {
|
||||
objectStore, err := storage.NewObjectStoreFromConfig(ctx, env.Config)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("initialize object store backend: %w", err)
|
||||
@@ -409,3 +409,34 @@ func ensureManifestIdentity(cfg *config.Config, m *manifest.Manifest) (bool, err
|
||||
func manifestPathFor(cfg *config.Config) string {
|
||||
return filepath.Join(cfg.Pipeline.Workspace.Root, "work", cfg.Session.SessionID, "manifest.json")
|
||||
}
|
||||
|
||||
func needsObjectStoreForRun(cfg *config.Config, stages []stage.Stage) bool {
|
||||
if cfg == nil || cfg.Pipeline == nil || cfg.Session == nil {
|
||||
return false
|
||||
}
|
||||
stageRequested := func(name string) bool {
|
||||
for _, s := range stages {
|
||||
if s != nil && s.Name() == name {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
if cfg.Session.Inputs.AudioS3 != nil && stageRequested("prepare") {
|
||||
return true
|
||||
}
|
||||
if !stageRequested("archive") {
|
||||
return false
|
||||
}
|
||||
if cfg.Pipeline.Archive == nil {
|
||||
return false
|
||||
}
|
||||
if cfg.Pipeline.Archive.Enabled != nil && !*cfg.Pipeline.Archive.Enabled {
|
||||
return false
|
||||
}
|
||||
if cfg.Pipeline.Archive.UploadRun != nil && !*cfg.Pipeline.Archive.UploadRun {
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
@@ -144,6 +144,15 @@ func TestExecuteStagesPlaceholderSuccessUpdatesManifest(t *testing.T) {
|
||||
}
|
||||
continue
|
||||
}
|
||||
if name == "archive" {
|
||||
if sr.Metadata == nil || sr.Metadata["stage"] != "archive" {
|
||||
t.Fatalf("archive metadata missing stage=archive: %#v", sr.Metadata)
|
||||
}
|
||||
if sr.Metadata["skipped"] != true {
|
||||
t.Fatalf("archive metadata missing skipped=true for test config without archive section: %#v", sr.Metadata)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if sr.Metadata == nil || sr.Metadata["placeholder"] != true {
|
||||
t.Fatalf("stage %q missing placeholder metadata", name)
|
||||
}
|
||||
@@ -315,7 +324,7 @@ func TestAdapterBackedStageFailureMarksManifestFailed(t *testing.T) {
|
||||
{name: "merge", env: &Env{Seriatim: &seriatim.FakeRunner{Err: errors.New("merge fail")}}},
|
||||
{name: "polish", env: &Env{Audita: &audita.FakeRunner{Err: errors.New("polish fail")}}},
|
||||
{name: "analyze", env: &Env{Scriptorium: &scriptorium.FakeRunner{RunErr: errors.New("analyze fail")}}},
|
||||
{name: "archive", env: &Env{Storage: &storage.FakeBackend{Err: errors.New("archive fail")}}},
|
||||
{name: "archive", env: &Env{ObjectStore: &storage.FakeBackend{UploadErr: errors.New("archive fail")}}},
|
||||
{name: "notify", env: &Env{Notifier: ¬ify.FakeSender{Err: errors.New("notify fail")}}},
|
||||
}
|
||||
|
||||
@@ -400,6 +409,41 @@ func TestAdapterBackedStageFailureMarksManifestFailed(t *testing.T) {
|
||||
},
|
||||
}
|
||||
}
|
||||
if tc.name == "archive" {
|
||||
cfg.Pipeline.Archive = &config.ArchiveConfig{
|
||||
Enabled: boolPtr(true),
|
||||
UploadRun: boolPtr(true),
|
||||
}
|
||||
cfg.Pipeline.Storage.S3 = &config.StorageS3Config{
|
||||
Bucket: "my-dnd-archive",
|
||||
RootPrefix: "dnd",
|
||||
}
|
||||
runID := "20260516T010203Z-0a1b2c3d"
|
||||
runWorkDir := filepath.Join(cfg.Pipeline.Workspace.Root, "work", cfg.Session.Campaign, cfg.Session.SessionID, runID)
|
||||
if err := os.MkdirAll(filepath.Join(runWorkDir, "inputs"), 0o755); err != nil {
|
||||
t.Fatalf("mkdir archive inputs dir: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(runWorkDir, "inputs", "session.yml"), []byte("session_id: 2026-05-03\n"), 0o644); err != nil {
|
||||
t.Fatalf("write archive fixture session.yml: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(runWorkDir, "manifest.json"), []byte("{}\n"), 0o644); err != nil {
|
||||
t.Fatalf("write archive fixture manifest.json: %v", err)
|
||||
}
|
||||
|
||||
seed := manifest.New(cfg.Session.SessionID, time.Now().UTC())
|
||||
seed.Campaign = cfg.Session.Campaign
|
||||
seed.RunID = runID
|
||||
seed.LocalWorkDir = runWorkDir
|
||||
seed.S3Bucket = "my-dnd-archive"
|
||||
seed.S3SessionPrefix = "dnd/campaigns/" + cfg.Session.Campaign + "/sessions/" + cfg.Session.SessionID + "/"
|
||||
seed.S3RunPrefix = seed.S3SessionPrefix + "runs/" + runID + "/"
|
||||
for _, name := range []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim", "analyze"} {
|
||||
seed.MarkStageSucceeded(name, time.Now().UTC(), nil)
|
||||
}
|
||||
if err := tc.env.ManifestStore.Save(context.Background(), manifestPathFor(cfg), seed); err != nil {
|
||||
t.Fatalf("seed archive manifest: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
_, runErr := executeStages(context.Background(), cfg, []stage.Stage{selected}, RunOptions{Env: tc.env})
|
||||
if runErr == nil {
|
||||
@@ -462,3 +506,8 @@ func mustWriteFile(t *testing.T, path, contents string) {
|
||||
t.Fatalf("WriteFile(%q): %v", path, err)
|
||||
}
|
||||
}
|
||||
|
||||
func boolPtr(v bool) *bool {
|
||||
p := v
|
||||
return &p
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user