Upload successful run records to storage

internal/stage/archive.go

@@ -0,0 +1,282 @@
+package stage
+
+import (
+	"context"
+	"fmt"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+
+	"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
+	"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
+	"gitea.maximumdirect.net/eric/narratio/internal/manifest"
+)
+
+type archiveStage struct{}
+
+var archivePrerequisiteStages = []string{
+	"prepare",
+	"transcribe",
+	"merge",
+	"polish",
+	"normalize",
+	"trim",
+	"analyze",
+}
+
+var archiveRunUploadDirs = []string{
+	"inputs",
+	"transcripts",
+	"artifacts",
+	"reports",
+	"config",
+	"logs",
+}
+
+func (archiveStage) Name() string { return "archive" }
+
+func (archiveStage) Declares() IODecl {
+	return IODecl{
+		Inputs: []artifacts.Ref{
+			{Kind: "manifest", Category: "input", RelativePath: "manifest.json"},
+		},
+	}
+}
+
+func (archiveStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*StageResult, error) {
+	if env == nil || env.Config == nil || env.Config.Pipeline == nil || env.Config.Session == nil {
+		return nil, fmt.Errorf("archive: resolved config must include pipeline and session")
+	}
+
+	if archiveDisabled(env) {
+		return &StageResult{
+			Metadata: map[string]any{
+				"stage":                "archive",
+				"skipped":              true,
+				"archive_enabled":      false,
+				"audio_upload_skipped": true,
+			},
+		}, nil
+	}
+	if archiveRunUploadDisabled(env) {
+		return &StageResult{
+			Metadata: map[string]any{
+				"stage":                "archive",
+				"skipped":              true,
+				"upload_run_enabled":   false,
+				"audio_upload_skipped": true,
+			},
+		}, nil
+	}
+
+	if err := validateArchivePrerequisites(m); err != nil {
+		return nil, fmt.Errorf("archive: %w", err)
+	}
+	if env.ObjectStore == nil {
+		return nil, fmt.Errorf("archive: remote object store backend is required when archive run upload is enabled")
+	}
+
+	workDir, err := archiveWorkDir(env, m)
+	if err != nil {
+		return nil, fmt.Errorf("archive: resolve local workdir: %w", err)
+	}
+	info, err := os.Stat(workDir)
+	if err != nil {
+		return nil, fmt.Errorf("archive: local workdir %q: %w", workDir, err)
+	}
+	if !info.IsDir() {
+		return nil, fmt.Errorf("archive: local workdir %q is not a directory", workDir)
+	}
+
+	runPrefix, err := archiveRunPrefix(env, m)
+	if err != nil {
+		return nil, fmt.Errorf("archive: resolve s3 run prefix: %w", err)
+	}
+	bucket := archiveBucket(env, m)
+	if bucket == "" {
+		return nil, fmt.Errorf("archive: resolve s3 bucket: bucket is required")
+	}
+
+	relFiles, err := collectArchiveRunFiles(workDir)
+	if err != nil {
+		return nil, fmt.Errorf("archive: collect run files: %w", err)
+	}
+
+	uploaded := make([]string, 0, len(relFiles))
+	for _, rel := range relFiles {
+		localPath := filepath.Join(workDir, filepath.FromSlash(rel))
+		key := artifacts.S3RunRelativeDestinationKey(runPrefix, rel)
+		if _, err := env.ObjectStore.Upload(ctx, localPath, key, storage.UploadOptions{}); err != nil {
+			return nil, fmt.Errorf("archive: upload %q to %q: %w", rel, key, err)
+		}
+		uploaded = append(uploaded, rel)
+	}
+
+	return &StageResult{
+		Metadata: map[string]any{
+			"stage":                "archive",
+			"uploaded":             true,
+			"s3_bucket":            bucket,
+			"s3_run_prefix":        runPrefix,
+			"files_uploaded":       len(uploaded),
+			"uploaded_paths":       uploaded,
+			"audio_upload_skipped": true,
+		},
+	}, nil
+}
+
+func archiveDisabled(env *Env) bool {
+	cfg := env.Config.Pipeline.Archive
+	if cfg == nil {
+		return true
+	}
+	return cfg.Enabled != nil && !*cfg.Enabled
+}
+
+func archiveRunUploadDisabled(env *Env) bool {
+	cfg := env.Config.Pipeline.Archive
+	if cfg == nil {
+		return true
+	}
+	return cfg.UploadRun != nil && !*cfg.UploadRun
+}
+
+func validateArchivePrerequisites(m *manifest.Manifest) error {
+	if m == nil {
+		return fmt.Errorf("manifest is required")
+	}
+	for _, stageName := range archivePrerequisiteStages {
+		sr := m.Stages[stageName]
+		if sr == nil {
+			return fmt.Errorf("prerequisite stage %q has not succeeded", stageName)
+		}
+		if sr.Status != manifest.StatusSucceeded {
+			return fmt.Errorf("prerequisite stage %q status is %q (want %q)", stageName, sr.Status, manifest.StatusSucceeded)
+		}
+	}
+	return nil
+}
+
+func archiveWorkDir(env *Env, m *manifest.Manifest) (string, error) {
+	workDir := strings.TrimSpace(m.LocalWorkDir)
+	if workDir != "" {
+		cleaned := filepath.Clean(workDir)
+		if info, err := os.Stat(cleaned); err == nil && info.IsDir() {
+			return cleaned, nil
+		}
+	}
+
+	sessionID := strings.TrimSpace(env.Config.Session.SessionID)
+	if sessionID == "" {
+		sessionID = strings.TrimSpace(m.SessionID)
+	}
+	campaign := strings.TrimSpace(env.Config.Session.Campaign)
+	if campaign == "" {
+		campaign = strings.TrimSpace(m.Campaign)
+	}
+	runID := strings.TrimSpace(m.RunID)
+	if runID == "" {
+		return "", fmt.Errorf("run id is required")
+	}
+	if campaign == "" || sessionID == "" {
+		return "", fmt.Errorf("campaign and session id are required")
+	}
+	runScoped := artifacts.SessionRunWorkDir(env.Config.Pipeline.Workspace.Root, campaign, sessionID, runID)
+	if info, err := os.Stat(runScoped); err == nil && info.IsDir() {
+		return runScoped, nil
+	}
+	legacy := artifacts.SessionWorkDir(env.Config.Pipeline.Workspace.Root, sessionID)
+	return legacy, nil
+}
+
+func archiveRunPrefix(env *Env, m *manifest.Manifest) (string, error) {
+	runPrefix := strings.TrimSpace(m.S3RunPrefix)
+	if runPrefix != "" {
+		return runPrefix, nil
+	}
+
+	sessionID := strings.TrimSpace(env.Config.Session.SessionID)
+	if sessionID == "" {
+		sessionID = strings.TrimSpace(m.SessionID)
+	}
+	campaign := strings.TrimSpace(env.Config.Session.Campaign)
+	if campaign == "" {
+		campaign = strings.TrimSpace(m.Campaign)
+	}
+	runID := strings.TrimSpace(m.RunID)
+	if runID == "" {
+		return "", fmt.Errorf("run id is required")
+	}
+	if env.Config.Pipeline.Storage.S3 == nil {
+		return "", fmt.Errorf("pipeline.storage.s3 configuration is required")
+	}
+	sessionPrefix := artifacts.S3SessionPrefix(env.Config.Pipeline.Storage.S3.RootPrefix, campaign, sessionID)
+	if strings.TrimSpace(sessionPrefix) == "" {
+		return "", fmt.Errorf("session prefix is required")
+	}
+	return artifacts.S3RunPrefix(sessionPrefix, runID), nil
+}
+
+func archiveBucket(env *Env, m *manifest.Manifest) string {
+	if m != nil && strings.TrimSpace(m.S3Bucket) != "" {
+		return strings.TrimSpace(m.S3Bucket)
+	}
+	if env == nil || env.Config == nil || env.Config.Pipeline == nil || env.Config.Pipeline.Storage.S3 == nil {
+		return ""
+	}
+	return strings.TrimSpace(env.Config.Pipeline.Storage.S3.Bucket)
+}
+
+func collectArchiveRunFiles(workDir string) ([]string, error) {
+	files := make([]string, 0, 64)
+
+	for _, dirName := range archiveRunUploadDirs {
+		fullDir := filepath.Join(workDir, dirName)
+		info, err := os.Stat(fullDir)
+		if err != nil {
+			if os.IsNotExist(err) {
+				continue
+			}
+			return nil, fmt.Errorf("stat %q: %w", fullDir, err)
+		}
+		if !info.IsDir() {
+			continue
+		}
+
+		if err := filepath.WalkDir(fullDir, func(path string, d fs.DirEntry, walkErr error) error {
+			if walkErr != nil {
+				return walkErr
+			}
+			if d.IsDir() {
+				return nil
+			}
+			rel, err := filepath.Rel(workDir, path)
+			if err != nil {
+				return fmt.Errorf("relative path from %q to %q: %w", workDir, path, err)
+			}
+			rel = filepath.ToSlash(rel)
+			files = append(files, rel)
+			return nil
+		}); err != nil {
+			return nil, fmt.Errorf("walk %q: %w", fullDir, err)
+		}
+	}
+
+	manifestPath := filepath.Join(workDir, "manifest.json")
+	manifestInfo, err := os.Stat(manifestPath)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return nil, fmt.Errorf("manifest.json not found in workdir %q", workDir)
+		}
+		return nil, fmt.Errorf("stat %q: %w", manifestPath, err)
+	}
+	if manifestInfo.IsDir() {
+		return nil, fmt.Errorf("manifest path %q is a directory", manifestPath)
+	}
+	files = append(files, "manifest.json")
+
+	sort.Strings(files)
+	return files, nil
+}

internal/stage/archive_test.go

@@ -0,0 +1,198 @@
+package stage
+
+import (
+	"context"
+	"errors"
+	"os"
+	"path/filepath"
+	"reflect"
+	"strings"
+	"testing"
+	"time"
+
+	"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
+	"gitea.maximumdirect.net/eric/narratio/internal/config"
+	"gitea.maximumdirect.net/eric/narratio/internal/manifest"
+)
+
+func TestArchiveSkipsWhenDisabled(t *testing.T) {
+	env, m, _ := archiveFixture(t)
+	env.Config.Pipeline.Archive.Enabled = boolPtr(false)
+
+	result, err := archiveStage{}.Run(context.Background(), env, m)
+	if err != nil {
+		t.Fatalf("Run() error = %v", err)
+	}
+	if result.Metadata["skipped"] != true {
+		t.Fatalf("metadata = %#v, want skipped=true", result.Metadata)
+	}
+	if len(env.ObjectStore.(*storage.FakeBackend).Objects) != 0 {
+		t.Fatalf("unexpected uploads when archive disabled")
+	}
+}
+
+func TestArchiveSkipsRunUploadWhenDisabled(t *testing.T) {
+	env, m, _ := archiveFixture(t)
+	env.Config.Pipeline.Archive.UploadRun = boolPtr(false)
+
+	result, err := archiveStage{}.Run(context.Background(), env, m)
+	if err != nil {
+		t.Fatalf("Run() error = %v", err)
+	}
+	if result.Metadata["skipped"] != true {
+		t.Fatalf("metadata = %#v, want skipped=true", result.Metadata)
+	}
+	if len(env.ObjectStore.(*storage.FakeBackend).Objects) != 0 {
+		t.Fatalf("unexpected uploads when upload_run disabled")
+	}
+}
+
+func TestArchiveFailsWhenPrerequisiteNotSucceeded(t *testing.T) {
+	env, m, _ := archiveFixture(t)
+	m.Stages["trim"].Status = manifest.StatusFailed
+
+	_, err := archiveStage{}.Run(context.Background(), env, m)
+	if err == nil || !strings.Contains(err.Error(), `prerequisite stage "trim"`) {
+		t.Fatalf("Run() error = %v, want prerequisite failure", err)
+	}
+}
+
+func TestArchiveUploadsRunRecordWithoutAudio(t *testing.T) {
+	env, m, workDir := archiveFixture(t)
+	fake := env.ObjectStore.(*storage.FakeBackend)
+
+	result, err := archiveStage{}.Run(context.Background(), env, m)
+	if err != nil {
+		t.Fatalf("Run() error = %v", err)
+	}
+
+	wantUploaded := []string{
+		"artifacts/session_recap.md",
+		"config/audita.generated.yml",
+		"inputs/session.yml",
+		"logs/audita.stderr.log",
+		"manifest.json",
+		"reports/audita.report.json",
+		"transcripts/raw/speaker.json",
+		"transcripts/trimmed.json",
+	}
+	if got := result.Metadata["uploaded_paths"]; !reflect.DeepEqual(got, wantUploaded) {
+		t.Fatalf("uploaded_paths = %#v, want %#v", got, wantUploaded)
+	}
+	if got := result.Metadata["files_uploaded"]; got != len(wantUploaded) {
+		t.Fatalf("files_uploaded = %#v, want %d", got, len(wantUploaded))
+	}
+
+	for _, rel := range wantUploaded {
+		key := m.S3RunPrefix + rel
+		if _, ok := fake.Objects[key]; !ok {
+			t.Fatalf("missing uploaded key %q", key)
+		}
+	}
+	audioKey := m.S3RunPrefix + "audio/speaker.flac"
+	if _, ok := fake.Objects[audioKey]; ok {
+		t.Fatalf("audio key %q should not be uploaded", audioKey)
+	}
+
+	// Ensure relative paths are preserved under runs/{run_id}/.
+	if _, err := os.Stat(filepath.Join(workDir, "transcripts", "raw", "speaker.json")); err != nil {
+		t.Fatalf("expected local fixture transcript file: %v", err)
+	}
+}
+
+func TestArchiveFailsWhenUploadFails(t *testing.T) {
+	env, m, _ := archiveFixture(t)
+	env.ObjectStore.(*storage.FakeBackend).UploadErr = errors.New("upload failed")
+
+	_, err := archiveStage{}.Run(context.Background(), env, m)
+	if err == nil || !strings.Contains(err.Error(), "upload") {
+		t.Fatalf("Run() error = %v, want upload failure", err)
+	}
+}
+
+func TestArchiveFailsWithoutObjectStore(t *testing.T) {
+	env, m, _ := archiveFixture(t)
+	env.ObjectStore = nil
+
+	_, err := archiveStage{}.Run(context.Background(), env, m)
+	if err == nil || !strings.Contains(err.Error(), "object store") {
+		t.Fatalf("Run() error = %v, want object store backend failure", err)
+	}
+}
+
+func TestArchiveDoesNotWriteCurrentPointers(t *testing.T) {
+	env, m, _ := archiveFixture(t)
+	fake := env.ObjectStore.(*storage.FakeBackend)
+
+	_, err := archiveStage{}.Run(context.Background(), env, m)
+	if err != nil {
+		t.Fatalf("Run() error = %v", err)
+	}
+
+	if _, ok := fake.Objects[m.S3SessionPrefix+"current/run_id.txt"]; ok {
+		t.Fatalf("unexpected current/run_id.txt upload")
+	}
+	if _, ok := fake.Objects[m.S3SessionPrefix+"current/manifest.json"]; ok {
+		t.Fatalf("unexpected current/manifest.json upload")
+	}
+}
+
+func archiveFixture(t *testing.T) (*Env, *manifest.Manifest, string) {
+	t.Helper()
+
+	root := t.TempDir()
+	runID := "20260516T010203Z-1a2b3c4d"
+	campaign := "forsaken"
+	sessionID := "2026-04-19"
+	workDir := filepath.Join(root, "work", campaign, sessionID, runID)
+
+	writeStageTestFile(t, filepath.Join(workDir, "inputs", "session.yml"), "session_id: 2026-04-19\n")
+	writeStageTestFile(t, filepath.Join(workDir, "transcripts", "raw", "speaker.json"), "{}\n")
+	writeStageTestFile(t, filepath.Join(workDir, "transcripts", "trimmed.json"), "{}\n")
+	writeStageTestFile(t, filepath.Join(workDir, "artifacts", "session_recap.md"), "# recap\n")
+	writeStageTestFile(t, filepath.Join(workDir, "reports", "audita.report.json"), "{}\n")
+	writeStageTestFile(t, filepath.Join(workDir, "config", "audita.generated.yml"), "key: value\n")
+	writeStageTestFile(t, filepath.Join(workDir, "logs", "audita.stderr.log"), "stderr\n")
+	writeStageTestFile(t, filepath.Join(workDir, "audio", "speaker.flac"), "flac")
+	writeStageTestFile(t, filepath.Join(workDir, "manifest.json"), "{}\n")
+
+	m := manifest.New(sessionID, time.Now().UTC())
+	m.Campaign = campaign
+	m.RunID = runID
+	m.LocalWorkDir = workDir
+	m.S3Bucket = "my-dnd-archive"
+	m.S3SessionPrefix = "dnd/campaigns/forsaken/sessions/2026-04-19/"
+	m.S3RunPrefix = m.S3SessionPrefix + "runs/" + runID + "/"
+	for _, name := range archivePrerequisiteStages {
+		m.MarkStageSucceeded(name, time.Now().UTC(), nil)
+	}
+
+	env := &Env{
+		Config: &config.Config{
+			Pipeline: &config.PipelineConfig{
+				Workspace: config.WorkspaceConfig{Root: root},
+				Storage: config.StorageConfig{
+					S3: &config.StorageS3Config{
+						Bucket:     "my-dnd-archive",
+						RootPrefix: "dnd",
+					},
+				},
+				Archive: &config.ArchiveConfig{
+					Enabled:   boolPtr(true),
+					UploadRun: boolPtr(true),
+				},
+			},
+			Session: &config.SessionConfig{
+				SessionID: sessionID,
+				Campaign:  campaign,
+			},
+		},
+		ObjectStore: &storage.FakeBackend{},
+	}
+	return env, m, workDir
+}
+
+func boolPtr(v bool) *bool {
+	p := v
+	return &p
+}

internal/stage/placeholders.go

@@ -3,10 +3,8 @@ package stage
 import (
 	"context"
 	"fmt"
-	"path/filepath"
 
 	"gitea.maximumdirect.net/eric/narratio/internal/adapters/notify"
-	"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
 	"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
 	"gitea.maximumdirect.net/eric/narratio/internal/manifest"
 )
@@ -47,25 +45,7 @@ func (s placeholderStage) Run(ctx context.Context, env *Env, m *manifest.Manifes
 		return result, nil
 	}
 
-	paths := env.ArtifactStore.SessionPaths(sessionID)
-
 	switch s.name {
-	case "archive":
-		if env.Storage != nil {
-			req := storage.ArchiveRequest{
-				SessionID:    sessionID,
-				ManifestPath: paths.ManifestPath,
-				Items: []storage.ArchiveItem{{
-					Kind:      "artifact",
-					LocalPath: filepath.Join(paths.ArtifactsDir, "session-log.md"),
-					RemoteKey: "sessions/" + sessionID + "/artifacts/session-log.md",
-				}},
-			}
-			_, err := env.Storage.Archive(ctx, req)
-			if err != nil {
-				return nil, fmt.Errorf("placeholder archive adapter call failed: %w", err)
-			}
-		}
 	case "notify":
 		if env.Notifier != nil {
 			req := notify.SendRequest{
@@ -95,7 +75,7 @@ func All() []Stage {
 		normalizeStage{},
 		trimStage{},
 		analyzeStage{},
-		placeholderStage{name: "archive"},
+		archiveStage{},
 		placeholderStage{name: "notify"},
 	}
 }

internal/stage/placeholders_test.go

@@ -49,7 +49,19 @@ func TestStagesReturnExpectedMetadata(t *testing.T) {
 		Config: &config.Config{
 			SessionPath:  sessionPath,
 			PipelinePath: pipelinePath,
-			Pipeline:     &config.PipelineConfig{Workspace: config.WorkspaceConfig{Root: root}},
+			Pipeline: &config.PipelineConfig{
+				Workspace: config.WorkspaceConfig{Root: root},
+				Storage: config.StorageConfig{
+					S3: &config.StorageS3Config{
+						Bucket:     "my-dnd-archive",
+						RootPrefix: "dnd",
+					},
+				},
+				Archive: &config.ArchiveConfig{
+					Enabled:   boolPtr(true),
+					UploadRun: boolPtr(true),
+				},
+			},
 			Session: &config.SessionConfig{
 				SessionID: "2026-05-03",
 				Inputs: config.SessionInputsConfig{
@@ -66,10 +78,24 @@ func TestStagesReturnExpectedMetadata(t *testing.T) {
 		Audita:        af,
 		Scriptorium:   sc,
 		Storage:       st,
+		ObjectStore:   st,
 		Notifier:      nf,
 	}
 
 	m := manifest.New("2026-05-03", time.Now().UTC())
+	for _, name := range []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim", "analyze"} {
+		m.MarkStageSucceeded(name, time.Now().UTC(), nil)
+	}
+	m.RunID = "20260516T000000Z-abcdef12"
+	m.Campaign = "sample-campaign"
+	m.LocalWorkDir = filepath.Join(root, "work", "sample-campaign", "2026-05-03", m.RunID)
+	m.S3RunPrefix = "dnd/campaigns/sample-campaign/sessions/2026-05-03/runs/" + m.RunID + "/"
+	m.S3Bucket = "my-dnd-archive"
+	if err := os.MkdirAll(filepath.Join(m.LocalWorkDir, "inputs"), 0o755); err != nil {
+		t.Fatalf("mkdir workdir inputs: %v", err)
+	}
+	writeStageTestFile(t, filepath.Join(m.LocalWorkDir, "inputs", "session.yml"), "session_id: 2026-05-03\n")
+	writeStageTestFile(t, filepath.Join(m.LocalWorkDir, "manifest.json"), "{}\n")
 	for _, s := range stages {
 		result, err := s.Run(context.Background(), env, m)
 		if err != nil {
@@ -150,6 +176,15 @@ func TestStagesReturnExpectedMetadata(t *testing.T) {
 			}
 			continue
 		}
+		if s.Name() == "archive" {
+			if result.Metadata["stage"] != "archive" {
+				t.Fatalf("archive metadata = %#v, want stage=archive", result.Metadata)
+			}
+			if result.Metadata["uploaded"] != true {
+				t.Fatalf("archive metadata = %#v, want uploaded=true", result.Metadata)
+			}
+			continue
+		}
 		if result.Metadata["placeholder"] != true {
 			t.Fatalf("stage %q missing placeholder metadata", s.Name())
 		}
@@ -167,8 +202,11 @@ func TestStagesReturnExpectedMetadata(t *testing.T) {
 	if len(sc.RunRequests) != 0 {
 		t.Fatalf("scriptorium run calls = %d, want 0 when scriptorium config is absent", len(sc.RunRequests))
 	}
-	if len(st.Requests) != 1 {
-		t.Fatalf("storage calls = %d, want 1", len(st.Requests))
+	if len(st.Requests) != 0 {
+		t.Fatalf("storage archive calls = %d, want 0", len(st.Requests))
+	}
+	if _, ok := st.Objects["dnd/campaigns/sample-campaign/sessions/2026-05-03/runs/"+m.RunID+"/manifest.json"]; !ok {
+		t.Fatalf("archive upload missing manifest key in fake object store")
 	}
 	if len(nf.Requests) != 1 {
 		t.Fatalf("notify calls = %d, want 1", len(nf.Requests))
@@ -181,7 +219,6 @@ func TestPlaceholderAdapterErrorPropagation(t *testing.T) {
 		env       *Env
 		wantErr   string
 	}{
-		{stageName: "archive", env: &Env{Storage: &storage.FakeBackend{Err: errors.New("sterr")}}, wantErr: "archive"},
 		{stageName: "notify", env: &Env{Notifier: &notify.FakeSender{Err: errors.New("nerr")}}, wantErr: "notify"},
 	}