From d01775b68a14b082c7086a1da65c2c5f82547fed Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Mon, 10 Aug 2026 01:43:43 +0000 Subject: [PATCH] Exclude staged Notarius bundles from publish uploads --- docs/roadmap/implementation.md | 2 +- internal/stage/publish.go | 28 +++++++++-------- internal/stage/publish_test.go | 56 +++++++++++++++++++++++++++++++++- 3 files changed, 71 insertions(+), 15 deletions(-) diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index c4eefef..563d0db 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -28,7 +28,7 @@ implementation sequence. | Stage 8 | Complete | | Stage 9 | Complete | | Stage 10 | Complete | -| Stage 11 | Pending | +| Stage 11 | Complete | | Stage 12 | Pending | | Stage 13 | Pending | | Stage 14 | Pending | diff --git a/internal/stage/publish.go b/internal/stage/publish.go index 68e8d51..907b94c 100644 --- a/internal/stage/publish.go +++ b/internal/stage/publish.go @@ -573,19 +573,7 @@ func collectPublishRunFiles(runRoot, manifestPath string) ([]publishUploadFile, if walkErr != nil { return walkErr } - if d.IsDir() { - if path == runRoot { - return nil - } - relDir, err := filepath.Rel(runRoot, path) - if err != nil { - return fmt.Errorf("relative dir from %q to %q: %w", runRoot, path, err) - } - relDir = filepath.ToSlash(relDir) - // Preserve existing behavior: audio is not uploaded in publish run record. - if relDir == "audio" || strings.HasPrefix(relDir, "audio/") { - return filepath.SkipDir - } + if path == runRoot { return nil } rel, err := filepath.Rel(runRoot, path) @@ -593,6 +581,15 @@ func collectPublishRunFiles(runRoot, manifestPath string) ([]publishUploadFile, return fmt.Errorf("relative path from %q to %q: %w", runRoot, path, err) } rel = filepath.ToSlash(rel) + if publishRunPathExcluded(rel) { + if d.IsDir() { + return filepath.SkipDir + } + return nil + } + if d.IsDir() { + return nil + } files = append(files, publishUploadFile{ RelativePath: rel, LocalPath: path, @@ -632,6 +629,11 @@ func collectPublishRunFiles(runRoot, manifestPath string) ([]publishUploadFile, return files, nil } +func publishRunPathExcluded(rel string) bool { + return rel == "audio" || strings.HasPrefix(rel, "audio/") || + rel == "extract/notarius-output" || strings.HasPrefix(rel, "extract/notarius-output/") +} + func collectPublishPreviousFiles(previousDir string) ([]publishUploadFile, error) { previousDir = filepath.Clean(strings.TrimSpace(previousDir)) if previousDir == "" { diff --git a/internal/stage/publish_test.go b/internal/stage/publish_test.go index ce8442c..db4539f 100644 --- a/internal/stage/publish_test.go +++ b/internal/stage/publish_test.go @@ -7,6 +7,7 @@ import ( "os" "path/filepath" "reflect" + "sort" "strings" "testing" "time" @@ -64,8 +65,27 @@ func TestPublishFailsWhenPrerequisiteNotSucceeded(t *testing.T) { } func TestPublishUploadsRunRecordPublishedOutputsAndCurrentPointer(t *testing.T) { - env, m, _ := publishFixture(t) + env, m, runRoot := publishFixture(t) fake := env.ObjectStore.(*storage.FakeBackend) + lanePath := configurePublishExtractionFixture(t, env, m) + durableBundleRoot := filepath.Dir(filepath.Dir(lanePath)) + runBundleRoot := filepath.Join(runRoot, "extract", "notarius-output", "notarius-run-1") + for rel, contents := range map[string]string{ + "index.json": `{"lanes":[]}`, + "manifest.json": `{"run_id":"notarius-run-1"}`, + "rejected.json": `[]`, + "warnings.json": `[]`, + "lanes/encounters.json": `{"encounters":[]}`, + "pipeline/chunk-map.json": `{"chunks":[]}`, + "pipeline/evidence-context.json": `{"evidence":[]}`, + "unknown/notes.txt": "internal bundle note\n", + } { + writeStageTestFile(t, filepath.Join(runBundleRoot, filepath.FromSlash(rel)), contents) + } + writeStageTestFile(t, filepath.Join(runRoot, "extract", "notarius.receipt.json"), `{"run_id":"notarius-run-1"}`) + writeStageTestFile(t, filepath.Join(runRoot, "extract", "notarius.stderr.log"), "notarius diagnostic\n") + writeStageTestFile(t, filepath.Join(runRoot, "extract", "notarius-output-copy", "keep.json"), "{}\n") + writeStageTestFile(t, filepath.Join(runRoot, "archive", "notarius-output", "keep.json"), "{}\n") result, err := publishStage{}.Run(context.Background(), env, m) if err != nil { @@ -76,6 +96,10 @@ func TestPublishUploadsRunRecordPublishedOutputsAndCurrentPointer(t *testing.T) sessionPrefix := m.S3SessionPrefix wantRunUploads := []string{ "analyze/outputs/artifacts/session_recap.md", + "archive/notarius-output/keep.json", + "extract/notarius-output-copy/keep.json", + "extract/notarius.receipt.json", + "extract/notarius.stderr.log", "merge/config/seriatim.generated.yml", "prepare/inputs/session.yml", "prepare/outputs/audio/speaker.flac", @@ -85,12 +109,24 @@ func TestPublishUploadsRunRecordPublishedOutputsAndCurrentPointer(t *testing.T) "transcribe/outputs/transcripts/raw/speaker.json", "trim/outputs/transcripts/final.trimmed.json", } + sort.Strings(wantRunUploads) for _, rel := range wantRunUploads { key := runPrefix + rel if _, ok := fake.Objects[key]; !ok { t.Fatalf("missing run upload key %q", key) } } + for key := range fake.Objects { + if strings.HasPrefix(key, runPrefix+"extract/notarius-output/") { + t.Fatalf("run-local Notarius bundle member was uploaded at %q", key) + } + } + for _, upload := range fake.Uploads { + localPath := filepath.Clean(upload.LocalPath) + if localPath == durableBundleRoot || strings.HasPrefix(localPath, durableBundleRoot+string(filepath.Separator)) { + t.Fatalf("durable Notarius bundle member was implicitly uploaded from %q", upload.LocalPath) + } + } trimmedKey := sessionPrefix + "transcripts/final.trimmed.json" recapKey := sessionPrefix + "artifacts/session_recap.md" @@ -135,6 +171,12 @@ func TestPublishUploadsRunRecordPublishedOutputsAndCurrentPointer(t *testing.T) if result.Metadata["previous_files_uploaded"] != 0 { t.Fatalf("metadata previous_files_uploaded = %#v, want 0", result.Metadata["previous_files_uploaded"]) } + if result.Metadata["run_files_uploaded"] != len(wantRunUploads) { + t.Fatalf("metadata run_files_uploaded = %#v, want %d", result.Metadata["run_files_uploaded"], len(wantRunUploads)) + } + if got := result.Metadata["run_uploaded_paths"]; !reflect.DeepEqual(got, wantRunUploads) { + t.Fatalf("metadata run_uploaded_paths = %#v, want %#v", got, wantRunUploads) + } } func TestPublishUploadsPreviousCacheWhenPresent(t *testing.T) { @@ -212,6 +254,7 @@ func TestPublishUploadsExplicitExtractionAndPreservesManifestMetadata(t *testing t.Fatalf("Run() error = %v", err) } fake := env.ObjectStore.(*storage.FakeBackend) + durableBundleRoot := filepath.Dir(filepath.Dir(lanePath)) publishedKey := m.S3SessionPrefix + "artifacts/encounters.json" if got := string(fake.Objects[publishedKey].Data); got != `{"encounters":[]}` { t.Fatalf("published extraction = %q", got) @@ -224,6 +267,12 @@ func TestPublishUploadsExplicitExtractionAndPreservesManifestMetadata(t *testing t.Fatalf("Notarius bundle member was implicitly published at %q", key) } } + for _, upload := range fake.Uploads { + localPath := filepath.Clean(upload.LocalPath) + if (localPath == durableBundleRoot || strings.HasPrefix(localPath, durableBundleRoot+string(filepath.Separator))) && localPath != filepath.Clean(lanePath) { + t.Fatalf("durable Notarius bundle member other than selected lane was uploaded from %q", upload.LocalPath) + } + } if result.Metadata["published_files_uploaded"] != 1 { t.Fatalf("published_files_uploaded = %#v, want 1", result.Metadata["published_files_uploaded"]) } @@ -723,6 +772,11 @@ func configurePublishExtractionFixture(t *testing.T, env *Env, m *manifest.Manif indexPath := filepath.Join(bundleRoot, "index.json") writeStageTestFile(t, lanePath, `{"encounters":[]}`) writeStageTestFile(t, indexPath, `{"lanes":[]}`) + writeStageTestFile(t, filepath.Join(bundleRoot, "manifest.json"), `{"run_id":"notarius-run-1"}`) + writeStageTestFile(t, filepath.Join(bundleRoot, "rejected.json"), `[]`) + writeStageTestFile(t, filepath.Join(bundleRoot, "warnings.json"), `[]`) + writeStageTestFile(t, filepath.Join(bundleRoot, "pipeline", "chunk-map.json"), `{"chunks":[]}`) + writeStageTestFile(t, filepath.Join(bundleRoot, "unknown", "notes.txt"), "internal bundle note\n") laneChecksum, err := artifacts.SHA256File(lanePath) if err != nil { t.Fatal(err)