eric/narratio
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 15 Wiki Activity

Exclude staged Notarius bundles from publish uploads

Browse Source
This commit is contained in:
Eric Rakestraw
2026-08-10 01:43:43 +00:00
parent 0d6f2dd0ce
commit d01775b68a
3 changed files with 71 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
docs/roadmap/implementation.md
View File

@@ -28,7 +28,7 @@ implementation sequence.
| Stage 8 | Complete |
| Stage 9 | Complete |
| Stage 10 | Complete |
| Stage 11 | Pending |
| Stage 11 | Complete |
| Stage 12 | Pending |
| Stage 13 | Pending |
| Stage 14 | Pending |

28
internal/stage/publish.go
View File

@@ -573,19 +573,7 @@ func collectPublishRunFiles(runRoot, manifestPath string) ([]publishUploadFile,
if walkErr != nil {
return walkErr
}
if d.IsDir() {
if path == runRoot {
return nil
}
relDir, err := filepath.Rel(runRoot, path)
if err != nil {
return fmt.Errorf("relative dir from %q to %q: %w", runRoot, path, err)
}
relDir = filepath.ToSlash(relDir)
// Preserve existing behavior: audio is not uploaded in publish run record.
if relDir == "audio" || strings.HasPrefix(relDir, "audio/") {
return filepath.SkipDir
}
if path == runRoot {
return nil
}
rel, err := filepath.Rel(runRoot, path)
@@ -593,6 +581,15 @@ func collectPublishRunFiles(runRoot, manifestPath string) ([]publishUploadFile,
return fmt.Errorf("relative path from %q to %q: %w", runRoot, path, err)
}
rel = filepath.ToSlash(rel)
if publishRunPathExcluded(rel) {
if d.IsDir() {
return filepath.SkipDir
}
return nil
}
if d.IsDir() {
return nil
}
files = append(files, publishUploadFile{
RelativePath: rel,
LocalPath: path,
@@ -632,6 +629,11 @@ func collectPublishRunFiles(runRoot, manifestPath string) ([]publishUploadFile,
return files, nil
}
func publishRunPathExcluded(rel string) bool {
return rel == "audio" || strings.HasPrefix(rel, "audio/") ||
rel == "extract/notarius-output" || strings.HasPrefix(rel, "extract/notarius-output/")
}
func collectPublishPreviousFiles(previousDir string) ([]publishUploadFile, error) {
previousDir = filepath.Clean(strings.TrimSpace(previousDir))
if previousDir == "" {

56
internal/stage/publish_test.go
View File

@@ -7,6 +7,7 @@ import (
"os"
"path/filepath"
"reflect"
"sort"
"strings"
"testing"
"time"
@@ -64,8 +65,27 @@ func TestPublishFailsWhenPrerequisiteNotSucceeded(t *testing.T) {
}
func TestPublishUploadsRunRecordPublishedOutputsAndCurrentPointer(t *testing.T) {
env, m, _ := publishFixture(t)
env, m, runRoot := publishFixture(t)
fake := env.ObjectStore.(*storage.FakeBackend)
lanePath := configurePublishExtractionFixture(t, env, m)
durableBundleRoot := filepath.Dir(filepath.Dir(lanePath))
runBundleRoot := filepath.Join(runRoot, "extract", "notarius-output", "notarius-run-1")
for rel, contents := range map[string]string{
"index.json": `{"lanes":[]}`,
"manifest.json": `{"run_id":"notarius-run-1"}`,
"rejected.json": `[]`,
"warnings.json": `[]`,
"lanes/encounters.json": `{"encounters":[]}`,
"pipeline/chunk-map.json": `{"chunks":[]}`,
"pipeline/evidence-context.json": `{"evidence":[]}`,
"unknown/notes.txt": "internal bundle note\n",
} {
writeStageTestFile(t, filepath.Join(runBundleRoot, filepath.FromSlash(rel)), contents)
}
writeStageTestFile(t, filepath.Join(runRoot, "extract", "notarius.receipt.json"), `{"run_id":"notarius-run-1"}`)
writeStageTestFile(t, filepath.Join(runRoot, "extract", "notarius.stderr.log"), "notarius diagnostic\n")
writeStageTestFile(t, filepath.Join(runRoot, "extract", "notarius-output-copy", "keep.json"), "{}\n")
writeStageTestFile(t, filepath.Join(runRoot, "archive", "notarius-output", "keep.json"), "{}\n")
result, err := publishStage{}.Run(context.Background(), env, m)
if err != nil {
@@ -76,6 +96,10 @@ func TestPublishUploadsRunRecordPublishedOutputsAndCurrentPointer(t *testing.T)
sessionPrefix := m.S3SessionPrefix
wantRunUploads := []string{
"analyze/outputs/artifacts/session_recap.md",
"archive/notarius-output/keep.json",
"extract/notarius-output-copy/keep.json",
"extract/notarius.receipt.json",
"extract/notarius.stderr.log",
"merge/config/seriatim.generated.yml",
"prepare/inputs/session.yml",
"prepare/outputs/audio/speaker.flac",
@@ -85,12 +109,24 @@ func TestPublishUploadsRunRecordPublishedOutputsAndCurrentPointer(t *testing.T)
"transcribe/outputs/transcripts/raw/speaker.json",
"trim/outputs/transcripts/final.trimmed.json",
}
sort.Strings(wantRunUploads)
for _, rel := range wantRunUploads {
key := runPrefix + rel
if _, ok := fake.Objects[key]; !ok {
t.Fatalf("missing run upload key %q", key)
}
}
for key := range fake.Objects {
if strings.HasPrefix(key, runPrefix+"extract/notarius-output/") {
t.Fatalf("run-local Notarius bundle member was uploaded at %q", key)
}
}
for _, upload := range fake.Uploads {
localPath := filepath.Clean(upload.LocalPath)
if localPath == durableBundleRoot || strings.HasPrefix(localPath, durableBundleRoot+string(filepath.Separator)) {
t.Fatalf("durable Notarius bundle member was implicitly uploaded from %q", upload.LocalPath)
}
}
trimmedKey := sessionPrefix + "transcripts/final.trimmed.json"
recapKey := sessionPrefix + "artifacts/session_recap.md"
@@ -135,6 +171,12 @@ func TestPublishUploadsRunRecordPublishedOutputsAndCurrentPointer(t *testing.T)
if result.Metadata["previous_files_uploaded"] != 0 {
t.Fatalf("metadata previous_files_uploaded = %#v, want 0", result.Metadata["previous_files_uploaded"])
}
if result.Metadata["run_files_uploaded"] != len(wantRunUploads) {
t.Fatalf("metadata run_files_uploaded = %#v, want %d", result.Metadata["run_files_uploaded"], len(wantRunUploads))
}
if got := result.Metadata["run_uploaded_paths"]; !reflect.DeepEqual(got, wantRunUploads) {
t.Fatalf("metadata run_uploaded_paths = %#v, want %#v", got, wantRunUploads)
}
}
func TestPublishUploadsPreviousCacheWhenPresent(t *testing.T) {
@@ -212,6 +254,7 @@ func TestPublishUploadsExplicitExtractionAndPreservesManifestMetadata(t *testing
t.Fatalf("Run() error = %v", err)
}
fake := env.ObjectStore.(*storage.FakeBackend)
durableBundleRoot := filepath.Dir(filepath.Dir(lanePath))
publishedKey := m.S3SessionPrefix + "artifacts/encounters.json"
if got := string(fake.Objects[publishedKey].Data); got != `{"encounters":[]}` {
t.Fatalf("published extraction = %q", got)
@@ -224,6 +267,12 @@ func TestPublishUploadsExplicitExtractionAndPreservesManifestMetadata(t *testing
t.Fatalf("Notarius bundle member was implicitly published at %q", key)
}
}
for _, upload := range fake.Uploads {
localPath := filepath.Clean(upload.LocalPath)
if (localPath == durableBundleRoot || strings.HasPrefix(localPath, durableBundleRoot+string(filepath.Separator))) && localPath != filepath.Clean(lanePath) {
t.Fatalf("durable Notarius bundle member other than selected lane was uploaded from %q", upload.LocalPath)
}
}
if result.Metadata["published_files_uploaded"] != 1 {
t.Fatalf("published_files_uploaded = %#v, want 1", result.Metadata["published_files_uploaded"])
}
@@ -723,6 +772,11 @@ func configurePublishExtractionFixture(t *testing.T, env *Env, m *manifest.Manif
indexPath := filepath.Join(bundleRoot, "index.json")
writeStageTestFile(t, lanePath, `{"encounters":[]}`)
writeStageTestFile(t, indexPath, `{"lanes":[]}`)
writeStageTestFile(t, filepath.Join(bundleRoot, "manifest.json"), `{"run_id":"notarius-run-1"}`)
writeStageTestFile(t, filepath.Join(bundleRoot, "rejected.json"), `[]`)
writeStageTestFile(t, filepath.Join(bundleRoot, "warnings.json"), `[]`)
writeStageTestFile(t, filepath.Join(bundleRoot, "pipeline", "chunk-map.json"), `{"chunks":[]}`)
writeStageTestFile(t, filepath.Join(bundleRoot, "unknown", "notes.txt"), "internal bundle note\n")
laneChecksum, err := artifacts.SHA256File(lanePath)
if err != nil {
t.Fatal(err)