eric/narratio
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 15 Wiki Activity

Add archive promotion locks

Browse Source
This commit is contained in:
Eric Rakestraw
2026-05-20 21:40:09 -05:00
parent 3aae4bbb12
commit 7111edeca4
9 changed files with 362 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

92
internal/stage/archive.go
View File

@@ -126,7 +126,14 @@ func (archiveStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*S
if err != nil {
return nil, fmt.Errorf("archive: build runtime artifact catalog: %w", err)
}
promotions, skippedOptional, err := resolveArchivePromotions(sessionPaths, m, runtimeCatalog, env.Config.Pipeline.Archive.PromoteArtifacts)
promotions, skippedOptional, lockedPromotions, err := resolveArchivePromotions(
sessionPaths,
m,
runtimeCatalog,
env.Config.Pipeline.Archive.PromoteArtifacts,
env.Config.Pipeline.Archive.Locks,
sessionPrefix,
)
if err != nil {
return nil, fmt.Errorf("archive: resolve promotion rules: %w", err)
}
@@ -166,6 +173,7 @@ func (archiveStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*S
promotedUploaded,
previousUploaded,
skippedOptional,
lockedPromotions,
currentManifestKey,
))
if err != nil {
@@ -204,6 +212,8 @@ func (archiveStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*S
"previous_files_uploaded": len(previousUploaded),
"previous_uploaded_paths": previousUploaded,
"skipped_optional_promotions": skippedOptional,
"locked_promotion_count": len(lockedPromotions),
"locked_promotions": lockedPromotionMetadata(lockedPromotions),
"current_manifest_key": currentManifestKey,
"current_run_id_key": currentRunPointerKey,
"current_pointer_written": true,
@@ -220,6 +230,16 @@ type archivePromotion struct {
Provenance string
}
type archiveLockedPromotion struct {
Source string
Dest string
RemoteKey string
Reason string
Required bool
LocalPath string
Provenance string
}
func archiveDisabled(env *Env) bool {
cfg := env.Config.Pipeline.Archive
if cfg == nil {
@@ -319,26 +339,53 @@ func resolveArchivePromotions(
m *manifest.Manifest,
catalog *artifacts.ArtifactCatalog,
rules []config.ArchivePromotionRule,
) ([]archivePromotion, []string, error) {
locks []config.ArchiveLockRule,
sessionPrefix string,
) ([]archivePromotion, []string, []archiveLockedPromotion, error) {
out := make([]archivePromotion, 0, len(rules))
skippedOptional := make([]string, 0)
lockedPromotions := make([]archiveLockedPromotion, 0)
lockSet := archiveLockSet(locks)
for _, rule := range rules {
source := strings.TrimSpace(rule.Source)
required := rule.Required == nil || *rule.Required
dest, err := resolveArchivePromotionDest(rule, catalog)
if err != nil {
return nil, nil, fmt.Errorf("source %q: %w", source, err)
return nil, nil, nil, fmt.Errorf("source %q: %w", source, err)
}
lock, locked := lockSet[source]
resolved, err := artifacts.ResolveSessionArtifactWithCatalog(paths, m, source, catalog)
if err != nil {
if locked {
lockedPromotions = append(lockedPromotions, archiveLockedPromotion{
Source: source,
Dest: dest,
RemoteKey: artifacts.S3PromotedArtifactKey(sessionPrefix, dest),
Reason: strings.TrimSpace(lock.Reason),
Required: required,
})
continue
}
if errors.Is(err, artifacts.ErrSessionArtifactNotFound) && !required {
skippedOptional = append(skippedOptional, dest)
continue
}
if errors.Is(err, artifacts.ErrSessionArtifactNotFound) {
return nil, nil, fmt.Errorf("required promotion source unavailable: %q", source)
return nil, nil, nil, fmt.Errorf("required promotion source unavailable: %q", source)
}
return nil, nil, fmt.Errorf("resolve source %q: %w", source, err)
return nil, nil, nil, fmt.Errorf("resolve source %q: %w", source, err)
}
if locked {
lockedPromotions = append(lockedPromotions, archiveLockedPromotion{
Source: source,
Dest: dest,
RemoteKey: artifacts.S3PromotedArtifactKey(sessionPrefix, dest),
Reason: strings.TrimSpace(lock.Reason),
Required: required,
LocalPath: resolved.Path,
Provenance: resolved.Provenance,
})
continue
}
out = append(out, archivePromotion{
Source: source,
@@ -348,7 +395,21 @@ func resolveArchivePromotions(
Provenance: resolved.Provenance,
})
}
return out, skippedOptional, nil
return out, skippedOptional, lockedPromotions, nil
}
func archiveLockSet(locks []config.ArchiveLockRule) map[string]config.ArchiveLockRule {
out := make(map[string]config.ArchiveLockRule, len(locks))
for _, lock := range locks {
source := strings.TrimSpace(lock.Source)
if source == "" {
continue
}
lock.Source = source
lock.Reason = strings.TrimSpace(lock.Reason)
out[source] = lock
}
return out
}
func resolveArchivePromotionDest(rule config.ArchivePromotionRule, catalog *artifacts.ArtifactCatalog) (string, error) {
@@ -663,6 +724,7 @@ func archiveMetadataPreview(
promotedUploaded []string,
previousUploaded []string,
skippedOptional []string,
lockedPromotions []archiveLockedPromotion,
currentManifestKey string,
) map[string]any {
return map[string]any{
@@ -677,9 +739,27 @@ func archiveMetadataPreview(
"previous_files_uploaded": len(previousUploaded),
"previous_uploaded_paths": append([]string(nil), previousUploaded...),
"skipped_optional_promotions": append([]string(nil), skippedOptional...),
"locked_promotion_count": len(lockedPromotions),
"locked_promotions": lockedPromotionMetadata(lockedPromotions),
"current_manifest_key": currentManifestKey,
"current_run_id_key": artifacts.S3CurrentRunPointerKey(sessionPrefix),
"current_pointer_written": false,
"audio_upload_skipped": true,
}
}
func lockedPromotionMetadata(locked []archiveLockedPromotion) []map[string]any {
out := make([]map[string]any, 0, len(locked))
for _, item := range locked {
out = append(out, map[string]any{
"source": item.Source,
"dest": item.Dest,
"remote_key": item.RemoteKey,
"reason": item.Reason,
"required": item.Required,
"local_path": item.LocalPath,
"provenance": item.Provenance,
})
}
return out
}

123
internal/stage/archive_test.go
View File

@@ -2,6 +2,7 @@ package stage
import (
"context"
"encoding/json"
"errors"
"os"
"path/filepath"
@@ -215,6 +216,128 @@ func TestArchiveSkipsOptionalMissingPromotion(t *testing.T) {
}
}
func TestArchiveSkipsLockedRequiredPromotionAndCommits(t *testing.T) {
env, m, _ := archiveFixture(t)
env.Config.Pipeline.Archive.Locks = []config.ArchiveLockRule{
{Source: "narratio.transcript.trimmed", Reason: "human reviewed"},
}
fake := env.ObjectStore.(*storage.FakeBackend)
result, err := archiveStage{}.Run(context.Background(), env, m)
if err != nil {
t.Fatalf("Run() error = %v", err)
}
trimmedKey := m.S3SessionPrefix + "transcripts/trimmed.json"
if _, ok := fake.Objects[trimmedKey]; ok {
t.Fatalf("locked promotion key %q should not be uploaded", trimmedKey)
}
recapKey := m.S3SessionPrefix + "artifacts/session_recap.md"
if _, ok := fake.Objects[recapKey]; !ok {
t.Fatalf("unlocked promotion key %q should be uploaded", recapKey)
}
runTrimmedKey := m.S3RunPrefix + "trim/outputs/transcripts/trimmed.json"
if _, ok := fake.Objects[runTrimmedKey]; !ok {
t.Fatalf("run-local locked source output %q should still be uploaded", runTrimmedKey)
}
currentRunIDKey := m.S3SessionPrefix + "current/run_id.txt"
if len(fake.Uploads) == 0 || fake.Uploads[len(fake.Uploads)-1].Key != currentRunIDKey {
t.Fatalf("last upload = %#v, want current run pointer %q", fake.Uploads, currentRunIDKey)
}
if result.Metadata["promoted_files_uploaded"] != 1 {
t.Fatalf("metadata promoted_files_uploaded = %#v, want 1", result.Metadata["promoted_files_uploaded"])
}
if result.Metadata["locked_promotion_count"] != 1 {
t.Fatalf("metadata locked_promotion_count = %#v, want 1", result.Metadata["locked_promotion_count"])
}
locked := result.Metadata["locked_promotions"].([]map[string]any)
if len(locked) != 1 {
t.Fatalf("locked_promotions = %#v, want one item", locked)
}
if locked[0]["source"] != "narratio.transcript.trimmed" ||
locked[0]["dest"] != "transcripts/trimmed.json" ||
locked[0]["remote_key"] != trimmedKey ||
locked[0]["reason"] != "human reviewed" ||
locked[0]["required"] != true ||
locked[0]["local_path"] == "" ||
locked[0]["provenance"] == "" {
t.Fatalf("locked promotion metadata = %#v", locked[0])
}
currentManifestKey := m.S3SessionPrefix + "current/manifest.json"
var current map[string]any
if err := json.Unmarshal(fake.Objects[currentManifestKey].Data, &current); err != nil {
t.Fatalf("unmarshal current manifest: %v", err)
}
stages := current["stages"].(map[string]any)
archive := stages["archive"].(map[string]any)
meta := archive["metadata"].(map[string]any)
if meta["locked_promotion_count"] != float64(1) {
t.Fatalf("current manifest locked_promotion_count = %#v, want 1", meta["locked_promotion_count"])
}
items := meta["locked_promotions"].([]any)
if len(items) != 1 {
t.Fatalf("current manifest locked_promotions = %#v, want one item", items)
}
}
func TestArchiveLockedRequiredMissingPromotionSucceeds(t *testing.T) {
env, m, _ := archiveFixture(t)
env.Config.Pipeline.Archive.PromoteArtifacts = []config.ArchivePromotionRule{
{Source: "narratio.transcript.merged", Dest: "transcripts/merged.json", Required: boolPtr(true)},
}
env.Config.Pipeline.Archive.Locks = []config.ArchiveLockRule{
{Source: "narratio.transcript.merged", Reason: "manual merge is locked"},
}
result, err := archiveStage{}.Run(context.Background(), env, m)
if err != nil {
t.Fatalf("Run() error = %v", err)
}
fake := env.ObjectStore.(*storage.FakeBackend)
mergedKey := m.S3SessionPrefix + "transcripts/merged.json"
if _, ok := fake.Objects[mergedKey]; ok {
t.Fatalf("locked missing promotion key %q should not be uploaded", mergedKey)
}
if _, ok := fake.Objects[m.S3SessionPrefix+"current/run_id.txt"]; !ok {
t.Fatalf("current run pointer should be written for locked missing promotion")
}
locked := result.Metadata["locked_promotions"].([]map[string]any)
if len(locked) != 1 {
t.Fatalf("locked_promotions = %#v, want one item", locked)
}
if locked[0]["local_path"] != "" || locked[0]["provenance"] != "" {
t.Fatalf("locked missing promotion metadata = %#v, want empty local path/provenance", locked[0])
}
}
func TestArchiveLockDoesNotOverwriteExistingPromotion(t *testing.T) {
env, m, _ := archiveFixture(t)
env.Config.Pipeline.Archive.Locks = []config.ArchiveLockRule{
{Source: "narratio.transcript.trimmed", Reason: "already published"},
}
fake := env.ObjectStore.(*storage.FakeBackend)
trimmedKey := m.S3SessionPrefix + "transcripts/trimmed.json"
fake.SeedObject(storage.FakeObject{Key: trimmedKey, Data: []byte("previously published\n")})
if _, err := (archiveStage{}).Run(context.Background(), env, m); err != nil {
t.Fatalf("Run() error = %v", err)
}
got := string(fake.Objects[trimmedKey].Data)
if got != "previously published\n" {
t.Fatalf("locked promotion object contents = %q, want existing object preserved", got)
}
for _, upload := range fake.Uploads {
if upload.Key == trimmedKey {
t.Fatalf("locked promotion key %q was uploaded", trimmedKey)
}
}
}
func TestArchiveFailsWhenRequiredPromotionMissing(t *testing.T) {
env, m, _ := archiveFixture(t)
env.Config.Pipeline.Archive.PromoteArtifacts = []config.ArchivePromotionRule{