Make remote publish locks generation-safe

This commit is contained in:
2026-08-10 20:17:54 +00:00
parent 361dbb4ca8
commit 0cf2cbfeb3
22 changed files with 560 additions and 101 deletions

View File

@@ -3,6 +3,7 @@ package stage
import (
"context"
"errors"
"io"
"os"
"path/filepath"
"sort"
@@ -1622,6 +1623,11 @@ func (s *analyzeObjectStoreTracker) List(context.Context, string) ([]storage.Obj
return nil, errors.New("unexpected object store list call")
}
func (s *analyzeObjectStoreTracker) Read(context.Context, string) (storage.ObjectInfo, io.ReadCloser, error) {
s.called = true
return storage.ObjectInfo{}, nil, errors.New("unexpected object store read call")
}
func (s *analyzeObjectStoreTracker) Download(context.Context, string, string) error {
s.called = true
return errors.New("unexpected object store download call")
@@ -1632,6 +1638,11 @@ func (s *analyzeObjectStoreTracker) Upload(context.Context, string, string, stor
return storage.ObjectInfo{}, errors.New("unexpected object store upload call")
}
func (s *analyzeObjectStoreTracker) UploadConditional(context.Context, io.Reader, string, storage.UploadOptions, storage.WriteCondition) (storage.ObjectInfo, error) {
s.called = true
return storage.ObjectInfo{}, errors.New("unexpected conditional object store upload call")
}
func (s *analyzeObjectStoreTracker) Exists(context.Context, string) (bool, error) {
s.called = true
return false, errors.New("unexpected object store exists call")

View File

@@ -387,6 +387,11 @@ func (s *preparePreviousCaptureStore) List(ctx context.Context, prefix string) (
return s.delegate.List(ctx, prefix)
}
func (s *preparePreviousCaptureStore) Read(ctx context.Context, key string) (storage.ObjectInfo, io.ReadCloser, error) {
s.downloadKeys = append(s.downloadKeys, key)
return s.delegate.Read(ctx, key)
}
func (s *preparePreviousCaptureStore) Download(ctx context.Context, key, localPath string) error {
s.downloadKeys = append(s.downloadKeys, key)
return s.delegate.Download(ctx, key, localPath)
@@ -401,6 +406,10 @@ func (s *preparePreviousCaptureStore) Upload(ctx context.Context, localPath, key
return s.delegate.Upload(ctx, localPath, key, opts)
}
func (s *preparePreviousCaptureStore) UploadConditional(ctx context.Context, source io.Reader, key string, opts storage.UploadOptions, condition storage.WriteCondition) (storage.ObjectInfo, error) {
return s.delegate.UploadConditional(ctx, source, key, opts, condition)
}
func (s *preparePreviousCaptureStore) Exists(ctx context.Context, key string) (bool, error) {
s.existsKeys = append(s.existsKeys, key)
return s.delegate.Exists(ctx, key)

View File

@@ -231,6 +231,9 @@ func (publishStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*S
if err != nil {
return nil, fmt.Errorf("publish: upload immutable commit %q: %w", commitKey, err)
}
if err := revalidatePublishCommitLocks(ctx, env, publishOutputs); err != nil {
return nil, fmt.Errorf("publish: revalidate locks before current commit selection: %w", err)
}
pointerData, err := artifacts.EncodeCurrentCommitPointer(artifacts.CurrentCommitPointer{
FormatVersion: artifacts.RemoteCommitFormatVersion,
Campaign: commit.Campaign,
@@ -271,6 +274,23 @@ func (publishStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*S
}, nil
}
func revalidatePublishCommitLocks(ctx context.Context, env *Env, outputs []publishOutput) error {
if env == nil || env.RevalidatePublishLocks == nil || len(outputs) == 0 {
return nil
}
locks, err := env.RevalidatePublishLocks(ctx)
if err != nil {
return err
}
lockSet := publishLockSet(locks)
for _, output := range outputs {
if lock, locked := lockSet[output.Source]; locked {
return fmt.Errorf("source %q is locked: %s", output.Source, strings.TrimSpace(lock.Reason))
}
}
return nil
}
type publishOutput struct {
Source string
Dest string

View File

@@ -885,6 +885,30 @@ func TestPublishRejectsConflictingImmutableObject(t *testing.T) {
}
}
func TestPublishDoesNotSelectCommitWhenLockAppearsAtCommitPoint(t *testing.T) {
env, m, _ := publishFixture(t)
fake := env.ObjectStore.(*storage.FakeBackend)
oldRunID := "20260515T010203Z-a1b2c3d4"
seedCommittedCurrentState(t, fake, m, oldRunID)
pointerKey := artifacts.S3CurrentCommitPointerKey(m.S3SessionPrefix)
priorPointer := append([]byte(nil), fake.Objects[pointerKey].Data...)
env.RevalidatePublishLocks = func(context.Context) ([]config.PublishLockRule, error) {
return []config.PublishLockRule{{Source: "narratio.transcript.final_trimmed", Reason: "manual review"}}, nil
}
_, err := (publishStage{}).Run(context.Background(), env, m)
if err == nil || !strings.Contains(err.Error(), "revalidate locks") || !strings.Contains(err.Error(), "manual review") {
t.Fatalf("Run() error = %v, want commit-point lock failure", err)
}
if got := fake.Objects[pointerKey].Data; !reflect.DeepEqual(got, priorPointer) {
t.Fatalf("current pointer changed after lock appeared: %q", got)
}
state, err := artifacts.LoadCurrentState(context.Background(), fake, m.S3SessionPrefix, artifacts.CurrentStateValidation{ValidateRunID: true})
if err != nil || state.RunID != oldRunID {
t.Fatalf("current state after lock loss = %#v, %v; want old run %q", state, err, oldRunID)
}
}
func TestPublishRejectsAmbiguousOrCollidingOutputMappings(t *testing.T) {
tests := []struct {
name string

View File

@@ -33,6 +33,10 @@ type Env struct {
Scriptorium scriptorium.Runner
ObjectStore storage.ObjectStore
Notifier notify.Sender
// RevalidatePublishLocks returns the effective lock set immediately before a
// publish commit selects a new remote snapshot.
RevalidatePublishLocks func(context.Context) ([]config.PublishLockRule, error)
}
// IODecl declares the intended input/output artifact kinds for a stage.