Make remote publish locks generation-safe
This commit is contained in:
@@ -3,6 +3,7 @@ package stage
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
@@ -1622,6 +1623,11 @@ func (s *analyzeObjectStoreTracker) List(context.Context, string) ([]storage.Obj
|
||||
return nil, errors.New("unexpected object store list call")
|
||||
}
|
||||
|
||||
func (s *analyzeObjectStoreTracker) Read(context.Context, string) (storage.ObjectInfo, io.ReadCloser, error) {
|
||||
s.called = true
|
||||
return storage.ObjectInfo{}, nil, errors.New("unexpected object store read call")
|
||||
}
|
||||
|
||||
func (s *analyzeObjectStoreTracker) Download(context.Context, string, string) error {
|
||||
s.called = true
|
||||
return errors.New("unexpected object store download call")
|
||||
@@ -1632,6 +1638,11 @@ func (s *analyzeObjectStoreTracker) Upload(context.Context, string, string, stor
|
||||
return storage.ObjectInfo{}, errors.New("unexpected object store upload call")
|
||||
}
|
||||
|
||||
func (s *analyzeObjectStoreTracker) UploadConditional(context.Context, io.Reader, string, storage.UploadOptions, storage.WriteCondition) (storage.ObjectInfo, error) {
|
||||
s.called = true
|
||||
return storage.ObjectInfo{}, errors.New("unexpected conditional object store upload call")
|
||||
}
|
||||
|
||||
func (s *analyzeObjectStoreTracker) Exists(context.Context, string) (bool, error) {
|
||||
s.called = true
|
||||
return false, errors.New("unexpected object store exists call")
|
||||
|
||||
@@ -387,6 +387,11 @@ func (s *preparePreviousCaptureStore) List(ctx context.Context, prefix string) (
|
||||
return s.delegate.List(ctx, prefix)
|
||||
}
|
||||
|
||||
func (s *preparePreviousCaptureStore) Read(ctx context.Context, key string) (storage.ObjectInfo, io.ReadCloser, error) {
|
||||
s.downloadKeys = append(s.downloadKeys, key)
|
||||
return s.delegate.Read(ctx, key)
|
||||
}
|
||||
|
||||
func (s *preparePreviousCaptureStore) Download(ctx context.Context, key, localPath string) error {
|
||||
s.downloadKeys = append(s.downloadKeys, key)
|
||||
return s.delegate.Download(ctx, key, localPath)
|
||||
@@ -401,6 +406,10 @@ func (s *preparePreviousCaptureStore) Upload(ctx context.Context, localPath, key
|
||||
return s.delegate.Upload(ctx, localPath, key, opts)
|
||||
}
|
||||
|
||||
func (s *preparePreviousCaptureStore) UploadConditional(ctx context.Context, source io.Reader, key string, opts storage.UploadOptions, condition storage.WriteCondition) (storage.ObjectInfo, error) {
|
||||
return s.delegate.UploadConditional(ctx, source, key, opts, condition)
|
||||
}
|
||||
|
||||
func (s *preparePreviousCaptureStore) Exists(ctx context.Context, key string) (bool, error) {
|
||||
s.existsKeys = append(s.existsKeys, key)
|
||||
return s.delegate.Exists(ctx, key)
|
||||
|
||||
@@ -231,6 +231,9 @@ func (publishStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*S
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("publish: upload immutable commit %q: %w", commitKey, err)
|
||||
}
|
||||
if err := revalidatePublishCommitLocks(ctx, env, publishOutputs); err != nil {
|
||||
return nil, fmt.Errorf("publish: revalidate locks before current commit selection: %w", err)
|
||||
}
|
||||
pointerData, err := artifacts.EncodeCurrentCommitPointer(artifacts.CurrentCommitPointer{
|
||||
FormatVersion: artifacts.RemoteCommitFormatVersion,
|
||||
Campaign: commit.Campaign,
|
||||
@@ -271,6 +274,23 @@ func (publishStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*S
|
||||
}, nil
|
||||
}
|
||||
|
||||
func revalidatePublishCommitLocks(ctx context.Context, env *Env, outputs []publishOutput) error {
|
||||
if env == nil || env.RevalidatePublishLocks == nil || len(outputs) == 0 {
|
||||
return nil
|
||||
}
|
||||
locks, err := env.RevalidatePublishLocks(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
lockSet := publishLockSet(locks)
|
||||
for _, output := range outputs {
|
||||
if lock, locked := lockSet[output.Source]; locked {
|
||||
return fmt.Errorf("source %q is locked: %s", output.Source, strings.TrimSpace(lock.Reason))
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type publishOutput struct {
|
||||
Source string
|
||||
Dest string
|
||||
|
||||
@@ -885,6 +885,30 @@ func TestPublishRejectsConflictingImmutableObject(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestPublishDoesNotSelectCommitWhenLockAppearsAtCommitPoint(t *testing.T) {
|
||||
env, m, _ := publishFixture(t)
|
||||
fake := env.ObjectStore.(*storage.FakeBackend)
|
||||
oldRunID := "20260515T010203Z-a1b2c3d4"
|
||||
seedCommittedCurrentState(t, fake, m, oldRunID)
|
||||
pointerKey := artifacts.S3CurrentCommitPointerKey(m.S3SessionPrefix)
|
||||
priorPointer := append([]byte(nil), fake.Objects[pointerKey].Data...)
|
||||
env.RevalidatePublishLocks = func(context.Context) ([]config.PublishLockRule, error) {
|
||||
return []config.PublishLockRule{{Source: "narratio.transcript.final_trimmed", Reason: "manual review"}}, nil
|
||||
}
|
||||
|
||||
_, err := (publishStage{}).Run(context.Background(), env, m)
|
||||
if err == nil || !strings.Contains(err.Error(), "revalidate locks") || !strings.Contains(err.Error(), "manual review") {
|
||||
t.Fatalf("Run() error = %v, want commit-point lock failure", err)
|
||||
}
|
||||
if got := fake.Objects[pointerKey].Data; !reflect.DeepEqual(got, priorPointer) {
|
||||
t.Fatalf("current pointer changed after lock appeared: %q", got)
|
||||
}
|
||||
state, err := artifacts.LoadCurrentState(context.Background(), fake, m.S3SessionPrefix, artifacts.CurrentStateValidation{ValidateRunID: true})
|
||||
if err != nil || state.RunID != oldRunID {
|
||||
t.Fatalf("current state after lock loss = %#v, %v; want old run %q", state, err, oldRunID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPublishRejectsAmbiguousOrCollidingOutputMappings(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
|
||||
@@ -33,6 +33,10 @@ type Env struct {
|
||||
Scriptorium scriptorium.Runner
|
||||
ObjectStore storage.ObjectStore
|
||||
Notifier notify.Sender
|
||||
|
||||
// RevalidatePublishLocks returns the effective lock set immediately before a
|
||||
// publish commit selects a new remote snapshot.
|
||||
RevalidatePublishLocks func(context.Context) ([]config.PublishLockRule, error)
|
||||
}
|
||||
|
||||
// IODecl declares the intended input/output artifact kinds for a stage.
|
||||
|
||||
Reference in New Issue
Block a user