eric/narratio
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 15 Wiki Activity

Make remote publish locks generation-safe

Browse Source
This commit is contained in:
Eric Rakestraw
2026-08-10 20:17:54 +00:00
parent 361dbb4ca8
commit 0cf2cbfeb3
22 changed files with 560 additions and 101 deletions
Download Patch File Download Diff File Expand all files Collapse all files

89
internal/app/remote_locks.go
View File

@@ -1,8 +1,11 @@
package app
import (
"bytes"
"context"
"errors"
"fmt"
"io"
"os"
"path/filepath"
"strings"
@@ -20,6 +23,8 @@ type effectiveLocks struct {
Key string
}
const remoteLockMutationAttempts = 4
func remoteLocksKey(cfg *config.Config) (string, error) {
if cfg == nil || cfg.Pipeline == nil || cfg.Session == nil {
return "", fmt.Errorf("resolved config is required")
@@ -35,32 +40,34 @@ func remoteLocksKey(cfg *config.Config) (string, error) {
return artifacts.S3SessionLocksKey(sessionPrefix), nil
}
func loadRemoteLockStore(ctx context.Context, cfg *config.Config, store storage.ObjectStore) (*config.PublishLockStore, string, error) {
func loadRemoteLockStore(ctx context.Context, cfg *config.Config, store storage.ObjectStore) (*config.PublishLockStore, string, string, error) {
key, err := remoteLocksKey(cfg)
if err != nil {
return nil, "", err
return nil, "", "", err
}
exists, err := store.Exists(ctx, key)
if store == nil {
return nil, key, "", fmt.Errorf("remote lock store is required")
}
info, body, err := store.Read(ctx, key)
if err != nil {
return nil, key, fmt.Errorf("check remote locks %q: %w", key, err)
if errors.Is(err, os.ErrNotExist) {
return &config.PublishLockStore{}, key, "", nil
}
return nil, key, "", fmt.Errorf("read remote locks %q: %w", key, err)
}
if !exists {
return &config.PublishLockStore{}, key, nil
defer body.Close()
if strings.TrimSpace(info.ETag) == "" {
return nil, key, "", fmt.Errorf("read remote locks %q: object has no generation", key)
}
tmp, err := storage.DownloadObjectToTemp(ctx, store, key, "narratio-locks-*.yml")
data, err := io.ReadAll(body)
if err != nil {
return nil, key, fmt.Errorf("download remote locks %q: %w", key, err)
}
defer func() { _ = os.Remove(tmp) }()
data, err := os.ReadFile(tmp)
if err != nil {
return nil, key, fmt.Errorf("read remote locks %q: %w", key, err)
return nil, key, "", fmt.Errorf("read remote locks %q: %w", key, err)
}
lockStore, err := config.LoadPublishLockStoreBytes("s3://"+s3BucketName(cfg.Pipeline)+"/"+key, data, cfg.Pipeline.Scriptorium, cfg.Pipeline.Notarius)
if err != nil {
return nil, key, err
return nil, key, "", err
}
return lockStore, key, nil
return lockStore, key, info.ETag, nil
}
func loadEffectiveLocks(ctx context.Context, cfg *config.Config, store storage.ObjectStore) (*effectiveLocks, error) {
@@ -71,7 +78,7 @@ func loadEffectiveLocks(ctx context.Context, cfg *config.Config, store storage.O
All: append([]config.PublishLockRule(nil), staticLocks...),
}, nil
}
lockStore, key, err := loadRemoteLockStore(ctx, cfg, store)
lockStore, key, _, err := loadRemoteLockStore(ctx, cfg, store)
if err != nil {
return nil, err
}
@@ -101,28 +108,38 @@ func applyEffectiveLocks(cfg *config.Config, locks []config.PublishLockRule) {
cfg.Pipeline.Publish.Locks = append([]config.PublishLockRule(nil), locks...)
}
func uploadRemoteLockStore(ctx context.Context, store storage.ObjectStore, key string, lockStore *config.PublishLockStore) error {
data, err := config.MarshalPublishLockStore(lockStore)
if err != nil {
func mutateRemoteLockStore(ctx context.Context, cfg *config.Config, store storage.ObjectStore, mutate func(*config.PublishLockStore) error) error {
for attempt := 0; attempt < remoteLockMutationAttempts; attempt++ {
if err := ctx.Err(); err != nil {
return err
}
lockStore, key, generation, err := loadRemoteLockStore(ctx, cfg, store)
if err != nil {
return err
}
if err := mutate(lockStore); err != nil {
return err
}
data, err := config.MarshalPublishLockStore(lockStore)
if err != nil {
return err
}
condition := storage.WriteCondition{MatchETag: generation}
if generation == "" {
condition = storage.WriteCondition{RequireAbsent: true}
}
_, err = store.UploadConditional(ctx, bytes.NewReader(data), key, storage.UploadOptions{ContentType: "application/x-yaml; charset=utf-8"}, condition)
if err == nil {
return nil
}
if !errors.Is(err, storage.ErrConditionNotMet) {
return fmt.Errorf("upload remote locks %q: %w", key, err)
}
}
if err := ctx.Err(); err != nil {
return err
}
tmp, err := os.CreateTemp("", "narratio-locks-upload-*.yml")
if err != nil {
return fmt.Errorf("create lock store temp file: %w", err)
}
tmpPath := tmp.Name()
defer func() { _ = os.Remove(tmpPath) }()
if _, err := tmp.Write(data); err != nil {
_ = tmp.Close()
return fmt.Errorf("write lock store temp file: %w", err)
}
if err := tmp.Close(); err != nil {
return fmt.Errorf("close lock store temp file: %w", err)
}
if _, err := store.Upload(ctx, tmpPath, key, storage.UploadOptions{ContentType: "application/x-yaml; charset=utf-8"}); err != nil {
return fmt.Errorf("upload remote locks %q: %w", key, err)
}
return nil
return fmt.Errorf("update remote locks: concurrent updates prevented a conditional write after %d attempts", remoteLockMutationAttempts)
}
func lockSourceSet(locks []config.PublishLockRule) map[string]config.PublishLockRule {