eric/narratio
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 15 Wiki Activity

Make remote publish locks generation-safe

Browse Source
This commit is contained in:
Eric Rakestraw
2026-08-10 20:17:54 +00:00
parent 361dbb4ca8
commit 0cf2cbfeb3
22 changed files with 560 additions and 101 deletions
Download Patch File Download Diff File Expand all files Collapse all files

118
internal/adapters/storage/fake.go
View File

@@ -1,6 +1,7 @@
package storage
import (
"bytes"
"context"
"crypto/sha256"
"encoding/hex"
@@ -10,11 +11,14 @@ import (
"path/filepath"
"sort"
"strings"
"sync"
"time"
)
// FakeBackend provides a deterministic in-memory object store for tests.
type FakeBackend struct {
mu sync.RWMutex
Objects map[string]FakeObject
Uploads []FakeUploadCall
Downloads []FakeDownloadCall
@@ -50,6 +54,12 @@ type FakeObject struct {
// SeedObject inserts or replaces an object in the fake object store.
func (f *FakeBackend) SeedObject(obj FakeObject) {
f.mu.Lock()
defer f.mu.Unlock()
f.seedObject(obj)
}
func (f *FakeBackend) seedObject(obj FakeObject) {
if f.Objects == nil {
f.Objects = map[string]FakeObject{}
}
@@ -72,6 +82,8 @@ func (f *FakeBackend) List(ctx context.Context, prefix string) ([]ObjectInfo, er
return nil, f.ListErr
}
f.mu.RLock()
defer f.mu.RUnlock()
normalizedPrefix := normalizeObjectKey(prefix)
keys := make([]string, 0, len(f.Objects))
for key := range f.Objects {
@@ -94,6 +106,28 @@ func (f *FakeBackend) List(ctx context.Context, prefix string) ([]ObjectInfo, er
return out, nil
}
// Read returns a stable object body and the generation observed with it.
func (f *FakeBackend) Read(ctx context.Context, key string) (ObjectInfo, io.ReadCloser, error) {
if err := ctx.Err(); err != nil {
return ObjectInfo{}, nil, err
}
if f.DownloadErr != nil {
return ObjectInfo{}, nil, f.DownloadErr
}
normalizedKey := normalizeObjectKey(key)
f.mu.RLock()
obj, ok := f.Objects[normalizedKey]
if ok {
obj.Data = append([]byte(nil), obj.Data...)
obj.Metadata = copyMetadata(obj.Metadata)
}
f.mu.RUnlock()
if !ok {
return ObjectInfo{}, nil, fmt.Errorf("read object %q: %w", normalizedKey, os.ErrNotExist)
}
return ObjectInfo{Key: obj.Key, Size: int64(len(obj.Data)), ETag: obj.ETag, LastModified: obj.LastModified}, io.NopCloser(bytes.NewReader(obj.Data)), nil
}
// DownloadTo writes one object to a caller-owned destination writer.
func (f *FakeBackend) DownloadTo(ctx context.Context, key string, destination io.Writer) error {
if err := ctx.Err(); err != nil {
@@ -106,12 +140,15 @@ func (f *FakeBackend) DownloadTo(ctx context.Context, key string, destination io
return fmt.Errorf("download object: destination writer is required")
}
obj, ok := f.Objects[normalizeObjectKey(key)]
if !ok {
return fmt.Errorf("download object %q: %w", key, os.ErrNotExist)
_, source, err := f.Read(ctx, key)
if err != nil {
return err
}
defer source.Close()
f.mu.Lock()
f.Downloads = append(f.Downloads, FakeDownloadCall{Key: normalizeObjectKey(key)})
if _, err := destination.Write(obj.Data); err != nil {
f.mu.Unlock()
if _, err := io.Copy(destination, source); err != nil {
return fmt.Errorf("download object %q: write destination: %w", key, err)
}
return nil
@@ -189,26 +226,71 @@ func (f *FakeBackend) uploadReader(ctx context.Context, source io.Reader, key st
ContentType: opts.ContentType,
},
}
f.mu.Lock()
f.Uploads = append(f.Uploads, call)
f.mu.Unlock()
if f.UploadHook != nil {
if err := f.UploadHook(call); err != nil {
return ObjectInfo{}, err
}
}
now := time.Now().UTC()
obj := FakeObject{
Key: normalizedKey,
Data: data,
Metadata: copyMetadata(opts.Metadata),
LastModified: &now,
return f.storeUploadedObject(normalizedKey, data, opts), nil
}
// UploadConditional atomically checks and replaces one mutable object.
func (f *FakeBackend) UploadConditional(ctx context.Context, source io.Reader, key string, opts UploadOptions, condition WriteCondition) (ObjectInfo, error) {
if err := ctx.Err(); err != nil {
return ObjectInfo{}, err
}
f.SeedObject(obj)
return ObjectInfo{
Key: normalizedKey,
Size: int64(len(data)),
ETag: fakeObjectETag(data),
LastModified: &now,
}, nil
if err := validateWriteCondition(condition); err != nil {
return ObjectInfo{}, err
}
if f.UploadErr != nil {
return ObjectInfo{}, f.UploadErr
}
if source == nil {
return ObjectInfo{}, fmt.Errorf("upload object: source is required")
}
normalizedKey := normalizeObjectKey(key)
if normalizedKey == "" {
return ObjectInfo{}, fmt.Errorf("upload object: key is required")
}
data, err := io.ReadAll(source)
if err != nil {
return ObjectInfo{}, fmt.Errorf("upload object %q: %w", normalizedKey, err)
}
call := FakeUploadCall{Key: normalizedKey, Options: UploadOptions{Metadata: copyMetadata(opts.Metadata), ContentType: opts.ContentType}}
f.mu.Lock()
f.Uploads = append(f.Uploads, call)
f.mu.Unlock()
if f.UploadHook != nil {
if err := f.UploadHook(call); err != nil {
return ObjectInfo{}, err
}
}
f.mu.Lock()
defer f.mu.Unlock()
existing, found := f.Objects[normalizedKey]
if condition.RequireAbsent && found {
return ObjectInfo{}, ErrConditionNotMet
}
if expected := strings.TrimSpace(condition.MatchETag); expected != "" && (!found || existing.ETag != expected) {
return ObjectInfo{}, ErrConditionNotMet
}
return f.storeUploadedObjectLocked(normalizedKey, data, opts), nil
}
func (f *FakeBackend) storeUploadedObject(key string, data []byte, opts UploadOptions) ObjectInfo {
f.mu.Lock()
defer f.mu.Unlock()
return f.storeUploadedObjectLocked(key, data, opts)
}
func (f *FakeBackend) storeUploadedObjectLocked(key string, data []byte, opts UploadOptions) ObjectInfo {
now := time.Now().UTC()
obj := FakeObject{Key: key, Data: append([]byte(nil), data...), Metadata: copyMetadata(opts.Metadata), LastModified: &now}
f.seedObject(obj)
return ObjectInfo{Key: key, Size: int64(len(data)), ETag: fakeObjectETag(data), LastModified: &now}
}
func fakeObjectETag(data []byte) string {
@@ -224,7 +306,9 @@ func (f *FakeBackend) Exists(ctx context.Context, key string) (bool, error) {
if f.ExistsErr != nil {
return false, f.ExistsErr
}
f.mu.RLock()
_, ok := f.Objects[normalizeObjectKey(key)]
f.mu.RUnlock()
return ok, nil
}

15
internal/adapters/storage/fake_test.go
View File

@@ -71,6 +71,21 @@ func TestFakeBackendUploadAndExists(t *testing.T) {
}
}
func TestFakeBackendConditionalUploadRejectsStaleGeneration(t *testing.T) {
fake := &FakeBackend{}
fake.SeedObject(FakeObject{Key: "locks.yml", Data: []byte("old")})
old := fake.Objects["locks.yml"].ETag
if _, err := fake.UploadConditional(context.Background(), strings.NewReader("new"), "locks.yml", UploadOptions{}, WriteCondition{MatchETag: old}); err != nil {
t.Fatalf("UploadConditional() error = %v", err)
}
if _, err := fake.UploadConditional(context.Background(), strings.NewReader("lost"), "locks.yml", UploadOptions{}, WriteCondition{MatchETag: old}); !errors.Is(err, ErrConditionNotMet) {
t.Fatalf("UploadConditional() error = %v, want ErrConditionNotMet", err)
}
if got := string(fake.Objects["locks.yml"].Data); got != "new" {
t.Fatalf("locks object = %q, want successful replacement preserved", got)
}
}
func TestFakeBackendObjectErrors(t *testing.T) {
fake := &FakeBackend{DownloadErr: errors.New("download fail"), UploadErr: errors.New("upload fail"), ListErr: errors.New("list fail"), ExistsErr: errors.New("exists fail")}

14
internal/adapters/storage/object_store.go
View File

@@ -2,10 +2,15 @@ package storage
import (
"context"
"errors"
"io"
"time"
)
// ErrConditionNotMet reports that an object changed or already existed before a
// conditional write could be committed.
var ErrConditionNotMet = errors.New("object write condition not met")
// ReaderUploader streams caller-owned, already-opened content to object storage.
// Callers retain source-selection and filesystem-confinement policy.
type ReaderUploader interface {
@@ -19,8 +24,10 @@ type ReaderUploader interface {
// infer Narratio session semantics and do not prepend root prefixes.
type ObjectStore interface {
List(ctx context.Context, prefix string) ([]ObjectInfo, error)
Read(ctx context.Context, key string) (ObjectInfo, io.ReadCloser, error)
Download(ctx context.Context, key, localPath string) error
Upload(ctx context.Context, localPath, key string, opts UploadOptions) (ObjectInfo, error)
UploadConditional(ctx context.Context, source io.Reader, key string, opts UploadOptions, condition WriteCondition) (ObjectInfo, error)
Exists(ctx context.Context, key string) (bool, error)
}
@@ -37,3 +44,10 @@ type UploadOptions struct {
Metadata map[string]string
ContentType string
}
// WriteCondition protects a mutable object update against a stale snapshot.
// Exactly one condition is required by UploadConditional.
type WriteCondition struct {
MatchETag string
RequireAbsent bool
}

104
internal/adapters/storage/s3_backend.go
View File

@@ -117,6 +117,7 @@ func (b *S3Backend) List(ctx context.Context, prefix string) ([]ObjectInfo, erro
normalizedPrefix := normalizeObjectKey(prefix)
out := make([]ObjectInfo, 0)
var token *string
seenTokens := map[string]struct{}{}
for {
resp, err := b.client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{
@@ -142,32 +143,56 @@ func (b *S3Backend) List(ctx context.Context, prefix string) ([]ObjectInfo, erro
})
}
if !valueOrFalseBool(resp.IsTruncated) || resp.NextContinuationToken == nil {
if !valueOrFalseBool(resp.IsTruncated) {
break
}
token = resp.NextContinuationToken
next := strings.TrimSpace(valueOrEmpty(resp.NextContinuationToken))
if next == "" {
return nil, fmt.Errorf("s3 list objects bucket %q prefix %q: truncated response has an empty continuation token", b.bucket, normalizedPrefix)
}
if _, repeated := seenTokens[next]; repeated {
return nil, fmt.Errorf("s3 list objects bucket %q prefix %q: truncated response repeated continuation token", b.bucket, normalizedPrefix)
}
seenTokens[next] = struct{}{}
token = &next
}
return out, nil
}
// Read retrieves an object together with the generation observed for its body.
func (b *S3Backend) Read(ctx context.Context, key string) (ObjectInfo, io.ReadCloser, error) {
normalizedKey := normalizeObjectKey(key)
resp, err := b.client.GetObject(ctx, &s3.GetObjectInput{Bucket: &b.bucket, Key: &normalizedKey})
if err != nil {
if isS3NotFound(err) {
return ObjectInfo{}, nil, fmt.Errorf("read object %q: %w", normalizedKey, os.ErrNotExist)
}
return ObjectInfo{}, nil, fmt.Errorf("read object %q: %w", normalizedKey, err)
}
var lastModified *time.Time
if resp.LastModified != nil {
t := *resp.LastModified
lastModified = &t
}
return ObjectInfo{
Key: normalizedKey, Size: valueOrZeroInt64(resp.ContentLength),
ETag: strings.Trim(valueOrEmpty(resp.ETag), "\""), LastModified: lastModified,
}, resp.Body, nil
}
// DownloadTo retrieves one object into the caller-owned destination writer.
func (b *S3Backend) DownloadTo(ctx context.Context, key string, destination io.Writer) error {
normalizedKey := normalizeObjectKey(key)
if destination == nil {
return fmt.Errorf("download object: destination writer is required")
}
resp, err := b.client.GetObject(ctx, &s3.GetObjectInput{
Bucket: &b.bucket,
Key: &normalizedKey,
})
_, body, err := b.Read(ctx, key)
if err != nil {
return fmt.Errorf("download object %q: %w", normalizedKey, err)
return fmt.Errorf("download object %q: %w", normalizeObjectKey(key), err)
}
defer resp.Body.Close()
defer body.Close()
if _, err := io.Copy(destination, resp.Body); err != nil {
return fmt.Errorf("download object %q: copy body: %w", normalizedKey, err)
if _, err := io.Copy(destination, body); err != nil {
return fmt.Errorf("download object %q: copy body: %w", normalizeObjectKey(key), err)
}
return nil
}
@@ -214,15 +239,24 @@ func (b *S3Backend) Upload(ctx context.Context, localPath, key string, opts Uplo
if err != nil {
return ObjectInfo{}, fmt.Errorf("upload object %q from %q: stat local file: %w", normalizedKey, localPath, err)
}
return b.uploadReader(ctx, file, key, opts, stat.Size())
return b.uploadReader(ctx, file, key, opts, stat.Size(), WriteCondition{})
}
// UploadReader sends caller-owned content to key.
func (b *S3Backend) UploadReader(ctx context.Context, source io.Reader, key string, opts UploadOptions) (ObjectInfo, error) {
return b.uploadReader(ctx, source, key, opts, 0)
return b.uploadReader(ctx, source, key, opts, 0, WriteCondition{})
}
func (b *S3Backend) uploadReader(ctx context.Context, source io.Reader, key string, opts UploadOptions, size int64) (ObjectInfo, error) {
// UploadConditional uploads a mutable object only when its observed generation
// still matches, or when no object exists yet.
func (b *S3Backend) UploadConditional(ctx context.Context, source io.Reader, key string, opts UploadOptions, condition WriteCondition) (ObjectInfo, error) {
if err := validateWriteCondition(condition); err != nil {
return ObjectInfo{}, err
}
return b.uploadReader(ctx, source, key, opts, 0, condition)
}
func (b *S3Backend) uploadReader(ctx context.Context, source io.Reader, key string, opts UploadOptions, size int64, condition WriteCondition) (ObjectInfo, error) {
normalizedKey := normalizeObjectKey(key)
if source == nil {
return ObjectInfo{}, fmt.Errorf("upload object: source is required")
@@ -241,9 +275,18 @@ func (b *S3Backend) uploadReader(ctx context.Context, source io.Reader, key stri
ct := strings.TrimSpace(opts.ContentType)
input.ContentType = &ct
}
if condition.RequireAbsent {
wildcard := "*"
input.IfNoneMatch = &wildcard
} else if expected := strings.TrimSpace(condition.MatchETag); expected != "" {
input.IfMatch = &expected
}
resp, err := b.client.PutObject(ctx, input)
if err != nil {
if isS3ConditionalConflict(err) {
return ObjectInfo{}, fmt.Errorf("upload object %q: %w", normalizedKey, ErrConditionNotMet)
}
return ObjectInfo{}, fmt.Errorf("upload object %q: %w", normalizedKey, err)
}
@@ -268,18 +311,43 @@ func (b *S3Backend) Exists(ctx context.Context, key string) (bool, error) {
return true, nil
}
if isS3NotFound(err) {
return false, nil
}
return false, fmt.Errorf("head object %q: %w", normalizedKey, err)
}
func validateWriteCondition(condition WriteCondition) error {
if condition.RequireAbsent == (strings.TrimSpace(condition.MatchETag) != "") {
return fmt.Errorf("conditional upload requires exactly one of MatchETag or RequireAbsent")
}
return nil
}
func isS3NotFound(err error) bool {
var notFound *types.NotFound
if errors.As(err, &notFound) {
return false, nil
return true
}
var apiErr smithy.APIError
if errors.As(err, &apiErr) {
switch apiErr.ErrorCode() {
case "NotFound", "NoSuchKey", "404":
return false, nil
return true
}
}
return false, fmt.Errorf("head object %q: %w", normalizedKey, err)
return false
}
func isS3ConditionalConflict(err error) bool {
var apiErr smithy.APIError
if errors.As(err, &apiErr) {
switch apiErr.ErrorCode() {
case "PreconditionFailed", "ConditionalRequestConflict", "412", "409":
return true
}
}
return false
}
func valueOrEmpty(v *string) string {

74
internal/adapters/storage/s3_backend_test.go
View File

@@ -2,6 +2,7 @@ package storage
import (
"context"
"errors"
"io"
"os"
"path/filepath"
@@ -17,8 +18,10 @@ import (
)
type fakeS3API struct {
listOut *s3.ListObjectsV2Output
listErr error
listOut *s3.ListObjectsV2Output
listOutputs []*s3.ListObjectsV2Output
listErr error
listCalls int
getBody io.ReadCloser
getErr error
@@ -28,23 +31,65 @@ type fakeS3API struct {
headErr error
lastList *s3.ListObjectsV2Input
lastGet *s3.GetObjectInput
lastPut *s3.PutObjectInput
lastHead *s3.HeadObjectInput
lastList *s3.ListObjectsV2Input
lastLists []*s3.ListObjectsV2Input
lastGet *s3.GetObjectInput
lastPut *s3.PutObjectInput
lastHead *s3.HeadObjectInput
}
func (f *fakeS3API) ListObjectsV2(_ context.Context, params *s3.ListObjectsV2Input, _ ...func(*s3.Options)) (*s3.ListObjectsV2Output, error) {
f.lastList = params
f.lastLists = append(f.lastLists, params)
if f.listErr != nil {
return nil, f.listErr
}
if f.listCalls < len(f.listOutputs) {
out := f.listOutputs[f.listCalls]
f.listCalls++
return out, nil
}
if f.listOut == nil {
return &s3.ListObjectsV2Output{}, nil
}
return f.listOut, nil
}
func TestS3BackendListPaginatesAndRejectsNonProgressingTokens(t *testing.T) {
t.Run("multiple pages", func(t *testing.T) {
client := &fakeS3API{listOutputs: []*s3.ListObjectsV2Output{
{Contents: []types.Object{{Key: strPtr("prefix/a"), Size: int64Ptr(1)}}, IsTruncated: boolPtr(true), NextContinuationToken: strPtr("next")},
{Contents: []types.Object{{Key: strPtr("prefix/b"), Size: int64Ptr(2)}}, IsTruncated: boolPtr(false)},
}}
items, err := (&S3Backend{bucket: "bucket-1", client: client}).List(context.Background(), "prefix/")
if err != nil {
t.Fatalf("List() error = %v", err)
}
if len(items) != 2 || items[0].Key != "prefix/a" || items[1].Key != "prefix/b" {
t.Fatalf("List() items = %#v", items)
}
if len(client.lastLists) != 2 || client.lastLists[1].ContinuationToken == nil || *client.lastLists[1].ContinuationToken != "next" {
t.Fatalf("continuation calls = %#v", client.lastLists)
}
})
for _, test := range []struct {
name string
outputs []*s3.ListObjectsV2Output
want string
}{
{name: "empty", outputs: []*s3.ListObjectsV2Output{{IsTruncated: boolPtr(true)}}, want: "empty continuation token"},
{name: "repeated", outputs: []*s3.ListObjectsV2Output{{IsTruncated: boolPtr(true), NextContinuationToken: strPtr("again")}, {IsTruncated: boolPtr(true), NextContinuationToken: strPtr("again")}}, want: "repeated continuation token"},
} {
t.Run(test.name, func(t *testing.T) {
_, err := (&S3Backend{bucket: "bucket-1", client: &fakeS3API{listOutputs: test.outputs}}).List(context.Background(), "prefix/")
if err == nil || !strings.Contains(err.Error(), test.want) || !strings.Contains(err.Error(), "bucket-1") || !strings.Contains(err.Error(), "prefix/") {
t.Fatalf("List() error = %v, want contextual %q", err, test.want)
}
})
}
}
func (f *fakeS3API) GetObject(_ context.Context, params *s3.GetObjectInput, _ ...func(*s3.Options)) (*s3.GetObjectOutput, error) {
f.lastGet = params
if f.getErr != nil {
@@ -160,6 +205,22 @@ func TestS3BackendUploadAndExists(t *testing.T) {
}
}
func TestS3BackendConditionalUploadUsesProviderPrecondition(t *testing.T) {
client := &fakeS3API{putOut: &s3.PutObjectOutput{ETag: strPtr(`"etag123"`)}}
backend := &S3Backend{bucket: "bucket-1", client: client}
if _, err := backend.UploadConditional(context.Background(), strings.NewReader("payload"), "locks.yml", UploadOptions{}, WriteCondition{MatchETag: "before"}); err != nil {
t.Fatalf("UploadConditional() error = %v", err)
}
if client.lastPut == nil || client.lastPut.IfMatch == nil || *client.lastPut.IfMatch != "before" || client.lastPut.IfNoneMatch != nil {
t.Fatalf("PutObject conditional input = %#v", client.lastPut)
}
client.putErr = &smithy.GenericAPIError{Code: "PreconditionFailed", Message: "changed"}
_, err := backend.UploadConditional(context.Background(), strings.NewReader("payload"), "locks.yml", UploadOptions{}, WriteCondition{RequireAbsent: true})
if !errors.Is(err, ErrConditionNotMet) {
t.Fatalf("UploadConditional() error = %v, want ErrConditionNotMet", err)
}
}
func TestS3BackendUploadMissingLocalFile(t *testing.T) {
backend := &S3Backend{bucket: "bucket-1", client: &fakeS3API{}}
_, err := backend.Upload(context.Background(), filepath.Join(t.TempDir(), "missing.txt"), "key.txt", UploadOptions{})
@@ -249,5 +310,6 @@ func TestNewS3BackendFromConfigFallsBackWhenCredentialEnvMissing(t *testing.T) {
func strPtr(v string) *string { return &v }
func int64Ptr(v int64) *int64 { return &v }
func boolPtr(v bool) *bool { return &v }
var _ s3API = (*fakeS3API)(nil)