eric/narratio
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 15 Wiki Activity

Make remote publish locks generation-safe

Browse Source
This commit is contained in:
Eric Rakestraw
2026-08-10 20:17:54 +00:00
parent 361dbb4ca8
commit 0cf2cbfeb3
22 changed files with 560 additions and 101 deletions
Download Patch File Download Diff File Expand all files Collapse all files

104
internal/adapters/storage/s3_backend.go
View File

@@ -117,6 +117,7 @@ func (b *S3Backend) List(ctx context.Context, prefix string) ([]ObjectInfo, erro
normalizedPrefix := normalizeObjectKey(prefix)
out := make([]ObjectInfo, 0)
var token *string
seenTokens := map[string]struct{}{}
for {
resp, err := b.client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{
@@ -142,32 +143,56 @@ func (b *S3Backend) List(ctx context.Context, prefix string) ([]ObjectInfo, erro
})
}
if !valueOrFalseBool(resp.IsTruncated) || resp.NextContinuationToken == nil {
if !valueOrFalseBool(resp.IsTruncated) {
break
}
token = resp.NextContinuationToken
next := strings.TrimSpace(valueOrEmpty(resp.NextContinuationToken))
if next == "" {
return nil, fmt.Errorf("s3 list objects bucket %q prefix %q: truncated response has an empty continuation token", b.bucket, normalizedPrefix)
}
if _, repeated := seenTokens[next]; repeated {
return nil, fmt.Errorf("s3 list objects bucket %q prefix %q: truncated response repeated continuation token", b.bucket, normalizedPrefix)
}
seenTokens[next] = struct{}{}
token = &next
}
return out, nil
}
// Read retrieves an object together with the generation observed for its body.
func (b *S3Backend) Read(ctx context.Context, key string) (ObjectInfo, io.ReadCloser, error) {
normalizedKey := normalizeObjectKey(key)
resp, err := b.client.GetObject(ctx, &s3.GetObjectInput{Bucket: &b.bucket, Key: &normalizedKey})
if err != nil {
if isS3NotFound(err) {
return ObjectInfo{}, nil, fmt.Errorf("read object %q: %w", normalizedKey, os.ErrNotExist)
}
return ObjectInfo{}, nil, fmt.Errorf("read object %q: %w", normalizedKey, err)
}
var lastModified *time.Time
if resp.LastModified != nil {
t := *resp.LastModified
lastModified = &t
}
return ObjectInfo{
Key: normalizedKey, Size: valueOrZeroInt64(resp.ContentLength),
ETag: strings.Trim(valueOrEmpty(resp.ETag), "\""), LastModified: lastModified,
}, resp.Body, nil
}
// DownloadTo retrieves one object into the caller-owned destination writer.
func (b *S3Backend) DownloadTo(ctx context.Context, key string, destination io.Writer) error {
normalizedKey := normalizeObjectKey(key)
if destination == nil {
return fmt.Errorf("download object: destination writer is required")
}
resp, err := b.client.GetObject(ctx, &s3.GetObjectInput{
Bucket: &b.bucket,
Key: &normalizedKey,
})
_, body, err := b.Read(ctx, key)
if err != nil {
return fmt.Errorf("download object %q: %w", normalizedKey, err)
return fmt.Errorf("download object %q: %w", normalizeObjectKey(key), err)
}
defer resp.Body.Close()
defer body.Close()
if _, err := io.Copy(destination, resp.Body); err != nil {
return fmt.Errorf("download object %q: copy body: %w", normalizedKey, err)
if _, err := io.Copy(destination, body); err != nil {
return fmt.Errorf("download object %q: copy body: %w", normalizeObjectKey(key), err)
}
return nil
}
@@ -214,15 +239,24 @@ func (b *S3Backend) Upload(ctx context.Context, localPath, key string, opts Uplo
if err != nil {
return ObjectInfo{}, fmt.Errorf("upload object %q from %q: stat local file: %w", normalizedKey, localPath, err)
}
return b.uploadReader(ctx, file, key, opts, stat.Size())
return b.uploadReader(ctx, file, key, opts, stat.Size(), WriteCondition{})
}
// UploadReader sends caller-owned content to key.
func (b *S3Backend) UploadReader(ctx context.Context, source io.Reader, key string, opts UploadOptions) (ObjectInfo, error) {
return b.uploadReader(ctx, source, key, opts, 0)
return b.uploadReader(ctx, source, key, opts, 0, WriteCondition{})
}
func (b *S3Backend) uploadReader(ctx context.Context, source io.Reader, key string, opts UploadOptions, size int64) (ObjectInfo, error) {
// UploadConditional uploads a mutable object only when its observed generation
// still matches, or when no object exists yet.
func (b *S3Backend) UploadConditional(ctx context.Context, source io.Reader, key string, opts UploadOptions, condition WriteCondition) (ObjectInfo, error) {
if err := validateWriteCondition(condition); err != nil {
return ObjectInfo{}, err
}
return b.uploadReader(ctx, source, key, opts, 0, condition)
}
func (b *S3Backend) uploadReader(ctx context.Context, source io.Reader, key string, opts UploadOptions, size int64, condition WriteCondition) (ObjectInfo, error) {
normalizedKey := normalizeObjectKey(key)
if source == nil {
return ObjectInfo{}, fmt.Errorf("upload object: source is required")
@@ -241,9 +275,18 @@ func (b *S3Backend) uploadReader(ctx context.Context, source io.Reader, key stri
ct := strings.TrimSpace(opts.ContentType)
input.ContentType = &ct
}
if condition.RequireAbsent {
wildcard := "*"
input.IfNoneMatch = &wildcard
} else if expected := strings.TrimSpace(condition.MatchETag); expected != "" {
input.IfMatch = &expected
}
resp, err := b.client.PutObject(ctx, input)
if err != nil {
if isS3ConditionalConflict(err) {
return ObjectInfo{}, fmt.Errorf("upload object %q: %w", normalizedKey, ErrConditionNotMet)
}
return ObjectInfo{}, fmt.Errorf("upload object %q: %w", normalizedKey, err)
}
@@ -268,18 +311,43 @@ func (b *S3Backend) Exists(ctx context.Context, key string) (bool, error) {
return true, nil
}
if isS3NotFound(err) {
return false, nil
}
return false, fmt.Errorf("head object %q: %w", normalizedKey, err)
}
func validateWriteCondition(condition WriteCondition) error {
if condition.RequireAbsent == (strings.TrimSpace(condition.MatchETag) != "") {
return fmt.Errorf("conditional upload requires exactly one of MatchETag or RequireAbsent")
}
return nil
}
func isS3NotFound(err error) bool {
var notFound *types.NotFound
if errors.As(err, &notFound) {
return false, nil
return true
}
var apiErr smithy.APIError
if errors.As(err, &apiErr) {
switch apiErr.ErrorCode() {
case "NotFound", "NoSuchKey", "404":
return false, nil
return true
}
}
return false, fmt.Errorf("head object %q: %w", normalizedKey, err)
return false
}
func isS3ConditionalConflict(err error) bool {
var apiErr smithy.APIError
if errors.As(err, &apiErr) {
switch apiErr.ErrorCode() {
case "PreconditionFailed", "ConditionalRequestConflict", "412", "409":
return true
}
}
return false
}
func valueOrEmpty(v *string) string {