eric/distributor
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 7 Wiki Activity

Add explicit force replacement workflow

Browse Source
This commit is contained in:
Eric Rakestraw
2026-05-31 17:29:20 +00:00
parent 7a174ce5f1
commit 48169dc8b4
28 changed files with 811 additions and 53 deletions
Download Patch File Download Diff File Expand all files Collapse all files

45
internal/adapters/local/backend.go
View File

@@ -267,6 +267,51 @@ func (b *Backend) DeleteManagedBundle(ctx context.Context, bundlePath string, ma
return nil
}
func (b *Backend) DeletePrefix(ctx context.Context, prefix string, opts storage.DeleteOptions) error {
if err := ctx.Err(); err != nil {
return err
}
nativePrefix, err := b.nativePath(prefix, true)
if err != nil {
return err
}
if err := b.rejectSymlinkAncestors(nativePrefix, false); err != nil {
return err
}
if prefix == "" {
entries, err := os.ReadDir(nativePrefix)
if err != nil {
if opts.IgnoreMissing && errors.Is(err, fs.ErrNotExist) {
return nil
}
return b.translateError(storage.OpDeletePrefix, prefix, err)
}
for _, entry := range entries {
if err := ctx.Err(); err != nil {
return err
}
child := filepath.Join(nativePrefix, entry.Name())
if err := os.RemoveAll(child); err != nil {
return b.translateError(storage.OpDeletePrefix, entry.Name(), err)
}
}
return nil
}
if _, err := os.Lstat(nativePrefix); err != nil {
if opts.IgnoreMissing && errors.Is(err, fs.ErrNotExist) {
return nil
}
return b.translateError(storage.OpDeletePrefix, prefix, err)
}
if err := os.RemoveAll(nativePrefix); err != nil {
return b.translateError(storage.OpDeletePrefix, prefix, err)
}
if opts.PruneEmptyDirs {
b.pruneEmptyParents(filepath.Dir(nativePrefix))
}
return nil
}
func (b *Backend) nativePath(logicalPath string, allowEmpty bool) (string, error) {
if logicalPath == "" {
if !allowEmpty {

24
internal/adapters/local/backend_test.go
View File

@@ -169,6 +169,30 @@ func TestBackendManagedDeletion(t *testing.T) {
}
}
func TestBackendDeletePrefixStaysWithinPrefix(t *testing.T) {
backend := newBackend(t)
mustWrite(t, backend, "bundle/report.md", "report")
mustWrite(t, backend, "bundle/nested/old.txt", "old")
mustWrite(t, backend, "bundle-sibling/keep.txt", "keep")
mustWrite(t, backend, "outside.txt", "outside")
if err := backend.DeletePrefix(context.Background(), "bundle", storage.DeleteOptions{IgnoreMissing: true, PruneEmptyDirs: true}); err != nil {
t.Fatalf("DeletePrefix() error = %v", err)
}
if _, err := backend.Stat(context.Background(), "bundle/report.md"); !storage.IsNotFound(err) {
t.Fatalf("deleted file stat error = %v, want not found", err)
}
if _, err := backend.Stat(context.Background(), "bundle/nested/old.txt"); !storage.IsNotFound(err) {
t.Fatalf("deleted nested file stat error = %v, want not found", err)
}
if _, err := backend.Stat(context.Background(), "bundle-sibling/keep.txt"); err != nil {
t.Fatalf("sibling stat error = %v", err)
}
if _, err := backend.Stat(context.Background(), "outside.txt"); err != nil {
t.Fatalf("outside stat error = %v", err)
}
}
func TestBackendHasAny(t *testing.T) {
backend := newBackend(t)
found, err := backend.HasAny(context.Background(), "missing")

86
internal/adapters/s3/backend.go
View File

@@ -242,33 +242,87 @@ func (b *Backend) DeleteManagedBundle(ctx context.Context, bundlePath string, ma
return err
}
for _, target := range targets {
key, err := b.objectKey(target, false)
if err := b.deleteObject(ctx, storage.OpDeleteManagedBundle, target, opts); err != nil {
return err
}
}
return nil
}
func (b *Backend) DeletePrefix(ctx context.Context, logicalPrefix string, opts storage.DeleteOptions) error {
if err := ctx.Err(); err != nil {
return err
}
if err := storage.ValidatePrefix(logicalPrefix); err != nil {
return err
}
found := false
if logicalPrefix != "" {
key, err := b.objectKey(logicalPrefix, false)
if err != nil {
return err
}
if key == b.prefix {
return storage.NewError(storage.OpDeleteManagedBundle, BackendName, target, storage.ErrInvalidPath, nil)
}
if !opts.IgnoreMissing {
_, err := b.client.HeadObject(ctx, &awss3.HeadObjectInput{
Bucket: aws.String(b.bucket),
Key: aws.String(key),
})
if err != nil {
return b.translateError(storage.OpDeleteManagedBundle, target, err)
_, err = b.client.HeadObject(ctx, &awss3.HeadObjectInput{
Bucket: aws.String(b.bucket),
Key: aws.String(key),
})
if err == nil {
found = true
if err := b.deleteObject(ctx, storage.OpDeletePrefix, logicalPrefix, storage.DeleteOptions{IgnoreMissing: false}); err != nil {
return err
}
} else if !isNotFound(err) {
return b.translateError(storage.OpDeletePrefix, logicalPrefix, err)
}
_, err = b.client.DeleteObject(ctx, &awss3.DeleteObjectInput{
}
var entries []storage.Entry
if err := b.walkObjects(ctx, logicalPrefix, storage.WalkOptions{Recursive: true}, func(entry storage.Entry) error {
if entry.Type == storage.EntryTypeFile {
entries = append(entries, entry)
}
return nil
}); err != nil {
return err
}
for _, entry := range entries {
found = true
if err := b.deleteObject(ctx, storage.OpDeletePrefix, entry.Path, storage.DeleteOptions{IgnoreMissing: true}); err != nil {
return err
}
}
if !found && !opts.IgnoreMissing {
return storage.NewError(storage.OpDeletePrefix, BackendName, logicalPrefix, storage.ErrNotFound, nil)
}
return nil
}
func (b *Backend) deleteObject(ctx context.Context, op, logicalPath string, opts storage.DeleteOptions) error {
key, err := b.objectKey(logicalPath, false)
if err != nil {
return err
}
if key == b.prefix {
return storage.NewError(op, BackendName, logicalPath, storage.ErrInvalidPath, nil)
}
if !opts.IgnoreMissing {
_, err := b.client.HeadObject(ctx, &awss3.HeadObjectInput{
Bucket: aws.String(b.bucket),
Key: aws.String(key),
})
if err != nil {
if opts.IgnoreMissing && isNotFound(err) {
continue
}
return b.translateError(storage.OpDeleteManagedBundle, target, err)
return b.translateError(op, logicalPath, err)
}
}
_, err = b.client.DeleteObject(ctx, &awss3.DeleteObjectInput{
Bucket: aws.String(b.bucket),
Key: aws.String(key),
})
if err != nil {
if opts.IgnoreMissing && isNotFound(err) {
return nil
}
return b.translateError(op, logicalPath, err)
}
return nil
}

28
internal/adapters/s3/backend_test.go
View File

@@ -191,6 +191,34 @@ func TestDeleteManagedBundleDeletesOnlyManagedTargets(t *testing.T) {
}
}
func TestDeletePrefixStaysWithinPrefix(t *testing.T) {
client := newFakeClient(map[string]string{
"root/bundle/report.md": "report",
"root/bundle/nested/old.txt": "old",
"root/bundle-sibling/keep.txt": "keep",
"root/outside.txt": "outside",
"other-root/bundle/report.md": "other",
"root/.distributor-prefix-marker": "marker",
})
backend := newTestBackend(t, "root", client)
if err := backend.DeletePrefix(context.Background(), "bundle", storage.DeleteOptions{IgnoreMissing: true}); err != nil {
t.Fatalf("DeletePrefix() error = %v", err)
}
for _, deleted := range []string{"root/bundle/report.md", "root/bundle/nested/old.txt"} {
if _, ok := client.objects[deleted]; ok {
t.Fatalf("%s still exists", deleted)
}
}
for _, kept := range []string{"root/bundle-sibling/keep.txt", "root/outside.txt", "other-root/bundle/report.md", "root/.distributor-prefix-marker"} {
if _, ok := client.objects[kept]; !ok {
t.Fatalf("%s was deleted", kept)
}
}
if got, want := sortedStrings(client.deleteKeys), []string{"root/bundle/nested/old.txt", "root/bundle/report.md"}; !equalStrings(got, want) {
t.Fatalf("deleted keys = %v, want %v", got, want)
}
}
func newTestBackend(t *testing.T, prefix string, client *fakeClient) *Backend {
t.Helper()
if client == nil {

76
internal/adapters/ssh/backend.go
View File

@@ -292,6 +292,82 @@ func (b *Backend) DeleteManagedBundle(ctx context.Context, bundlePath string, ma
return nil
}
func (b *Backend) DeletePrefix(ctx context.Context, prefix string, opts storage.DeleteOptions) error {
if err := ctx.Err(); err != nil {
return err
}
if err := storage.ValidatePrefix(prefix); err != nil {
return err
}
var entries []storage.Entry
if prefix != "" {
entry, err := b.Stat(ctx, prefix)
if err != nil {
if opts.IgnoreMissing && storage.IsNotFound(err) {
return nil
}
return err
}
if entry.Type != storage.EntryTypeDirectory {
return b.deleteEntry(ctx, entry, opts)
}
entries = append(entries, entry)
}
if err := b.Walk(ctx, prefix, storage.WalkOptions{Recursive: true}, func(entry storage.Entry) error {
entries = append(entries, entry)
return nil
}); err != nil {
return err
}
if prefix != "" && len(entries) == 1 {
if err := b.deleteEntry(ctx, entries[0], opts); err != nil {
return err
}
if opts.PruneEmptyDirs {
b.pruneEmptyParents(parentOf(prefix))
}
return nil
}
sort.Slice(entries, func(i, j int) bool {
return strings.Count(entries[i].Path, "/") > strings.Count(entries[j].Path, "/")
})
for _, entry := range entries {
if entry.Path == "" {
continue
}
if err := b.deleteEntry(ctx, entry, storage.DeleteOptions{IgnoreMissing: true}); err != nil {
return err
}
}
if opts.PruneEmptyDirs {
b.pruneEmptyParents(parentOf(prefix))
}
return nil
}
func (b *Backend) deleteEntry(ctx context.Context, entry storage.Entry, opts storage.DeleteOptions) error {
if err := ctx.Err(); err != nil {
return err
}
nativePath, err := b.nativePath(entry.Path, false)
if err != nil {
return err
}
var removeErr error
if entry.Type == storage.EntryTypeDirectory {
removeErr = b.client.RemoveDirectory(nativePath)
} else {
removeErr = b.client.Remove(nativePath)
}
if removeErr != nil {
if opts.IgnoreMissing && isNotExist(removeErr) {
return nil
}
return b.translateError(storage.OpDeletePrefix, entry.Path, removeErr)
}
return nil
}
func (b *Backend) walkDirectory(ctx context.Context, logicalPrefix, nativePrefix string, opts storage.WalkOptions, emit func(storage.Entry) error) error {
entries, err := b.client.ReadDir(nativePrefix)
if err != nil {