package s3 import ( "context" "io" "sort" "strings" "testing" "gitea.maximumdirect.net/eric/distributor/internal/storage" "github.com/aws/aws-sdk-go-v2/aws" awss3 "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/aws/aws-sdk-go-v2/service/s3/types" ) func TestKeyAndPrefixNormalization(t *testing.T) { backend := newTestBackend(t, "root/prefix", nil) key, err := backend.objectKey("bundle/report.md", false) if err != nil { t.Fatalf("objectKey() error = %v", err) } if got, want := key, "root/prefix/bundle/report.md"; got != want { t.Fatalf("objectKey() = %q, want %q", got, want) } listPrefix, err := backend.listPrefix("bundle") if err != nil { t.Fatalf("listPrefix() error = %v", err) } if got, want := listPrefix, "root/prefix/bundle/"; got != want { t.Fatalf("listPrefix() = %q, want %q", got, want) } } func TestPathTraversalRejected(t *testing.T) { backend := newTestBackend(t, "", nil) for _, logicalPath := range []string{"/absolute", "../escape", "a/../b", `a\b`} { t.Run(logicalPath, func(t *testing.T) { if _, err := backend.objectKey(logicalPath, false); err == nil || !storage.IsInvalidPath(err) { t.Fatalf("objectKey() error = %v, want invalid path", err) } }) } } func TestContentType(t *testing.T) { tests := map[string]string{ "report.md": "text/markdown; charset=utf-8", "report.html": "text/html; charset=utf-8", "state.json": "application/json", "summary.txt": "text/plain; charset=utf-8", "data.bin": "application/octet-stream", } for path, want := range tests { if got := ContentType(path); got != want { t.Fatalf("ContentType(%q) = %q, want %q", path, got, want) } } } func TestStatRequiresExactObject(t *testing.T) { client := newFakeClient(map[string]string{"root/dir/file.txt": "data"}) backend := newTestBackend(t, "root", client) _, err := backend.Stat(context.Background(), "dir") if err == nil || !storage.IsNotFound(err) { t.Fatalf("Stat() error = %v, want not found", err) } } func TestWalkIncludesExactObjectAndDescendants(t *testing.T) { client := newFakeClient(map[string]string{ "root/bundle": "marker", "root/bundle/report.md": "report", }) backend := newTestBackend(t, "root", client) entries, err := storage.List(context.Background(), backend, "bundle", storage.WalkOptions{Recursive: true}) if err != nil { t.Fatalf("List() error = %v", err) } paths := entryPaths(entries) if got, want := paths, []string{"bundle", "bundle/report.md"}; !equalStrings(got, want) { t.Fatalf("paths = %v, want %v", got, want) } } func TestWalkExactObjectHonorsLimitBeforeListingDescendants(t *testing.T) { client := newFakeClient(map[string]string{ "root/bundle": "marker", "root/bundle/report.md": "report", }) backend := newTestBackend(t, "root", client) var entries []storage.Entry err := backend.Walk(context.Background(), "bundle", storage.WalkOptions{Recursive: true, Limit: 1}, func(entry storage.Entry) error { entries = append(entries, entry) return nil }) if err != nil { t.Fatalf("Walk() error = %v", err) } paths := entryPaths(entries) if got, want := paths, []string{"bundle"}; !equalStrings(got, want) { t.Fatalf("paths = %v, want %v", got, want) } if len(client.tokens) != 0 { t.Fatalf("list calls = %d, want none", len(client.tokens)) } } func TestHasAnyWithExactObjectStopsBeforeListingDescendants(t *testing.T) { client := newFakeClient(map[string]string{ "root/bundle": "marker", "root/bundle/report.md": "report", }) backend := newTestBackend(t, "root", client) found, err := backend.HasAny(context.Background(), "bundle") if err != nil { t.Fatalf("HasAny() error = %v", err) } if !found { t.Fatal("HasAny() = false, want true") } if len(client.tokens) != 0 { t.Fatalf("list calls = %d, want none", len(client.tokens)) } } func TestWriteFromChecksOverwriteBeforePut(t *testing.T) { client := newFakeClient(map[string]string{"root/report.md": "old"}) backend := newTestBackend(t, "root", client) _, err := backend.WriteFile(context.Background(), "report.md", []byte("new"), storage.WriteOptions{}) if err == nil || !storage.IsAlreadyExists(err) { t.Fatalf("WriteFile() error = %v, want already exists", err) } if len(client.putKeys) != 0 { t.Fatalf("put keys = %v, want none", client.putKeys) } } func TestWriteFromPutsNewObjectWithContentType(t *testing.T) { client := newFakeClient(nil) backend := newTestBackend(t, "root", client) entry, err := backend.WriteFile(context.Background(), "report.html", []byte("

ok

"), storage.WriteOptions{}) if err != nil { t.Fatalf("WriteFile() error = %v", err) } if entry.Path != "report.html" || entry.Size != 9 || entry.Type != storage.EntryTypeFile { t.Fatalf("entry = %#v", entry) } if got, want := client.objects["root/report.html"], "

ok

"; got != want { t.Fatalf("object = %q, want %q", got, want) } if got, want := client.contentTypes["root/report.html"], "text/html; charset=utf-8"; got != want { t.Fatalf("content type = %q, want %q", got, want) } } func TestWalkUsesPagination(t *testing.T) { client := newFakeClient(nil) client.listPages = []awss3.ListObjectsV2Output{ { Contents: []types.Object{{Key: aws.String("root/a.txt"), Size: aws.Int64(1)}}, IsTruncated: aws.Bool(true), NextContinuationToken: aws.String("next"), }, { Contents: []types.Object{{Key: aws.String("root/b.txt"), Size: aws.Int64(2)}}, IsTruncated: aws.Bool(false), }, } backend := newTestBackend(t, "root", client) entries, err := storage.List(context.Background(), backend, "", storage.WalkOptions{Recursive: true}) if err != nil { t.Fatalf("List() error = %v", err) } paths := entryPaths(entries) if got, want := paths, []string{"a.txt", "b.txt"}; !equalStrings(got, want) { t.Fatalf("paths = %v, want %v", got, want) } if got, want := client.tokens, []string{"", "next"}; !equalStrings(got, want) { t.Fatalf("tokens = %v, want %v", got, want) } } func TestHasAnyStopsAfterFirstPage(t *testing.T) { client := newFakeClient(nil) client.listPages = []awss3.ListObjectsV2Output{ { Contents: []types.Object{{Key: aws.String("root/a.txt"), Size: aws.Int64(1)}}, IsTruncated: aws.Bool(true), NextContinuationToken: aws.String("next"), }, { Contents: []types.Object{{Key: aws.String("root/b.txt"), Size: aws.Int64(2)}}, IsTruncated: aws.Bool(false), }, } backend := newTestBackend(t, "root", client) found, err := backend.HasAny(context.Background(), "") if err != nil { t.Fatalf("HasAny() error = %v", err) } if !found { t.Fatal("HasAny() = false, want true") } if got, want := len(client.tokens), 1; got != want { t.Fatalf("list calls = %d, want %d", got, want) } } func TestWalkNonRecursiveUsesPrefixBoundary(t *testing.T) { client := newFakeClient(map[string]string{ "base/dir/file.txt": "nested", "base/file.txt": "file", "baseball/file.txt": "wrong", }) backend := newTestBackend(t, "base", client) entries, err := storage.List(context.Background(), backend, "", storage.WalkOptions{}) if err != nil { t.Fatalf("List() error = %v", err) } paths := entryPaths(entries) if got, want := paths, []string{"dir", "file.txt"}; !equalStrings(got, want) { t.Fatalf("paths = %v, want %v", got, want) } } func TestDeleteManagedBundleDeletesOnlyManagedTargets(t *testing.T) { client := newFakeClient(map[string]string{ "root/report.md": "report", "root/.distributor.json": "state", "root/keep.txt": "keep", }) backend := newTestBackend(t, "root", client) err := backend.DeleteManagedBundle(context.Background(), "", []string{"report.md"}, storage.DeleteOptions{IgnoreMissing: true}) if err != nil { t.Fatalf("DeleteManagedBundle() error = %v", err) } if _, ok := client.objects["root/report.md"]; ok { t.Fatal("managed output still exists") } if _, ok := client.objects["root/.distributor.json"]; ok { t.Fatal("state file still exists") } if _, ok := client.objects["root/keep.txt"]; !ok { t.Fatal("unmanaged object was deleted") } if got, want := sortedStrings(client.deleteKeys), []string{"root/.distributor.json", "root/report.md"}; !equalStrings(got, want) { t.Fatalf("deleted keys = %v, want %v", got, want) } } func TestDeletePrefixStaysWithinPrefix(t *testing.T) { client := newFakeClient(map[string]string{ "root/bundle/report.md": "report", "root/bundle/nested/old.txt": "old", "root/bundle-sibling/keep.txt": "keep", "root/outside.txt": "outside", "other-root/bundle/report.md": "other", "root/.distributor-prefix-marker": "marker", }) backend := newTestBackend(t, "root", client) if err := backend.DeletePrefix(context.Background(), "bundle", storage.DeleteOptions{IgnoreMissing: true}); err != nil { t.Fatalf("DeletePrefix() error = %v", err) } for _, deleted := range []string{"root/bundle/report.md", "root/bundle/nested/old.txt"} { if _, ok := client.objects[deleted]; ok { t.Fatalf("%s still exists", deleted) } } for _, kept := range []string{"root/bundle-sibling/keep.txt", "root/outside.txt", "other-root/bundle/report.md", "root/.distributor-prefix-marker"} { if _, ok := client.objects[kept]; !ok { t.Fatalf("%s was deleted", kept) } } if got, want := sortedStrings(client.deleteKeys), []string{"root/bundle/nested/old.txt", "root/bundle/report.md"}; !equalStrings(got, want) { t.Fatalf("deleted keys = %v, want %v", got, want) } } func newTestBackend(t *testing.T, prefix string, client *fakeClient) *Backend { t.Helper() if client == nil { client = newFakeClient(nil) } backend, err := NewWithClient(client, Options{ Endpoint: "https://s3.example.com", Bucket: "reports", Prefix: prefix, Region: DefaultRegion, ForcePathStyle: true, }) if err != nil { t.Fatalf("NewWithClient() error = %v", err) } return backend } type fakeClient struct { objects map[string]string contentTypes map[string]string listPages []awss3.ListObjectsV2Output tokens []string putKeys []string deleteKeys []string } func newFakeClient(objects map[string]string) *fakeClient { copied := make(map[string]string) for key, value := range objects { copied[key] = value } return &fakeClient{ objects: copied, contentTypes: make(map[string]string), } } func (c *fakeClient) HeadObject(ctx context.Context, input *awss3.HeadObjectInput, optFns ...func(*awss3.Options)) (*awss3.HeadObjectOutput, error) { if err := ctx.Err(); err != nil { return nil, err } value, ok := c.objects[aws.ToString(input.Key)] if !ok { return nil, &types.NotFound{} } return &awss3.HeadObjectOutput{ContentLength: aws.Int64(int64(len(value)))}, nil } func (c *fakeClient) GetObject(ctx context.Context, input *awss3.GetObjectInput, optFns ...func(*awss3.Options)) (*awss3.GetObjectOutput, error) { if err := ctx.Err(); err != nil { return nil, err } value, ok := c.objects[aws.ToString(input.Key)] if !ok { return nil, &types.NotFound{} } return &awss3.GetObjectOutput{ Body: io.NopCloser(stringsReader(value)), ContentLength: aws.Int64(int64(len(value))), }, nil } func (c *fakeClient) PutObject(ctx context.Context, input *awss3.PutObjectInput, optFns ...func(*awss3.Options)) (*awss3.PutObjectOutput, error) { if err := ctx.Err(); err != nil { return nil, err } data, err := io.ReadAll(input.Body) if err != nil { return nil, err } key := aws.ToString(input.Key) c.objects[key] = string(data) c.contentTypes[key] = aws.ToString(input.ContentType) c.putKeys = append(c.putKeys, key) return &awss3.PutObjectOutput{}, nil } func (c *fakeClient) ListObjectsV2(ctx context.Context, input *awss3.ListObjectsV2Input, optFns ...func(*awss3.Options)) (*awss3.ListObjectsV2Output, error) { if err := ctx.Err(); err != nil { return nil, err } c.tokens = append(c.tokens, aws.ToString(input.ContinuationToken)) if len(c.listPages) > 0 { index := len(c.tokens) - 1 if index >= len(c.listPages) { return &awss3.ListObjectsV2Output{IsTruncated: aws.Bool(false)}, nil } page := c.listPages[index] return &page, nil } return c.dynamicList(input), nil } func (c *fakeClient) DeleteObject(ctx context.Context, input *awss3.DeleteObjectInput, optFns ...func(*awss3.Options)) (*awss3.DeleteObjectOutput, error) { if err := ctx.Err(); err != nil { return nil, err } key := aws.ToString(input.Key) delete(c.objects, key) c.deleteKeys = append(c.deleteKeys, key) return &awss3.DeleteObjectOutput{}, nil } func (c *fakeClient) dynamicList(input *awss3.ListObjectsV2Input) *awss3.ListObjectsV2Output { prefix := aws.ToString(input.Prefix) delimiter := aws.ToString(input.Delimiter) var contents []types.Object commonPrefixes := make(map[string]struct{}) for key, value := range c.objects { if !strings.HasPrefix(key, prefix) { continue } remainder := strings.TrimPrefix(key, prefix) if delimiter != "" { if index := strings.Index(remainder, delimiter); index >= 0 { commonPrefixes[prefix+remainder[:index+1]] = struct{}{} continue } } contents = append(contents, types.Object{Key: aws.String(key), Size: aws.Int64(int64(len(value)))}) } sort.Slice(contents, func(i, j int) bool { return aws.ToString(contents[i].Key) < aws.ToString(contents[j].Key) }) prefixes := make([]types.CommonPrefix, 0, len(commonPrefixes)) for prefix := range commonPrefixes { prefixes = append(prefixes, types.CommonPrefix{Prefix: aws.String(prefix)}) } sort.Slice(prefixes, func(i, j int) bool { return aws.ToString(prefixes[i].Prefix) < aws.ToString(prefixes[j].Prefix) }) return &awss3.ListObjectsV2Output{ Contents: contents, CommonPrefixes: prefixes, IsTruncated: aws.Bool(false), } } func stringsReader(value string) io.Reader { return strings.NewReader(value) } func entryPaths(entries []storage.Entry) []string { paths := make([]string, 0, len(entries)) for _, entry := range entries { paths = append(paths, entry.Path) } return paths } func sortedStrings(values []string) []string { copied := append([]string(nil), values...) sort.Strings(copied) return copied } func equalStrings(a, b []string) bool { if len(a) != len(b) { return false } for index := range a { if a[index] != b[index] { return false } } return true }