eric/distributor
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 7 Wiki Activity

Route upload client by pipeline

Browse Source
This commit is contained in:
Eric Rakestraw
2026-06-08 04:37:33 +00:00
parent 1c5d7198e3
commit ce43a6044a
4 changed files with 142 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files

104
pkg/upload/client_test.go
View File

@@ -47,7 +47,7 @@ func TestNewClientValidatesOptions(t *testing.T) {
if err != nil {
t.Fatalf("NewClient() error = %v", err)
}
if got, want := client.uploadURL(), "http://127.0.0.1:8080/base/upload"; got != want {
if got, want := client.uploadURL("reports.daily"), "http://127.0.0.1:8080/base/v1/pipelines/reports.daily/upload"; got != want {
t.Fatalf("upload URL = %q, want %q", got, want)
}
if client.httpClient == nil || client.httpClient.Timeout == 0 {
@@ -65,7 +65,7 @@ func TestUploadBundleSendsCallerKeyAndManifestArchive(t *testing.T) {
}
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if got, want := r.URL.Path, "/upload"; got != want {
if got, want := r.URL.Path, "/v1/pipelines/reports.daily/upload"; got != want {
t.Fatalf("path = %q, want %q", got, want)
}
if got, want := r.Header.Get("Authorization"), "Bearer secret-token"; got != want {
@@ -93,6 +93,7 @@ func TestUploadBundleSendsCallerKeyAndManifestArchive(t *testing.T) {
t.Fatalf("NewClient() error = %v", err)
}
result, err := client.UploadBundle(context.Background(), UploadBundleOptions{
PipelineID: "reports.daily",
Root: root,
IdempotencyKey: "producer.retry:one",
})
@@ -112,6 +113,9 @@ func TestUploadFilesBuildsTemporaryBundleWithoutTouchingSources(t *testing.T) {
}
tempDir := t.TempDir()
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if got, want := r.URL.Path, "/v1/pipelines/reports.files/upload"; got != want {
t.Fatalf("path = %q, want %q", got, want)
}
entries := readArchiveEntries(t, r.Body)
if got := string(entries["manifest.json"]); !strings.Contains(got, `"id": "reports.from.files"`) {
t.Fatalf("manifest = %s, want uploaded id", got)
@@ -131,7 +135,8 @@ func TestUploadFilesBuildsTemporaryBundleWithoutTouchingSources(t *testing.T) {
t.Fatalf("NewClient() error = %v", err)
}
_, err = client.UploadFiles(context.Background(), UploadFilesOptions{
ID: "reports.from.files",
PipelineID: "reports.files",
ID: "reports.from.files",
Files: []sourcebundle.BundleFile{{
SourcePath: sourcePath,
Path: "reports/report.md",
@@ -153,6 +158,77 @@ func TestUploadFilesBuildsTemporaryBundleWithoutTouchingSources(t *testing.T) {
}
}
func TestUploadMethodsRequirePipelineIDBeforeLocalWork(t *testing.T) {
var requests atomic.Int64
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
requests.Add(1)
t.Fatal("server should not receive request")
}))
defer server.Close()
client, err := NewClient(ClientOptions{Endpoint: server.URL, Token: "secret", HTTPClient: server.Client()})
if err != nil {
t.Fatalf("NewClient() error = %v", err)
}
missingRoot := filepath.Join(t.TempDir(), "missing")
if _, err := client.UploadBundle(context.Background(), UploadBundleOptions{Root: missingRoot}); err == nil || !strings.Contains(err.Error(), "pipeline id is required") {
t.Fatalf("UploadBundle() error = %v, want missing pipeline id", err)
}
tempDir := t.TempDir()
sourcePath := filepath.Join(t.TempDir(), "report.md")
if err := os.WriteFile(sourcePath, []byte("data"), 0o600); err != nil {
t.Fatalf("write source: %v", err)
}
if _, err := client.UploadFiles(context.Background(), UploadFilesOptions{
ID: "reports.from.files",
Files: []sourcebundle.BundleFile{{
SourcePath: sourcePath,
Path: "report.md",
}},
TempDir: tempDir,
}); err == nil || !strings.Contains(err.Error(), "pipeline id is required") {
t.Fatalf("UploadFiles() error = %v, want missing pipeline id", err)
}
entries, err := os.ReadDir(tempDir)
if err != nil {
t.Fatalf("read temp dir: %v", err)
}
if len(entries) != 0 {
t.Fatalf("temp dir entries = %d, want no local bundle work", len(entries))
}
if got := requests.Load(); got != 0 {
t.Fatalf("requests = %d, want 0", got)
}
}
func TestUploadMethodsRejectInvalidPipelineIDBeforeHTTPRequest(t *testing.T) {
root := writeTestBundle(t, "reports.daily", []testFile{{path: "report.md", data: "data"}})
var requests atomic.Int64
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
requests.Add(1)
t.Fatal("server should not receive request")
}))
defer server.Close()
client, err := NewClient(ClientOptions{Endpoint: server.URL, Token: "secret", HTTPClient: server.Client()})
if err != nil {
t.Fatalf("NewClient() error = %v", err)
}
for _, pipelineID := range []string{".reports", "reports/daily", "reports daily"} {
t.Run(pipelineID, func(t *testing.T) {
_, err := client.UploadBundle(context.Background(), UploadBundleOptions{PipelineID: pipelineID, Root: root})
if err == nil || !strings.Contains(err.Error(), "pipeline id must be a slug-like identifier") {
t.Fatalf("UploadBundle() error = %v, want invalid pipeline id", err)
}
})
}
if got := requests.Load(); got != 0 {
t.Fatalf("requests = %d, want 0", got)
}
}
func TestUploadBundleValidationFailurePreventsHTTPRequest(t *testing.T) {
root := writeTestBundle(t, "reports.daily", []testFile{{path: "report.md", data: "original"}})
if err := os.WriteFile(filepath.Join(root, "report.md"), []byte("changed"), 0o600); err != nil {
@@ -169,7 +245,7 @@ func TestUploadBundleValidationFailurePreventsHTTPRequest(t *testing.T) {
if err != nil {
t.Fatalf("NewClient() error = %v", err)
}
if _, err := client.UploadBundle(context.Background(), UploadBundleOptions{Root: root}); err == nil {
if _, err := client.UploadBundle(context.Background(), UploadBundleOptions{PipelineID: "reports", Root: root}); err == nil {
t.Fatal("UploadBundle() error = nil, want validation error")
}
if got := requests.Load(); got != 0 {
@@ -193,7 +269,7 @@ func TestUploadBundleCanDisableLocalValidation(t *testing.T) {
if err != nil {
t.Fatalf("NewClient() error = %v", err)
}
if _, err := client.UploadBundle(context.Background(), UploadBundleOptions{Root: root, DisableValidation: true}); err != nil {
if _, err := client.UploadBundle(context.Background(), UploadBundleOptions{PipelineID: "reports", Root: root, DisableValidation: true}); err != nil {
t.Fatalf("UploadBundle() error = %v", err)
}
if got := requests.Load(); got != 1 {
@@ -206,6 +282,9 @@ func TestGeneratedIdempotencyKeyIsReusedAcrossRetry(t *testing.T) {
var attempts atomic.Int64
var keys []string
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if got, want := r.URL.Path, "/v1/pipelines/reports/upload"; got != want {
t.Fatalf("path = %q, want %q", got, want)
}
keys = append(keys, r.Header.Get(idempotencyKeyHeader))
if attempts.Add(1) == 1 {
writeJSONError(w, http.StatusServiceUnavailable, "busy", false)
@@ -224,7 +303,7 @@ func TestGeneratedIdempotencyKeyIsReusedAcrossRetry(t *testing.T) {
if err != nil {
t.Fatalf("NewClient() error = %v", err)
}
if _, err := client.UploadBundle(context.Background(), UploadBundleOptions{Root: root}); err != nil {
if _, err := client.UploadBundle(context.Background(), UploadBundleOptions{PipelineID: "reports", Root: root}); err != nil {
t.Fatalf("UploadBundle() error = %v", err)
}
if got, want := attempts.Load(), int64(2); got != want {
@@ -276,7 +355,7 @@ func TestUploadResponseParsingAndNoRetryStatuses(t *testing.T) {
if err != nil {
t.Fatalf("NewClient() error = %v", err)
}
_, err = client.UploadBundle(context.Background(), UploadBundleOptions{Root: root, IdempotencyKey: "key"})
_, err = client.UploadBundle(context.Background(), UploadBundleOptions{PipelineID: "reports", Root: root, IdempotencyKey: "key"})
if err == nil {
t.Fatal("UploadBundle() error = nil, want error")
}
@@ -309,7 +388,7 @@ func TestTokenRedactedFromHTTPError(t *testing.T) {
if err != nil {
t.Fatalf("NewClient() error = %v", err)
}
_, err = client.UploadBundle(context.Background(), UploadBundleOptions{Root: root, IdempotencyKey: "key"})
_, err = client.UploadBundle(context.Background(), UploadBundleOptions{PipelineID: "reports", Root: root, IdempotencyKey: "key"})
if err == nil {
t.Fatal("UploadBundle() error = nil, want error")
}
@@ -330,6 +409,9 @@ func TestNetworkRetryUsesSameIdempotencyKey(t *testing.T) {
Token: "secret",
HTTPClient: &http.Client{Transport: roundTripFunc(func(request *http.Request) (*http.Response, error) {
keys = append(keys, request.Header.Get(idempotencyKeyHeader))
if got, want := request.URL.Path, "/v1/pipelines/reports/upload"; got != want {
t.Fatalf("path = %q, want %q", got, want)
}
if attempts.Add(1) == 1 {
return nil, temporaryNetworkError{}
}
@@ -346,7 +428,7 @@ func TestNetworkRetryUsesSameIdempotencyKey(t *testing.T) {
if err != nil {
t.Fatalf("NewClient() error = %v", err)
}
result, err := client.UploadBundle(context.Background(), UploadBundleOptions{Root: root, IdempotencyKey: "network-retry"})
result, err := client.UploadBundle(context.Background(), UploadBundleOptions{PipelineID: "reports", Root: root, IdempotencyKey: "network-retry"})
if err != nil {
t.Fatalf("UploadBundle() error = %v", err)
}
@@ -381,7 +463,7 @@ func TestContextCancellationDuringRetryBackoff(t *testing.T) {
if err != nil {
t.Fatalf("NewClient() error = %v", err)
}
_, err = client.UploadBundle(ctx, UploadBundleOptions{Root: root, IdempotencyKey: "cancel"})
_, err = client.UploadBundle(ctx, UploadBundleOptions{PipelineID: "reports", Root: root, IdempotencyKey: "cancel"})
if !errors.Is(err, context.Canceled) {
t.Fatalf("UploadBundle() error = %v, want context.Canceled", err)
}
@@ -446,7 +528,7 @@ func TestInvalidCallerIdempotencyKeyPreventsHTTPRequest(t *testing.T) {
if err != nil {
t.Fatalf("NewClient() error = %v", err)
}
if _, err := client.UploadBundle(context.Background(), UploadBundleOptions{Root: root, IdempotencyKey: "bad key"}); err == nil {
if _, err := client.UploadBundle(context.Background(), UploadBundleOptions{PipelineID: "reports", Root: root, IdempotencyKey: "bad key"}); err == nil {
t.Fatal("UploadBundle() error = nil, want invalid key error")
}
if got := requests.Load(); got != 0 {