eric/distributor
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 7 Wiki Activity

Add HTTP upload idempotency support

Browse Source
This commit is contained in:
Eric Rakestraw
2026-06-04 14:03:26 +00:00
parent 1a402e6cfa
commit a15722571f
9 changed files with 603 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files

75
internal/app/upload_http_integration_test.go
View File

@@ -97,6 +97,62 @@ func TestHTTPUploadInvalidArchiveIsRejectedWithoutRunID(t *testing.T) {
assertDirectoryEmpty(t, destination)
}
func TestHTTPUploadIdempotencyReturnsOriginalRunForSameBundle(t *testing.T) {
destination := t.TempDir()
coordinator := NewUploadCoordinator(context.Background(), httpUploadIntegrationConfig(t, []httpUploadPipelineSpec{{
id: "reports",
tokenEnv: "REPORTS_TOKEN",
stagingPath: filepath.Join(t.TempDir(), "reports"),
destinations: []string{destination},
}}, 4, 1))
handler := uploadHTTPHandler{
coordinator: coordinator,
tokens: map[string]string{"reports-secret": "reports"},
}
server := httptest.NewServer(handler)
defer server.Close()
firstRunID := submitHTTPUploadWithKey(t, server, "reports-secret", ingest.ContentTypeTar, "same-key", bundleArchive(t, false, testutil.BundleOptions{}))
waitForHTTPUploadStatus(t, server, firstRunID, UploadStatusSucceeded)
secondRunID := submitHTTPUploadWithKey(t, server, "reports-secret", ingest.ContentTypeGzip, "same-key", bundleArchive(t, true, testutil.BundleOptions{}))
if secondRunID != firstRunID {
t.Fatalf("second run id = %q, want original %q", secondRunID, firstRunID)
}
if got := coordinator.QueueDepth(); got != 0 {
t.Fatalf("queue depth = %d, want no duplicate run queued", got)
}
}
func TestHTTPUploadIdempotencyReturnsConflictForDifferentBundle(t *testing.T) {
destination := t.TempDir()
coordinator := NewUploadCoordinator(context.Background(), httpUploadIntegrationConfig(t, []httpUploadPipelineSpec{{
id: "reports",
tokenEnv: "REPORTS_TOKEN",
stagingPath: filepath.Join(t.TempDir(), "reports"),
destinations: []string{destination},
}}, 4, 1))
handler := uploadHTTPHandler{
coordinator: coordinator,
tokens: map[string]string{"reports-secret": "reports"},
}
server := httptest.NewServer(handler)
defer server.Close()
firstRunID := submitHTTPUploadWithKey(t, server, "reports-secret", ingest.ContentTypeTar, "same-key", bundleArchive(t, false, testutil.BundleOptions{}))
waitForHTTPUploadStatus(t, server, firstRunID, UploadStatusSucceeded)
status, body := postHTTPUploadWithKey(t, server, "reports-secret", ingest.ContentTypeTar, "same-key", bundleArchive(t, false, testutil.BundleOptions{
ID: "weather.daily.brentwood.2026-05-31",
}))
if status != http.StatusConflict {
t.Fatalf("POST /upload status = %d, want %d; body = %s", status, http.StatusConflict, body)
}
if strings.Contains(body, "reports-secret") {
t.Fatalf("conflict response exposed token: %s", body)
}
}
func TestHTTPUploadOversizedArchiveIsRejectedWithoutRunID(t *testing.T) {
destination := t.TempDir()
stagingPath := filepath.Join(t.TempDir(), "reports")
@@ -278,6 +334,17 @@ func httpUploadIntegrationConfig(t *testing.T, pipelines []httpUploadPipelineSpe
func submitHTTPUpload(t *testing.T, server *httptest.Server, token, contentType string, body []byte) UploadRunID {
t.Helper()
status, responseBody := postHTTPUpload(t, server, token, contentType, body)
return decodeAcceptedHTTPUpload(t, status, responseBody)
}
func submitHTTPUploadWithKey(t *testing.T, server *httptest.Server, token, contentType, key string, body []byte) UploadRunID {
t.Helper()
status, responseBody := postHTTPUploadWithKey(t, server, token, contentType, key, body)
return decodeAcceptedHTTPUpload(t, status, responseBody)
}
func decodeAcceptedHTTPUpload(t *testing.T, status int, responseBody string) UploadRunID {
t.Helper()
if status != http.StatusAccepted {
t.Fatalf("POST /upload status = %d, want %d; body = %s", status, http.StatusAccepted, responseBody)
}
@@ -292,6 +359,11 @@ func submitHTTPUpload(t *testing.T, server *httptest.Server, token, contentType
}
func postHTTPUpload(t *testing.T, server *httptest.Server, token, contentType string, body []byte) (int, string) {
t.Helper()
return postHTTPUploadWithKey(t, server, token, contentType, "", body)
}
func postHTTPUploadWithKey(t *testing.T, server *httptest.Server, token, contentType, key string, body []byte) (int, string) {
t.Helper()
request, err := http.NewRequest(http.MethodPost, server.URL+"/upload", bytes.NewReader(body))
if err != nil {
@@ -299,6 +371,9 @@ func postHTTPUpload(t *testing.T, server *httptest.Server, token, contentType st
}
request.Header.Set("Authorization", "Bearer "+token)
request.Header.Set("Content-Type", contentType)
if key != "" {
request.Header.Set("Idempotency-Key", key)
}
response, err := server.Client().Do(request)
if err != nil {
t.Fatalf("POST /upload error = %v", err)