eric/distributor
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 7 Wiki Activity

Stage uploads before accepting HTTP runs

Browse Source
This commit is contained in:
Eric Rakestraw
2026-06-04 00:14:32 +00:00
parent 1340418a2b
commit 0d346dcdf5
12 changed files with 248 additions and 162 deletions
Download Patch File Download Diff File Expand all files Collapse all files

97
internal/app/upload_http_integration_test.go
View File

@@ -13,6 +13,7 @@ import (
"net/http/httptest"
"os"
"path/filepath"
"strings"
"testing"
"time"
@@ -68,32 +69,65 @@ func TestHTTPUploadPublishesTarAndGzipFanout(t *testing.T) {
}
}
func TestHTTPUploadInvalidArchiveFailsWithoutPublishing(t *testing.T) {
func TestHTTPUploadInvalidArchiveIsRejectedWithoutRunID(t *testing.T) {
destination := t.TempDir()
cfg := httpUploadIntegrationConfig(t, []httpUploadPipelineSpec{{
coordinator := NewUploadCoordinator(context.Background(), httpUploadIntegrationConfig(t, []httpUploadPipelineSpec{{
id: "reports",
tokenEnv: "REPORTS_TOKEN",
stagingPath: filepath.Join(t.TempDir(), "reports"),
destinations: []string{destination},
}}, 4, 1)
handler, err := newUploadHTTPHandler(context.Background(), cfg, uploadHTTPTestEnvironment(map[string]string{
"REPORTS_TOKEN": "reports-secret",
}))
if err != nil {
t.Fatalf("newUploadHTTPHandler() error = %v", err)
}}, 4, 1))
handler := uploadHTTPHandler{
coordinator: coordinator,
tokens: map[string]string{"reports-secret": "reports"},
}
server := httptest.NewServer(handler)
defer server.Close()
runID := submitHTTPUpload(t, server, "reports-secret", ingest.ContentTypeTar, []byte("not a tar archive"))
record := waitForHTTPUploadStatus(t, server, runID, UploadStatusFailed)
status, body := postHTTPUpload(t, server, "reports-secret", ingest.ContentTypeTar, []byte("not a tar archive"))
if status != http.StatusBadRequest {
t.Fatalf("POST /upload status = %d, want %d; body = %s", status, http.StatusBadRequest, body)
}
if strings.Contains(body, "run_id") || strings.Contains(body, "reports-secret") {
t.Fatalf("invalid archive response exposed run id or token: %s", body)
}
if got := coordinator.QueueDepth(); got != 0 {
t.Fatalf("queue depth = %d, want 0", got)
}
assertDirectoryEmpty(t, destination)
}
if record.Error == "" {
t.Fatal("failed status error is empty")
func TestHTTPUploadOversizedArchiveIsRejectedWithoutRunID(t *testing.T) {
destination := t.TempDir()
stagingPath := filepath.Join(t.TempDir(), "reports")
cfg := httpUploadIntegrationConfig(t, []httpUploadPipelineSpec{{
id: "reports",
tokenEnv: "REPORTS_TOKEN",
stagingPath: stagingPath,
destinations: []string{destination},
}}, 4, 1)
size := config.ByteSize(4)
cfg.Server.HTTP.MaxUploadSize = &size
cfg.Pipelines[0].Source.Upload.MaxUploadSize = &size
coordinator := NewUploadCoordinator(context.Background(), cfg)
handler := uploadHTTPHandler{
coordinator: coordinator,
tokens: map[string]string{"reports-secret": "reports"},
}
if record.Report != nil {
t.Fatalf("failed staging report = %#v, want nil", record.Report)
server := httptest.NewServer(handler)
defer server.Close()
status, body := postHTTPUpload(t, server, "reports-secret", ingest.ContentTypeTar, bundleArchive(t, false, testutil.BundleOptions{}))
if status != http.StatusRequestEntityTooLarge {
t.Fatalf("POST /upload status = %d, want %d; body = %s", status, http.StatusRequestEntityTooLarge, body)
}
if strings.Contains(body, "run_id") || strings.Contains(body, "reports-secret") {
t.Fatalf("oversized response exposed run id or token: %s", body)
}
if got := coordinator.QueueDepth(); got != 0 {
t.Fatalf("queue depth = %d, want 0", got)
}
assertDirectoryEmpty(t, stagingPath)
assertDirectoryEmpty(t, destination)
}
@@ -122,7 +156,6 @@ func TestHTTPUploadSamePipelineRequestsSerialize(t *testing.T) {
handler := uploadHTTPHandler{
coordinator: coordinator,
tokens: map[string]string{"reports-secret": "reports"},
limits: map[string]int64{"reports": 1024},
}
server := httptest.NewServer(handler)
defer server.Close()
@@ -177,10 +210,6 @@ func TestHTTPUploadDifferentPipelinesRunConcurrently(t *testing.T) {
"one-secret": "reports-one",
"two-secret": "reports-two",
},
limits: map[string]int64{
"reports-one": 1024,
"reports-two": 1024,
},
}
server := httptest.NewServer(handler)
defer server.Close()
@@ -247,6 +276,22 @@ func httpUploadIntegrationConfig(t *testing.T, pipelines []httpUploadPipelineSpe
}
func submitHTTPUpload(t *testing.T, server *httptest.Server, token, contentType string, body []byte) UploadRunID {
t.Helper()
status, responseBody := postHTTPUpload(t, server, token, contentType, body)
if status != http.StatusAccepted {
t.Fatalf("POST /upload status = %d, want %d; body = %s", status, http.StatusAccepted, responseBody)
}
var accepted uploadAcceptedResponse
if err := json.Unmarshal([]byte(responseBody), &accepted); err != nil {
t.Fatalf("decode accepted response: %v", err)
}
if accepted.RunID == "" || accepted.Status != UploadStatusAccepted {
t.Fatalf("accepted response = %#v, want run id and accepted status", accepted)
}
return accepted.RunID
}
func postHTTPUpload(t *testing.T, server *httptest.Server, token, contentType string, body []byte) (int, string) {
t.Helper()
request, err := http.NewRequest(http.MethodPost, server.URL+"/upload", bytes.NewReader(body))
if err != nil {
@@ -259,17 +304,11 @@ func submitHTTPUpload(t *testing.T, server *httptest.Server, token, contentType
t.Fatalf("POST /upload error = %v", err)
}
defer response.Body.Close()
if response.StatusCode != http.StatusAccepted {
t.Fatalf("POST /upload status = %d, want %d", response.StatusCode, http.StatusAccepted)
data, err := io.ReadAll(response.Body)
if err != nil {
t.Fatalf("read response body: %v", err)
}
var accepted uploadAcceptedResponse
if err := json.NewDecoder(response.Body).Decode(&accepted); err != nil {
t.Fatalf("decode accepted response: %v", err)
}
if accepted.RunID == "" || accepted.Status != UploadStatusAccepted {
t.Fatalf("accepted response = %#v, want run id and accepted status", accepted)
}
return accepted.RunID
return response.StatusCode, string(data)
}
func waitForHTTPUploadStatus(t *testing.T, server *httptest.Server, runID UploadRunID, status UploadStatus) UploadRunRecord {