eric/distributor
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 7 Wiki Activity

Route uploads by pipeline path

Browse Source
This commit is contained in:
Eric Rakestraw
2026-06-08 04:32:20 +00:00
parent 033b2e5015
commit 9d4694c6d8
5 changed files with 261 additions and 69 deletions
Download Patch File Download Diff File Expand all files Collapse all files

67
internal/app/upload_http_integration_test.go
View File

@@ -78,15 +78,16 @@ func TestHTTPUploadInvalidArchiveIsRejectedWithoutRunID(t *testing.T) {
destinations: []string{destination},
}}, 4, 1))
handler := uploadHTTPHandler{
coordinator: coordinator,
tokens: map[string]string{"reports-secret": "reports"},
coordinator: coordinator,
tokens: map[string]resolvedUploadToken{"reports-secret": uploadHTTPTestToken("reports-reporter", "reports-secret", "reports")},
uploadPipelines: pipelineIDSet([]string{"reports"}),
}
server := httptest.NewServer(handler)
defer server.Close()
status, body := postHTTPUpload(t, server, "reports-secret", ingest.ContentTypeTar, []byte("not a tar archive"))
if status != http.StatusBadRequest {
t.Fatalf("POST /upload status = %d, want %d; body = %s", status, http.StatusBadRequest, body)
t.Fatalf("POST upload status = %d, want %d; body = %s", status, http.StatusBadRequest, body)
}
if strings.Contains(body, "run_id") || strings.Contains(body, "reports-secret") {
t.Fatalf("invalid archive response exposed run id or token: %s", body)
@@ -106,8 +107,9 @@ func TestHTTPUploadIdempotencyReturnsOriginalRunForSameBundle(t *testing.T) {
destinations: []string{destination},
}}, 4, 1))
handler := uploadHTTPHandler{
coordinator: coordinator,
tokens: map[string]string{"reports-secret": "reports"},
coordinator: coordinator,
tokens: map[string]resolvedUploadToken{"reports-secret": uploadHTTPTestToken("reports-reporter", "reports-secret", "reports")},
uploadPipelines: pipelineIDSet([]string{"reports"}),
}
server := httptest.NewServer(handler)
defer server.Close()
@@ -133,8 +135,9 @@ func TestHTTPUploadIdempotencyReturnsConflictForDifferentBundle(t *testing.T) {
destinations: []string{destination},
}}, 4, 1))
handler := uploadHTTPHandler{
coordinator: coordinator,
tokens: map[string]string{"reports-secret": "reports"},
coordinator: coordinator,
tokens: map[string]resolvedUploadToken{"reports-secret": uploadHTTPTestToken("reports-reporter", "reports-secret", "reports")},
uploadPipelines: pipelineIDSet([]string{"reports"}),
}
server := httptest.NewServer(handler)
defer server.Close()
@@ -146,7 +149,7 @@ func TestHTTPUploadIdempotencyReturnsConflictForDifferentBundle(t *testing.T) {
ID: "weather.daily.brentwood.2026-05-31",
}))
if status != http.StatusConflict {
t.Fatalf("POST /upload status = %d, want %d; body = %s", status, http.StatusConflict, body)
t.Fatalf("POST upload status = %d, want %d; body = %s", status, http.StatusConflict, body)
}
if strings.Contains(body, "reports-secret") {
t.Fatalf("conflict response exposed token: %s", body)
@@ -167,15 +170,16 @@ func TestHTTPUploadOversizedArchiveIsRejectedWithoutRunID(t *testing.T) {
cfg.Pipelines[0].Source.Upload.MaxUploadSize = &size
coordinator := NewUploadCoordinator(context.Background(), cfg)
handler := uploadHTTPHandler{
coordinator: coordinator,
tokens: map[string]string{"reports-secret": "reports"},
coordinator: coordinator,
tokens: map[string]resolvedUploadToken{"reports-secret": uploadHTTPTestToken("reports-reporter", "reports-secret", "reports")},
uploadPipelines: pipelineIDSet([]string{"reports"}),
}
server := httptest.NewServer(handler)
defer server.Close()
status, body := postHTTPUpload(t, server, "reports-secret", ingest.ContentTypeTar, bundleArchive(t, false, testutil.BundleOptions{}))
if status != http.StatusRequestEntityTooLarge {
t.Fatalf("POST /upload status = %d, want %d; body = %s", status, http.StatusRequestEntityTooLarge, body)
t.Fatalf("POST upload status = %d, want %d; body = %s", status, http.StatusRequestEntityTooLarge, body)
}
if strings.Contains(body, "run_id") || strings.Contains(body, "reports-secret") {
t.Fatalf("oversized response exposed run id or token: %s", body)
@@ -210,8 +214,9 @@ func TestHTTPUploadSamePipelineRequestsSerialize(t *testing.T) {
},
})
handler := uploadHTTPHandler{
coordinator: coordinator,
tokens: map[string]string{"reports-secret": "reports"},
coordinator: coordinator,
tokens: map[string]resolvedUploadToken{"reports-secret": uploadHTTPTestToken("reports-reporter", "reports-secret", "reports")},
uploadPipelines: pipelineIDSet([]string{"reports"}),
}
server := httptest.NewServer(handler)
defer server.Close()
@@ -262,16 +267,17 @@ func TestHTTPUploadDifferentPipelinesRunConcurrently(t *testing.T) {
})
handler := uploadHTTPHandler{
coordinator: coordinator,
tokens: map[string]string{
"one-secret": "reports-one",
"two-secret": "reports-two",
tokens: map[string]resolvedUploadToken{
"one-secret": uploadHTTPTestToken("reports-one-reporter", "one-secret", "reports-one"),
"two-secret": uploadHTTPTestToken("reports-two-reporter", "two-secret", "reports-two"),
},
uploadPipelines: pipelineIDSet([]string{"reports-one", "reports-two"}),
}
server := httptest.NewServer(handler)
defer server.Close()
firstRunID := submitHTTPUpload(t, server, "one-secret", ingest.ContentTypeTar, []byte("first"))
secondRunID := submitHTTPUpload(t, server, "two-secret", ingest.ContentTypeTar, []byte("second"))
firstRunID := submitHTTPUploadToPipeline(t, server, "reports-one", "one-secret", ingest.ContentTypeTar, []byte("first"))
secondRunID := submitHTTPUploadToPipeline(t, server, "reports-two", "two-secret", ingest.ContentTypeTar, []byte("second"))
waitForStartedPipelines(t, started, "reports-one", "reports-two")
waitForHTTPUploadStatus(t, server, firstRunID, UploadStatusRunning)
waitForHTTPUploadStatus(t, server, secondRunID, UploadStatusRunning)
@@ -337,7 +343,12 @@ func httpUploadIntegrationConfig(t *testing.T, pipelines []httpUploadPipelineSpe
func submitHTTPUpload(t *testing.T, server *httptest.Server, token, contentType string, body []byte) UploadRunID {
t.Helper()
status, responseBody := postHTTPUpload(t, server, token, contentType, body)
return submitHTTPUploadToPipeline(t, server, "reports", token, contentType, body)
}
func submitHTTPUploadToPipeline(t *testing.T, server *httptest.Server, pipelineID, token, contentType string, body []byte) UploadRunID {
t.Helper()
status, responseBody := postHTTPUploadToPipeline(t, server, pipelineID, token, contentType, body)
return decodeAcceptedHTTPUpload(t, status, responseBody)
}
@@ -350,7 +361,7 @@ func submitHTTPUploadWithKey(t *testing.T, server *httptest.Server, token, conte
func decodeAcceptedHTTPUpload(t *testing.T, status int, responseBody string) UploadRunID {
t.Helper()
if status != http.StatusAccepted {
t.Fatalf("POST /upload status = %d, want %d; body = %s", status, http.StatusAccepted, responseBody)
t.Fatalf("POST upload status = %d, want %d; body = %s", status, http.StatusAccepted, responseBody)
}
var accepted uploadAcceptedResponse
if err := json.Unmarshal([]byte(responseBody), &accepted); err != nil {
@@ -364,12 +375,22 @@ func decodeAcceptedHTTPUpload(t *testing.T, status int, responseBody string) Upl
func postHTTPUpload(t *testing.T, server *httptest.Server, token, contentType string, body []byte) (int, string) {
t.Helper()
return postHTTPUploadWithKey(t, server, token, contentType, "", body)
return postHTTPUploadToPipeline(t, server, "reports", token, contentType, body)
}
func postHTTPUploadToPipeline(t *testing.T, server *httptest.Server, pipelineID, token, contentType string, body []byte) (int, string) {
t.Helper()
return postHTTPUploadWithKeyToPipeline(t, server, pipelineID, token, contentType, "", body)
}
func postHTTPUploadWithKey(t *testing.T, server *httptest.Server, token, contentType, key string, body []byte) (int, string) {
t.Helper()
request, err := http.NewRequest(http.MethodPost, server.URL+"/upload", bytes.NewReader(body))
return postHTTPUploadWithKeyToPipeline(t, server, "reports", token, contentType, key, body)
}
func postHTTPUploadWithKeyToPipeline(t *testing.T, server *httptest.Server, pipelineID, token, contentType, key string, body []byte) (int, string) {
t.Helper()
request, err := http.NewRequest(http.MethodPost, server.URL+"/v1/pipelines/"+pipelineID+"/upload", bytes.NewReader(body))
if err != nil {
t.Fatalf("NewRequest() error = %v", err)
}
@@ -380,7 +401,7 @@ func postHTTPUploadWithKey(t *testing.T, server *httptest.Server, token, content
}
response, err := server.Client().Do(request)
if err != nil {
t.Fatalf("POST /upload error = %v", err)
t.Fatalf("POST upload error = %v", err)
}
defer response.Body.Close()
data, err := io.ReadAll(response.Body)