eric/distributor
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 7 Wiki Activity

Route uploads by pipeline path

Browse Source
This commit is contained in:
Eric Rakestraw
2026-06-08 04:32:20 +00:00
parent 033b2e5015
commit 9d4694c6d8
5 changed files with 261 additions and 69 deletions
Download Patch File Download Diff File Expand all files Collapse all files

154
internal/app/upload_http_test.go
View File

@@ -75,6 +75,39 @@ func TestResolveUploadTokensFailsForMissingAndDuplicateTokens(t *testing.T) {
}
}
func TestResolveUploadTokensAllowsMultiplePipelines(t *testing.T) {
cfg := uploadHTTPTestConfig()
cfg.Pipelines = append(cfg.Pipelines, config.Pipeline{
ID: "weekly",
Source: config.Backend{
Backend: config.BackendHTTPUpload,
},
Destinations: cfg.Pipelines[0].Destinations,
})
cfg.UploadTokens[0].AllowPipelines = []string{"reports", "weekly"}
config.ApplyDefaults(&cfg)
tokens, err := resolveUploadTokens(cfg, uploadHTTPTestEnvironment(map[string]string{
"UPLOAD_TOKEN": "secret",
}))
if err != nil {
t.Fatalf("resolveUploadTokens() error = %v", err)
}
token, ok := tokens["secret"]
if !ok {
t.Fatal("resolved token missing")
}
if token.ID != "reporter" || token.Value != "secret" {
t.Fatalf("resolved token = %#v, want id and value", token)
}
if _, ok := token.AllowedPipelines["reports"]; !ok {
t.Fatalf("allowed pipelines = %#v, want reports", token.AllowedPipelines)
}
if _, ok := token.AllowedPipelines["weekly"]; !ok {
t.Fatalf("allowed pipelines = %#v, want weekly", token.AllowedPipelines)
}
}
func TestNewUploadHTTPHandlerAcceptsDefaultedConfig(t *testing.T) {
cfg := uploadHTTPTestConfig()
cfg.Server.HTTP.Bind = ""
@@ -113,10 +146,11 @@ func TestUploadHTTPHandlerAuthenticatesAndAcceptsUpload(t *testing.T) {
return UploadRunRecord{ID: "reports.20260603T120000Z.abcdef12", Status: UploadStatusAccepted}, nil
},
},
tokens: map[string]string{"valid-token": "reports"},
tokens: map[string]resolvedUploadToken{"valid-token": uploadHTTPTestToken("reporter", "valid-token", "reports")},
uploadPipelines: pipelineIDSet([]string{"reports"}),
}
recorder := httptest.NewRecorder()
request := httptest.NewRequest(http.MethodPost, "/upload", strings.NewReader("archive"))
request := httptest.NewRequest(http.MethodPost, "/v1/pipelines/reports/upload", strings.NewReader("archive"))
request.Header.Set("Authorization", "Bearer valid-token")
request.Header.Set("Content-Type", "application/x-tar")
request.Header.Set("Idempotency-Key", "producer.retry:20260603")
@@ -129,6 +163,9 @@ func TestUploadHTTPHandlerAuthenticatesAndAcceptsUpload(t *testing.T) {
if submitted.PipelineID != "reports" {
t.Fatalf("submitted pipeline = %q, want reports", submitted.PipelineID)
}
if submitted.TokenID != "reporter" {
t.Fatalf("submitted token id = %q, want reporter", submitted.TokenID)
}
if submitted.IdempotencyKey != "producer.retry:20260603" {
t.Fatalf("submitted idempotency key = %q, want producer.retry:20260603", submitted.IdempotencyKey)
}
@@ -146,13 +183,14 @@ func TestUploadHTTPHandlerAuthenticatesAndAcceptsUpload(t *testing.T) {
func TestUploadHTTPHandlerRejectsUnauthorizedRequests(t *testing.T) {
handler := uploadHTTPHandler{
coordinator: fakeUploadCoordinator{},
tokens: map[string]string{"valid-token": "reports"},
coordinator: fakeUploadCoordinator{},
tokens: map[string]resolvedUploadToken{"valid-token": uploadHTTPTestToken("reporter", "valid-token", "reports")},
uploadPipelines: pipelineIDSet([]string{"reports"}),
}
for _, authHeader := range []string{"", "Bearer wrong-token"} {
for _, authHeader := range []string{"", "Basic valid-token", "Bearer", "Bearer wrong-token"} {
recorder := httptest.NewRecorder()
request := httptest.NewRequest(http.MethodPost, "/upload", strings.NewReader("archive"))
request := httptest.NewRequest(http.MethodPost, "/v1/pipelines/reports/upload", strings.NewReader("archive"))
request.Header.Set("Authorization", authHeader)
request.Header.Set("Content-Type", "application/x-tar")
@@ -167,7 +205,73 @@ func TestUploadHTTPHandlerRejectsUnauthorizedRequests(t *testing.T) {
}
}
func TestUploadHTTPHandlerRejectsUnsupportedContentTypeInvalidKeyAndPipelineID(t *testing.T) {
func TestUploadHTTPHandlerRejectsForbiddenPipeline(t *testing.T) {
handler := uploadHTTPHandler{
coordinator: fakeUploadCoordinator{
submit: func(context.Context, UploadRequest) (UploadRunRecord, error) {
t.Fatal("Submit should not be called")
return UploadRunRecord{}, nil
},
},
tokens: map[string]resolvedUploadToken{"valid-token": uploadHTTPTestToken("reporter", "valid-token", "reports")},
uploadPipelines: pipelineIDSet([]string{"reports", "private"}),
}
recorder := httptest.NewRecorder()
request := httptest.NewRequest(http.MethodPost, "/v1/pipelines/private/upload", strings.NewReader("archive"))
request.Header.Set("Authorization", "Bearer valid-token")
request.Header.Set("Content-Type", "application/x-tar")
handler.ServeHTTP(recorder, request)
if recorder.Code != http.StatusForbidden {
t.Fatalf("status = %d, want %d; body = %q", recorder.Code, http.StatusForbidden, recorder.Body.String())
}
if strings.Contains(recorder.Body.String(), "valid-token") {
t.Fatalf("forbidden response exposed token: %q", recorder.Body.String())
}
}
func TestUploadHTTPHandlerRejectsInvalidPathAndRemovedLegacyUpload(t *testing.T) {
handler := uploadHTTPHandler{
coordinator: fakeUploadCoordinator{
submit: func(context.Context, UploadRequest) (UploadRunRecord, error) {
t.Fatal("Submit should not be called")
return UploadRunRecord{}, nil
},
},
tokens: map[string]resolvedUploadToken{"valid-token": uploadHTTPTestToken("reporter", "valid-token", "reports")},
uploadPipelines: pipelineIDSet([]string{"reports"}),
}
tests := []struct {
name string
url string
wantStatus int
}{
{name: "legacy upload", url: "/upload", wantStatus: http.StatusNotFound},
{name: "missing pipeline", url: "/v1/pipelines//upload", wantStatus: http.StatusNotFound},
{name: "extra segment", url: "/v1/pipelines/reports/upload/extra", wantStatus: http.StatusNotFound},
{name: "invalid pipeline id", url: "/v1/pipelines/.reports/upload", wantStatus: http.StatusBadRequest},
{name: "pipeline query", url: "/v1/pipelines/reports/upload?pipeline=other", wantStatus: http.StatusBadRequest},
{name: "pipeline id query", url: "/v1/pipelines/reports/upload?pipeline_id=other", wantStatus: http.StatusBadRequest},
{name: "unknown upload pipeline", url: "/v1/pipelines/missing/upload", wantStatus: http.StatusNotFound},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
recorder := httptest.NewRecorder()
request := httptest.NewRequest(http.MethodPost, tt.url, strings.NewReader("archive"))
request.Header.Set("Authorization", "Bearer valid-token")
request.Header.Set("Content-Type", "application/x-tar")
handler.ServeHTTP(recorder, request)
if recorder.Code != tt.wantStatus {
t.Fatalf("status = %d, want %d; body = %q", recorder.Code, tt.wantStatus, recorder.Body.String())
}
})
}
}
func TestUploadHTTPHandlerRejectsUnsupportedContentTypeAndInvalidKey(t *testing.T) {
tests := []struct {
name string
url string
@@ -178,14 +282,14 @@ func TestUploadHTTPHandlerRejectsUnsupportedContentTypeInvalidKeyAndPipelineID(t
}{
{
name: "unsupported content type",
url: "/upload",
url: "/v1/pipelines/reports/upload",
contentType: "application/zip",
body: strings.NewReader("archive"),
wantStatus: http.StatusUnsupportedMediaType,
},
{
name: "invalid key syntax",
url: "/upload",
url: "/v1/pipelines/reports/upload",
contentType: "application/x-tar",
keyValues: []string{"bad key"},
body: strings.NewReader("archive"),
@@ -193,7 +297,7 @@ func TestUploadHTTPHandlerRejectsUnsupportedContentTypeInvalidKeyAndPipelineID(t
},
{
name: "empty key",
url: "/upload",
url: "/v1/pipelines/reports/upload",
contentType: "application/x-tar",
keyValues: []string{""},
body: strings.NewReader("archive"),
@@ -201,7 +305,7 @@ func TestUploadHTTPHandlerRejectsUnsupportedContentTypeInvalidKeyAndPipelineID(t
},
{
name: "too long key",
url: "/upload",
url: "/v1/pipelines/reports/upload",
contentType: "application/x-tar",
keyValues: []string{strings.Repeat("a", 129)},
body: strings.NewReader("archive"),
@@ -209,19 +313,12 @@ func TestUploadHTTPHandlerRejectsUnsupportedContentTypeInvalidKeyAndPipelineID(t
},
{
name: "multiple keys",
url: "/upload",
url: "/v1/pipelines/reports/upload",
contentType: "application/x-tar",
keyValues: []string{"one", "two"},
body: strings.NewReader("archive"),
wantStatus: http.StatusBadRequest,
},
{
name: "submitted pipeline id",
url: "/upload?pipeline_id=reports",
contentType: "application/x-tar",
body: strings.NewReader("archive"),
wantStatus: http.StatusBadRequest,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
@@ -232,7 +329,8 @@ func TestUploadHTTPHandlerRejectsUnsupportedContentTypeInvalidKeyAndPipelineID(t
return UploadRunRecord{}, nil
},
},
tokens: map[string]string{"valid-token": "reports"},
tokens: map[string]resolvedUploadToken{"valid-token": uploadHTTPTestToken("reporter", "valid-token", "reports")},
uploadPipelines: pipelineIDSet([]string{"reports"}),
}
recorder := httptest.NewRecorder()
request := httptest.NewRequest(http.MethodPost, tt.url, tt.body)
@@ -274,10 +372,11 @@ func TestUploadHTTPHandlerMapsSubmitErrors(t *testing.T) {
return UploadRunRecord{}, tt.err
},
},
tokens: map[string]string{"valid-token": "reports"},
tokens: map[string]resolvedUploadToken{"valid-token": uploadHTTPTestToken("reporter", "valid-token", "reports")},
uploadPipelines: pipelineIDSet([]string{"reports"}),
}
recorder := httptest.NewRecorder()
request := httptest.NewRequest(http.MethodPost, "/upload", strings.NewReader("archive"))
request := httptest.NewRequest(http.MethodPost, "/v1/pipelines/reports/upload", strings.NewReader("archive"))
request.Header.Set("Authorization", "Bearer valid-token")
request.Header.Set("Content-Type", "application/x-tar")
@@ -310,7 +409,8 @@ func TestUploadHTTPHandlerRunStatusAndHealth(t *testing.T) {
}, true
},
},
tokens: map[string]string{"valid-token": "reports"},
tokens: map[string]resolvedUploadToken{"valid-token": uploadHTTPTestToken("reporter", "valid-token", "reports")},
uploadPipelines: pipelineIDSet([]string{"reports"}),
}
recorder := httptest.NewRecorder()
@@ -392,3 +492,11 @@ func uploadHTTPTestEnvironment(values map[string]string) config.Environment {
return "", false
})
}
func uploadHTTPTestToken(id, value string, pipelines ...string) resolvedUploadToken {
return resolvedUploadToken{
ID: id,
Value: value,
AllowedPipelines: pipelineIDSet(pipelines),
}
}