eric/distributor
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 7 Wiki Activity

Add HTTP upload idempotency support

Browse Source
This commit is contained in:
Eric Rakestraw
2026-06-04 14:03:26 +00:00
parent 1a402e6cfa
commit a15722571f
9 changed files with 603 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files

59
internal/app/upload_http_test.go
View File

@@ -97,7 +97,6 @@ func TestUploadHTTPHandlerAuthenticatesAndAcceptsUpload(t *testing.T) {
var submitted UploadRequest
handler := uploadHTTPHandler{
coordinator: fakeUploadCoordinator{
canAccept: true,
submit: func(_ context.Context, request UploadRequest) (UploadRunRecord, error) {
submitted = request
body, err := io.ReadAll(request.Body)
@@ -116,6 +115,7 @@ func TestUploadHTTPHandlerAuthenticatesAndAcceptsUpload(t *testing.T) {
request := httptest.NewRequest(http.MethodPost, "/upload", strings.NewReader("archive"))
request.Header.Set("Authorization", "Bearer valid-token")
request.Header.Set("Content-Type", "application/x-tar")
request.Header.Set("Idempotency-Key", "producer.retry:20260603")
handler.ServeHTTP(recorder, request)
@@ -125,6 +125,9 @@ func TestUploadHTTPHandlerAuthenticatesAndAcceptsUpload(t *testing.T) {
if submitted.PipelineID != "reports" {
t.Fatalf("submitted pipeline = %q, want reports", submitted.PipelineID)
}
if submitted.IdempotencyKey != "producer.retry:20260603" {
t.Fatalf("submitted idempotency key = %q, want producer.retry:20260603", submitted.IdempotencyKey)
}
var response uploadAcceptedResponse
if err := json.Unmarshal(recorder.Body.Bytes(), &response); err != nil {
t.Fatalf("decode response: %v", err)
@@ -139,7 +142,7 @@ func TestUploadHTTPHandlerAuthenticatesAndAcceptsUpload(t *testing.T) {
func TestUploadHTTPHandlerRejectsUnauthorizedRequests(t *testing.T) {
handler := uploadHTTPHandler{
coordinator: fakeUploadCoordinator{canAccept: true},
coordinator: fakeUploadCoordinator{},
tokens: map[string]string{"valid-token": "reports"},
}
@@ -160,34 +163,56 @@ func TestUploadHTTPHandlerRejectsUnauthorizedRequests(t *testing.T) {
}
}
func TestUploadHTTPHandlerRejectsUnsupportedOversizedFullQueueAndPipelineID(t *testing.T) {
func TestUploadHTTPHandlerRejectsUnsupportedContentTypeInvalidKeyAndPipelineID(t *testing.T) {
tests := []struct {
name string
canAccept bool
url string
contentType string
keyValues []string
body io.Reader
wantStatus int
}{
{
name: "unsupported content type",
canAccept: true,
url: "/upload",
contentType: "application/zip",
body: strings.NewReader("archive"),
wantStatus: http.StatusUnsupportedMediaType,
},
{
name: "full queue",
canAccept: false,
name: "invalid key syntax",
url: "/upload",
contentType: "application/x-tar",
body: &countingReader{reader: strings.NewReader("archive")},
wantStatus: http.StatusServiceUnavailable,
keyValues: []string{"bad key"},
body: strings.NewReader("archive"),
wantStatus: http.StatusBadRequest,
},
{
name: "empty key",
url: "/upload",
contentType: "application/x-tar",
keyValues: []string{""},
body: strings.NewReader("archive"),
wantStatus: http.StatusBadRequest,
},
{
name: "too long key",
url: "/upload",
contentType: "application/x-tar",
keyValues: []string{strings.Repeat("a", 129)},
body: strings.NewReader("archive"),
wantStatus: http.StatusBadRequest,
},
{
name: "multiple keys",
url: "/upload",
contentType: "application/x-tar",
keyValues: []string{"one", "two"},
body: strings.NewReader("archive"),
wantStatus: http.StatusBadRequest,
},
{
name: "submitted pipeline id",
canAccept: true,
url: "/upload?pipeline_id=reports",
contentType: "application/x-tar",
body: strings.NewReader("archive"),
@@ -198,7 +223,6 @@ func TestUploadHTTPHandlerRejectsUnsupportedOversizedFullQueueAndPipelineID(t *t
t.Run(tt.name, func(t *testing.T) {
handler := uploadHTTPHandler{
coordinator: fakeUploadCoordinator{
canAccept: tt.canAccept,
submit: func(context.Context, UploadRequest) (UploadRunRecord, error) {
t.Fatal("Submit should not be called")
return UploadRunRecord{}, nil
@@ -210,15 +234,15 @@ func TestUploadHTTPHandlerRejectsUnsupportedOversizedFullQueueAndPipelineID(t *t
request := httptest.NewRequest(http.MethodPost, tt.url, tt.body)
request.Header.Set("Authorization", "Bearer valid-token")
request.Header.Set("Content-Type", tt.contentType)
for _, value := range tt.keyValues {
request.Header.Add("Idempotency-Key", value)
}
handler.ServeHTTP(recorder, request)
if recorder.Code != tt.wantStatus {
t.Fatalf("status = %d, want %d; body = %q", recorder.Code, tt.wantStatus, recorder.Body.String())
}
if reader, ok := tt.body.(*countingReader); ok && reader.reads != 0 {
t.Fatalf("full queue read body %d time(s), want zero", reader.reads)
}
})
}
}
@@ -228,9 +252,13 @@ func TestUploadHTTPHandlerMapsSubmitErrors(t *testing.T) {
name string
err error
wantStatus int
wantBody string
}{
{name: "oversized", err: ingest.ErrUploadTooLarge, wantStatus: http.StatusRequestEntityTooLarge},
{name: "unsupported", err: ingest.ErrUnsupportedContentType, wantStatus: http.StatusUnsupportedMediaType},
{name: "full queue", err: UploadQueueFullError{QueueSize: 1}, wantStatus: http.StatusServiceUnavailable},
{name: "idempotency conflict", err: UploadIdempotencyConflictError{}, wantStatus: http.StatusConflict, wantBody: "different source manifest"},
{name: "idempotency in progress", err: UploadIdempotencyConflictError{Retryable: true}, wantStatus: http.StatusConflict, wantBody: `"retryable":true`},
{name: "malformed", err: errors.New("malformed archive"), wantStatus: http.StatusBadRequest},
}
for _, tt := range tests {
@@ -254,6 +282,9 @@ func TestUploadHTTPHandlerMapsSubmitErrors(t *testing.T) {
if recorder.Code != tt.wantStatus {
t.Fatalf("status = %d, want %d; body = %q", recorder.Code, tt.wantStatus, recorder.Body.String())
}
if tt.wantBody != "" && !strings.Contains(recorder.Body.String(), tt.wantBody) {
t.Fatalf("body = %q, want substring %q", recorder.Body.String(), tt.wantBody)
}
})
}
}