Stage uploads before accepting HTTP runs

internal/app/upload_http.go

@@ -1,13 +1,10 @@
 package app
 
 import (
-	"bytes"
 	"context"
 	"encoding/json"
 	"errors"
 	"fmt"
-	"io"
-	"mime"
 	"net/http"
 	"strings"
 
@@ -24,7 +21,6 @@ type uploadCoordinator interface {
 type uploadHTTPHandler struct {
 	coordinator uploadCoordinator
 	tokens      map[string]string
-	limits      map[string]int64
 }
 
 type uploadAcceptedResponse struct {
@@ -38,20 +34,18 @@ type httpErrorResponse struct {
 
 func newUploadHTTPHandler(ctx context.Context, cfg config.Config, environment config.Environment) (http.Handler, error) {
 	config.ApplyDefaults(&cfg)
-	tokens, limits, err := resolveUploadTokens(cfg, environment)
+	tokens, err := resolveUploadTokens(cfg, environment)
 	if err != nil {
 		return nil, err
 	}
 	return uploadHTTPHandler{
 		coordinator: NewUploadCoordinator(ctx, cfg),
 		tokens:      tokens,
-		limits:      limits,
 	}, nil
 }
 
-func resolveUploadTokens(cfg config.Config, environment config.Environment) (map[string]string, map[string]int64, error) {
+func resolveUploadTokens(cfg config.Config, environment config.Environment) (map[string]string, error) {
 	tokens := make(map[string]string)
-	limits := make(map[string]int64)
 	for _, pipeline := range cfg.Pipelines {
 		if pipeline.Source.Backend != config.BackendHTTPUpload {
 			continue
@@ -59,18 +53,17 @@ func resolveUploadTokens(cfg config.Config, environment config.Environment) (map
 		tokenName := pipeline.Source.Upload.TokenEnv
 		token, ok := environment.Lookup(tokenName)
 		if !ok {
-			return nil, nil, fmt.Errorf("upload token environment variable %s is not set", tokenName)
+			return nil, fmt.Errorf("upload token environment variable %s is not set", tokenName)
 		}
 		if token == "" {
-			return nil, nil, fmt.Errorf("upload token environment variable %s is empty", tokenName)
+			return nil, fmt.Errorf("upload token environment variable %s is empty", tokenName)
 		}
 		if existing, exists := tokens[token]; exists {
-			return nil, nil, fmt.Errorf("upload token environment variables for pipelines %s and %s resolve to the same value", existing, pipeline.ID)
+			return nil, fmt.Errorf("upload token environment variables for pipelines %s and %s resolve to the same value", existing, pipeline.ID)
 		}
 		tokens[token] = pipeline.ID
-		limits[pipeline.ID] = int64(*pipeline.Source.Upload.MaxUploadSize)
 	}
-	return tokens, limits, nil
+	return tokens, nil
 }
 
 func (handler uploadHTTPHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
@@ -101,7 +94,7 @@ func (handler uploadHTTPHandler) handleUpload(w http.ResponseWriter, r *http.Req
 		return
 	}
 	contentType := r.Header.Get("Content-Type")
-	if !supportedUploadContentType(contentType) {
+	if err := ingest.ValidateContentType(contentType); err != nil {
 		writeHTTPError(w, http.StatusUnsupportedMediaType, "unsupported content type")
 		return
 	}
@@ -109,19 +102,10 @@ func (handler uploadHTTPHandler) handleUpload(w http.ResponseWriter, r *http.Req
 		writeHTTPError(w, http.StatusServiceUnavailable, "upload queue is full")
 		return
 	}
-	body, err := readUploadBody(r.Body, handler.limits[pipelineID])
-	if err != nil {
-		if errors.Is(err, ingest.ErrUploadTooLarge) {
-			writeHTTPError(w, http.StatusRequestEntityTooLarge, "upload exceeds maximum size")
-			return
-		}
-		writeHTTPError(w, http.StatusBadRequest, "read upload body failed")
-		return
-	}
 	record, err := handler.coordinator.Submit(r.Context(), UploadRequest{
 		PipelineID:  pipelineID,
 		ContentType: contentType,
-		Body:        bytes.NewReader(body),
+		Body:        r.Body,
 	})
 	if err != nil {
 		writeUploadSubmitError(w, err)
@@ -160,31 +144,6 @@ func (handler uploadHTTPHandler) authenticate(header string) (string, bool) {
 	return pipelineID, ok
 }
 
-func supportedUploadContentType(contentType string) bool {
-	mediaType, _, err := mime.ParseMediaType(contentType)
-	if err != nil {
-		mediaType = contentType
-	}
-	switch mediaType {
-	case ingest.ContentTypeTar, ingest.ContentTypeGzip, ingest.ContentTypeXGzip:
-		return true
-	default:
-		return false
-	}
-}
-
-func readUploadBody(body io.Reader, maxSize int64) ([]byte, error) {
-	limited := &io.LimitedReader{R: body, N: maxSize + 1}
-	data, err := io.ReadAll(limited)
-	if err != nil {
-		return nil, err
-	}
-	if int64(len(data)) > maxSize {
-		return nil, ingest.ErrUploadTooLarge
-	}
-	return data, nil
-}
-
 func writeUploadSubmitError(w http.ResponseWriter, err error) {
 	switch {
 	case IsUploadQueueFull(err):