Add HTTP upload server and serve command

This commit is contained in:
2026-06-03 15:22:52 +00:00
parent f0c10210eb
commit 6beef58dbf
12 changed files with 942 additions and 10 deletions

62
internal/app/serve.go Normal file
View File

@@ -0,0 +1,62 @@
package app
import (
"context"
"errors"
"fmt"
"net"
"net/http"
"gitea.maximumdirect.net/eric/distributor/internal/config"
)
type ServeOptions struct {
ConfigPath string
}
func Serve(ctx context.Context, options ServeOptions) error {
if ctx == nil {
ctx = context.Background()
}
if err := ctx.Err(); err != nil {
return err
}
configPath := options.ConfigPath
if configPath == "" {
configPath = config.DefaultConfigPath
}
cfg, err := config.LoadFile(configPath)
if err != nil {
return err
}
secretLoad, err := config.LoadSecretEnvironment(cfg.Secrets.Directory, nil)
if err != nil {
return err
}
handler, err := newUploadHTTPHandler(ctx, cfg, secretLoad.Environment)
if err != nil {
return err
}
listener, err := net.Listen("tcp", cfg.Server.HTTP.Bind)
if err != nil {
return fmt.Errorf("bind HTTP server %q: %w", cfg.Server.HTTP.Bind, err)
}
defer listener.Close()
server := &http.Server{Handler: handler}
shutdownDone := make(chan struct{})
go func() {
defer close(shutdownDone)
<-ctx.Done()
_ = server.Shutdown(context.Background())
}()
err = server.Serve(listener)
if errors.Is(err, http.ErrServerClosed) {
<-shutdownDone
return nil
}
return err
}

View File

@@ -201,6 +201,13 @@ func (coordinator *UploadCoordinator) Expire() []UploadRunRecord {
return coordinator.expireLocked(coordinator.now().UTC())
}
func (coordinator *UploadCoordinator) CanAccept() bool {
coordinator.mu.Lock()
defer coordinator.mu.Unlock()
coordinator.expireLocked(coordinator.now().UTC())
return len(coordinator.pending) < coordinator.queueSize
}
func (coordinator *UploadCoordinator) QueueDepth() int {
coordinator.mu.Lock()
defer coordinator.mu.Unlock()

209
internal/app/upload_http.go Normal file
View File

@@ -0,0 +1,209 @@
package app
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"mime"
"net/http"
"strings"
"gitea.maximumdirect.net/eric/distributor/internal/config"
"gitea.maximumdirect.net/eric/distributor/internal/ingest"
)
type uploadCoordinator interface {
CanAccept() bool
Submit(context.Context, UploadRequest) (UploadRunRecord, error)
Status(UploadRunID) (UploadRunRecord, bool)
}
type uploadHTTPHandler struct {
coordinator uploadCoordinator
tokens map[string]string
limits map[string]int64
}
type uploadAcceptedResponse struct {
RunID UploadRunID `json:"run_id"`
Status UploadStatus `json:"status"`
}
type httpErrorResponse struct {
Error string `json:"error"`
}
func newUploadHTTPHandler(ctx context.Context, cfg config.Config, environment config.Environment) (http.Handler, error) {
config.ApplyDefaults(&cfg)
tokens, limits, err := resolveUploadTokens(cfg, environment)
if err != nil {
return nil, err
}
return uploadHTTPHandler{
coordinator: NewUploadCoordinator(ctx, cfg),
tokens: tokens,
limits: limits,
}, nil
}
func resolveUploadTokens(cfg config.Config, environment config.Environment) (map[string]string, map[string]int64, error) {
tokens := make(map[string]string)
limits := make(map[string]int64)
for _, pipeline := range cfg.Pipelines {
if pipeline.Source.Backend != config.BackendHTTPUpload {
continue
}
tokenName := pipeline.Source.Upload.TokenEnv
token, ok := environment.Lookup(tokenName)
if !ok {
return nil, nil, fmt.Errorf("upload token environment variable %s is not set", tokenName)
}
if token == "" {
return nil, nil, fmt.Errorf("upload token environment variable %s is empty", tokenName)
}
if existing, exists := tokens[token]; exists {
return nil, nil, fmt.Errorf("upload token environment variables for pipelines %s and %s resolve to the same value", existing, pipeline.ID)
}
tokens[token] = pipeline.ID
limits[pipeline.ID] = int64(*pipeline.Source.Upload.MaxUploadSize)
}
return tokens, limits, nil
}
func (handler uploadHTTPHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
switch {
case r.Method == http.MethodGet && r.URL.Path == "/healthz":
handler.handleHealth(w)
case r.Method == http.MethodPost && r.URL.Path == "/upload":
handler.handleUpload(w, r)
case r.Method == http.MethodGet && strings.HasPrefix(r.URL.Path, "/runs/"):
handler.handleRunStatus(w, r)
default:
writeHTTPError(w, http.StatusNotFound, "not found")
}
}
func (handler uploadHTTPHandler) handleHealth(w http.ResponseWriter) {
writeJSON(w, http.StatusOK, map[string]string{"status": "ok"})
}
func (handler uploadHTTPHandler) handleUpload(w http.ResponseWriter, r *http.Request) {
if r.URL.Query().Has("pipeline") || r.URL.Query().Has("pipeline_id") {
writeHTTPError(w, http.StatusBadRequest, "pipeline id is not accepted")
return
}
pipelineID, ok := handler.authenticate(r.Header.Get("Authorization"))
if !ok {
writeHTTPError(w, http.StatusUnauthorized, "unauthorized")
return
}
contentType := r.Header.Get("Content-Type")
if !supportedUploadContentType(contentType) {
writeHTTPError(w, http.StatusUnsupportedMediaType, "unsupported content type")
return
}
if !handler.coordinator.CanAccept() {
writeHTTPError(w, http.StatusServiceUnavailable, "upload queue is full")
return
}
body, err := readUploadBody(r.Body, handler.limits[pipelineID])
if err != nil {
if errors.Is(err, ingest.ErrUploadTooLarge) {
writeHTTPError(w, http.StatusRequestEntityTooLarge, "upload exceeds maximum size")
return
}
writeHTTPError(w, http.StatusBadRequest, "read upload body failed")
return
}
record, err := handler.coordinator.Submit(r.Context(), UploadRequest{
PipelineID: pipelineID,
ContentType: contentType,
Body: bytes.NewReader(body),
})
if err != nil {
writeUploadSubmitError(w, err)
return
}
writeJSON(w, http.StatusAccepted, uploadAcceptedResponse{
RunID: record.ID,
Status: UploadStatusAccepted,
})
}
func (handler uploadHTTPHandler) handleRunStatus(w http.ResponseWriter, r *http.Request) {
rawRunID := strings.TrimPrefix(r.URL.Path, "/runs/")
if rawRunID == "" || strings.Contains(rawRunID, "/") {
writeHTTPError(w, http.StatusNotFound, "not found")
return
}
record, ok := handler.coordinator.Status(UploadRunID(rawRunID))
if !ok {
writeHTTPError(w, http.StatusNotFound, "run not found")
return
}
writeJSON(w, http.StatusOK, record)
}
func (handler uploadHTTPHandler) authenticate(header string) (string, bool) {
const prefix = "Bearer "
if !strings.HasPrefix(header, prefix) {
return "", false
}
token := strings.TrimSpace(strings.TrimPrefix(header, prefix))
if token == "" {
return "", false
}
pipelineID, ok := handler.tokens[token]
return pipelineID, ok
}
func supportedUploadContentType(contentType string) bool {
mediaType, _, err := mime.ParseMediaType(contentType)
if err != nil {
mediaType = contentType
}
switch mediaType {
case ingest.ContentTypeTar, ingest.ContentTypeGzip, ingest.ContentTypeXGzip:
return true
default:
return false
}
}
func readUploadBody(body io.Reader, maxSize int64) ([]byte, error) {
limited := &io.LimitedReader{R: body, N: maxSize + 1}
data, err := io.ReadAll(limited)
if err != nil {
return nil, err
}
if int64(len(data)) > maxSize {
return nil, ingest.ErrUploadTooLarge
}
return data, nil
}
func writeUploadSubmitError(w http.ResponseWriter, err error) {
switch {
case IsUploadQueueFull(err):
writeHTTPError(w, http.StatusServiceUnavailable, "upload queue is full")
case errors.Is(err, ingest.ErrUploadTooLarge):
writeHTTPError(w, http.StatusRequestEntityTooLarge, "upload exceeds maximum size")
case errors.Is(err, ingest.ErrUnsupportedContentType):
writeHTTPError(w, http.StatusUnsupportedMediaType, "unsupported content type")
default:
writeHTTPError(w, http.StatusBadRequest, "upload rejected")
}
}
func writeHTTPError(w http.ResponseWriter, status int, message string) {
writeJSON(w, status, httpErrorResponse{Error: message})
}
func writeJSON(w http.ResponseWriter, status int, value any) {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(status)
_ = json.NewEncoder(w).Encode(value)
}

View File

@@ -0,0 +1,331 @@
package app
import (
"context"
"encoding/json"
"errors"
"io"
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
"gitea.maximumdirect.net/eric/distributor/internal/config"
)
type fakeUploadCoordinator struct {
canAccept bool
submit func(context.Context, UploadRequest) (UploadRunRecord, error)
status func(UploadRunID) (UploadRunRecord, bool)
}
func (fake fakeUploadCoordinator) CanAccept() bool {
return fake.canAccept
}
func (fake fakeUploadCoordinator) Submit(ctx context.Context, request UploadRequest) (UploadRunRecord, error) {
if fake.submit == nil {
return UploadRunRecord{}, errors.New("unexpected submit")
}
return fake.submit(ctx, request)
}
func (fake fakeUploadCoordinator) Status(runID UploadRunID) (UploadRunRecord, bool) {
if fake.status == nil {
return UploadRunRecord{}, false
}
return fake.status(runID)
}
func TestResolveUploadTokensFailsForMissingAndDuplicateTokens(t *testing.T) {
cfg := uploadHTTPTestConfig()
_, _, err := resolveUploadTokens(cfg, config.NewEnvironment(nil, func(string) (string, bool) {
return "", false
}))
if err == nil || !strings.Contains(err.Error(), "UPLOAD_TOKEN") {
t.Fatalf("resolveUploadTokens() error = %v, want missing UPLOAD_TOKEN", err)
}
cfg.Pipelines = append(cfg.Pipelines, config.Pipeline{
ID: "weekly",
Source: config.Backend{
Backend: config.BackendHTTPUpload,
Upload: config.HTTPUpload{TokenEnv: "OTHER_UPLOAD_TOKEN"},
},
Destinations: cfg.Pipelines[0].Destinations,
})
config.ApplyDefaults(&cfg)
secret := "super-secret-token"
_, _, err = resolveUploadTokens(cfg, uploadHTTPTestEnvironment(map[string]string{
"UPLOAD_TOKEN": secret,
"OTHER_UPLOAD_TOKEN": secret,
}))
if err == nil {
t.Fatal("resolveUploadTokens() error = nil, want duplicate token error")
}
if strings.Contains(err.Error(), secret) {
t.Fatalf("duplicate token error exposed secret value: %q", err)
}
}
func TestNewUploadHTTPHandlerAcceptsDefaultedConfig(t *testing.T) {
cfg := uploadHTTPTestConfig()
cfg.Server.HTTP.Bind = ""
cfg.Server.HTTP.StagingRoot = ""
cfg.Server.HTTP.MaxUploadSize = nil
cfg.Server.HTTP.QueueSize = 0
cfg.Server.HTTP.MaxConcurrency = 0
cfg.Server.HTTP.Retention = nil
cfg.Pipelines[0].Source.Upload.StagingPath = ""
cfg.Pipelines[0].Source.Upload.MaxUploadSize = nil
handler, err := newUploadHTTPHandler(context.Background(), cfg, uploadHTTPTestEnvironment(map[string]string{
"UPLOAD_TOKEN": "secret",
}))
if err != nil {
t.Fatalf("newUploadHTTPHandler() error = %v", err)
}
if handler == nil {
t.Fatal("newUploadHTTPHandler() = nil")
}
}
func TestUploadHTTPHandlerAuthenticatesAndAcceptsUpload(t *testing.T) {
var submitted UploadRequest
handler := uploadHTTPHandler{
coordinator: fakeUploadCoordinator{
canAccept: true,
submit: func(_ context.Context, request UploadRequest) (UploadRunRecord, error) {
submitted = request
body, err := io.ReadAll(request.Body)
if err != nil {
t.Fatalf("read submitted body: %v", err)
}
if string(body) != "archive" {
t.Fatalf("submitted body = %q, want archive", body)
}
return UploadRunRecord{ID: "reports.20260603T120000Z.abcdef12", Status: UploadStatusAccepted}, nil
},
},
tokens: map[string]string{"valid-token": "reports"},
limits: map[string]int64{"reports": 1024},
}
recorder := httptest.NewRecorder()
request := httptest.NewRequest(http.MethodPost, "/upload", strings.NewReader("archive"))
request.Header.Set("Authorization", "Bearer valid-token")
request.Header.Set("Content-Type", "application/x-tar")
handler.ServeHTTP(recorder, request)
if recorder.Code != http.StatusAccepted {
t.Fatalf("status = %d, want %d; body = %q", recorder.Code, http.StatusAccepted, recorder.Body.String())
}
if submitted.PipelineID != "reports" {
t.Fatalf("submitted pipeline = %q, want reports", submitted.PipelineID)
}
var response uploadAcceptedResponse
if err := json.Unmarshal(recorder.Body.Bytes(), &response); err != nil {
t.Fatalf("decode response: %v", err)
}
if response.RunID != "reports.20260603T120000Z.abcdef12" || response.Status != UploadStatusAccepted {
t.Fatalf("response = %#v, want accepted run id", response)
}
if strings.Contains(recorder.Body.String(), "valid-token") {
t.Fatalf("response exposed token: %q", recorder.Body.String())
}
}
func TestUploadHTTPHandlerRejectsUnauthorizedRequests(t *testing.T) {
handler := uploadHTTPHandler{
coordinator: fakeUploadCoordinator{canAccept: true},
tokens: map[string]string{"valid-token": "reports"},
limits: map[string]int64{"reports": 1024},
}
for _, authHeader := range []string{"", "Bearer wrong-token"} {
recorder := httptest.NewRecorder()
request := httptest.NewRequest(http.MethodPost, "/upload", strings.NewReader("archive"))
request.Header.Set("Authorization", authHeader)
request.Header.Set("Content-Type", "application/x-tar")
handler.ServeHTTP(recorder, request)
if recorder.Code != http.StatusUnauthorized {
t.Fatalf("auth %q status = %d, want %d", authHeader, recorder.Code, http.StatusUnauthorized)
}
if strings.Contains(recorder.Body.String(), "valid-token") || strings.Contains(recorder.Body.String(), "wrong-token") {
t.Fatalf("unauthorized response exposed token: %q", recorder.Body.String())
}
}
}
func TestUploadHTTPHandlerRejectsUnsupportedOversizedFullQueueAndPipelineID(t *testing.T) {
tests := []struct {
name string
canAccept bool
url string
contentType string
body io.Reader
wantStatus int
}{
{
name: "unsupported content type",
canAccept: true,
url: "/upload",
contentType: "application/zip",
body: strings.NewReader("archive"),
wantStatus: http.StatusUnsupportedMediaType,
},
{
name: "oversized",
canAccept: true,
url: "/upload",
contentType: "application/x-tar",
body: strings.NewReader("too-large"),
wantStatus: http.StatusRequestEntityTooLarge,
},
{
name: "full queue",
canAccept: false,
url: "/upload",
contentType: "application/x-tar",
body: &countingReader{reader: strings.NewReader("archive")},
wantStatus: http.StatusServiceUnavailable,
},
{
name: "submitted pipeline id",
canAccept: true,
url: "/upload?pipeline_id=reports",
contentType: "application/x-tar",
body: strings.NewReader("archive"),
wantStatus: http.StatusBadRequest,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
handler := uploadHTTPHandler{
coordinator: fakeUploadCoordinator{
canAccept: tt.canAccept,
submit: func(context.Context, UploadRequest) (UploadRunRecord, error) {
t.Fatal("Submit should not be called")
return UploadRunRecord{}, nil
},
},
tokens: map[string]string{"valid-token": "reports"},
limits: map[string]int64{"reports": 4},
}
recorder := httptest.NewRecorder()
request := httptest.NewRequest(http.MethodPost, tt.url, tt.body)
request.Header.Set("Authorization", "Bearer valid-token")
request.Header.Set("Content-Type", tt.contentType)
handler.ServeHTTP(recorder, request)
if recorder.Code != tt.wantStatus {
t.Fatalf("status = %d, want %d; body = %q", recorder.Code, tt.wantStatus, recorder.Body.String())
}
if reader, ok := tt.body.(*countingReader); ok && reader.reads != 0 {
t.Fatalf("full queue read body %d time(s), want zero", reader.reads)
}
})
}
}
func TestUploadHTTPHandlerRunStatusAndHealth(t *testing.T) {
finishedAt := time.Date(2026, 6, 3, 12, 0, 0, 0, time.UTC)
handler := uploadHTTPHandler{
coordinator: fakeUploadCoordinator{
canAccept: true,
status: func(runID UploadRunID) (UploadRunRecord, bool) {
if runID != "reports.20260603T120000Z.abcdef12" {
return UploadRunRecord{}, false
}
return UploadRunRecord{
ID: runID,
PipelineID: "reports",
Status: UploadStatusSucceeded,
FinishedAt: &finishedAt,
}, true
},
},
tokens: map[string]string{"valid-token": "reports"},
limits: map[string]int64{"reports": 1024},
}
recorder := httptest.NewRecorder()
handler.ServeHTTP(recorder, httptest.NewRequest(http.MethodGet, "/healthz", nil))
if recorder.Code != http.StatusOK {
t.Fatalf("health status = %d, want %d", recorder.Code, http.StatusOK)
}
recorder = httptest.NewRecorder()
handler.ServeHTTP(recorder, httptest.NewRequest(http.MethodGet, "/runs/reports.20260603T120000Z.abcdef12", nil))
if recorder.Code != http.StatusOK {
t.Fatalf("run status = %d, want %d; body = %q", recorder.Code, http.StatusOK, recorder.Body.String())
}
var record UploadRunRecord
if err := json.Unmarshal(recorder.Body.Bytes(), &record); err != nil {
t.Fatalf("decode run status: %v", err)
}
if record.ID != "reports.20260603T120000Z.abcdef12" || record.Status != UploadStatusSucceeded {
t.Fatalf("record = %#v, want succeeded run status", record)
}
recorder = httptest.NewRecorder()
handler.ServeHTTP(recorder, httptest.NewRequest(http.MethodGet, "/runs/unknown", nil))
if recorder.Code != http.StatusNotFound {
t.Fatalf("unknown run status = %d, want %d", recorder.Code, http.StatusNotFound)
}
}
type countingReader struct {
reader io.Reader
reads int
}
func (reader *countingReader) Read(data []byte) (int, error) {
reader.reads++
return reader.reader.Read(data)
}
func uploadHTTPTestConfig() config.Config {
size := config.ByteSize(1024)
retention := config.Duration(24 * time.Hour)
cfg := config.Config{
Server: config.Server{HTTP: config.HTTPServer{
Bind: config.DefaultHTTPBind,
StagingRoot: "/tmp/distributor-test",
MaxUploadSize: &size,
QueueSize: 2,
MaxConcurrency: 1,
Retention: &retention,
}},
Pipelines: []config.Pipeline{{
ID: "reports",
Source: config.Backend{
Backend: config.BackendHTTPUpload,
Upload: config.HTTPUpload{
TokenEnv: "UPLOAD_TOKEN",
StagingPath: "/tmp/distributor-test/reports",
MaxUploadSize: &size,
},
},
Destinations: []config.Destination{{
ID: "local",
Backend: config.BackendLocal,
Path: "/tmp/distributor-output",
Publish: &config.PublishPolicy{Source: true},
}},
}},
}
config.ApplyDefaults(&cfg)
return cfg
}
func uploadHTTPTestEnvironment(values map[string]string) config.Environment {
return config.NewEnvironment(values, func(string) (string, bool) {
return "", false
})
}

View File

@@ -29,6 +29,8 @@ func Execute(ctx context.Context, args []string, stdout, stderr io.Writer) int {
return versionCommand(ctx, args[1:], stdout, stderr)
case "run":
return runCommand(ctx, args[1:], stdout, stderr)
case "serve":
return serveCommand(ctx, args[1:], stdout, stderr)
case "validate":
return validateCommand(ctx, args[1:], stdout, stderr)
case "inspect":
@@ -51,6 +53,7 @@ Usage:
Commands:
version Print version information
run Run configured distribution pipelines
serve Run the HTTP upload server
validate Validate a source bundle or bundle tree
inspect Inspect bundles or distributor state
manifest Create source bundle manifests

View File

@@ -11,6 +11,7 @@ import (
"strings"
"testing"
"gitea.maximumdirect.net/eric/distributor/internal/app"
"gitea.maximumdirect.net/eric/distributor/internal/storage"
"gitea.maximumdirect.net/eric/distributor/internal/testutil"
producerbundle "gitea.maximumdirect.net/eric/distributor/pkg/bundle"
@@ -92,6 +93,34 @@ func TestExecuteVersionJSON(t *testing.T) {
}
}
func TestExecuteServeParsesConfig(t *testing.T) {
originalServeApp := serveApp
defer func() {
serveApp = originalServeApp
}()
var gotOptions app.ServeOptions
serveApp = func(_ context.Context, options app.ServeOptions) error {
gotOptions = options
return nil
}
var stdout, stderr bytes.Buffer
code := Execute(context.Background(), []string{"serve", "--config", "config.yml"}, &stdout, &stderr)
if code != exitOK {
t.Fatalf("exit code = %d, want %d; stderr = %q", code, exitOK, stderr.String())
}
if gotOptions.ConfigPath != "config.yml" {
t.Fatalf("ConfigPath = %q, want config.yml", gotOptions.ConfigPath)
}
if stdout.Len() != 0 {
t.Fatalf("stdout = %q, want empty", stdout.String())
}
if stderr.Len() != 0 {
t.Fatalf("stderr = %q, want empty", stderr.String())
}
}
func TestExecuteRejectsInvalidFormat(t *testing.T) {
var stdout, stderr bytes.Buffer

46
internal/cli/serve.go Normal file
View File

@@ -0,0 +1,46 @@
package cli
import (
"context"
"flag"
"fmt"
"io"
"gitea.maximumdirect.net/eric/distributor/internal/app"
)
var serveApp = app.Serve
func serveCommand(ctx context.Context, args []string, stdout, stderr io.Writer) int {
if hasHelp(args) {
printServeHelp(stdout)
return exitOK
}
flags := flag.NewFlagSet("serve", flag.ContinueOnError)
flags.SetOutput(stderr)
configPath := flags.String("config", "", "path to config file")
if err := flags.Parse(args); err != nil {
return exitUsage
}
if rejectPositionalArgs(stderr, "serve", flags.Args()) {
return exitUsage
}
if err := serveApp(ctx, app.ServeOptions{ConfigPath: *configPath}); err != nil {
return fail(stderr, err)
}
return exitOK
}
func printServeHelp(w io.Writer) {
fmt.Fprint(w, `Usage:
distributor serve --config <path>
Options:
--config <path> Path to config file
Serve loads configured HTTP upload sources, resolves upload tokens through the
configured secret environment, and starts the HTTP upload API.
`)
}