Add HTTP upload server and serve command
This commit is contained in:
62
internal/app/serve.go
Normal file
62
internal/app/serve.go
Normal file
@@ -0,0 +1,62 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/http"
|
||||
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/config"
|
||||
)
|
||||
|
||||
type ServeOptions struct {
|
||||
ConfigPath string
|
||||
}
|
||||
|
||||
func Serve(ctx context.Context, options ServeOptions) error {
|
||||
if ctx == nil {
|
||||
ctx = context.Background()
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
configPath := options.ConfigPath
|
||||
if configPath == "" {
|
||||
configPath = config.DefaultConfigPath
|
||||
}
|
||||
cfg, err := config.LoadFile(configPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
secretLoad, err := config.LoadSecretEnvironment(cfg.Secrets.Directory, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
handler, err := newUploadHTTPHandler(ctx, cfg, secretLoad.Environment)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
listener, err := net.Listen("tcp", cfg.Server.HTTP.Bind)
|
||||
if err != nil {
|
||||
return fmt.Errorf("bind HTTP server %q: %w", cfg.Server.HTTP.Bind, err)
|
||||
}
|
||||
defer listener.Close()
|
||||
|
||||
server := &http.Server{Handler: handler}
|
||||
shutdownDone := make(chan struct{})
|
||||
go func() {
|
||||
defer close(shutdownDone)
|
||||
<-ctx.Done()
|
||||
_ = server.Shutdown(context.Background())
|
||||
}()
|
||||
|
||||
err = server.Serve(listener)
|
||||
if errors.Is(err, http.ErrServerClosed) {
|
||||
<-shutdownDone
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
@@ -201,6 +201,13 @@ func (coordinator *UploadCoordinator) Expire() []UploadRunRecord {
|
||||
return coordinator.expireLocked(coordinator.now().UTC())
|
||||
}
|
||||
|
||||
func (coordinator *UploadCoordinator) CanAccept() bool {
|
||||
coordinator.mu.Lock()
|
||||
defer coordinator.mu.Unlock()
|
||||
coordinator.expireLocked(coordinator.now().UTC())
|
||||
return len(coordinator.pending) < coordinator.queueSize
|
||||
}
|
||||
|
||||
func (coordinator *UploadCoordinator) QueueDepth() int {
|
||||
coordinator.mu.Lock()
|
||||
defer coordinator.mu.Unlock()
|
||||
|
||||
209
internal/app/upload_http.go
Normal file
209
internal/app/upload_http.go
Normal file
@@ -0,0 +1,209 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"mime"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/config"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/ingest"
|
||||
)
|
||||
|
||||
type uploadCoordinator interface {
|
||||
CanAccept() bool
|
||||
Submit(context.Context, UploadRequest) (UploadRunRecord, error)
|
||||
Status(UploadRunID) (UploadRunRecord, bool)
|
||||
}
|
||||
|
||||
type uploadHTTPHandler struct {
|
||||
coordinator uploadCoordinator
|
||||
tokens map[string]string
|
||||
limits map[string]int64
|
||||
}
|
||||
|
||||
type uploadAcceptedResponse struct {
|
||||
RunID UploadRunID `json:"run_id"`
|
||||
Status UploadStatus `json:"status"`
|
||||
}
|
||||
|
||||
type httpErrorResponse struct {
|
||||
Error string `json:"error"`
|
||||
}
|
||||
|
||||
func newUploadHTTPHandler(ctx context.Context, cfg config.Config, environment config.Environment) (http.Handler, error) {
|
||||
config.ApplyDefaults(&cfg)
|
||||
tokens, limits, err := resolveUploadTokens(cfg, environment)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return uploadHTTPHandler{
|
||||
coordinator: NewUploadCoordinator(ctx, cfg),
|
||||
tokens: tokens,
|
||||
limits: limits,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func resolveUploadTokens(cfg config.Config, environment config.Environment) (map[string]string, map[string]int64, error) {
|
||||
tokens := make(map[string]string)
|
||||
limits := make(map[string]int64)
|
||||
for _, pipeline := range cfg.Pipelines {
|
||||
if pipeline.Source.Backend != config.BackendHTTPUpload {
|
||||
continue
|
||||
}
|
||||
tokenName := pipeline.Source.Upload.TokenEnv
|
||||
token, ok := environment.Lookup(tokenName)
|
||||
if !ok {
|
||||
return nil, nil, fmt.Errorf("upload token environment variable %s is not set", tokenName)
|
||||
}
|
||||
if token == "" {
|
||||
return nil, nil, fmt.Errorf("upload token environment variable %s is empty", tokenName)
|
||||
}
|
||||
if existing, exists := tokens[token]; exists {
|
||||
return nil, nil, fmt.Errorf("upload token environment variables for pipelines %s and %s resolve to the same value", existing, pipeline.ID)
|
||||
}
|
||||
tokens[token] = pipeline.ID
|
||||
limits[pipeline.ID] = int64(*pipeline.Source.Upload.MaxUploadSize)
|
||||
}
|
||||
return tokens, limits, nil
|
||||
}
|
||||
|
||||
func (handler uploadHTTPHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
switch {
|
||||
case r.Method == http.MethodGet && r.URL.Path == "/healthz":
|
||||
handler.handleHealth(w)
|
||||
case r.Method == http.MethodPost && r.URL.Path == "/upload":
|
||||
handler.handleUpload(w, r)
|
||||
case r.Method == http.MethodGet && strings.HasPrefix(r.URL.Path, "/runs/"):
|
||||
handler.handleRunStatus(w, r)
|
||||
default:
|
||||
writeHTTPError(w, http.StatusNotFound, "not found")
|
||||
}
|
||||
}
|
||||
|
||||
func (handler uploadHTTPHandler) handleHealth(w http.ResponseWriter) {
|
||||
writeJSON(w, http.StatusOK, map[string]string{"status": "ok"})
|
||||
}
|
||||
|
||||
func (handler uploadHTTPHandler) handleUpload(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Query().Has("pipeline") || r.URL.Query().Has("pipeline_id") {
|
||||
writeHTTPError(w, http.StatusBadRequest, "pipeline id is not accepted")
|
||||
return
|
||||
}
|
||||
pipelineID, ok := handler.authenticate(r.Header.Get("Authorization"))
|
||||
if !ok {
|
||||
writeHTTPError(w, http.StatusUnauthorized, "unauthorized")
|
||||
return
|
||||
}
|
||||
contentType := r.Header.Get("Content-Type")
|
||||
if !supportedUploadContentType(contentType) {
|
||||
writeHTTPError(w, http.StatusUnsupportedMediaType, "unsupported content type")
|
||||
return
|
||||
}
|
||||
if !handler.coordinator.CanAccept() {
|
||||
writeHTTPError(w, http.StatusServiceUnavailable, "upload queue is full")
|
||||
return
|
||||
}
|
||||
body, err := readUploadBody(r.Body, handler.limits[pipelineID])
|
||||
if err != nil {
|
||||
if errors.Is(err, ingest.ErrUploadTooLarge) {
|
||||
writeHTTPError(w, http.StatusRequestEntityTooLarge, "upload exceeds maximum size")
|
||||
return
|
||||
}
|
||||
writeHTTPError(w, http.StatusBadRequest, "read upload body failed")
|
||||
return
|
||||
}
|
||||
record, err := handler.coordinator.Submit(r.Context(), UploadRequest{
|
||||
PipelineID: pipelineID,
|
||||
ContentType: contentType,
|
||||
Body: bytes.NewReader(body),
|
||||
})
|
||||
if err != nil {
|
||||
writeUploadSubmitError(w, err)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusAccepted, uploadAcceptedResponse{
|
||||
RunID: record.ID,
|
||||
Status: UploadStatusAccepted,
|
||||
})
|
||||
}
|
||||
|
||||
func (handler uploadHTTPHandler) handleRunStatus(w http.ResponseWriter, r *http.Request) {
|
||||
rawRunID := strings.TrimPrefix(r.URL.Path, "/runs/")
|
||||
if rawRunID == "" || strings.Contains(rawRunID, "/") {
|
||||
writeHTTPError(w, http.StatusNotFound, "not found")
|
||||
return
|
||||
}
|
||||
record, ok := handler.coordinator.Status(UploadRunID(rawRunID))
|
||||
if !ok {
|
||||
writeHTTPError(w, http.StatusNotFound, "run not found")
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, record)
|
||||
}
|
||||
|
||||
func (handler uploadHTTPHandler) authenticate(header string) (string, bool) {
|
||||
const prefix = "Bearer "
|
||||
if !strings.HasPrefix(header, prefix) {
|
||||
return "", false
|
||||
}
|
||||
token := strings.TrimSpace(strings.TrimPrefix(header, prefix))
|
||||
if token == "" {
|
||||
return "", false
|
||||
}
|
||||
pipelineID, ok := handler.tokens[token]
|
||||
return pipelineID, ok
|
||||
}
|
||||
|
||||
func supportedUploadContentType(contentType string) bool {
|
||||
mediaType, _, err := mime.ParseMediaType(contentType)
|
||||
if err != nil {
|
||||
mediaType = contentType
|
||||
}
|
||||
switch mediaType {
|
||||
case ingest.ContentTypeTar, ingest.ContentTypeGzip, ingest.ContentTypeXGzip:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func readUploadBody(body io.Reader, maxSize int64) ([]byte, error) {
|
||||
limited := &io.LimitedReader{R: body, N: maxSize + 1}
|
||||
data, err := io.ReadAll(limited)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if int64(len(data)) > maxSize {
|
||||
return nil, ingest.ErrUploadTooLarge
|
||||
}
|
||||
return data, nil
|
||||
}
|
||||
|
||||
func writeUploadSubmitError(w http.ResponseWriter, err error) {
|
||||
switch {
|
||||
case IsUploadQueueFull(err):
|
||||
writeHTTPError(w, http.StatusServiceUnavailable, "upload queue is full")
|
||||
case errors.Is(err, ingest.ErrUploadTooLarge):
|
||||
writeHTTPError(w, http.StatusRequestEntityTooLarge, "upload exceeds maximum size")
|
||||
case errors.Is(err, ingest.ErrUnsupportedContentType):
|
||||
writeHTTPError(w, http.StatusUnsupportedMediaType, "unsupported content type")
|
||||
default:
|
||||
writeHTTPError(w, http.StatusBadRequest, "upload rejected")
|
||||
}
|
||||
}
|
||||
|
||||
func writeHTTPError(w http.ResponseWriter, status int, message string) {
|
||||
writeJSON(w, status, httpErrorResponse{Error: message})
|
||||
}
|
||||
|
||||
func writeJSON(w http.ResponseWriter, status int, value any) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(status)
|
||||
_ = json.NewEncoder(w).Encode(value)
|
||||
}
|
||||
331
internal/app/upload_http_test.go
Normal file
331
internal/app/upload_http_test.go
Normal file
@@ -0,0 +1,331 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/config"
|
||||
)
|
||||
|
||||
type fakeUploadCoordinator struct {
|
||||
canAccept bool
|
||||
submit func(context.Context, UploadRequest) (UploadRunRecord, error)
|
||||
status func(UploadRunID) (UploadRunRecord, bool)
|
||||
}
|
||||
|
||||
func (fake fakeUploadCoordinator) CanAccept() bool {
|
||||
return fake.canAccept
|
||||
}
|
||||
|
||||
func (fake fakeUploadCoordinator) Submit(ctx context.Context, request UploadRequest) (UploadRunRecord, error) {
|
||||
if fake.submit == nil {
|
||||
return UploadRunRecord{}, errors.New("unexpected submit")
|
||||
}
|
||||
return fake.submit(ctx, request)
|
||||
}
|
||||
|
||||
func (fake fakeUploadCoordinator) Status(runID UploadRunID) (UploadRunRecord, bool) {
|
||||
if fake.status == nil {
|
||||
return UploadRunRecord{}, false
|
||||
}
|
||||
return fake.status(runID)
|
||||
}
|
||||
|
||||
func TestResolveUploadTokensFailsForMissingAndDuplicateTokens(t *testing.T) {
|
||||
cfg := uploadHTTPTestConfig()
|
||||
|
||||
_, _, err := resolveUploadTokens(cfg, config.NewEnvironment(nil, func(string) (string, bool) {
|
||||
return "", false
|
||||
}))
|
||||
if err == nil || !strings.Contains(err.Error(), "UPLOAD_TOKEN") {
|
||||
t.Fatalf("resolveUploadTokens() error = %v, want missing UPLOAD_TOKEN", err)
|
||||
}
|
||||
|
||||
cfg.Pipelines = append(cfg.Pipelines, config.Pipeline{
|
||||
ID: "weekly",
|
||||
Source: config.Backend{
|
||||
Backend: config.BackendHTTPUpload,
|
||||
Upload: config.HTTPUpload{TokenEnv: "OTHER_UPLOAD_TOKEN"},
|
||||
},
|
||||
Destinations: cfg.Pipelines[0].Destinations,
|
||||
})
|
||||
config.ApplyDefaults(&cfg)
|
||||
secret := "super-secret-token"
|
||||
_, _, err = resolveUploadTokens(cfg, uploadHTTPTestEnvironment(map[string]string{
|
||||
"UPLOAD_TOKEN": secret,
|
||||
"OTHER_UPLOAD_TOKEN": secret,
|
||||
}))
|
||||
if err == nil {
|
||||
t.Fatal("resolveUploadTokens() error = nil, want duplicate token error")
|
||||
}
|
||||
if strings.Contains(err.Error(), secret) {
|
||||
t.Fatalf("duplicate token error exposed secret value: %q", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewUploadHTTPHandlerAcceptsDefaultedConfig(t *testing.T) {
|
||||
cfg := uploadHTTPTestConfig()
|
||||
cfg.Server.HTTP.Bind = ""
|
||||
cfg.Server.HTTP.StagingRoot = ""
|
||||
cfg.Server.HTTP.MaxUploadSize = nil
|
||||
cfg.Server.HTTP.QueueSize = 0
|
||||
cfg.Server.HTTP.MaxConcurrency = 0
|
||||
cfg.Server.HTTP.Retention = nil
|
||||
cfg.Pipelines[0].Source.Upload.StagingPath = ""
|
||||
cfg.Pipelines[0].Source.Upload.MaxUploadSize = nil
|
||||
|
||||
handler, err := newUploadHTTPHandler(context.Background(), cfg, uploadHTTPTestEnvironment(map[string]string{
|
||||
"UPLOAD_TOKEN": "secret",
|
||||
}))
|
||||
if err != nil {
|
||||
t.Fatalf("newUploadHTTPHandler() error = %v", err)
|
||||
}
|
||||
if handler == nil {
|
||||
t.Fatal("newUploadHTTPHandler() = nil")
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadHTTPHandlerAuthenticatesAndAcceptsUpload(t *testing.T) {
|
||||
var submitted UploadRequest
|
||||
handler := uploadHTTPHandler{
|
||||
coordinator: fakeUploadCoordinator{
|
||||
canAccept: true,
|
||||
submit: func(_ context.Context, request UploadRequest) (UploadRunRecord, error) {
|
||||
submitted = request
|
||||
body, err := io.ReadAll(request.Body)
|
||||
if err != nil {
|
||||
t.Fatalf("read submitted body: %v", err)
|
||||
}
|
||||
if string(body) != "archive" {
|
||||
t.Fatalf("submitted body = %q, want archive", body)
|
||||
}
|
||||
return UploadRunRecord{ID: "reports.20260603T120000Z.abcdef12", Status: UploadStatusAccepted}, nil
|
||||
},
|
||||
},
|
||||
tokens: map[string]string{"valid-token": "reports"},
|
||||
limits: map[string]int64{"reports": 1024},
|
||||
}
|
||||
recorder := httptest.NewRecorder()
|
||||
request := httptest.NewRequest(http.MethodPost, "/upload", strings.NewReader("archive"))
|
||||
request.Header.Set("Authorization", "Bearer valid-token")
|
||||
request.Header.Set("Content-Type", "application/x-tar")
|
||||
|
||||
handler.ServeHTTP(recorder, request)
|
||||
|
||||
if recorder.Code != http.StatusAccepted {
|
||||
t.Fatalf("status = %d, want %d; body = %q", recorder.Code, http.StatusAccepted, recorder.Body.String())
|
||||
}
|
||||
if submitted.PipelineID != "reports" {
|
||||
t.Fatalf("submitted pipeline = %q, want reports", submitted.PipelineID)
|
||||
}
|
||||
var response uploadAcceptedResponse
|
||||
if err := json.Unmarshal(recorder.Body.Bytes(), &response); err != nil {
|
||||
t.Fatalf("decode response: %v", err)
|
||||
}
|
||||
if response.RunID != "reports.20260603T120000Z.abcdef12" || response.Status != UploadStatusAccepted {
|
||||
t.Fatalf("response = %#v, want accepted run id", response)
|
||||
}
|
||||
if strings.Contains(recorder.Body.String(), "valid-token") {
|
||||
t.Fatalf("response exposed token: %q", recorder.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadHTTPHandlerRejectsUnauthorizedRequests(t *testing.T) {
|
||||
handler := uploadHTTPHandler{
|
||||
coordinator: fakeUploadCoordinator{canAccept: true},
|
||||
tokens: map[string]string{"valid-token": "reports"},
|
||||
limits: map[string]int64{"reports": 1024},
|
||||
}
|
||||
|
||||
for _, authHeader := range []string{"", "Bearer wrong-token"} {
|
||||
recorder := httptest.NewRecorder()
|
||||
request := httptest.NewRequest(http.MethodPost, "/upload", strings.NewReader("archive"))
|
||||
request.Header.Set("Authorization", authHeader)
|
||||
request.Header.Set("Content-Type", "application/x-tar")
|
||||
|
||||
handler.ServeHTTP(recorder, request)
|
||||
|
||||
if recorder.Code != http.StatusUnauthorized {
|
||||
t.Fatalf("auth %q status = %d, want %d", authHeader, recorder.Code, http.StatusUnauthorized)
|
||||
}
|
||||
if strings.Contains(recorder.Body.String(), "valid-token") || strings.Contains(recorder.Body.String(), "wrong-token") {
|
||||
t.Fatalf("unauthorized response exposed token: %q", recorder.Body.String())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadHTTPHandlerRejectsUnsupportedOversizedFullQueueAndPipelineID(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
canAccept bool
|
||||
url string
|
||||
contentType string
|
||||
body io.Reader
|
||||
wantStatus int
|
||||
}{
|
||||
{
|
||||
name: "unsupported content type",
|
||||
canAccept: true,
|
||||
url: "/upload",
|
||||
contentType: "application/zip",
|
||||
body: strings.NewReader("archive"),
|
||||
wantStatus: http.StatusUnsupportedMediaType,
|
||||
},
|
||||
{
|
||||
name: "oversized",
|
||||
canAccept: true,
|
||||
url: "/upload",
|
||||
contentType: "application/x-tar",
|
||||
body: strings.NewReader("too-large"),
|
||||
wantStatus: http.StatusRequestEntityTooLarge,
|
||||
},
|
||||
{
|
||||
name: "full queue",
|
||||
canAccept: false,
|
||||
url: "/upload",
|
||||
contentType: "application/x-tar",
|
||||
body: &countingReader{reader: strings.NewReader("archive")},
|
||||
wantStatus: http.StatusServiceUnavailable,
|
||||
},
|
||||
{
|
||||
name: "submitted pipeline id",
|
||||
canAccept: true,
|
||||
url: "/upload?pipeline_id=reports",
|
||||
contentType: "application/x-tar",
|
||||
body: strings.NewReader("archive"),
|
||||
wantStatus: http.StatusBadRequest,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
handler := uploadHTTPHandler{
|
||||
coordinator: fakeUploadCoordinator{
|
||||
canAccept: tt.canAccept,
|
||||
submit: func(context.Context, UploadRequest) (UploadRunRecord, error) {
|
||||
t.Fatal("Submit should not be called")
|
||||
return UploadRunRecord{}, nil
|
||||
},
|
||||
},
|
||||
tokens: map[string]string{"valid-token": "reports"},
|
||||
limits: map[string]int64{"reports": 4},
|
||||
}
|
||||
recorder := httptest.NewRecorder()
|
||||
request := httptest.NewRequest(http.MethodPost, tt.url, tt.body)
|
||||
request.Header.Set("Authorization", "Bearer valid-token")
|
||||
request.Header.Set("Content-Type", tt.contentType)
|
||||
|
||||
handler.ServeHTTP(recorder, request)
|
||||
|
||||
if recorder.Code != tt.wantStatus {
|
||||
t.Fatalf("status = %d, want %d; body = %q", recorder.Code, tt.wantStatus, recorder.Body.String())
|
||||
}
|
||||
if reader, ok := tt.body.(*countingReader); ok && reader.reads != 0 {
|
||||
t.Fatalf("full queue read body %d time(s), want zero", reader.reads)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadHTTPHandlerRunStatusAndHealth(t *testing.T) {
|
||||
finishedAt := time.Date(2026, 6, 3, 12, 0, 0, 0, time.UTC)
|
||||
handler := uploadHTTPHandler{
|
||||
coordinator: fakeUploadCoordinator{
|
||||
canAccept: true,
|
||||
status: func(runID UploadRunID) (UploadRunRecord, bool) {
|
||||
if runID != "reports.20260603T120000Z.abcdef12" {
|
||||
return UploadRunRecord{}, false
|
||||
}
|
||||
return UploadRunRecord{
|
||||
ID: runID,
|
||||
PipelineID: "reports",
|
||||
Status: UploadStatusSucceeded,
|
||||
FinishedAt: &finishedAt,
|
||||
}, true
|
||||
},
|
||||
},
|
||||
tokens: map[string]string{"valid-token": "reports"},
|
||||
limits: map[string]int64{"reports": 1024},
|
||||
}
|
||||
|
||||
recorder := httptest.NewRecorder()
|
||||
handler.ServeHTTP(recorder, httptest.NewRequest(http.MethodGet, "/healthz", nil))
|
||||
if recorder.Code != http.StatusOK {
|
||||
t.Fatalf("health status = %d, want %d", recorder.Code, http.StatusOK)
|
||||
}
|
||||
|
||||
recorder = httptest.NewRecorder()
|
||||
handler.ServeHTTP(recorder, httptest.NewRequest(http.MethodGet, "/runs/reports.20260603T120000Z.abcdef12", nil))
|
||||
if recorder.Code != http.StatusOK {
|
||||
t.Fatalf("run status = %d, want %d; body = %q", recorder.Code, http.StatusOK, recorder.Body.String())
|
||||
}
|
||||
var record UploadRunRecord
|
||||
if err := json.Unmarshal(recorder.Body.Bytes(), &record); err != nil {
|
||||
t.Fatalf("decode run status: %v", err)
|
||||
}
|
||||
if record.ID != "reports.20260603T120000Z.abcdef12" || record.Status != UploadStatusSucceeded {
|
||||
t.Fatalf("record = %#v, want succeeded run status", record)
|
||||
}
|
||||
|
||||
recorder = httptest.NewRecorder()
|
||||
handler.ServeHTTP(recorder, httptest.NewRequest(http.MethodGet, "/runs/unknown", nil))
|
||||
if recorder.Code != http.StatusNotFound {
|
||||
t.Fatalf("unknown run status = %d, want %d", recorder.Code, http.StatusNotFound)
|
||||
}
|
||||
}
|
||||
|
||||
type countingReader struct {
|
||||
reader io.Reader
|
||||
reads int
|
||||
}
|
||||
|
||||
func (reader *countingReader) Read(data []byte) (int, error) {
|
||||
reader.reads++
|
||||
return reader.reader.Read(data)
|
||||
}
|
||||
|
||||
func uploadHTTPTestConfig() config.Config {
|
||||
size := config.ByteSize(1024)
|
||||
retention := config.Duration(24 * time.Hour)
|
||||
cfg := config.Config{
|
||||
Server: config.Server{HTTP: config.HTTPServer{
|
||||
Bind: config.DefaultHTTPBind,
|
||||
StagingRoot: "/tmp/distributor-test",
|
||||
MaxUploadSize: &size,
|
||||
QueueSize: 2,
|
||||
MaxConcurrency: 1,
|
||||
Retention: &retention,
|
||||
}},
|
||||
Pipelines: []config.Pipeline{{
|
||||
ID: "reports",
|
||||
Source: config.Backend{
|
||||
Backend: config.BackendHTTPUpload,
|
||||
Upload: config.HTTPUpload{
|
||||
TokenEnv: "UPLOAD_TOKEN",
|
||||
StagingPath: "/tmp/distributor-test/reports",
|
||||
MaxUploadSize: &size,
|
||||
},
|
||||
},
|
||||
Destinations: []config.Destination{{
|
||||
ID: "local",
|
||||
Backend: config.BackendLocal,
|
||||
Path: "/tmp/distributor-output",
|
||||
Publish: &config.PublishPolicy{Source: true},
|
||||
}},
|
||||
}},
|
||||
}
|
||||
config.ApplyDefaults(&cfg)
|
||||
return cfg
|
||||
}
|
||||
|
||||
func uploadHTTPTestEnvironment(values map[string]string) config.Environment {
|
||||
return config.NewEnvironment(values, func(string) (string, bool) {
|
||||
return "", false
|
||||
})
|
||||
}
|
||||
@@ -29,6 +29,8 @@ func Execute(ctx context.Context, args []string, stdout, stderr io.Writer) int {
|
||||
return versionCommand(ctx, args[1:], stdout, stderr)
|
||||
case "run":
|
||||
return runCommand(ctx, args[1:], stdout, stderr)
|
||||
case "serve":
|
||||
return serveCommand(ctx, args[1:], stdout, stderr)
|
||||
case "validate":
|
||||
return validateCommand(ctx, args[1:], stdout, stderr)
|
||||
case "inspect":
|
||||
@@ -51,6 +53,7 @@ Usage:
|
||||
Commands:
|
||||
version Print version information
|
||||
run Run configured distribution pipelines
|
||||
serve Run the HTTP upload server
|
||||
validate Validate a source bundle or bundle tree
|
||||
inspect Inspect bundles or distributor state
|
||||
manifest Create source bundle manifests
|
||||
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/app"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/storage"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/testutil"
|
||||
producerbundle "gitea.maximumdirect.net/eric/distributor/pkg/bundle"
|
||||
@@ -92,6 +93,34 @@ func TestExecuteVersionJSON(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteServeParsesConfig(t *testing.T) {
|
||||
originalServeApp := serveApp
|
||||
defer func() {
|
||||
serveApp = originalServeApp
|
||||
}()
|
||||
var gotOptions app.ServeOptions
|
||||
serveApp = func(_ context.Context, options app.ServeOptions) error {
|
||||
gotOptions = options
|
||||
return nil
|
||||
}
|
||||
var stdout, stderr bytes.Buffer
|
||||
|
||||
code := Execute(context.Background(), []string{"serve", "--config", "config.yml"}, &stdout, &stderr)
|
||||
|
||||
if code != exitOK {
|
||||
t.Fatalf("exit code = %d, want %d; stderr = %q", code, exitOK, stderr.String())
|
||||
}
|
||||
if gotOptions.ConfigPath != "config.yml" {
|
||||
t.Fatalf("ConfigPath = %q, want config.yml", gotOptions.ConfigPath)
|
||||
}
|
||||
if stdout.Len() != 0 {
|
||||
t.Fatalf("stdout = %q, want empty", stdout.String())
|
||||
}
|
||||
if stderr.Len() != 0 {
|
||||
t.Fatalf("stderr = %q, want empty", stderr.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteRejectsInvalidFormat(t *testing.T) {
|
||||
var stdout, stderr bytes.Buffer
|
||||
|
||||
|
||||
46
internal/cli/serve.go
Normal file
46
internal/cli/serve.go
Normal file
@@ -0,0 +1,46 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"context"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/app"
|
||||
)
|
||||
|
||||
var serveApp = app.Serve
|
||||
|
||||
func serveCommand(ctx context.Context, args []string, stdout, stderr io.Writer) int {
|
||||
if hasHelp(args) {
|
||||
printServeHelp(stdout)
|
||||
return exitOK
|
||||
}
|
||||
|
||||
flags := flag.NewFlagSet("serve", flag.ContinueOnError)
|
||||
flags.SetOutput(stderr)
|
||||
configPath := flags.String("config", "", "path to config file")
|
||||
if err := flags.Parse(args); err != nil {
|
||||
return exitUsage
|
||||
}
|
||||
if rejectPositionalArgs(stderr, "serve", flags.Args()) {
|
||||
return exitUsage
|
||||
}
|
||||
|
||||
if err := serveApp(ctx, app.ServeOptions{ConfigPath: *configPath}); err != nil {
|
||||
return fail(stderr, err)
|
||||
}
|
||||
return exitOK
|
||||
}
|
||||
|
||||
func printServeHelp(w io.Writer) {
|
||||
fmt.Fprint(w, `Usage:
|
||||
distributor serve --config <path>
|
||||
|
||||
Options:
|
||||
--config <path> Path to config file
|
||||
|
||||
Serve loads configured HTTP upload sources, resolves upload tokens through the
|
||||
configured secret environment, and starts the HTTP upload API.
|
||||
`)
|
||||
}
|
||||
Reference in New Issue
Block a user