package app import ( "context" "encoding/json" "errors" "io" "net/http" "net/http/httptest" "strings" "testing" "time" "gitea.maximumdirect.net/eric/distributor/internal/config" "gitea.maximumdirect.net/eric/distributor/internal/ingest" ) type fakeUploadCoordinator struct { canAccept bool submit func(context.Context, UploadRequest) (UploadRunRecord, error) status func(UploadRunID) (UploadRunRecord, bool) } func (fake fakeUploadCoordinator) CanAccept() bool { return fake.canAccept } func (fake fakeUploadCoordinator) Submit(ctx context.Context, request UploadRequest) (UploadRunRecord, error) { if fake.submit == nil { return UploadRunRecord{}, errors.New("unexpected submit") } return fake.submit(ctx, request) } func (fake fakeUploadCoordinator) Status(runID UploadRunID) (UploadRunRecord, bool) { if fake.status == nil { return UploadRunRecord{}, false } return fake.status(runID) } func TestResolveUploadTokensFailsForMissingAndDuplicateTokens(t *testing.T) { cfg := uploadHTTPTestConfig() _, err := resolveUploadTokens(cfg, config.NewEnvironment(nil, func(string) (string, bool) { return "", false })) if err == nil || !strings.Contains(err.Error(), "UPLOAD_TOKEN") { t.Fatalf("resolveUploadTokens() error = %v, want missing UPLOAD_TOKEN", err) } cfg.Pipelines = append(cfg.Pipelines, config.Pipeline{ ID: "weekly", Source: config.Backend{ Backend: config.BackendHTTPUpload, }, Destinations: cfg.Pipelines[0].Destinations, }) cfg.UploadTokens = append(cfg.UploadTokens, config.UploadToken{ ID: "weekly-reporter", TokenEnv: "OTHER_UPLOAD_TOKEN", AllowPipelines: []string{"weekly"}, }) config.ApplyDefaults(&cfg) secret := "super-secret-token" _, err = resolveUploadTokens(cfg, uploadHTTPTestEnvironment(map[string]string{ "UPLOAD_TOKEN": secret, "OTHER_UPLOAD_TOKEN": secret, })) if err == nil { t.Fatal("resolveUploadTokens() error = nil, want duplicate token error") } if strings.Contains(err.Error(), secret) { t.Fatalf("duplicate token error exposed secret value: %q", err) } } func TestResolveUploadTokensAllowsMultiplePipelines(t *testing.T) { cfg := uploadHTTPTestConfig() cfg.Pipelines = append(cfg.Pipelines, config.Pipeline{ ID: "weekly", Source: config.Backend{ Backend: config.BackendHTTPUpload, }, Destinations: cfg.Pipelines[0].Destinations, }) cfg.UploadTokens[0].AllowPipelines = []string{"reports", "weekly"} config.ApplyDefaults(&cfg) tokens, err := resolveUploadTokens(cfg, uploadHTTPTestEnvironment(map[string]string{ "UPLOAD_TOKEN": "secret", })) if err != nil { t.Fatalf("resolveUploadTokens() error = %v", err) } token, ok := tokens["secret"] if !ok { t.Fatal("resolved token missing") } if token.ID != "reporter" || token.Value != "secret" { t.Fatalf("resolved token = %#v, want id and value", token) } if _, ok := token.AllowedPipelines["reports"]; !ok { t.Fatalf("allowed pipelines = %#v, want reports", token.AllowedPipelines) } if _, ok := token.AllowedPipelines["weekly"]; !ok { t.Fatalf("allowed pipelines = %#v, want weekly", token.AllowedPipelines) } } func TestNewUploadHTTPHandlerAcceptsDefaultedConfig(t *testing.T) { cfg := uploadHTTPTestConfig() cfg.Server.HTTP.Bind = "" cfg.Server.HTTP.StagingRoot = "" cfg.Server.HTTP.MaxUploadSize = nil cfg.Server.HTTP.QueueSize = 0 cfg.Server.HTTP.MaxConcurrency = 0 cfg.Server.HTTP.Retention = nil cfg.Pipelines[0].Source.Upload.StagingPath = "" cfg.Pipelines[0].Source.Upload.MaxUploadSize = nil handler, err := newUploadHTTPHandler(context.Background(), cfg, uploadHTTPTestEnvironment(map[string]string{ "UPLOAD_TOKEN": "secret", })) if err != nil { t.Fatalf("newUploadHTTPHandler() error = %v", err) } if handler == nil { t.Fatal("newUploadHTTPHandler() = nil") } } func TestUploadHTTPHandlerAuthenticatesAndAcceptsUpload(t *testing.T) { var submitted UploadRequest handler := uploadHTTPHandler{ coordinator: fakeUploadCoordinator{ submit: func(_ context.Context, request UploadRequest) (UploadRunRecord, error) { submitted = request body, err := io.ReadAll(request.Body) if err != nil { t.Fatalf("read submitted body: %v", err) } if string(body) != "archive" { t.Fatalf("submitted body = %q, want archive", body) } return UploadRunRecord{ID: "reports.20260603T120000Z.abcdef12", Status: UploadStatusAccepted}, nil }, }, tokens: map[string]resolvedUploadToken{"valid-token": uploadHTTPTestToken("reporter", "valid-token", "reports")}, uploadPipelines: pipelineIDSet([]string{"reports"}), } recorder := httptest.NewRecorder() request := httptest.NewRequest(http.MethodPost, "/v1/pipelines/reports/upload", strings.NewReader("archive")) request.Header.Set("Authorization", "Bearer valid-token") request.Header.Set("Content-Type", "application/x-tar") request.Header.Set("Idempotency-Key", "producer.retry:20260603") handler.ServeHTTP(recorder, request) if recorder.Code != http.StatusAccepted { t.Fatalf("status = %d, want %d; body = %q", recorder.Code, http.StatusAccepted, recorder.Body.String()) } if submitted.PipelineID != "reports" { t.Fatalf("submitted pipeline = %q, want reports", submitted.PipelineID) } if submitted.TokenID != "reporter" { t.Fatalf("submitted token id = %q, want reporter", submitted.TokenID) } if submitted.IdempotencyKey != "producer.retry:20260603" { t.Fatalf("submitted idempotency key = %q, want producer.retry:20260603", submitted.IdempotencyKey) } var response uploadAcceptedResponse if err := json.Unmarshal(recorder.Body.Bytes(), &response); err != nil { t.Fatalf("decode response: %v", err) } if response.RunID != "reports.20260603T120000Z.abcdef12" || response.Status != UploadStatusAccepted { t.Fatalf("response = %#v, want accepted run id", response) } if strings.Contains(recorder.Body.String(), "valid-token") { t.Fatalf("response exposed token: %q", recorder.Body.String()) } } func TestUploadHTTPHandlerRejectsUnauthorizedRequests(t *testing.T) { handler := uploadHTTPHandler{ coordinator: fakeUploadCoordinator{}, tokens: map[string]resolvedUploadToken{"valid-token": uploadHTTPTestToken("reporter", "valid-token", "reports")}, uploadPipelines: pipelineIDSet([]string{"reports"}), } for _, authHeader := range []string{"", "Basic valid-token", "Bearer", "Bearer wrong-token"} { recorder := httptest.NewRecorder() request := httptest.NewRequest(http.MethodPost, "/v1/pipelines/reports/upload", strings.NewReader("archive")) request.Header.Set("Authorization", authHeader) request.Header.Set("Content-Type", "application/x-tar") handler.ServeHTTP(recorder, request) if recorder.Code != http.StatusUnauthorized { t.Fatalf("auth %q status = %d, want %d", authHeader, recorder.Code, http.StatusUnauthorized) } if strings.Contains(recorder.Body.String(), "valid-token") || strings.Contains(recorder.Body.String(), "wrong-token") { t.Fatalf("unauthorized response exposed token: %q", recorder.Body.String()) } } } func TestUploadHTTPHandlerRejectsForbiddenPipeline(t *testing.T) { handler := uploadHTTPHandler{ coordinator: fakeUploadCoordinator{ submit: func(context.Context, UploadRequest) (UploadRunRecord, error) { t.Fatal("Submit should not be called") return UploadRunRecord{}, nil }, }, tokens: map[string]resolvedUploadToken{"valid-token": uploadHTTPTestToken("reporter", "valid-token", "reports")}, uploadPipelines: pipelineIDSet([]string{"reports", "private"}), } recorder := httptest.NewRecorder() request := httptest.NewRequest(http.MethodPost, "/v1/pipelines/private/upload", strings.NewReader("archive")) request.Header.Set("Authorization", "Bearer valid-token") request.Header.Set("Content-Type", "application/x-tar") handler.ServeHTTP(recorder, request) if recorder.Code != http.StatusForbidden { t.Fatalf("status = %d, want %d; body = %q", recorder.Code, http.StatusForbidden, recorder.Body.String()) } if strings.Contains(recorder.Body.String(), "valid-token") { t.Fatalf("forbidden response exposed token: %q", recorder.Body.String()) } } func TestUploadHTTPHandlerRejectsInvalidPathAndRemovedLegacyUpload(t *testing.T) { handler := uploadHTTPHandler{ coordinator: fakeUploadCoordinator{ submit: func(context.Context, UploadRequest) (UploadRunRecord, error) { t.Fatal("Submit should not be called") return UploadRunRecord{}, nil }, }, tokens: map[string]resolvedUploadToken{"valid-token": uploadHTTPTestToken("reporter", "valid-token", "reports")}, uploadPipelines: pipelineIDSet([]string{"reports"}), } tests := []struct { name string url string wantStatus int }{ {name: "legacy upload", url: "/upload", wantStatus: http.StatusNotFound}, {name: "missing pipeline", url: "/v1/pipelines//upload", wantStatus: http.StatusNotFound}, {name: "extra segment", url: "/v1/pipelines/reports/upload/extra", wantStatus: http.StatusNotFound}, {name: "invalid pipeline id", url: "/v1/pipelines/.reports/upload", wantStatus: http.StatusBadRequest}, {name: "pipeline query", url: "/v1/pipelines/reports/upload?pipeline=other", wantStatus: http.StatusBadRequest}, {name: "pipeline id query", url: "/v1/pipelines/reports/upload?pipeline_id=other", wantStatus: http.StatusBadRequest}, {name: "unknown upload pipeline", url: "/v1/pipelines/missing/upload", wantStatus: http.StatusNotFound}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { recorder := httptest.NewRecorder() request := httptest.NewRequest(http.MethodPost, tt.url, strings.NewReader("archive")) request.Header.Set("Authorization", "Bearer valid-token") request.Header.Set("Content-Type", "application/x-tar") handler.ServeHTTP(recorder, request) if recorder.Code != tt.wantStatus { t.Fatalf("status = %d, want %d; body = %q", recorder.Code, tt.wantStatus, recorder.Body.String()) } }) } } func TestUploadHTTPHandlerRejectsUnsupportedContentTypeAndInvalidKey(t *testing.T) { tests := []struct { name string url string contentType string keyValues []string body io.Reader wantStatus int }{ { name: "unsupported content type", url: "/v1/pipelines/reports/upload", contentType: "application/zip", body: strings.NewReader("archive"), wantStatus: http.StatusUnsupportedMediaType, }, { name: "invalid key syntax", url: "/v1/pipelines/reports/upload", contentType: "application/x-tar", keyValues: []string{"bad key"}, body: strings.NewReader("archive"), wantStatus: http.StatusBadRequest, }, { name: "empty key", url: "/v1/pipelines/reports/upload", contentType: "application/x-tar", keyValues: []string{""}, body: strings.NewReader("archive"), wantStatus: http.StatusBadRequest, }, { name: "too long key", url: "/v1/pipelines/reports/upload", contentType: "application/x-tar", keyValues: []string{strings.Repeat("a", 129)}, body: strings.NewReader("archive"), wantStatus: http.StatusBadRequest, }, { name: "multiple keys", url: "/v1/pipelines/reports/upload", contentType: "application/x-tar", keyValues: []string{"one", "two"}, body: strings.NewReader("archive"), wantStatus: http.StatusBadRequest, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { handler := uploadHTTPHandler{ coordinator: fakeUploadCoordinator{ submit: func(context.Context, UploadRequest) (UploadRunRecord, error) { t.Fatal("Submit should not be called") return UploadRunRecord{}, nil }, }, tokens: map[string]resolvedUploadToken{"valid-token": uploadHTTPTestToken("reporter", "valid-token", "reports")}, uploadPipelines: pipelineIDSet([]string{"reports"}), } recorder := httptest.NewRecorder() request := httptest.NewRequest(http.MethodPost, tt.url, tt.body) request.Header.Set("Authorization", "Bearer valid-token") request.Header.Set("Content-Type", tt.contentType) for _, value := range tt.keyValues { request.Header.Add("Idempotency-Key", value) } handler.ServeHTTP(recorder, request) if recorder.Code != tt.wantStatus { t.Fatalf("status = %d, want %d; body = %q", recorder.Code, tt.wantStatus, recorder.Body.String()) } }) } } func TestUploadHTTPHandlerMapsSubmitErrors(t *testing.T) { tests := []struct { name string err error wantStatus int wantBody string }{ {name: "oversized", err: ingest.ErrUploadTooLarge, wantStatus: http.StatusRequestEntityTooLarge}, {name: "unsupported", err: ingest.ErrUnsupportedContentType, wantStatus: http.StatusUnsupportedMediaType}, {name: "full queue", err: UploadQueueFullError{QueueSize: 1}, wantStatus: http.StatusServiceUnavailable}, {name: "idempotency conflict", err: UploadIdempotencyConflictError{}, wantStatus: http.StatusConflict, wantBody: "different source manifest"}, {name: "idempotency in progress", err: UploadIdempotencyConflictError{Retryable: true}, wantStatus: http.StatusConflict, wantBody: `"retryable":true`}, {name: "malformed", err: errors.New("malformed archive"), wantStatus: http.StatusBadRequest}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { handler := uploadHTTPHandler{ coordinator: fakeUploadCoordinator{ canAccept: true, submit: func(context.Context, UploadRequest) (UploadRunRecord, error) { return UploadRunRecord{}, tt.err }, }, tokens: map[string]resolvedUploadToken{"valid-token": uploadHTTPTestToken("reporter", "valid-token", "reports")}, uploadPipelines: pipelineIDSet([]string{"reports"}), } recorder := httptest.NewRecorder() request := httptest.NewRequest(http.MethodPost, "/v1/pipelines/reports/upload", strings.NewReader("archive")) request.Header.Set("Authorization", "Bearer valid-token") request.Header.Set("Content-Type", "application/x-tar") handler.ServeHTTP(recorder, request) if recorder.Code != tt.wantStatus { t.Fatalf("status = %d, want %d; body = %q", recorder.Code, tt.wantStatus, recorder.Body.String()) } if tt.wantBody != "" && !strings.Contains(recorder.Body.String(), tt.wantBody) { t.Fatalf("body = %q, want substring %q", recorder.Body.String(), tt.wantBody) } }) } } func TestUploadHTTPHandlerRunStatusAndHealth(t *testing.T) { finishedAt := time.Date(2026, 6, 3, 12, 0, 0, 0, time.UTC) handler := uploadHTTPHandler{ coordinator: fakeUploadCoordinator{ canAccept: true, status: func(runID UploadRunID) (UploadRunRecord, bool) { if runID != "reports.20260603T120000Z.abcdef12" { return UploadRunRecord{}, false } return UploadRunRecord{ ID: runID, PipelineID: "reports", Status: UploadStatusSucceeded, FinishedAt: &finishedAt, }, true }, }, tokens: map[string]resolvedUploadToken{"valid-token": uploadHTTPTestToken("reporter", "valid-token", "reports")}, uploadPipelines: pipelineIDSet([]string{"reports"}), } recorder := httptest.NewRecorder() handler.ServeHTTP(recorder, httptest.NewRequest(http.MethodGet, "/healthz", nil)) if recorder.Code != http.StatusOK { t.Fatalf("health status = %d, want %d", recorder.Code, http.StatusOK) } recorder = httptest.NewRecorder() handler.ServeHTTP(recorder, httptest.NewRequest(http.MethodGet, "/runs/reports.20260603T120000Z.abcdef12", nil)) if recorder.Code != http.StatusOK { t.Fatalf("run status = %d, want %d; body = %q", recorder.Code, http.StatusOK, recorder.Body.String()) } var record UploadRunRecord if err := json.Unmarshal(recorder.Body.Bytes(), &record); err != nil { t.Fatalf("decode run status: %v", err) } if record.ID != "reports.20260603T120000Z.abcdef12" || record.Status != UploadStatusSucceeded { t.Fatalf("record = %#v, want succeeded run status", record) } recorder = httptest.NewRecorder() handler.ServeHTTP(recorder, httptest.NewRequest(http.MethodGet, "/runs/unknown", nil)) if recorder.Code != http.StatusNotFound { t.Fatalf("unknown run status = %d, want %d", recorder.Code, http.StatusNotFound) } } type countingReader struct { reader io.Reader reads int } func (reader *countingReader) Read(data []byte) (int, error) { reader.reads++ return reader.reader.Read(data) } func uploadHTTPTestConfig() config.Config { size := config.ByteSize(1024) retention := config.Duration(24 * time.Hour) cfg := config.Config{ Server: config.Server{HTTP: config.HTTPServer{ Bind: config.DefaultHTTPBind, StagingRoot: "/tmp/distributor-test", MaxUploadSize: &size, QueueSize: 2, MaxConcurrency: 1, Retention: &retention, }}, Pipelines: []config.Pipeline{{ ID: "reports", Source: config.Backend{ Backend: config.BackendHTTPUpload, Upload: config.HTTPUpload{ StagingPath: "/tmp/distributor-test/reports", MaxUploadSize: &size, }, }, Destinations: []config.Destination{{ ID: "local", Backend: config.BackendLocal, Path: "/tmp/distributor-output", Publish: &config.PublishPolicy{Source: true}, }}, }}, UploadTokens: []config.UploadToken{{ ID: "reporter", TokenEnv: "UPLOAD_TOKEN", AllowPipelines: []string{"reports"}, }}, } config.ApplyDefaults(&cfg) return cfg } func uploadHTTPTestEnvironment(values map[string]string) config.Environment { return config.NewEnvironment(values, func(string) (string, bool) { return "", false }) } func uploadHTTPTestToken(id, value string, pipelines ...string) resolvedUploadToken { return resolvedUploadToken{ ID: id, Value: value, AllowedPipelines: pipelineIDSet(pipelines), } }