diff --git a/examples/upload-client/main.go b/examples/upload-client/main.go index c0c9cdb..35a2151 100644 --- a/examples/upload-client/main.go +++ b/examples/upload-client/main.go @@ -23,6 +23,13 @@ func main() { if len(os.Args) > 1 { bundleRoot = os.Args[1] } + pipelineID := os.Getenv("DISTRIBUTOR_EXAMPLE_UPLOAD_PIPELINE_ID") + if pipelineID == "" { + pipelineID = "example-http-upload" + } + if len(os.Args) > 2 { + pipelineID = os.Args[2] + } idempotencyKey := os.Getenv("DISTRIBUTOR_EXAMPLE_UPLOAD_IDEMPOTENCY_KEY") client, err := upload.NewClient(upload.ClientOptions{ @@ -35,7 +42,10 @@ func main() { ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() - opts := upload.UploadBundleOptions{Root: bundleRoot} + opts := upload.UploadBundleOptions{ + PipelineID: pipelineID, + Root: bundleRoot, + } if idempotencyKey != "" { opts.IdempotencyKey = idempotencyKey } diff --git a/internal/app/upload_http_integration_test.go b/internal/app/upload_http_integration_test.go index 6e1c98c..3fa0241 100644 --- a/internal/app/upload_http_integration_test.go +++ b/internal/app/upload_http_integration_test.go @@ -21,6 +21,7 @@ import ( "gitea.maximumdirect.net/eric/distributor/internal/ingest" "gitea.maximumdirect.net/eric/distributor/internal/storage" "gitea.maximumdirect.net/eric/distributor/internal/testutil" + clientupload "gitea.maximumdirect.net/eric/distributor/pkg/upload" ) func TestHTTPUploadPublishesTarAndGzipFanout(t *testing.T) { @@ -290,6 +291,199 @@ func TestHTTPUploadDifferentPipelinesRunConcurrently(t *testing.T) { waitForHTTPUploadStatus(t, server, secondRunID, UploadStatusSucceeded) } +func TestHTTPUploadOneTokenCanUploadToMultiplePipelines(t *testing.T) { + firstDestination := t.TempDir() + secondDestination := t.TempDir() + cfg := httpUploadIntegrationConfig(t, []httpUploadPipelineSpec{ + { + id: "reports-one", + tokenEnv: "SHARED_UPLOAD_TOKEN", + stagingPath: filepath.Join(t.TempDir(), "reports-one"), + destinations: []string{firstDestination}, + }, + { + id: "reports-two", + tokenEnv: "SHARED_UPLOAD_TOKEN", + stagingPath: filepath.Join(t.TempDir(), "reports-two"), + destinations: []string{secondDestination}, + }, + }, 4, 1) + cfg.UploadTokens = []config.UploadToken{{ + ID: "shared-reporter", + TokenEnv: "SHARED_UPLOAD_TOKEN", + AllowPipelines: []string{"reports-one", "reports-two"}, + }} + handler, err := newUploadHTTPHandler(context.Background(), cfg, uploadHTTPTestEnvironment(map[string]string{ + "SHARED_UPLOAD_TOKEN": "shared-secret", + })) + if err != nil { + t.Fatalf("newUploadHTTPHandler() error = %v", err) + } + server := httptest.NewServer(handler) + defer server.Close() + + firstRunID := submitHTTPUploadToPipeline(t, server, "reports-one", "shared-secret", ingest.ContentTypeTar, bundleArchive(t, false, testutil.BundleOptions{ + ID: "reports.one.2026-06-08", + })) + secondRunID := submitHTTPUploadToPipeline(t, server, "reports-two", "shared-secret", ingest.ContentTypeTar, bundleArchive(t, false, testutil.BundleOptions{ + ID: "reports.two.2026-06-08", + })) + + first := waitForHTTPUploadStatus(t, server, firstRunID, UploadStatusSucceeded) + second := waitForHTTPUploadStatus(t, server, secondRunID, UploadStatusSucceeded) + if first.PipelineID != "reports-one" || second.PipelineID != "reports-two" { + t.Fatalf("statuses pipeline = %q/%q, want reports-one/reports-two", first.PipelineID, second.PipelineID) + } + assertPublishedBundle(t, firstDestination) + assertPublishedBundle(t, secondDestination) +} + +func TestHTTPUploadMultipleTokensCanUploadToOnePipeline(t *testing.T) { + destination := t.TempDir() + cfg := httpUploadIntegrationConfig(t, []httpUploadPipelineSpec{{ + id: "reports", + tokenEnv: "FIRST_UPLOAD_TOKEN", + stagingPath: filepath.Join(t.TempDir(), "reports"), + destinations: []string{destination}, + }}, 4, 1) + cfg.UploadTokens = []config.UploadToken{ + {ID: "first-reporter", TokenEnv: "FIRST_UPLOAD_TOKEN", AllowPipelines: []string{"reports"}}, + {ID: "second-reporter", TokenEnv: "SECOND_UPLOAD_TOKEN", AllowPipelines: []string{"reports"}}, + } + handler, err := newUploadHTTPHandler(context.Background(), cfg, uploadHTTPTestEnvironment(map[string]string{ + "FIRST_UPLOAD_TOKEN": "first-secret", + "SECOND_UPLOAD_TOKEN": "second-secret", + })) + if err != nil { + t.Fatalf("newUploadHTTPHandler() error = %v", err) + } + server := httptest.NewServer(handler) + defer server.Close() + + firstRunID := submitHTTPUpload(t, server, "first-secret", ingest.ContentTypeTar, bundleArchive(t, false, testutil.BundleOptions{})) + secondRunID := submitHTTPUpload(t, server, "second-secret", ingest.ContentTypeTar, bundleArchive(t, false, testutil.BundleOptions{})) + + first := waitForHTTPUploadStatus(t, server, firstRunID, UploadStatusSucceeded) + second := waitForHTTPUploadStatus(t, server, secondRunID, UploadStatusSucceeded) + if first.PipelineID != "reports" || second.PipelineID != "reports" { + t.Fatalf("statuses pipeline = %q/%q, want reports/reports", first.PipelineID, second.PipelineID) + } + assertPublishedBundle(t, destination) +} + +func TestHTTPUploadRejectsDisallowedPipelineAndLegacyUploadWithoutQueueing(t *testing.T) { + coordinator := NewUploadCoordinator(context.Background(), httpUploadIntegrationConfig(t, []httpUploadPipelineSpec{ + { + id: "reports", + tokenEnv: "REPORTS_TOKEN", + stagingPath: filepath.Join(t.TempDir(), "reports"), + destinations: []string{t.TempDir()}, + }, + { + id: "private", + tokenEnv: "PRIVATE_TOKEN", + stagingPath: filepath.Join(t.TempDir(), "private"), + destinations: []string{t.TempDir()}, + }, + }, 4, 1)) + handler := uploadHTTPHandler{ + coordinator: coordinator, + tokens: map[string]resolvedUploadToken{ + "reports-secret": uploadHTTPTestToken("reports-reporter", "reports-secret", "reports"), + }, + uploadPipelines: pipelineIDSet([]string{"reports", "private"}), + } + server := httptest.NewServer(handler) + defer server.Close() + + status, body := postHTTPUploadToPipeline(t, server, "private", "reports-secret", ingest.ContentTypeTar, []byte("archive")) + if status != http.StatusForbidden { + t.Fatalf("disallowed upload status = %d, want %d; body = %s", status, http.StatusForbidden, body) + } + status, body = postLegacyHTTPUpload(t, server, "reports-secret", ingest.ContentTypeTar, []byte("archive")) + if status != http.StatusNotFound { + t.Fatalf("legacy upload status = %d, want %d; body = %s", status, http.StatusNotFound, body) + } + if got := coordinator.QueueDepth(); got != 0 { + t.Fatalf("queue depth = %d, want 0", got) + } +} + +func TestHTTPUploadPublishesThroughSelectedPipeline(t *testing.T) { + firstDestination := t.TempDir() + secondDestination := t.TempDir() + cfg := httpUploadIntegrationConfig(t, []httpUploadPipelineSpec{ + { + id: "reports-one", + tokenEnv: "SHARED_UPLOAD_TOKEN", + stagingPath: filepath.Join(t.TempDir(), "reports-one"), + destinations: []string{firstDestination}, + }, + { + id: "reports-two", + tokenEnv: "SHARED_UPLOAD_TOKEN", + stagingPath: filepath.Join(t.TempDir(), "reports-two"), + destinations: []string{secondDestination}, + }, + }, 4, 1) + cfg.UploadTokens = []config.UploadToken{{ + ID: "shared-reporter", + TokenEnv: "SHARED_UPLOAD_TOKEN", + AllowPipelines: []string{"reports-one", "reports-two"}, + }} + handler, err := newUploadHTTPHandler(context.Background(), cfg, uploadHTTPTestEnvironment(map[string]string{ + "SHARED_UPLOAD_TOKEN": "shared-secret", + })) + if err != nil { + t.Fatalf("newUploadHTTPHandler() error = %v", err) + } + server := httptest.NewServer(handler) + defer server.Close() + + bundleRoot := t.TempDir() + testutil.WriteSourceBundle(t, bundleRoot, "", testutil.BundleOptions{ + ID: "reports.selected.2026-06-08", + }) + client, err := clientupload.NewClient(clientupload.ClientOptions{ + Endpoint: server.URL, + Token: "shared-secret", + HTTPClient: server.Client(), + }) + if err != nil { + t.Fatalf("NewClient() error = %v", err) + } + result, err := client.UploadBundle(context.Background(), clientupload.UploadBundleOptions{ + PipelineID: "reports-two", + Root: bundleRoot, + }) + if err != nil { + t.Fatalf("UploadBundle() error = %v", err) + } + runID := UploadRunID(result.RunID) + record := waitForHTTPUploadStatus(t, server, runID, UploadStatusSucceeded) + + if record.PipelineID != "reports-two" { + t.Fatalf("record pipeline = %q, want reports-two", record.PipelineID) + } + if record.Report == nil { + t.Fatal("completed status report = nil, want run report") + } + if got, want := len(record.Report.Pipelines), 1; got != want { + t.Fatalf("report pipeline count = %d, want %d", got, want) + } + if record.Report.Pipelines[0].ID != "reports-two" { + t.Fatalf("report pipeline = %q, want reports-two", record.Report.Pipelines[0].ID) + } + if got, want := len(record.Report.Actions), 1; got != want { + t.Fatalf("report action count = %d, want %d", got, want) + } + if record.Report.Actions[0].PipelineID != "reports-two" { + t.Fatalf("action pipeline = %q, want reports-two", record.Report.Actions[0].PipelineID) + } + assertDirectoryEmpty(t, firstDestination) + assertPublishedBundle(t, secondDestination) +} + type httpUploadPipelineSpec struct { id string tokenEnv string @@ -411,6 +605,26 @@ func postHTTPUploadWithKeyToPipeline(t *testing.T, server *httptest.Server, pipe return response.StatusCode, string(data) } +func postLegacyHTTPUpload(t *testing.T, server *httptest.Server, token, contentType string, body []byte) (int, string) { + t.Helper() + request, err := http.NewRequest(http.MethodPost, server.URL+"/upload", bytes.NewReader(body)) + if err != nil { + t.Fatalf("NewRequest() error = %v", err) + } + request.Header.Set("Authorization", "Bearer "+token) + request.Header.Set("Content-Type", contentType) + response, err := server.Client().Do(request) + if err != nil { + t.Fatalf("POST legacy upload error = %v", err) + } + defer response.Body.Close() + data, err := io.ReadAll(response.Body) + if err != nil { + t.Fatalf("read response body: %v", err) + } + return response.StatusCode, string(data) +} + func waitForHTTPUploadStatus(t *testing.T, server *httptest.Server, runID UploadRunID, status UploadStatus) UploadRunRecord { t.Helper() deadline := time.Now().Add(3 * time.Second)