eric/distributor
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 7 Wiki Activity

Add upload pipeline integration coverage

Browse Source
This commit is contained in:
Eric Rakestraw
2026-06-08 04:41:05 +00:00
parent ce43a6044a
commit bd5892d1f2
2 changed files with 225 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
examples/upload-client/main.go
View File

@@ -23,6 +23,13 @@ func main() {
if len(os.Args) > 1 {
bundleRoot = os.Args[1]
}
pipelineID := os.Getenv("DISTRIBUTOR_EXAMPLE_UPLOAD_PIPELINE_ID")
if pipelineID == "" {
pipelineID = "example-http-upload"
}
if len(os.Args) > 2 {
pipelineID = os.Args[2]
}
idempotencyKey := os.Getenv("DISTRIBUTOR_EXAMPLE_UPLOAD_IDEMPOTENCY_KEY")
client, err := upload.NewClient(upload.ClientOptions{
@@ -35,7 +42,10 @@ func main() {
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
opts := upload.UploadBundleOptions{Root: bundleRoot}
opts := upload.UploadBundleOptions{
PipelineID: pipelineID,
Root: bundleRoot,
}
if idempotencyKey != "" {
opts.IdempotencyKey = idempotencyKey
}

214
internal/app/upload_http_integration_test.go
View File

@@ -21,6 +21,7 @@ import (
"gitea.maximumdirect.net/eric/distributor/internal/ingest"
"gitea.maximumdirect.net/eric/distributor/internal/storage"
"gitea.maximumdirect.net/eric/distributor/internal/testutil"
clientupload "gitea.maximumdirect.net/eric/distributor/pkg/upload"
)
func TestHTTPUploadPublishesTarAndGzipFanout(t *testing.T) {
@@ -290,6 +291,199 @@ func TestHTTPUploadDifferentPipelinesRunConcurrently(t *testing.T) {
waitForHTTPUploadStatus(t, server, secondRunID, UploadStatusSucceeded)
}
func TestHTTPUploadOneTokenCanUploadToMultiplePipelines(t *testing.T) {
firstDestination := t.TempDir()
secondDestination := t.TempDir()
cfg := httpUploadIntegrationConfig(t, []httpUploadPipelineSpec{
{
id: "reports-one",
tokenEnv: "SHARED_UPLOAD_TOKEN",
stagingPath: filepath.Join(t.TempDir(), "reports-one"),
destinations: []string{firstDestination},
},
{
id: "reports-two",
tokenEnv: "SHARED_UPLOAD_TOKEN",
stagingPath: filepath.Join(t.TempDir(), "reports-two"),
destinations: []string{secondDestination},
},
}, 4, 1)
cfg.UploadTokens = []config.UploadToken{{
ID: "shared-reporter",
TokenEnv: "SHARED_UPLOAD_TOKEN",
AllowPipelines: []string{"reports-one", "reports-two"},
}}
handler, err := newUploadHTTPHandler(context.Background(), cfg, uploadHTTPTestEnvironment(map[string]string{
"SHARED_UPLOAD_TOKEN": "shared-secret",
}))
if err != nil {
t.Fatalf("newUploadHTTPHandler() error = %v", err)
}
server := httptest.NewServer(handler)
defer server.Close()
firstRunID := submitHTTPUploadToPipeline(t, server, "reports-one", "shared-secret", ingest.ContentTypeTar, bundleArchive(t, false, testutil.BundleOptions{
ID: "reports.one.2026-06-08",
}))
secondRunID := submitHTTPUploadToPipeline(t, server, "reports-two", "shared-secret", ingest.ContentTypeTar, bundleArchive(t, false, testutil.BundleOptions{
ID: "reports.two.2026-06-08",
}))
first := waitForHTTPUploadStatus(t, server, firstRunID, UploadStatusSucceeded)
second := waitForHTTPUploadStatus(t, server, secondRunID, UploadStatusSucceeded)
if first.PipelineID != "reports-one" || second.PipelineID != "reports-two" {
t.Fatalf("statuses pipeline = %q/%q, want reports-one/reports-two", first.PipelineID, second.PipelineID)
}
assertPublishedBundle(t, firstDestination)
assertPublishedBundle(t, secondDestination)
}
func TestHTTPUploadMultipleTokensCanUploadToOnePipeline(t *testing.T) {
destination := t.TempDir()
cfg := httpUploadIntegrationConfig(t, []httpUploadPipelineSpec{{
id: "reports",
tokenEnv: "FIRST_UPLOAD_TOKEN",
stagingPath: filepath.Join(t.TempDir(), "reports"),
destinations: []string{destination},
}}, 4, 1)
cfg.UploadTokens = []config.UploadToken{
{ID: "first-reporter", TokenEnv: "FIRST_UPLOAD_TOKEN", AllowPipelines: []string{"reports"}},
{ID: "second-reporter", TokenEnv: "SECOND_UPLOAD_TOKEN", AllowPipelines: []string{"reports"}},
}
handler, err := newUploadHTTPHandler(context.Background(), cfg, uploadHTTPTestEnvironment(map[string]string{
"FIRST_UPLOAD_TOKEN": "first-secret",
"SECOND_UPLOAD_TOKEN": "second-secret",
}))
if err != nil {
t.Fatalf("newUploadHTTPHandler() error = %v", err)
}
server := httptest.NewServer(handler)
defer server.Close()
firstRunID := submitHTTPUpload(t, server, "first-secret", ingest.ContentTypeTar, bundleArchive(t, false, testutil.BundleOptions{}))
secondRunID := submitHTTPUpload(t, server, "second-secret", ingest.ContentTypeTar, bundleArchive(t, false, testutil.BundleOptions{}))
first := waitForHTTPUploadStatus(t, server, firstRunID, UploadStatusSucceeded)
second := waitForHTTPUploadStatus(t, server, secondRunID, UploadStatusSucceeded)
if first.PipelineID != "reports" || second.PipelineID != "reports" {
t.Fatalf("statuses pipeline = %q/%q, want reports/reports", first.PipelineID, second.PipelineID)
}
assertPublishedBundle(t, destination)
}
func TestHTTPUploadRejectsDisallowedPipelineAndLegacyUploadWithoutQueueing(t *testing.T) {
coordinator := NewUploadCoordinator(context.Background(), httpUploadIntegrationConfig(t, []httpUploadPipelineSpec{
{
id: "reports",
tokenEnv: "REPORTS_TOKEN",
stagingPath: filepath.Join(t.TempDir(), "reports"),
destinations: []string{t.TempDir()},
},
{
id: "private",
tokenEnv: "PRIVATE_TOKEN",
stagingPath: filepath.Join(t.TempDir(), "private"),
destinations: []string{t.TempDir()},
},
}, 4, 1))
handler := uploadHTTPHandler{
coordinator: coordinator,
tokens: map[string]resolvedUploadToken{
"reports-secret": uploadHTTPTestToken("reports-reporter", "reports-secret", "reports"),
},
uploadPipelines: pipelineIDSet([]string{"reports", "private"}),
}
server := httptest.NewServer(handler)
defer server.Close()
status, body := postHTTPUploadToPipeline(t, server, "private", "reports-secret", ingest.ContentTypeTar, []byte("archive"))
if status != http.StatusForbidden {
t.Fatalf("disallowed upload status = %d, want %d; body = %s", status, http.StatusForbidden, body)
}
status, body = postLegacyHTTPUpload(t, server, "reports-secret", ingest.ContentTypeTar, []byte("archive"))
if status != http.StatusNotFound {
t.Fatalf("legacy upload status = %d, want %d; body = %s", status, http.StatusNotFound, body)
}
if got := coordinator.QueueDepth(); got != 0 {
t.Fatalf("queue depth = %d, want 0", got)
}
}
func TestHTTPUploadPublishesThroughSelectedPipeline(t *testing.T) {
firstDestination := t.TempDir()
secondDestination := t.TempDir()
cfg := httpUploadIntegrationConfig(t, []httpUploadPipelineSpec{
{
id: "reports-one",
tokenEnv: "SHARED_UPLOAD_TOKEN",
stagingPath: filepath.Join(t.TempDir(), "reports-one"),
destinations: []string{firstDestination},
},
{
id: "reports-two",
tokenEnv: "SHARED_UPLOAD_TOKEN",
stagingPath: filepath.Join(t.TempDir(), "reports-two"),
destinations: []string{secondDestination},
},
}, 4, 1)
cfg.UploadTokens = []config.UploadToken{{
ID: "shared-reporter",
TokenEnv: "SHARED_UPLOAD_TOKEN",
AllowPipelines: []string{"reports-one", "reports-two"},
}}
handler, err := newUploadHTTPHandler(context.Background(), cfg, uploadHTTPTestEnvironment(map[string]string{
"SHARED_UPLOAD_TOKEN": "shared-secret",
}))
if err != nil {
t.Fatalf("newUploadHTTPHandler() error = %v", err)
}
server := httptest.NewServer(handler)
defer server.Close()
bundleRoot := t.TempDir()
testutil.WriteSourceBundle(t, bundleRoot, "", testutil.BundleOptions{
ID: "reports.selected.2026-06-08",
})
client, err := clientupload.NewClient(clientupload.ClientOptions{
Endpoint: server.URL,
Token: "shared-secret",
HTTPClient: server.Client(),
})
if err != nil {
t.Fatalf("NewClient() error = %v", err)
}
result, err := client.UploadBundle(context.Background(), clientupload.UploadBundleOptions{
PipelineID: "reports-two",
Root: bundleRoot,
})
if err != nil {
t.Fatalf("UploadBundle() error = %v", err)
}
runID := UploadRunID(result.RunID)
record := waitForHTTPUploadStatus(t, server, runID, UploadStatusSucceeded)
if record.PipelineID != "reports-two" {
t.Fatalf("record pipeline = %q, want reports-two", record.PipelineID)
}
if record.Report == nil {
t.Fatal("completed status report = nil, want run report")
}
if got, want := len(record.Report.Pipelines), 1; got != want {
t.Fatalf("report pipeline count = %d, want %d", got, want)
}
if record.Report.Pipelines[0].ID != "reports-two" {
t.Fatalf("report pipeline = %q, want reports-two", record.Report.Pipelines[0].ID)
}
if got, want := len(record.Report.Actions), 1; got != want {
t.Fatalf("report action count = %d, want %d", got, want)
}
if record.Report.Actions[0].PipelineID != "reports-two" {
t.Fatalf("action pipeline = %q, want reports-two", record.Report.Actions[0].PipelineID)
}
assertDirectoryEmpty(t, firstDestination)
assertPublishedBundle(t, secondDestination)
}
type httpUploadPipelineSpec struct {
id string
tokenEnv string
@@ -411,6 +605,26 @@ func postHTTPUploadWithKeyToPipeline(t *testing.T, server *httptest.Server, pipe
return response.StatusCode, string(data)
}
func postLegacyHTTPUpload(t *testing.T, server *httptest.Server, token, contentType string, body []byte) (int, string) {
t.Helper()
request, err := http.NewRequest(http.MethodPost, server.URL+"/upload", bytes.NewReader(body))
if err != nil {
t.Fatalf("NewRequest() error = %v", err)
}
request.Header.Set("Authorization", "Bearer "+token)
request.Header.Set("Content-Type", contentType)
response, err := server.Client().Do(request)
if err != nil {
t.Fatalf("POST legacy upload error = %v", err)
}
defer response.Body.Close()
data, err := io.ReadAll(response.Body)
if err != nil {
t.Fatalf("read response body: %v", err)
}
return response.StatusCode, string(data)
}
func waitForHTTPUploadStatus(t *testing.T, server *httptest.Server, runID UploadRunID, status UploadStatus) UploadRunRecord {
t.Helper()
deadline := time.Now().Add(3 * time.Second)