eric/distributor
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 7 Wiki Activity

Add upload token config validation

Browse Source
This commit is contained in:
Eric Rakestraw
2026-06-08 04:27:49 +00:00
parent 4fa7d1ebb5
commit 033b2e5015
11 changed files with 278 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files

149
internal/config/load_test.go
View File

@@ -256,13 +256,17 @@ pipelines:
- id: weather-daily
source:
backend: http_upload
token_env: WEATHER_DAILY_UPLOAD_TOKEN
staging_path: /srv/distributor/staging/weather-daily
max_upload_size: 32MB
destinations:
- id: archive
backend: local
path: /archive
upload_tokens:
- id: weather-reporter
token_env: WEATHER_DAILY_UPLOAD_TOKEN
allow_pipelines:
- weather-daily
`)
server := cfg.Server.HTTP
@@ -289,9 +293,6 @@ pipelines:
if got, want := source.Backend, BackendHTTPUpload; got != want {
t.Fatalf("source.backend = %q, want %q", got, want)
}
if got, want := source.Upload.TokenEnv, "WEATHER_DAILY_UPLOAD_TOKEN"; got != want {
t.Fatalf("source.token_env = %q, want %q", got, want)
}
if got, want := source.Upload.StagingPath, "/srv/distributor/staging/weather-daily"; got != want {
t.Fatalf("source.staging_path = %q, want %q", got, want)
}
@@ -309,11 +310,15 @@ pipelines:
- id: weather-daily
source:
backend: http_upload
token_env: WEATHER_DAILY_UPLOAD_TOKEN
destinations:
- id: archive
backend: local
path: /archive
upload_tokens:
- id: weather-reporter
token_env: WEATHER_DAILY_UPLOAD_TOKEN
allow_pipelines:
- weather-daily
`)
source := cfg.Pipelines[0].Source
@@ -325,6 +330,120 @@ pipelines:
}
}
func TestLoadFileAcceptsHTTPUploadTokens(t *testing.T) {
tests := map[string]string{
"valid multi pipeline token": `
pipelines:
- id: weather-daily
source:
backend: http_upload
destinations:
- id: archive
backend: local
path: /archive/weather
- id: calendar-daily
source:
backend: http_upload
destinations:
- id: archive
backend: local
path: /archive/calendar
upload_tokens:
- id: reporter
token_env: REPORTER_UPLOAD_TOKEN
allow_pipelines:
- weather-daily
- calendar-daily
`,
"multiple tokens for one pipeline": `
pipelines:
- id: reports
source:
backend: http_upload
destinations:
- id: archive
backend: local
path: /archive
upload_tokens:
- id: reporter-a
token_env: REPORTER_A_UPLOAD_TOKEN
allow_pipelines:
- reports
- id: reporter-b
token_env: REPORTER_B_UPLOAD_TOKEN
allow_pipelines:
- reports
`,
}
for name, body := range tests {
t.Run(name, func(t *testing.T) {
loadConfig(t, body)
})
}
}
func TestLoadFileRejectsInvalidUploadTokens(t *testing.T) {
tests := map[string]struct {
body string
want string
}{
"missing token list": {
body: `pipelines: [{id: reports, source: {backend: http_upload}, destinations: [{id: archive, backend: local, path: /archive}]}]`,
want: "upload_tokens is required",
},
"duplicate token ids": {
body: `upload_tokens: [{id: reporter, token_env: ONE_UPLOAD_TOKEN, allow_pipelines: [reports]}, {id: reporter, token_env: TWO_UPLOAD_TOKEN, allow_pipelines: [reports]}]
pipelines: [{id: reports, source: {backend: http_upload}, destinations: [{id: archive, backend: local, path: /archive}]}]`,
want: "upload token id reporter is duplicated",
},
"duplicate allowlist entries": {
body: `upload_tokens: [{id: reporter, token_env: UPLOAD_TOKEN, allow_pipelines: [reports, reports]}]
pipelines: [{id: reports, source: {backend: http_upload}, destinations: [{id: archive, backend: local, path: /archive}]}]`,
want: "allow_pipelines contains duplicate pipeline id reports",
},
"unknown allowed pipeline id": {
body: `upload_tokens: [{id: reporter, token_env: UPLOAD_TOKEN, allow_pipelines: [missing]}]
pipelines: [{id: reports, source: {backend: http_upload}, destinations: [{id: archive, backend: local, path: /archive}]}]`,
want: "references unknown pipeline missing",
},
"non upload allowed pipeline id": {
body: `upload_tokens: [{id: reporter, token_env: UPLOAD_TOKEN, allow_pipelines: [reports]}, {id: uploader, token_env: OTHER_UPLOAD_TOKEN, allow_pipelines: [upload]}]
pipelines: [{id: reports, source: {backend: local, path: /source}, destinations: [{id: archive, backend: local, path: /archive}]}, {id: upload, source: {backend: http_upload}, destinations: [{id: archive, backend: local, path: /archive-upload}]}]`,
want: "references non-http_upload pipeline reports",
},
"upload pipeline not allowed": {
body: `upload_tokens: [{id: reporter, token_env: UPLOAD_TOKEN, allow_pipelines: [reports]}]
pipelines: [{id: reports, source: {backend: http_upload}, destinations: [{id: archive, backend: local, path: /archive}]}, {id: other, source: {backend: http_upload}, destinations: [{id: archive, backend: local, path: /archive-other}]}]`,
want: "http_upload pipeline other is not allowed by any upload token",
},
"missing token id": {
body: `upload_tokens: [{token_env: UPLOAD_TOKEN, allow_pipelines: [reports]}]
pipelines: [{id: reports, source: {backend: http_upload}, destinations: [{id: archive, backend: local, path: /archive}]}]`,
want: "upload_tokens[0].id is required",
},
"invalid token id": {
body: `upload_tokens: [{id: ".reporter", token_env: UPLOAD_TOKEN, allow_pipelines: [reports]}]
pipelines: [{id: reports, source: {backend: http_upload}, destinations: [{id: archive, backend: local, path: /archive}]}]`,
want: "upload_tokens[0].id must be a slug-like identifier",
},
"missing token env": {
body: `upload_tokens: [{id: reporter, allow_pipelines: [reports]}]
pipelines: [{id: reports, source: {backend: http_upload}, destinations: [{id: archive, backend: local, path: /archive}]}]`,
want: "upload_tokens[0].token_env is required",
},
"missing allowlist": {
body: `upload_tokens: [{id: reporter, token_env: UPLOAD_TOKEN}]
pipelines: [{id: reports, source: {backend: http_upload}, destinations: [{id: archive, backend: local, path: /archive}]}]`,
want: "upload_tokens[0].allow_pipelines is required",
},
}
for name, tt := range tests {
t.Run(name, func(t *testing.T) {
assertLoadError(t, tt.body, tt.want)
})
}
}
func TestLoadFileValidBackendConfigs(t *testing.T) {
tests := map[string]string{
"local": `
@@ -515,16 +634,22 @@ func TestLoadFileRejectsInvalidS3Config(t *testing.T) {
func TestLoadFileRejectsInvalidHTTPUploadConfig(t *testing.T) {
tests := map[string]string{
"server size": `server: {http: {max_upload_size: 20XB}}`,
"source size": `pipelines: [{id: reports, source: {backend: http_upload, token_env: UPLOAD_TOKEN, max_upload_size: 20XB}, destinations: [{id: archive, backend: local, path: /archive}]}]`,
"zero source size": `pipelines: [{id: reports, source: {backend: http_upload, token_env: UPLOAD_TOKEN, max_upload_size: 0B}, destinations: [{id: archive, backend: local, path: /archive}]}]`,
"server size": `server: {http: {max_upload_size: 20XB}}`,
"source size": `upload_tokens: [{id: reporter, token_env: UPLOAD_TOKEN, allow_pipelines: [reports]}]
pipelines: [{id: reports, source: {backend: http_upload, max_upload_size: 20XB}, destinations: [{id: archive, backend: local, path: /archive}]}]`,
"zero source size": `upload_tokens: [{id: reporter, token_env: UPLOAD_TOKEN, allow_pipelines: [reports]}]
pipelines: [{id: reports, source: {backend: http_upload, max_upload_size: 0B}, destinations: [{id: archive, backend: local, path: /archive}]}]`,
"server duration": `server: {http: {retention: forever}}`,
"zero server duration": `server: {http: {retention: 0s}}`,
"missing token env": `pipelines: [{id: reports, source: {backend: http_upload}, destinations: [{id: archive, backend: local, path: /archive}]}]`,
"missing upload tokens": `pipelines: [{id: reports, source: {backend: http_upload}, destinations: [{id: archive, backend: local, path: /archive}]}]`,
"destination http upload": `pipelines: [{id: reports, source: {backend: local, path: /source}, destinations: [{id: ingest, backend: http_upload}]}]`,
"literal token": `pipelines: [{id: reports, source: {backend: http_upload, token: secret, token_env: UPLOAD_TOKEN}, destinations: [{id: archive, backend: local, path: /archive}]}]`,
"unknown server field": `server: {http: {surprise: true}}`,
"unknown source field": `pipelines: [{id: reports, source: {backend: http_upload, token_env: UPLOAD_TOKEN, surprise: true}, destinations: [{id: archive, backend: local, path: /archive}]}]`,
"literal token": `upload_tokens: [{id: reporter, token: secret, token_env: UPLOAD_TOKEN, allow_pipelines: [reports]}]
pipelines: [{id: reports, source: {backend: http_upload}, destinations: [{id: archive, backend: local, path: /archive}]}]`,
"unknown server field": `server: {http: {surprise: true}}`,
"legacy source token env": `upload_tokens: [{id: reporter, token_env: UPLOAD_TOKEN, allow_pipelines: [reports]}]
pipelines: [{id: reports, source: {backend: http_upload, token_env: UPLOAD_TOKEN}, destinations: [{id: archive, backend: local, path: /archive}]}]`,
"unknown source field": `upload_tokens: [{id: reporter, token_env: UPLOAD_TOKEN, allow_pipelines: [reports]}]
pipelines: [{id: reports, source: {backend: http_upload, surprise: true}, destinations: [{id: archive, backend: local, path: /archive}]}]`,
}
for name, body := range tests {
t.Run(name, func(t *testing.T) {