eric/distributor
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 7 Wiki Activity

Add upload token config validation

Browse Source
This commit is contained in:
Eric Rakestraw
2026-06-08 04:27:49 +00:00
parent 4fa7d1ebb5
commit 033b2e5015
11 changed files with 278 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
internal/config/backend_view_test.go
View File

@@ -53,12 +53,15 @@ func TestBackendViewValidationKeepsHTTPUploadSourceOnly(t *testing.T) {
ID: "reports",
Source: Backend{
Backend: BackendHTTPUpload,
Upload: HTTPUpload{TokenEnv: "UPLOAD_TOKEN"},
},
Destinations: []Destination{{
ID: "archive",
Backend: BackendHTTPUpload,
}},
}}, UploadTokens: []UploadToken{{
ID: "reporter",
TokenEnv: "UPLOAD_TOKEN",
AllowPipelines: []string{"reports"},
}}}
ApplyDefaults(&cfg)

14
internal/config/config.go
View File

@@ -1,9 +1,10 @@
package config
type Config struct {
Server Server `yaml:"server"`
Secrets Secrets `yaml:"secrets"`
Pipelines []Pipeline `yaml:"pipelines"`
Server Server `yaml:"server"`
Secrets Secrets `yaml:"secrets"`
UploadTokens []UploadToken `yaml:"upload_tokens"`
Pipelines []Pipeline `yaml:"pipelines"`
}
type Server struct {
@@ -23,6 +24,12 @@ type Secrets struct {
Directory string `yaml:"directory"`
}
type UploadToken struct {
ID string `yaml:"id"`
TokenEnv string `yaml:"token_env"`
AllowPipelines []string `yaml:"allow_pipelines"`
}
type Pipeline struct {
ID string `yaml:"id"`
Source Backend `yaml:"source"`
@@ -68,7 +75,6 @@ type Backend struct {
}
type HTTPUpload struct {
TokenEnv string `yaml:"token_env"`
StagingPath string `yaml:"staging_path"`
MaxUploadSize *ByteSize `yaml:"max_upload_size"`
}

149
internal/config/load_test.go
View File

@@ -256,13 +256,17 @@ pipelines:
- id: weather-daily
source:
backend: http_upload
token_env: WEATHER_DAILY_UPLOAD_TOKEN
staging_path: /srv/distributor/staging/weather-daily
max_upload_size: 32MB
destinations:
- id: archive
backend: local
path: /archive
upload_tokens:
- id: weather-reporter
token_env: WEATHER_DAILY_UPLOAD_TOKEN
allow_pipelines:
- weather-daily
`)
server := cfg.Server.HTTP
@@ -289,9 +293,6 @@ pipelines:
if got, want := source.Backend, BackendHTTPUpload; got != want {
t.Fatalf("source.backend = %q, want %q", got, want)
}
if got, want := source.Upload.TokenEnv, "WEATHER_DAILY_UPLOAD_TOKEN"; got != want {
t.Fatalf("source.token_env = %q, want %q", got, want)
}
if got, want := source.Upload.StagingPath, "/srv/distributor/staging/weather-daily"; got != want {
t.Fatalf("source.staging_path = %q, want %q", got, want)
}
@@ -309,11 +310,15 @@ pipelines:
- id: weather-daily
source:
backend: http_upload
token_env: WEATHER_DAILY_UPLOAD_TOKEN
destinations:
- id: archive
backend: local
path: /archive
upload_tokens:
- id: weather-reporter
token_env: WEATHER_DAILY_UPLOAD_TOKEN
allow_pipelines:
- weather-daily
`)
source := cfg.Pipelines[0].Source
@@ -325,6 +330,120 @@ pipelines:
}
}
func TestLoadFileAcceptsHTTPUploadTokens(t *testing.T) {
tests := map[string]string{
"valid multi pipeline token": `
pipelines:
- id: weather-daily
source:
backend: http_upload
destinations:
- id: archive
backend: local
path: /archive/weather
- id: calendar-daily
source:
backend: http_upload
destinations:
- id: archive
backend: local
path: /archive/calendar
upload_tokens:
- id: reporter
token_env: REPORTER_UPLOAD_TOKEN
allow_pipelines:
- weather-daily
- calendar-daily
`,
"multiple tokens for one pipeline": `
pipelines:
- id: reports
source:
backend: http_upload
destinations:
- id: archive
backend: local
path: /archive
upload_tokens:
- id: reporter-a
token_env: REPORTER_A_UPLOAD_TOKEN
allow_pipelines:
- reports
- id: reporter-b
token_env: REPORTER_B_UPLOAD_TOKEN
allow_pipelines:
- reports
`,
}
for name, body := range tests {
t.Run(name, func(t *testing.T) {
loadConfig(t, body)
})
}
}
func TestLoadFileRejectsInvalidUploadTokens(t *testing.T) {
tests := map[string]struct {
body string
want string
}{
"missing token list": {
body: `pipelines: [{id: reports, source: {backend: http_upload}, destinations: [{id: archive, backend: local, path: /archive}]}]`,
want: "upload_tokens is required",
},
"duplicate token ids": {
body: `upload_tokens: [{id: reporter, token_env: ONE_UPLOAD_TOKEN, allow_pipelines: [reports]}, {id: reporter, token_env: TWO_UPLOAD_TOKEN, allow_pipelines: [reports]}]
pipelines: [{id: reports, source: {backend: http_upload}, destinations: [{id: archive, backend: local, path: /archive}]}]`,
want: "upload token id reporter is duplicated",
},
"duplicate allowlist entries": {
body: `upload_tokens: [{id: reporter, token_env: UPLOAD_TOKEN, allow_pipelines: [reports, reports]}]
pipelines: [{id: reports, source: {backend: http_upload}, destinations: [{id: archive, backend: local, path: /archive}]}]`,
want: "allow_pipelines contains duplicate pipeline id reports",
},
"unknown allowed pipeline id": {
body: `upload_tokens: [{id: reporter, token_env: UPLOAD_TOKEN, allow_pipelines: [missing]}]
pipelines: [{id: reports, source: {backend: http_upload}, destinations: [{id: archive, backend: local, path: /archive}]}]`,
want: "references unknown pipeline missing",
},
"non upload allowed pipeline id": {
body: `upload_tokens: [{id: reporter, token_env: UPLOAD_TOKEN, allow_pipelines: [reports]}, {id: uploader, token_env: OTHER_UPLOAD_TOKEN, allow_pipelines: [upload]}]
pipelines: [{id: reports, source: {backend: local, path: /source}, destinations: [{id: archive, backend: local, path: /archive}]}, {id: upload, source: {backend: http_upload}, destinations: [{id: archive, backend: local, path: /archive-upload}]}]`,
want: "references non-http_upload pipeline reports",
},
"upload pipeline not allowed": {
body: `upload_tokens: [{id: reporter, token_env: UPLOAD_TOKEN, allow_pipelines: [reports]}]
pipelines: [{id: reports, source: {backend: http_upload}, destinations: [{id: archive, backend: local, path: /archive}]}, {id: other, source: {backend: http_upload}, destinations: [{id: archive, backend: local, path: /archive-other}]}]`,
want: "http_upload pipeline other is not allowed by any upload token",
},
"missing token id": {
body: `upload_tokens: [{token_env: UPLOAD_TOKEN, allow_pipelines: [reports]}]
pipelines: [{id: reports, source: {backend: http_upload}, destinations: [{id: archive, backend: local, path: /archive}]}]`,
want: "upload_tokens[0].id is required",
},
"invalid token id": {
body: `upload_tokens: [{id: ".reporter", token_env: UPLOAD_TOKEN, allow_pipelines: [reports]}]
pipelines: [{id: reports, source: {backend: http_upload}, destinations: [{id: archive, backend: local, path: /archive}]}]`,
want: "upload_tokens[0].id must be a slug-like identifier",
},
"missing token env": {
body: `upload_tokens: [{id: reporter, allow_pipelines: [reports]}]
pipelines: [{id: reports, source: {backend: http_upload}, destinations: [{id: archive, backend: local, path: /archive}]}]`,
want: "upload_tokens[0].token_env is required",
},
"missing allowlist": {
body: `upload_tokens: [{id: reporter, token_env: UPLOAD_TOKEN}]
pipelines: [{id: reports, source: {backend: http_upload}, destinations: [{id: archive, backend: local, path: /archive}]}]`,
want: "upload_tokens[0].allow_pipelines is required",
},
}
for name, tt := range tests {
t.Run(name, func(t *testing.T) {
assertLoadError(t, tt.body, tt.want)
})
}
}
func TestLoadFileValidBackendConfigs(t *testing.T) {
tests := map[string]string{
"local": `
@@ -515,16 +634,22 @@ func TestLoadFileRejectsInvalidS3Config(t *testing.T) {
func TestLoadFileRejectsInvalidHTTPUploadConfig(t *testing.T) {
tests := map[string]string{
"server size": `server: {http: {max_upload_size: 20XB}}`,
"source size": `pipelines: [{id: reports, source: {backend: http_upload, token_env: UPLOAD_TOKEN, max_upload_size: 20XB}, destinations: [{id: archive, backend: local, path: /archive}]}]`,
"zero source size": `pipelines: [{id: reports, source: {backend: http_upload, token_env: UPLOAD_TOKEN, max_upload_size: 0B}, destinations: [{id: archive, backend: local, path: /archive}]}]`,
"server size": `server: {http: {max_upload_size: 20XB}}`,
"source size": `upload_tokens: [{id: reporter, token_env: UPLOAD_TOKEN, allow_pipelines: [reports]}]
pipelines: [{id: reports, source: {backend: http_upload, max_upload_size: 20XB}, destinations: [{id: archive, backend: local, path: /archive}]}]`,
"zero source size": `upload_tokens: [{id: reporter, token_env: UPLOAD_TOKEN, allow_pipelines: [reports]}]
pipelines: [{id: reports, source: {backend: http_upload, max_upload_size: 0B}, destinations: [{id: archive, backend: local, path: /archive}]}]`,
"server duration": `server: {http: {retention: forever}}`,
"zero server duration": `server: {http: {retention: 0s}}`,
"missing token env": `pipelines: [{id: reports, source: {backend: http_upload}, destinations: [{id: archive, backend: local, path: /archive}]}]`,
"missing upload tokens": `pipelines: [{id: reports, source: {backend: http_upload}, destinations: [{id: archive, backend: local, path: /archive}]}]`,
"destination http upload": `pipelines: [{id: reports, source: {backend: local, path: /source}, destinations: [{id: ingest, backend: http_upload}]}]`,
"literal token": `pipelines: [{id: reports, source: {backend: http_upload, token: secret, token_env: UPLOAD_TOKEN}, destinations: [{id: archive, backend: local, path: /archive}]}]`,
"unknown server field": `server: {http: {surprise: true}}`,
"unknown source field": `pipelines: [{id: reports, source: {backend: http_upload, token_env: UPLOAD_TOKEN, surprise: true}, destinations: [{id: archive, backend: local, path: /archive}]}]`,
"literal token": `upload_tokens: [{id: reporter, token: secret, token_env: UPLOAD_TOKEN, allow_pipelines: [reports]}]
pipelines: [{id: reports, source: {backend: http_upload}, destinations: [{id: archive, backend: local, path: /archive}]}]`,
"unknown server field": `server: {http: {surprise: true}}`,
"legacy source token env": `upload_tokens: [{id: reporter, token_env: UPLOAD_TOKEN, allow_pipelines: [reports]}]
pipelines: [{id: reports, source: {backend: http_upload, token_env: UPLOAD_TOKEN}, destinations: [{id: archive, backend: local, path: /archive}]}]`,
"unknown source field": `upload_tokens: [{id: reporter, token_env: UPLOAD_TOKEN, allow_pipelines: [reports]}]
pipelines: [{id: reports, source: {backend: http_upload, surprise: true}, destinations: [{id: archive, backend: local, path: /archive}]}]`,
}
for name, body := range tests {
t.Run(name, func(t *testing.T) {

73
internal/config/validate.go
View File

@@ -29,6 +29,7 @@ func Validate(cfg Config) error {
}
pipelineIDs := make(map[string]struct{}, len(cfg.Pipelines))
uploadPipelineIDs := make(map[string]struct{})
for pipelineIndex, pipeline := range cfg.Pipelines {
pipelineContext := fmt.Sprintf("pipelines[%d]", pipelineIndex)
if pipeline.ID == "" {
@@ -42,6 +43,9 @@ func Validate(cfg Config) error {
}
errs = validateSourceBackend(errs, pipelineContext+".source", pipeline.Source)
if pipeline.Source.Backend == BackendHTTPUpload && pipeline.ID != "" {
uploadPipelineIDs[pipeline.ID] = struct{}{}
}
errs = validateValidationPolicy(errs, pipelineContext+".validation", pipeline.Validation)
if len(pipeline.Destinations) == 0 {
errs = append(errs, pipelineContext+".destinations is required")
@@ -68,6 +72,8 @@ func Validate(cfg Config) error {
}
}
errs = validateUploadTokens(errs, cfg.UploadTokens, pipelineIDs, uploadPipelineIDs)
if len(errs) > 0 {
return errs
}
@@ -112,9 +118,6 @@ func validateDestinationBackend(errs ValidationErrors, context string, destinati
}
func validateHTTPUploadSource(errs ValidationErrors, context string, upload HTTPUpload) ValidationErrors {
if upload.TokenEnv == "" {
errs = append(errs, context+".token_env is required for http_upload backend")
}
if upload.StagingPath == "" {
errs = append(errs, context+".staging_path is required for http_upload backend")
}
@@ -124,6 +127,70 @@ func validateHTTPUploadSource(errs ValidationErrors, context string, upload HTTP
return errs
}
func validateUploadTokens(errs ValidationErrors, tokens []UploadToken, pipelineIDs, uploadPipelineIDs map[string]struct{}) ValidationErrors {
if len(uploadPipelineIDs) == 0 {
if len(tokens) > 0 {
errs = append(errs, "upload_tokens must reference configured http_upload pipelines")
}
return errs
}
if len(tokens) == 0 {
return append(errs, "upload_tokens is required when any pipeline source backend is http_upload")
}
tokenIDs := make(map[string]struct{}, len(tokens))
allowedUploadPipelineIDs := make(map[string]struct{}, len(uploadPipelineIDs))
for tokenIndex, token := range tokens {
context := fmt.Sprintf("upload_tokens[%d]", tokenIndex)
if token.ID == "" {
errs = append(errs, context+".id is required")
} else if !idPattern.MatchString(token.ID) {
errs = append(errs, context+".id must be a slug-like identifier")
} else if _, exists := tokenIDs[token.ID]; exists {
errs = append(errs, "upload token id "+token.ID+" is duplicated")
} else {
tokenIDs[token.ID] = struct{}{}
}
if token.TokenEnv == "" {
errs = append(errs, context+".token_env is required")
}
if len(token.AllowPipelines) == 0 {
errs = append(errs, context+".allow_pipelines is required")
}
seenAllowed := make(map[string]struct{}, len(token.AllowPipelines))
for allowIndex, pipelineID := range token.AllowPipelines {
allowContext := fmt.Sprintf("%s.allow_pipelines[%d]", context, allowIndex)
if pipelineID == "" {
errs = append(errs, allowContext+" is required")
continue
}
if _, exists := seenAllowed[pipelineID]; exists {
errs = append(errs, context+".allow_pipelines contains duplicate pipeline id "+pipelineID)
continue
}
seenAllowed[pipelineID] = struct{}{}
if _, exists := pipelineIDs[pipelineID]; !exists {
errs = append(errs, allowContext+" references unknown pipeline "+pipelineID)
continue
}
if _, exists := uploadPipelineIDs[pipelineID]; !exists {
errs = append(errs, allowContext+" references non-http_upload pipeline "+pipelineID)
continue
}
allowedUploadPipelineIDs[pipelineID] = struct{}{}
}
}
for pipelineID := range uploadPipelineIDs {
if _, exists := allowedUploadPipelineIDs[pipelineID]; !exists {
errs = append(errs, "http_upload pipeline "+pipelineID+" is not allowed by any upload token")
}
}
return errs
}
func validateBackend(errs ValidationErrors, context string, backend backendView) ValidationErrors {
switch backend.Backend {
case "":