diff --git a/docs/config.md b/docs/config.md index 79f0f28..4b799f9 100644 --- a/docs/config.md +++ b/docs/config.md @@ -10,7 +10,7 @@ If `--config` is omitted, `run` uses: /usr/local/etc/distributor/config.yml ``` -Config parsing rejects unknown YAML fields. The executable backends are `local`, `ssh`, and `s3`. +Config parsing rejects unknown YAML fields. The executable `run` backends are `local`, `ssh`, and `s3`. The schema also accepts `http_upload` as a source-only ingestion backend configuration. ## Minimal Local Config @@ -31,6 +31,14 @@ This publishes source files only. It uses the default validation and transfer po ## Production-Oriented Local Config ```yaml +server: + http: + bind: 127.0.0.1:8080 + staging_root: /var/spool/distributor + max_upload_size: 20MB + queue_size: 16 + max_concurrency: 1 + retention: 24h pipelines: - id: reports source: @@ -52,6 +60,38 @@ pipelines: on_conflict: fail ``` +## HTTP Upload Source Configuration + +HTTP upload sources are configured as pipeline sources only. They are not valid destination backends. + +```yaml +server: + http: + bind: 127.0.0.1:8080 + staging_root: /var/spool/distributor + max_upload_size: 20MB + queue_size: 16 + max_concurrency: 1 + retention: 24h +pipelines: + - id: weather-daily + source: + backend: http_upload + token_env: WEATHER_DAILY_UPLOAD_TOKEN + staging_path: /var/spool/distributor/weather-daily + max_upload_size: 20MB + destinations: + - id: archive + backend: local + path: /srv/reports/archive +``` + +`source.token_env` is required and names the environment variable or `secrets.directory` file that provides the bearer token. Literal upload tokens are not supported in YAML. + +`source.staging_path` is optional. When omitted, it defaults to `/`. + +`source.max_upload_size` is optional. When omitted, it defaults to `server.http.max_upload_size`. + ## HTML Publication To publish generated sidecar HTML from Markdown files: @@ -143,6 +183,12 @@ Output URLs are built from `links.base_url`, the destination bundle path, and th Top level: +- `server.http.bind`: optional HTTP bind address; defaults to `127.0.0.1:8080`. +- `server.http.staging_root`: optional root for default HTTP upload staging paths; defaults to `/var/spool/distributor`. +- `server.http.max_upload_size`: optional default upload size limit; defaults to `20MB`. +- `server.http.queue_size`: optional HTTP upload admission queue size; defaults to `16`. +- `server.http.max_concurrency`: optional HTTP upload worker concurrency; defaults to `1`. +- `server.http.retention`: optional completed upload retention duration; defaults to `24h`. - `secrets.directory`: optional credential secrets directory. - `pipelines`: required non-empty list. @@ -170,6 +216,9 @@ Source backend: - `force_path_style`: optional for `s3`; defaults to `true`. Set `false` only for services that require virtual-host addressing. - `credentials.access_key_id_env`: optional S3 credential environment variable name. - `credentials.secret_access_key_env`: optional S3 credential environment variable name. +- `token_env`: required for `http_upload`; names the token environment variable or secret-file name. +- `staging_path`: optional for `http_upload`; defaults below `server.http.staging_root` using the pipeline id. +- `max_upload_size`: optional for `http_upload`; defaults to `server.http.max_upload_size`. Destination: @@ -187,6 +236,20 @@ Accepted backend names: - `local`: executable; requires `path`. - `ssh`: executable; requires `host` and `path`. - `s3`: executable; requires `endpoint` and `bucket`. +- `http_upload`: source-only configuration; requires `token_env`. + +## Size And Duration Values + +Upload size fields use an integer plus one of the supported binary-size suffixes: + +- `B` +- `KB` +- `MB` +- `GB` + +Suffix multipliers use powers of 1024. Size values must be greater than zero after defaults are applied. + +HTTP retention uses Go-style duration strings such as `24h`, `90m`, or `168h`. Retention must be greater than zero after defaults are applied. ## SSH Backend @@ -266,6 +329,14 @@ Defaults are applied after YAML decoding and before validation: - SSH `host_key_policy: accept-new` - S3 `region: us-east-1` - S3 `force_path_style: true` +- `server.http.bind: 127.0.0.1:8080` +- `server.http.staging_root: /var/spool/distributor` +- `server.http.max_upload_size: 20MB` +- `server.http.queue_size: 16` +- `server.http.max_concurrency: 1` +- `server.http.retention: 24h` +- `source.staging_path: /var/spool/distributor/` for `http_upload` +- `source.max_upload_size: server.http.max_upload_size` for `http_upload` - `transform.markdown_to_html.mode: sidecar` when a Markdown-to-HTML transform block is present and mode is omitted - `publish.source: true` - `publish.html: false` diff --git a/docs/internal/config.md b/docs/internal/config.md index 42d7960..80d7828 100644 --- a/docs/internal/config.md +++ b/docs/internal/config.md @@ -6,7 +6,7 @@ ## Inputs and outputs -Input is a YAML file containing optional `secrets` and required `pipelines`. Output is a `Config` value with defaults applied and validation completed. Load failures include the config path and whether the failure occurred during file loading, YAML parsing, or validation. +Input is a YAML file containing optional `server`, optional `secrets`, and required `pipelines`. Output is a `Config` value with defaults applied and validation completed. Load failures include the config path and whether the failure occurred during file loading, YAML parsing, or validation. ## Loading flow @@ -20,6 +20,14 @@ Known-field checking rejects misspelled or unknown YAML keys before defaults and Defaults are applied in `ApplyDefaults`: +- HTTP server `bind` defaults to `127.0.0.1:8080`; +- HTTP server `staging_root` defaults to `/var/spool/distributor`; +- HTTP server `max_upload_size` defaults to `20MB`; +- HTTP server `queue_size` defaults to `16`; +- HTTP server `max_concurrency` defaults to `1`; +- HTTP server `retention` defaults to `24h`; +- `http_upload` source `staging_path` defaults to `/`; +- `http_upload` source `max_upload_size` defaults to `server.http.max_upload_size`; - pipeline validation defaults `on_digest_mismatch` to `fail`; - SSH backend `port` defaults to `22`; - SSH backend `host_key_policy` defaults to `accept-new`; @@ -34,7 +42,11 @@ Defaults are applied in `ApplyDefaults`: ## Validation responsibilities -Validation requires at least one pipeline, slug-like unique pipeline ids, one source per pipeline, at least one destination, slug-like unique destination ids within each pipeline, backend-specific required fields, valid validation policy, valid publish and transform combinations, valid destination path mapping mode, valid destination link config, and valid transfer actions. +Validation requires positive HTTP server limits and retention, at least one pipeline, slug-like unique pipeline ids, one source per pipeline, at least one destination, slug-like unique destination ids within each pipeline, backend-specific required fields, valid validation policy, valid publish and transform combinations, valid destination path mapping mode, valid destination link config, and valid transfer actions. + +HTTP upload sources require `token_env`, a staging path after defaults, and a positive maximum upload size. Literal token fields are not part of the YAML schema. The `http_upload` backend is accepted only for sources and rejected for destinations. + +Upload size values are parsed from strings with `B`, `KB`, `MB`, or `GB` suffixes using 1024 multipliers. Retention values are parsed with `time.ParseDuration`. Explicit zero values fail validation; omitted values receive defaults before validation. Transfer validation accepts `replace` for `on_destination_newer` and `on_conflict`, but publish planning honors those destructive actions only when the current run explicitly requests force. @@ -46,12 +58,14 @@ Destination links are optional. When a `links` block is present, `base_url` is r ## Executable support boundary -Config validation accepts `local`, `ssh`, and `s3` backend shapes. Runtime execution opens all three through `internal/app`. +Config validation accepts `local`, `ssh`, `s3`, and source-only `http_upload` backend shapes. Runtime execution opens `local`, `ssh`, and `s3` through `internal/app`. SSH config uses structured fields: `host`, optional `user`, optional `port`, `path`, optional `ssh_key_file`, optional `known_hosts`, and optional `host_key_policy`. `host_key_policy` accepts YAML booleans and strings and normalizes `true`/`strict`, `accept-new`, and `false`/`off`. S3 config requires `endpoint` and `bucket`, normalizes optional `prefix`, defaults `region` to `us-east-1`, and defaults omitted `force_path_style` to `true` while preserving explicit `false`. +HTTP upload config is source-only. Config owns its YAML shape, defaulting, size and duration parsing, and validation. The config package does not resolve `token_env`, authenticate requests, stage uploads, or execute HTTP upload sources. + ## Secrets and credential resolution `secrets.directory` points to a directory of credential files. `LoadSecretEnvironment` reads regular files and symlinks to regular files, rejects invalid filenames, trims exactly one trailing LF or CRLF, and returns an `Environment` resolver plus conflict metadata. diff --git a/internal/config/config.go b/internal/config/config.go index 5ce25b2..c4e02fe 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -1,10 +1,24 @@ package config type Config struct { + Server Server `yaml:"server"` Secrets Secrets `yaml:"secrets"` Pipelines []Pipeline `yaml:"pipelines"` } +type Server struct { + HTTP HTTPServer `yaml:"http"` +} + +type HTTPServer struct { + Bind string `yaml:"bind"` + StagingRoot string `yaml:"staging_root"` + MaxUploadSize *ByteSize `yaml:"max_upload_size"` + QueueSize int `yaml:"queue_size"` + MaxConcurrency int `yaml:"max_concurrency"` + Retention *Duration `yaml:"retention"` +} + type Secrets struct { Directory string `yaml:"directory"` } @@ -50,6 +64,13 @@ type Backend struct { ForcePath *bool `yaml:"force_path_style"` Creds Credentials `yaml:"credentials"` SSH SSH `yaml:",inline"` + Upload HTTPUpload `yaml:",inline"` +} + +type HTTPUpload struct { + TokenEnv string `yaml:"token_env"` + StagingPath string `yaml:"staging_path"` + MaxUploadSize *ByteSize `yaml:"max_upload_size"` } type SSH struct { diff --git a/internal/config/defaults.go b/internal/config/defaults.go index 4e479a3..5cd25e4 100644 --- a/internal/config/defaults.go +++ b/internal/config/defaults.go @@ -1,13 +1,19 @@ package config -import "gitea.maximumdirect.net/eric/distributor/internal/transform" +import ( + "path/filepath" + "time" + + "gitea.maximumdirect.net/eric/distributor/internal/transform" +) const DefaultConfigPath = "/usr/local/etc/distributor/config.yml" const ( - BackendLocal = "local" - BackendSSH = "ssh" - BackendS3 = "s3" + BackendLocal = "local" + BackendSSH = "ssh" + BackendS3 = "s3" + BackendHTTPUpload = "http_upload" ) const ( @@ -38,10 +44,23 @@ const ( const DefaultS3Region = "us-east-1" +const ( + DefaultHTTPBind = "127.0.0.1:8080" + DefaultHTTPStagingRoot = "/var/spool/distributor" + DefaultHTTPMaxUploadSize = ByteSize(20 * 1024 * 1024) + DefaultHTTPQueueSize = 16 + DefaultHTTPMaxConcurrency = 1 + DefaultHTTPRetention = Duration(24 * time.Hour) +) + func ApplyDefaults(cfg *Config) { + applyHTTPServerDefaults(&cfg.Server.HTTP) for pipelineIndex := range cfg.Pipelines { pipeline := &cfg.Pipelines[pipelineIndex] applyBackendDefaults(&pipeline.Source) + if pipeline.Source.Backend == BackendHTTPUpload { + applyHTTPUploadDefaults(&pipeline.Source.Upload, pipeline.ID, cfg.Server.HTTP) + } if pipeline.Validation.OnDigestMismatch == "" { pipeline.Validation.OnDigestMismatch = ValidationActionFail } @@ -76,6 +95,44 @@ func ApplyDefaults(cfg *Config) { } } +func applyHTTPServerDefaults(server *HTTPServer) { + if server.Bind == "" { + server.Bind = DefaultHTTPBind + } + if server.StagingRoot == "" { + server.StagingRoot = DefaultHTTPStagingRoot + } + if server.MaxUploadSize == nil { + server.MaxUploadSize = byteSize(DefaultHTTPMaxUploadSize) + } + if server.QueueSize == 0 { + server.QueueSize = DefaultHTTPQueueSize + } + if server.MaxConcurrency == 0 { + server.MaxConcurrency = DefaultHTTPMaxConcurrency + } + if server.Retention == nil { + server.Retention = duration(DefaultHTTPRetention) + } +} + +func applyHTTPUploadDefaults(upload *HTTPUpload, pipelineID string, server HTTPServer) { + if upload.StagingPath == "" && pipelineID != "" { + upload.StagingPath = filepath.Join(server.StagingRoot, pipelineID) + } + if upload.MaxUploadSize == nil && server.MaxUploadSize != nil { + upload.MaxUploadSize = byteSize(*server.MaxUploadSize) + } +} + +func byteSize(value ByteSize) *ByteSize { + return &value +} + +func duration(value Duration) *Duration { + return &value +} + func applyBackendDefaults(backend *Backend) { if backend.Backend == BackendSSH { if backend.Port == 0 { diff --git a/internal/config/load_test.go b/internal/config/load_test.go index a687e4d..213e805 100644 --- a/internal/config/load_test.go +++ b/internal/config/load_test.go @@ -208,6 +208,123 @@ pipelines: } } +func TestLoadFileDefaultsHTTPServerConfig(t *testing.T) { + cfg := loadConfig(t, ` +pipelines: + - id: reports + source: + backend: local + path: /source + destinations: + - id: archive + backend: local + path: /archive +`) + + server := cfg.Server.HTTP + if got, want := server.Bind, DefaultHTTPBind; got != want { + t.Fatalf("server.http.bind = %q, want %q", got, want) + } + if got, want := server.StagingRoot, DefaultHTTPStagingRoot; got != want { + t.Fatalf("server.http.staging_root = %q, want %q", got, want) + } + if got, want := *server.MaxUploadSize, DefaultHTTPMaxUploadSize; got != want { + t.Fatalf("server.http.max_upload_size = %s, want %s", got, want) + } + if got, want := server.QueueSize, DefaultHTTPQueueSize; got != want { + t.Fatalf("server.http.queue_size = %d, want %d", got, want) + } + if got, want := server.MaxConcurrency, DefaultHTTPMaxConcurrency; got != want { + t.Fatalf("server.http.max_concurrency = %d, want %d", got, want) + } + if got, want := *server.Retention, DefaultHTTPRetention; got != want { + t.Fatalf("server.http.retention = %s, want %s", got, want) + } +} + +func TestLoadFileAcceptsHTTPUploadSourceConfig(t *testing.T) { + cfg := loadConfig(t, ` +server: + http: + bind: 127.0.0.1:9090 + staging_root: /srv/distributor/staging + max_upload_size: 64MB + queue_size: 32 + max_concurrency: 2 + retention: 48h +pipelines: + - id: weather-daily + source: + backend: http_upload + token_env: WEATHER_DAILY_UPLOAD_TOKEN + staging_path: /srv/distributor/staging/weather-daily + max_upload_size: 32MB + destinations: + - id: archive + backend: local + path: /archive +`) + + server := cfg.Server.HTTP + if got, want := server.Bind, "127.0.0.1:9090"; got != want { + t.Fatalf("server.http.bind = %q, want %q", got, want) + } + if got, want := server.StagingRoot, "/srv/distributor/staging"; got != want { + t.Fatalf("server.http.staging_root = %q, want %q", got, want) + } + if got, want := *server.MaxUploadSize, ByteSize(64*1024*1024); got != want { + t.Fatalf("server.http.max_upload_size = %s, want %s", got, want) + } + if got, want := server.QueueSize, 32; got != want { + t.Fatalf("server.http.queue_size = %d, want %d", got, want) + } + if got, want := server.MaxConcurrency, 2; got != want { + t.Fatalf("server.http.max_concurrency = %d, want %d", got, want) + } + if got, want := server.Retention.String(), "48h0m0s"; got != want { + t.Fatalf("server.http.retention = %s, want %s", got, want) + } + + source := cfg.Pipelines[0].Source + if got, want := source.Backend, BackendHTTPUpload; got != want { + t.Fatalf("source.backend = %q, want %q", got, want) + } + if got, want := source.Upload.TokenEnv, "WEATHER_DAILY_UPLOAD_TOKEN"; got != want { + t.Fatalf("source.token_env = %q, want %q", got, want) + } + if got, want := source.Upload.StagingPath, "/srv/distributor/staging/weather-daily"; got != want { + t.Fatalf("source.staging_path = %q, want %q", got, want) + } + if got, want := *source.Upload.MaxUploadSize, ByteSize(32*1024*1024); got != want { + t.Fatalf("source.max_upload_size = %s, want %s", got, want) + } +} + +func TestLoadFileDefaultsHTTPUploadSourceConfig(t *testing.T) { + cfg := loadConfig(t, ` +server: + http: + max_upload_size: 12MB +pipelines: + - id: weather-daily + source: + backend: http_upload + token_env: WEATHER_DAILY_UPLOAD_TOKEN + destinations: + - id: archive + backend: local + path: /archive +`) + + source := cfg.Pipelines[0].Source + if got, want := source.Upload.StagingPath, "/var/spool/distributor/weather-daily"; got != want { + t.Fatalf("source.staging_path = %q, want %q", got, want) + } + if got, want := *source.Upload.MaxUploadSize, ByteSize(12*1024*1024); got != want { + t.Fatalf("source.max_upload_size = %s, want %s", got, want) + } +} + func TestLoadFileValidBackendConfigs(t *testing.T) { tests := map[string]string{ "local": ` @@ -396,6 +513,26 @@ func TestLoadFileRejectsInvalidS3Config(t *testing.T) { } } +func TestLoadFileRejectsInvalidHTTPUploadConfig(t *testing.T) { + tests := map[string]string{ + "server size": `server: {http: {max_upload_size: 20XB}}`, + "source size": `pipelines: [{id: reports, source: {backend: http_upload, token_env: UPLOAD_TOKEN, max_upload_size: 20XB}, destinations: [{id: archive, backend: local, path: /archive}]}]`, + "zero source size": `pipelines: [{id: reports, source: {backend: http_upload, token_env: UPLOAD_TOKEN, max_upload_size: 0B}, destinations: [{id: archive, backend: local, path: /archive}]}]`, + "server duration": `server: {http: {retention: forever}}`, + "zero server duration": `server: {http: {retention: 0s}}`, + "missing token env": `pipelines: [{id: reports, source: {backend: http_upload}, destinations: [{id: archive, backend: local, path: /archive}]}]`, + "destination http upload": `pipelines: [{id: reports, source: {backend: local, path: /source}, destinations: [{id: ingest, backend: http_upload}]}]`, + "literal token": `pipelines: [{id: reports, source: {backend: http_upload, token: secret, token_env: UPLOAD_TOKEN}, destinations: [{id: archive, backend: local, path: /archive}]}]`, + "unknown server field": `server: {http: {surprise: true}}`, + "unknown source field": `pipelines: [{id: reports, source: {backend: http_upload, token_env: UPLOAD_TOKEN, surprise: true}, destinations: [{id: archive, backend: local, path: /archive}]}]`, + } + for name, body := range tests { + t.Run(name, func(t *testing.T) { + assertLoadError(t, body, "") + }) + } +} + func TestLoadFileDefaultsSSHConfig(t *testing.T) { cfg := loadConfig(t, ` pipelines: diff --git a/internal/config/quantity.go b/internal/config/quantity.go new file mode 100644 index 0000000..7222d96 --- /dev/null +++ b/internal/config/quantity.go @@ -0,0 +1,117 @@ +package config + +import ( + "fmt" + "strconv" + "strings" + "time" + + "gopkg.in/yaml.v3" +) + +type ByteSize int64 + +type Duration time.Duration + +func (size *ByteSize) UnmarshalYAML(value *yaml.Node) error { + if value.Kind != yaml.ScalarNode || value.Tag != "!!str" { + return fmt.Errorf("size must be a string with B, KB, MB, or GB suffix") + } + + var raw string + if err := value.Decode(&raw); err != nil { + return err + } + parsed, err := ParseByteSize(raw) + if err != nil { + return err + } + *size = parsed + return nil +} + +func (size ByteSize) String() string { + value := int64(size) + if value == 0 { + return "0B" + } + units := []struct { + suffix string + multiplier int64 + }{ + {suffix: "GB", multiplier: 1024 * 1024 * 1024}, + {suffix: "MB", multiplier: 1024 * 1024}, + {suffix: "KB", multiplier: 1024}, + {suffix: "B", multiplier: 1}, + } + for _, unit := range units { + if value%unit.multiplier == 0 { + return strconv.FormatInt(value/unit.multiplier, 10) + unit.suffix + } + } + return strconv.FormatInt(value, 10) + "B" +} + +func ParseByteSize(raw string) (ByteSize, error) { + value := strings.TrimSpace(raw) + if value == "" { + return 0, fmt.Errorf("size is required") + } + + units := []struct { + suffix string + multiplier int64 + }{ + {suffix: "GB", multiplier: 1024 * 1024 * 1024}, + {suffix: "MB", multiplier: 1024 * 1024}, + {suffix: "KB", multiplier: 1024}, + {suffix: "B", multiplier: 1}, + } + for _, unit := range units { + number, ok := strings.CutSuffix(value, unit.suffix) + if !ok { + continue + } + if strings.TrimSpace(number) != number || number == "" { + return 0, fmt.Errorf("size must be an integer followed by B, KB, MB, or GB") + } + parsed, err := strconv.ParseInt(number, 10, 64) + if err != nil { + return 0, fmt.Errorf("size must be an integer followed by B, KB, MB, or GB") + } + if parsed < 0 { + return 0, fmt.Errorf("size must be non-negative") + } + const maxInt64 = int64(1<<63 - 1) + if parsed > 0 && parsed > maxInt64/unit.multiplier { + return 0, fmt.Errorf("size is too large") + } + return ByteSize(parsed * unit.multiplier), nil + } + return 0, fmt.Errorf("size must use B, KB, MB, or GB suffix") +} + +func (duration *Duration) UnmarshalYAML(value *yaml.Node) error { + if value.Kind != yaml.ScalarNode || value.Tag != "!!str" { + return fmt.Errorf("duration must be a string duration") + } + + var raw string + if err := value.Decode(&raw); err != nil { + return err + } + parsed, err := time.ParseDuration(raw) + if err != nil { + return fmt.Errorf("duration must be a valid duration: %w", err) + } + *duration = Duration(parsed) + return nil +} + +func (duration Duration) String() string { + return time.Duration(duration).String() +} + +func (duration Duration) AsDuration() time.Duration { + return time.Duration(duration) +} diff --git a/internal/config/validate.go b/internal/config/validate.go index 8968bed..af93a24 100644 --- a/internal/config/validate.go +++ b/internal/config/validate.go @@ -22,6 +22,8 @@ func (e ValidationErrors) Error() string { func Validate(cfg Config) error { var errs ValidationErrors + errs = validateHTTPServer(errs, "server.http", cfg.Server.HTTP) + if len(cfg.Pipelines) == 0 { errs = append(errs, "pipelines is required") } @@ -72,14 +74,56 @@ func Validate(cfg Config) error { return nil } +func validateHTTPServer(errs ValidationErrors, context string, server HTTPServer) ValidationErrors { + if server.Bind == "" { + errs = append(errs, context+".bind is required") + } + if server.StagingRoot == "" { + errs = append(errs, context+".staging_root is required") + } + if server.MaxUploadSize == nil || *server.MaxUploadSize <= 0 { + errs = append(errs, context+".max_upload_size must be greater than zero") + } + if server.QueueSize <= 0 { + errs = append(errs, context+".queue_size must be greater than zero") + } + if server.MaxConcurrency <= 0 { + errs = append(errs, context+".max_concurrency must be greater than zero") + } + if server.Retention == nil || *server.Retention <= 0 { + errs = append(errs, context+".retention must be greater than zero") + } + return errs +} + func validateSourceBackend(errs ValidationErrors, context string, backend Backend) ValidationErrors { + if backend.Backend == BackendHTTPUpload { + return validateHTTPUploadSource(errs, context, backend.Upload) + } return validateBackend(errs, context, backend.Backend, backend.Host, backend.Port, backend.Path, backend.Endpoint, backend.Bucket, backend.Prefix, backend.SSH.HostKeyPolicy, backend.Creds) } func validateDestinationBackend(errs ValidationErrors, context string, destination Destination) ValidationErrors { + if destination.Backend == BackendHTTPUpload { + errs = append(errs, context+".backend "+BackendHTTPUpload+" is only supported for sources") + return errs + } return validateBackend(errs, context, destination.Backend, destination.Host, destination.Port, destination.Path, destination.Endpoint, destination.Bucket, destination.Prefix, destination.SSH.HostKeyPolicy, destination.Creds) } +func validateHTTPUploadSource(errs ValidationErrors, context string, upload HTTPUpload) ValidationErrors { + if upload.TokenEnv == "" { + errs = append(errs, context+".token_env is required for http_upload backend") + } + if upload.StagingPath == "" { + errs = append(errs, context+".staging_path is required for http_upload backend") + } + if upload.MaxUploadSize == nil || *upload.MaxUploadSize <= 0 { + errs = append(errs, context+".max_upload_size must be greater than zero") + } + return errs +} + func validateBackend(errs ValidationErrors, context, backend, host string, port int, path, endpoint, bucket, prefix string, hostKeyPolicy HostKeyPolicy, creds Credentials) ValidationErrors { switch backend { case "":