Add HTTP upload configuration support
This commit is contained in:
@@ -10,7 +10,7 @@ If `--config` is omitted, `run` uses:
|
||||
/usr/local/etc/distributor/config.yml
|
||||
```
|
||||
|
||||
Config parsing rejects unknown YAML fields. The executable backends are `local`, `ssh`, and `s3`.
|
||||
Config parsing rejects unknown YAML fields. The executable `run` backends are `local`, `ssh`, and `s3`. The schema also accepts `http_upload` as a source-only ingestion backend configuration.
|
||||
|
||||
## Minimal Local Config
|
||||
|
||||
@@ -31,6 +31,14 @@ This publishes source files only. It uses the default validation and transfer po
|
||||
## Production-Oriented Local Config
|
||||
|
||||
```yaml
|
||||
server:
|
||||
http:
|
||||
bind: 127.0.0.1:8080
|
||||
staging_root: /var/spool/distributor
|
||||
max_upload_size: 20MB
|
||||
queue_size: 16
|
||||
max_concurrency: 1
|
||||
retention: 24h
|
||||
pipelines:
|
||||
- id: reports
|
||||
source:
|
||||
@@ -52,6 +60,38 @@ pipelines:
|
||||
on_conflict: fail
|
||||
```
|
||||
|
||||
## HTTP Upload Source Configuration
|
||||
|
||||
HTTP upload sources are configured as pipeline sources only. They are not valid destination backends.
|
||||
|
||||
```yaml
|
||||
server:
|
||||
http:
|
||||
bind: 127.0.0.1:8080
|
||||
staging_root: /var/spool/distributor
|
||||
max_upload_size: 20MB
|
||||
queue_size: 16
|
||||
max_concurrency: 1
|
||||
retention: 24h
|
||||
pipelines:
|
||||
- id: weather-daily
|
||||
source:
|
||||
backend: http_upload
|
||||
token_env: WEATHER_DAILY_UPLOAD_TOKEN
|
||||
staging_path: /var/spool/distributor/weather-daily
|
||||
max_upload_size: 20MB
|
||||
destinations:
|
||||
- id: archive
|
||||
backend: local
|
||||
path: /srv/reports/archive
|
||||
```
|
||||
|
||||
`source.token_env` is required and names the environment variable or `secrets.directory` file that provides the bearer token. Literal upload tokens are not supported in YAML.
|
||||
|
||||
`source.staging_path` is optional. When omitted, it defaults to `<server.http.staging_root>/<pipeline id>`.
|
||||
|
||||
`source.max_upload_size` is optional. When omitted, it defaults to `server.http.max_upload_size`.
|
||||
|
||||
## HTML Publication
|
||||
|
||||
To publish generated sidecar HTML from Markdown files:
|
||||
@@ -143,6 +183,12 @@ Output URLs are built from `links.base_url`, the destination bundle path, and th
|
||||
|
||||
Top level:
|
||||
|
||||
- `server.http.bind`: optional HTTP bind address; defaults to `127.0.0.1:8080`.
|
||||
- `server.http.staging_root`: optional root for default HTTP upload staging paths; defaults to `/var/spool/distributor`.
|
||||
- `server.http.max_upload_size`: optional default upload size limit; defaults to `20MB`.
|
||||
- `server.http.queue_size`: optional HTTP upload admission queue size; defaults to `16`.
|
||||
- `server.http.max_concurrency`: optional HTTP upload worker concurrency; defaults to `1`.
|
||||
- `server.http.retention`: optional completed upload retention duration; defaults to `24h`.
|
||||
- `secrets.directory`: optional credential secrets directory.
|
||||
- `pipelines`: required non-empty list.
|
||||
|
||||
@@ -170,6 +216,9 @@ Source backend:
|
||||
- `force_path_style`: optional for `s3`; defaults to `true`. Set `false` only for services that require virtual-host addressing.
|
||||
- `credentials.access_key_id_env`: optional S3 credential environment variable name.
|
||||
- `credentials.secret_access_key_env`: optional S3 credential environment variable name.
|
||||
- `token_env`: required for `http_upload`; names the token environment variable or secret-file name.
|
||||
- `staging_path`: optional for `http_upload`; defaults below `server.http.staging_root` using the pipeline id.
|
||||
- `max_upload_size`: optional for `http_upload`; defaults to `server.http.max_upload_size`.
|
||||
|
||||
Destination:
|
||||
|
||||
@@ -187,6 +236,20 @@ Accepted backend names:
|
||||
- `local`: executable; requires `path`.
|
||||
- `ssh`: executable; requires `host` and `path`.
|
||||
- `s3`: executable; requires `endpoint` and `bucket`.
|
||||
- `http_upload`: source-only configuration; requires `token_env`.
|
||||
|
||||
## Size And Duration Values
|
||||
|
||||
Upload size fields use an integer plus one of the supported binary-size suffixes:
|
||||
|
||||
- `B`
|
||||
- `KB`
|
||||
- `MB`
|
||||
- `GB`
|
||||
|
||||
Suffix multipliers use powers of 1024. Size values must be greater than zero after defaults are applied.
|
||||
|
||||
HTTP retention uses Go-style duration strings such as `24h`, `90m`, or `168h`. Retention must be greater than zero after defaults are applied.
|
||||
|
||||
## SSH Backend
|
||||
|
||||
@@ -266,6 +329,14 @@ Defaults are applied after YAML decoding and before validation:
|
||||
- SSH `host_key_policy: accept-new`
|
||||
- S3 `region: us-east-1`
|
||||
- S3 `force_path_style: true`
|
||||
- `server.http.bind: 127.0.0.1:8080`
|
||||
- `server.http.staging_root: /var/spool/distributor`
|
||||
- `server.http.max_upload_size: 20MB`
|
||||
- `server.http.queue_size: 16`
|
||||
- `server.http.max_concurrency: 1`
|
||||
- `server.http.retention: 24h`
|
||||
- `source.staging_path: /var/spool/distributor/<pipeline id>` for `http_upload`
|
||||
- `source.max_upload_size: server.http.max_upload_size` for `http_upload`
|
||||
- `transform.markdown_to_html.mode: sidecar` when a Markdown-to-HTML transform block is present and mode is omitted
|
||||
- `publish.source: true`
|
||||
- `publish.html: false`
|
||||
|
||||
@@ -6,7 +6,7 @@
|
||||
|
||||
## Inputs and outputs
|
||||
|
||||
Input is a YAML file containing optional `secrets` and required `pipelines`. Output is a `Config` value with defaults applied and validation completed. Load failures include the config path and whether the failure occurred during file loading, YAML parsing, or validation.
|
||||
Input is a YAML file containing optional `server`, optional `secrets`, and required `pipelines`. Output is a `Config` value with defaults applied and validation completed. Load failures include the config path and whether the failure occurred during file loading, YAML parsing, or validation.
|
||||
|
||||
## Loading flow
|
||||
|
||||
@@ -20,6 +20,14 @@ Known-field checking rejects misspelled or unknown YAML keys before defaults and
|
||||
|
||||
Defaults are applied in `ApplyDefaults`:
|
||||
|
||||
- HTTP server `bind` defaults to `127.0.0.1:8080`;
|
||||
- HTTP server `staging_root` defaults to `/var/spool/distributor`;
|
||||
- HTTP server `max_upload_size` defaults to `20MB`;
|
||||
- HTTP server `queue_size` defaults to `16`;
|
||||
- HTTP server `max_concurrency` defaults to `1`;
|
||||
- HTTP server `retention` defaults to `24h`;
|
||||
- `http_upload` source `staging_path` defaults to `<server.http.staging_root>/<pipeline id>`;
|
||||
- `http_upload` source `max_upload_size` defaults to `server.http.max_upload_size`;
|
||||
- pipeline validation defaults `on_digest_mismatch` to `fail`;
|
||||
- SSH backend `port` defaults to `22`;
|
||||
- SSH backend `host_key_policy` defaults to `accept-new`;
|
||||
@@ -34,7 +42,11 @@ Defaults are applied in `ApplyDefaults`:
|
||||
|
||||
## Validation responsibilities
|
||||
|
||||
Validation requires at least one pipeline, slug-like unique pipeline ids, one source per pipeline, at least one destination, slug-like unique destination ids within each pipeline, backend-specific required fields, valid validation policy, valid publish and transform combinations, valid destination path mapping mode, valid destination link config, and valid transfer actions.
|
||||
Validation requires positive HTTP server limits and retention, at least one pipeline, slug-like unique pipeline ids, one source per pipeline, at least one destination, slug-like unique destination ids within each pipeline, backend-specific required fields, valid validation policy, valid publish and transform combinations, valid destination path mapping mode, valid destination link config, and valid transfer actions.
|
||||
|
||||
HTTP upload sources require `token_env`, a staging path after defaults, and a positive maximum upload size. Literal token fields are not part of the YAML schema. The `http_upload` backend is accepted only for sources and rejected for destinations.
|
||||
|
||||
Upload size values are parsed from strings with `B`, `KB`, `MB`, or `GB` suffixes using 1024 multipliers. Retention values are parsed with `time.ParseDuration`. Explicit zero values fail validation; omitted values receive defaults before validation.
|
||||
|
||||
Transfer validation accepts `replace` for `on_destination_newer` and `on_conflict`, but publish planning honors those destructive actions only when the current run explicitly requests force.
|
||||
|
||||
@@ -46,12 +58,14 @@ Destination links are optional. When a `links` block is present, `base_url` is r
|
||||
|
||||
## Executable support boundary
|
||||
|
||||
Config validation accepts `local`, `ssh`, and `s3` backend shapes. Runtime execution opens all three through `internal/app`.
|
||||
Config validation accepts `local`, `ssh`, `s3`, and source-only `http_upload` backend shapes. Runtime execution opens `local`, `ssh`, and `s3` through `internal/app`.
|
||||
|
||||
SSH config uses structured fields: `host`, optional `user`, optional `port`, `path`, optional `ssh_key_file`, optional `known_hosts`, and optional `host_key_policy`. `host_key_policy` accepts YAML booleans and strings and normalizes `true`/`strict`, `accept-new`, and `false`/`off`.
|
||||
|
||||
S3 config requires `endpoint` and `bucket`, normalizes optional `prefix`, defaults `region` to `us-east-1`, and defaults omitted `force_path_style` to `true` while preserving explicit `false`.
|
||||
|
||||
HTTP upload config is source-only. Config owns its YAML shape, defaulting, size and duration parsing, and validation. The config package does not resolve `token_env`, authenticate requests, stage uploads, or execute HTTP upload sources.
|
||||
|
||||
## Secrets and credential resolution
|
||||
|
||||
`secrets.directory` points to a directory of credential files. `LoadSecretEnvironment` reads regular files and symlinks to regular files, rejects invalid filenames, trims exactly one trailing LF or CRLF, and returns an `Environment` resolver plus conflict metadata.
|
||||
|
||||
@@ -1,10 +1,24 @@
|
||||
package config
|
||||
|
||||
type Config struct {
|
||||
Server Server `yaml:"server"`
|
||||
Secrets Secrets `yaml:"secrets"`
|
||||
Pipelines []Pipeline `yaml:"pipelines"`
|
||||
}
|
||||
|
||||
type Server struct {
|
||||
HTTP HTTPServer `yaml:"http"`
|
||||
}
|
||||
|
||||
type HTTPServer struct {
|
||||
Bind string `yaml:"bind"`
|
||||
StagingRoot string `yaml:"staging_root"`
|
||||
MaxUploadSize *ByteSize `yaml:"max_upload_size"`
|
||||
QueueSize int `yaml:"queue_size"`
|
||||
MaxConcurrency int `yaml:"max_concurrency"`
|
||||
Retention *Duration `yaml:"retention"`
|
||||
}
|
||||
|
||||
type Secrets struct {
|
||||
Directory string `yaml:"directory"`
|
||||
}
|
||||
@@ -50,6 +64,13 @@ type Backend struct {
|
||||
ForcePath *bool `yaml:"force_path_style"`
|
||||
Creds Credentials `yaml:"credentials"`
|
||||
SSH SSH `yaml:",inline"`
|
||||
Upload HTTPUpload `yaml:",inline"`
|
||||
}
|
||||
|
||||
type HTTPUpload struct {
|
||||
TokenEnv string `yaml:"token_env"`
|
||||
StagingPath string `yaml:"staging_path"`
|
||||
MaxUploadSize *ByteSize `yaml:"max_upload_size"`
|
||||
}
|
||||
|
||||
type SSH struct {
|
||||
|
||||
@@ -1,6 +1,11 @@
|
||||
package config
|
||||
|
||||
import "gitea.maximumdirect.net/eric/distributor/internal/transform"
|
||||
import (
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/transform"
|
||||
)
|
||||
|
||||
const DefaultConfigPath = "/usr/local/etc/distributor/config.yml"
|
||||
|
||||
@@ -8,6 +13,7 @@ const (
|
||||
BackendLocal = "local"
|
||||
BackendSSH = "ssh"
|
||||
BackendS3 = "s3"
|
||||
BackendHTTPUpload = "http_upload"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -38,10 +44,23 @@ const (
|
||||
|
||||
const DefaultS3Region = "us-east-1"
|
||||
|
||||
const (
|
||||
DefaultHTTPBind = "127.0.0.1:8080"
|
||||
DefaultHTTPStagingRoot = "/var/spool/distributor"
|
||||
DefaultHTTPMaxUploadSize = ByteSize(20 * 1024 * 1024)
|
||||
DefaultHTTPQueueSize = 16
|
||||
DefaultHTTPMaxConcurrency = 1
|
||||
DefaultHTTPRetention = Duration(24 * time.Hour)
|
||||
)
|
||||
|
||||
func ApplyDefaults(cfg *Config) {
|
||||
applyHTTPServerDefaults(&cfg.Server.HTTP)
|
||||
for pipelineIndex := range cfg.Pipelines {
|
||||
pipeline := &cfg.Pipelines[pipelineIndex]
|
||||
applyBackendDefaults(&pipeline.Source)
|
||||
if pipeline.Source.Backend == BackendHTTPUpload {
|
||||
applyHTTPUploadDefaults(&pipeline.Source.Upload, pipeline.ID, cfg.Server.HTTP)
|
||||
}
|
||||
if pipeline.Validation.OnDigestMismatch == "" {
|
||||
pipeline.Validation.OnDigestMismatch = ValidationActionFail
|
||||
}
|
||||
@@ -76,6 +95,44 @@ func ApplyDefaults(cfg *Config) {
|
||||
}
|
||||
}
|
||||
|
||||
func applyHTTPServerDefaults(server *HTTPServer) {
|
||||
if server.Bind == "" {
|
||||
server.Bind = DefaultHTTPBind
|
||||
}
|
||||
if server.StagingRoot == "" {
|
||||
server.StagingRoot = DefaultHTTPStagingRoot
|
||||
}
|
||||
if server.MaxUploadSize == nil {
|
||||
server.MaxUploadSize = byteSize(DefaultHTTPMaxUploadSize)
|
||||
}
|
||||
if server.QueueSize == 0 {
|
||||
server.QueueSize = DefaultHTTPQueueSize
|
||||
}
|
||||
if server.MaxConcurrency == 0 {
|
||||
server.MaxConcurrency = DefaultHTTPMaxConcurrency
|
||||
}
|
||||
if server.Retention == nil {
|
||||
server.Retention = duration(DefaultHTTPRetention)
|
||||
}
|
||||
}
|
||||
|
||||
func applyHTTPUploadDefaults(upload *HTTPUpload, pipelineID string, server HTTPServer) {
|
||||
if upload.StagingPath == "" && pipelineID != "" {
|
||||
upload.StagingPath = filepath.Join(server.StagingRoot, pipelineID)
|
||||
}
|
||||
if upload.MaxUploadSize == nil && server.MaxUploadSize != nil {
|
||||
upload.MaxUploadSize = byteSize(*server.MaxUploadSize)
|
||||
}
|
||||
}
|
||||
|
||||
func byteSize(value ByteSize) *ByteSize {
|
||||
return &value
|
||||
}
|
||||
|
||||
func duration(value Duration) *Duration {
|
||||
return &value
|
||||
}
|
||||
|
||||
func applyBackendDefaults(backend *Backend) {
|
||||
if backend.Backend == BackendSSH {
|
||||
if backend.Port == 0 {
|
||||
|
||||
@@ -208,6 +208,123 @@ pipelines:
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadFileDefaultsHTTPServerConfig(t *testing.T) {
|
||||
cfg := loadConfig(t, `
|
||||
pipelines:
|
||||
- id: reports
|
||||
source:
|
||||
backend: local
|
||||
path: /source
|
||||
destinations:
|
||||
- id: archive
|
||||
backend: local
|
||||
path: /archive
|
||||
`)
|
||||
|
||||
server := cfg.Server.HTTP
|
||||
if got, want := server.Bind, DefaultHTTPBind; got != want {
|
||||
t.Fatalf("server.http.bind = %q, want %q", got, want)
|
||||
}
|
||||
if got, want := server.StagingRoot, DefaultHTTPStagingRoot; got != want {
|
||||
t.Fatalf("server.http.staging_root = %q, want %q", got, want)
|
||||
}
|
||||
if got, want := *server.MaxUploadSize, DefaultHTTPMaxUploadSize; got != want {
|
||||
t.Fatalf("server.http.max_upload_size = %s, want %s", got, want)
|
||||
}
|
||||
if got, want := server.QueueSize, DefaultHTTPQueueSize; got != want {
|
||||
t.Fatalf("server.http.queue_size = %d, want %d", got, want)
|
||||
}
|
||||
if got, want := server.MaxConcurrency, DefaultHTTPMaxConcurrency; got != want {
|
||||
t.Fatalf("server.http.max_concurrency = %d, want %d", got, want)
|
||||
}
|
||||
if got, want := *server.Retention, DefaultHTTPRetention; got != want {
|
||||
t.Fatalf("server.http.retention = %s, want %s", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadFileAcceptsHTTPUploadSourceConfig(t *testing.T) {
|
||||
cfg := loadConfig(t, `
|
||||
server:
|
||||
http:
|
||||
bind: 127.0.0.1:9090
|
||||
staging_root: /srv/distributor/staging
|
||||
max_upload_size: 64MB
|
||||
queue_size: 32
|
||||
max_concurrency: 2
|
||||
retention: 48h
|
||||
pipelines:
|
||||
- id: weather-daily
|
||||
source:
|
||||
backend: http_upload
|
||||
token_env: WEATHER_DAILY_UPLOAD_TOKEN
|
||||
staging_path: /srv/distributor/staging/weather-daily
|
||||
max_upload_size: 32MB
|
||||
destinations:
|
||||
- id: archive
|
||||
backend: local
|
||||
path: /archive
|
||||
`)
|
||||
|
||||
server := cfg.Server.HTTP
|
||||
if got, want := server.Bind, "127.0.0.1:9090"; got != want {
|
||||
t.Fatalf("server.http.bind = %q, want %q", got, want)
|
||||
}
|
||||
if got, want := server.StagingRoot, "/srv/distributor/staging"; got != want {
|
||||
t.Fatalf("server.http.staging_root = %q, want %q", got, want)
|
||||
}
|
||||
if got, want := *server.MaxUploadSize, ByteSize(64*1024*1024); got != want {
|
||||
t.Fatalf("server.http.max_upload_size = %s, want %s", got, want)
|
||||
}
|
||||
if got, want := server.QueueSize, 32; got != want {
|
||||
t.Fatalf("server.http.queue_size = %d, want %d", got, want)
|
||||
}
|
||||
if got, want := server.MaxConcurrency, 2; got != want {
|
||||
t.Fatalf("server.http.max_concurrency = %d, want %d", got, want)
|
||||
}
|
||||
if got, want := server.Retention.String(), "48h0m0s"; got != want {
|
||||
t.Fatalf("server.http.retention = %s, want %s", got, want)
|
||||
}
|
||||
|
||||
source := cfg.Pipelines[0].Source
|
||||
if got, want := source.Backend, BackendHTTPUpload; got != want {
|
||||
t.Fatalf("source.backend = %q, want %q", got, want)
|
||||
}
|
||||
if got, want := source.Upload.TokenEnv, "WEATHER_DAILY_UPLOAD_TOKEN"; got != want {
|
||||
t.Fatalf("source.token_env = %q, want %q", got, want)
|
||||
}
|
||||
if got, want := source.Upload.StagingPath, "/srv/distributor/staging/weather-daily"; got != want {
|
||||
t.Fatalf("source.staging_path = %q, want %q", got, want)
|
||||
}
|
||||
if got, want := *source.Upload.MaxUploadSize, ByteSize(32*1024*1024); got != want {
|
||||
t.Fatalf("source.max_upload_size = %s, want %s", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadFileDefaultsHTTPUploadSourceConfig(t *testing.T) {
|
||||
cfg := loadConfig(t, `
|
||||
server:
|
||||
http:
|
||||
max_upload_size: 12MB
|
||||
pipelines:
|
||||
- id: weather-daily
|
||||
source:
|
||||
backend: http_upload
|
||||
token_env: WEATHER_DAILY_UPLOAD_TOKEN
|
||||
destinations:
|
||||
- id: archive
|
||||
backend: local
|
||||
path: /archive
|
||||
`)
|
||||
|
||||
source := cfg.Pipelines[0].Source
|
||||
if got, want := source.Upload.StagingPath, "/var/spool/distributor/weather-daily"; got != want {
|
||||
t.Fatalf("source.staging_path = %q, want %q", got, want)
|
||||
}
|
||||
if got, want := *source.Upload.MaxUploadSize, ByteSize(12*1024*1024); got != want {
|
||||
t.Fatalf("source.max_upload_size = %s, want %s", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadFileValidBackendConfigs(t *testing.T) {
|
||||
tests := map[string]string{
|
||||
"local": `
|
||||
@@ -396,6 +513,26 @@ func TestLoadFileRejectsInvalidS3Config(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadFileRejectsInvalidHTTPUploadConfig(t *testing.T) {
|
||||
tests := map[string]string{
|
||||
"server size": `server: {http: {max_upload_size: 20XB}}`,
|
||||
"source size": `pipelines: [{id: reports, source: {backend: http_upload, token_env: UPLOAD_TOKEN, max_upload_size: 20XB}, destinations: [{id: archive, backend: local, path: /archive}]}]`,
|
||||
"zero source size": `pipelines: [{id: reports, source: {backend: http_upload, token_env: UPLOAD_TOKEN, max_upload_size: 0B}, destinations: [{id: archive, backend: local, path: /archive}]}]`,
|
||||
"server duration": `server: {http: {retention: forever}}`,
|
||||
"zero server duration": `server: {http: {retention: 0s}}`,
|
||||
"missing token env": `pipelines: [{id: reports, source: {backend: http_upload}, destinations: [{id: archive, backend: local, path: /archive}]}]`,
|
||||
"destination http upload": `pipelines: [{id: reports, source: {backend: local, path: /source}, destinations: [{id: ingest, backend: http_upload}]}]`,
|
||||
"literal token": `pipelines: [{id: reports, source: {backend: http_upload, token: secret, token_env: UPLOAD_TOKEN}, destinations: [{id: archive, backend: local, path: /archive}]}]`,
|
||||
"unknown server field": `server: {http: {surprise: true}}`,
|
||||
"unknown source field": `pipelines: [{id: reports, source: {backend: http_upload, token_env: UPLOAD_TOKEN, surprise: true}, destinations: [{id: archive, backend: local, path: /archive}]}]`,
|
||||
}
|
||||
for name, body := range tests {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
assertLoadError(t, body, "")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadFileDefaultsSSHConfig(t *testing.T) {
|
||||
cfg := loadConfig(t, `
|
||||
pipelines:
|
||||
|
||||
117
internal/config/quantity.go
Normal file
117
internal/config/quantity.go
Normal file
@@ -0,0 +1,117 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
type ByteSize int64
|
||||
|
||||
type Duration time.Duration
|
||||
|
||||
func (size *ByteSize) UnmarshalYAML(value *yaml.Node) error {
|
||||
if value.Kind != yaml.ScalarNode || value.Tag != "!!str" {
|
||||
return fmt.Errorf("size must be a string with B, KB, MB, or GB suffix")
|
||||
}
|
||||
|
||||
var raw string
|
||||
if err := value.Decode(&raw); err != nil {
|
||||
return err
|
||||
}
|
||||
parsed, err := ParseByteSize(raw)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
*size = parsed
|
||||
return nil
|
||||
}
|
||||
|
||||
func (size ByteSize) String() string {
|
||||
value := int64(size)
|
||||
if value == 0 {
|
||||
return "0B"
|
||||
}
|
||||
units := []struct {
|
||||
suffix string
|
||||
multiplier int64
|
||||
}{
|
||||
{suffix: "GB", multiplier: 1024 * 1024 * 1024},
|
||||
{suffix: "MB", multiplier: 1024 * 1024},
|
||||
{suffix: "KB", multiplier: 1024},
|
||||
{suffix: "B", multiplier: 1},
|
||||
}
|
||||
for _, unit := range units {
|
||||
if value%unit.multiplier == 0 {
|
||||
return strconv.FormatInt(value/unit.multiplier, 10) + unit.suffix
|
||||
}
|
||||
}
|
||||
return strconv.FormatInt(value, 10) + "B"
|
||||
}
|
||||
|
||||
func ParseByteSize(raw string) (ByteSize, error) {
|
||||
value := strings.TrimSpace(raw)
|
||||
if value == "" {
|
||||
return 0, fmt.Errorf("size is required")
|
||||
}
|
||||
|
||||
units := []struct {
|
||||
suffix string
|
||||
multiplier int64
|
||||
}{
|
||||
{suffix: "GB", multiplier: 1024 * 1024 * 1024},
|
||||
{suffix: "MB", multiplier: 1024 * 1024},
|
||||
{suffix: "KB", multiplier: 1024},
|
||||
{suffix: "B", multiplier: 1},
|
||||
}
|
||||
for _, unit := range units {
|
||||
number, ok := strings.CutSuffix(value, unit.suffix)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
if strings.TrimSpace(number) != number || number == "" {
|
||||
return 0, fmt.Errorf("size must be an integer followed by B, KB, MB, or GB")
|
||||
}
|
||||
parsed, err := strconv.ParseInt(number, 10, 64)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("size must be an integer followed by B, KB, MB, or GB")
|
||||
}
|
||||
if parsed < 0 {
|
||||
return 0, fmt.Errorf("size must be non-negative")
|
||||
}
|
||||
const maxInt64 = int64(1<<63 - 1)
|
||||
if parsed > 0 && parsed > maxInt64/unit.multiplier {
|
||||
return 0, fmt.Errorf("size is too large")
|
||||
}
|
||||
return ByteSize(parsed * unit.multiplier), nil
|
||||
}
|
||||
return 0, fmt.Errorf("size must use B, KB, MB, or GB suffix")
|
||||
}
|
||||
|
||||
func (duration *Duration) UnmarshalYAML(value *yaml.Node) error {
|
||||
if value.Kind != yaml.ScalarNode || value.Tag != "!!str" {
|
||||
return fmt.Errorf("duration must be a string duration")
|
||||
}
|
||||
|
||||
var raw string
|
||||
if err := value.Decode(&raw); err != nil {
|
||||
return err
|
||||
}
|
||||
parsed, err := time.ParseDuration(raw)
|
||||
if err != nil {
|
||||
return fmt.Errorf("duration must be a valid duration: %w", err)
|
||||
}
|
||||
*duration = Duration(parsed)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (duration Duration) String() string {
|
||||
return time.Duration(duration).String()
|
||||
}
|
||||
|
||||
func (duration Duration) AsDuration() time.Duration {
|
||||
return time.Duration(duration)
|
||||
}
|
||||
@@ -22,6 +22,8 @@ func (e ValidationErrors) Error() string {
|
||||
func Validate(cfg Config) error {
|
||||
var errs ValidationErrors
|
||||
|
||||
errs = validateHTTPServer(errs, "server.http", cfg.Server.HTTP)
|
||||
|
||||
if len(cfg.Pipelines) == 0 {
|
||||
errs = append(errs, "pipelines is required")
|
||||
}
|
||||
@@ -72,14 +74,56 @@ func Validate(cfg Config) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateHTTPServer(errs ValidationErrors, context string, server HTTPServer) ValidationErrors {
|
||||
if server.Bind == "" {
|
||||
errs = append(errs, context+".bind is required")
|
||||
}
|
||||
if server.StagingRoot == "" {
|
||||
errs = append(errs, context+".staging_root is required")
|
||||
}
|
||||
if server.MaxUploadSize == nil || *server.MaxUploadSize <= 0 {
|
||||
errs = append(errs, context+".max_upload_size must be greater than zero")
|
||||
}
|
||||
if server.QueueSize <= 0 {
|
||||
errs = append(errs, context+".queue_size must be greater than zero")
|
||||
}
|
||||
if server.MaxConcurrency <= 0 {
|
||||
errs = append(errs, context+".max_concurrency must be greater than zero")
|
||||
}
|
||||
if server.Retention == nil || *server.Retention <= 0 {
|
||||
errs = append(errs, context+".retention must be greater than zero")
|
||||
}
|
||||
return errs
|
||||
}
|
||||
|
||||
func validateSourceBackend(errs ValidationErrors, context string, backend Backend) ValidationErrors {
|
||||
if backend.Backend == BackendHTTPUpload {
|
||||
return validateHTTPUploadSource(errs, context, backend.Upload)
|
||||
}
|
||||
return validateBackend(errs, context, backend.Backend, backend.Host, backend.Port, backend.Path, backend.Endpoint, backend.Bucket, backend.Prefix, backend.SSH.HostKeyPolicy, backend.Creds)
|
||||
}
|
||||
|
||||
func validateDestinationBackend(errs ValidationErrors, context string, destination Destination) ValidationErrors {
|
||||
if destination.Backend == BackendHTTPUpload {
|
||||
errs = append(errs, context+".backend "+BackendHTTPUpload+" is only supported for sources")
|
||||
return errs
|
||||
}
|
||||
return validateBackend(errs, context, destination.Backend, destination.Host, destination.Port, destination.Path, destination.Endpoint, destination.Bucket, destination.Prefix, destination.SSH.HostKeyPolicy, destination.Creds)
|
||||
}
|
||||
|
||||
func validateHTTPUploadSource(errs ValidationErrors, context string, upload HTTPUpload) ValidationErrors {
|
||||
if upload.TokenEnv == "" {
|
||||
errs = append(errs, context+".token_env is required for http_upload backend")
|
||||
}
|
||||
if upload.StagingPath == "" {
|
||||
errs = append(errs, context+".staging_path is required for http_upload backend")
|
||||
}
|
||||
if upload.MaxUploadSize == nil || *upload.MaxUploadSize <= 0 {
|
||||
errs = append(errs, context+".max_upload_size must be greater than zero")
|
||||
}
|
||||
return errs
|
||||
}
|
||||
|
||||
func validateBackend(errs ValidationErrors, context, backend, host string, port int, path, endpoint, bucket, prefix string, hostKeyPolicy HostKeyPolicy, creds Credentials) ValidationErrors {
|
||||
switch backend {
|
||||
case "":
|
||||
|
||||
Reference in New Issue
Block a user