Add distributor notification config validation

This commit is contained in:
2026-06-07 23:18:39 +00:00
parent 7c8d9191c1
commit a2ba6f5382
7 changed files with 544 additions and 0 deletions

View File

@@ -76,6 +76,35 @@ become the environment variable value and overwrite any existing value. One
trailing LF or CRLF is stripped. Subdirectories, symlinks, invalid filenames, trailing LF or CRLF is stripped. Subdirectories, symlinks, invalid filenames,
missing directories, and unreadable files fail config loading. missing directories, and unreadable files fail config loading.
### `notify`
`notify.distributor` is validated configuration for distributor notification.
It is disabled by default. This configuration does not add CLI flags.
- `enabled`: whether distributor notification config is active. Default:
`false`.
- `endpoint`: absolute distributor endpoint URL. Required when enabled.
Default: `https://distributor.example.com`.
- `token_env`: environment variable name that will contain the distributor
upload token. Required when enabled. Default: `DISTRIBUTOR_UPLOAD_TOKEN`.
- `timeout`: distributor operation timeout. Must be greater than zero when
enabled. Default: `30s`.
- `failure_policy`: must be `error` when enabled. Default: `error`.
- `bundle_id_template`: template for distributor bundle IDs. Default:
`weatherreporter.{location_id}.{report_id}.{run_id}`.
- `idempotency_key_template`: template for distributor idempotency keys.
Default: `{bundle_id}`.
- `report_path_template`: template for the Markdown report path inside the
distributor bundle. Default: `{batch_output_name}`.
Supported template variables are `location_id`, `report_id`, `run_id`,
`artifact_group`, and `batch_output_name`. `idempotency_key_template` may also
use `bundle_id`.
Rendered report paths must be relative paths with `/` separators. They must not
contain backslashes, empty path segments, `.`, `..`, `manifest.json`, or
`.distributor.json`.
### `missing_source` ### `missing_source`
- `default`: missing-source behavior for optional sources. One of `error`, `warn`, or `none`. Default: `warn`. - `default`: missing-source behavior for optional sources. One of `error`, `warn`, or `none`. Default: `warn`.

View File

@@ -14,6 +14,17 @@ location:
secrets: secrets:
directory: "" directory: ""
notify:
distributor:
enabled: false
endpoint: https://distributor.example.com
token_env: DISTRIBUTOR_UPLOAD_TOKEN
timeout: 30s
failure_policy: error
bundle_id_template: "weatherreporter.{location_id}.{report_id}.{run_id}"
idempotency_key_template: "{bundle_id}"
report_path_template: "{batch_output_name}"
missing_source: missing_source:
default: warn default: warn
sources: sources:

View File

@@ -5,17 +5,21 @@ package config
import "time" import "time"
type MissingSourcePolicy string type MissingSourcePolicy string
type NotifyFailurePolicy string
const ( const (
MissingSourceError MissingSourcePolicy = "error" MissingSourceError MissingSourcePolicy = "error"
MissingSourceWarn MissingSourcePolicy = "warn" MissingSourceWarn MissingSourcePolicy = "warn"
MissingSourceNone MissingSourcePolicy = "none" MissingSourceNone MissingSourcePolicy = "none"
NotifyFailureError NotifyFailurePolicy = "error"
) )
type Config struct { type Config struct {
WeatherAPI WeatherAPIConfig `yaml:"weather_api"` WeatherAPI WeatherAPIConfig `yaml:"weather_api"`
Location LocationConfig `yaml:"location"` Location LocationConfig `yaml:"location"`
Secrets SecretsConfig `yaml:"secrets"` Secrets SecretsConfig `yaml:"secrets"`
Notify NotifyConfig `yaml:"notify"`
MissingSource MissingSourceConfig `yaml:"missing_source"` MissingSource MissingSourceConfig `yaml:"missing_source"`
Scriptorium ScriptoriumConfig `yaml:"scriptorium"` Scriptorium ScriptoriumConfig `yaml:"scriptorium"`
Workspace WorkspaceConfig `yaml:"workspace"` Workspace WorkspaceConfig `yaml:"workspace"`
@@ -42,6 +46,21 @@ type SecretsConfig struct {
Directory string `yaml:"directory"` Directory string `yaml:"directory"`
} }
type NotifyConfig struct {
Distributor DistributorNotifyConfig `yaml:"distributor"`
}
type DistributorNotifyConfig struct {
Enabled bool `yaml:"enabled"`
Endpoint string `yaml:"endpoint"`
TokenEnv string `yaml:"token_env"`
Timeout time.Duration `yaml:"timeout"`
FailurePolicy NotifyFailurePolicy `yaml:"failure_policy"`
BundleIDTemplate string `yaml:"bundle_id_template"`
IdempotencyKeyTemplate string `yaml:"idempotency_key_template"`
ReportPathTemplate string `yaml:"report_path_template"`
}
type MissingSourceConfig struct { type MissingSourceConfig struct {
Default MissingSourcePolicy `yaml:"default"` Default MissingSourcePolicy `yaml:"default"`
Sources map[string]MissingSourcePolicy `yaml:"sources"` Sources map[string]MissingSourcePolicy `yaml:"sources"`

View File

@@ -29,6 +29,30 @@ func TestDefaults(t *testing.T) {
if cfg.Secrets.Directory != "" { if cfg.Secrets.Directory != "" {
t.Fatalf("Secrets.Directory = %q, want empty", cfg.Secrets.Directory) t.Fatalf("Secrets.Directory = %q, want empty", cfg.Secrets.Directory)
} }
if cfg.Notify.Distributor.Enabled {
t.Fatalf("Notify.Distributor.Enabled = true, want false")
}
if cfg.Notify.Distributor.Endpoint != "https://distributor.example.com" {
t.Fatalf("Notify.Distributor.Endpoint = %q, want default endpoint", cfg.Notify.Distributor.Endpoint)
}
if cfg.Notify.Distributor.TokenEnv != "DISTRIBUTOR_UPLOAD_TOKEN" {
t.Fatalf("Notify.Distributor.TokenEnv = %q, want DISTRIBUTOR_UPLOAD_TOKEN", cfg.Notify.Distributor.TokenEnv)
}
if cfg.Notify.Distributor.Timeout != 30*time.Second {
t.Fatalf("Notify.Distributor.Timeout = %s, want 30s", cfg.Notify.Distributor.Timeout)
}
if cfg.Notify.Distributor.FailurePolicy != NotifyFailureError {
t.Fatalf("Notify.Distributor.FailurePolicy = %q, want error", cfg.Notify.Distributor.FailurePolicy)
}
if cfg.Notify.Distributor.BundleIDTemplate != "weatherreporter.{location_id}.{report_id}.{run_id}" {
t.Fatalf("Notify.Distributor.BundleIDTemplate = %q, want default", cfg.Notify.Distributor.BundleIDTemplate)
}
if cfg.Notify.Distributor.IdempotencyKeyTemplate != "{bundle_id}" {
t.Fatalf("Notify.Distributor.IdempotencyKeyTemplate = %q, want default", cfg.Notify.Distributor.IdempotencyKeyTemplate)
}
if cfg.Notify.Distributor.ReportPathTemplate != "{batch_output_name}" {
t.Fatalf("Notify.Distributor.ReportPathTemplate = %q, want default", cfg.Notify.Distributor.ReportPathTemplate)
}
if cfg.MissingSource.Default != MissingSourceWarn { if cfg.MissingSource.Default != MissingSourceWarn {
t.Fatalf("MissingSource.Default = %q, want warn", cfg.MissingSource.Default) t.Fatalf("MissingSource.Default = %q, want warn", cfg.MissingSource.Default)
} }
@@ -116,6 +140,256 @@ func TestLoadAppliesOverrides(t *testing.T) {
} }
} }
func TestDisabledDistributorNotifyAcceptsOmittedFields(t *testing.T) {
dir := t.TempDir()
path := filepath.Join(dir, "config.yml")
if err := os.WriteFile(path, []byte("notify:\n distributor:\n enabled: false\n"), 0o600); err != nil {
t.Fatalf("write config fixture: %v", err)
}
cfg, err := LoadFile(path)
if err != nil {
t.Fatalf("LoadFile() error = %v", err)
}
if cfg.Notify.Distributor.Enabled {
t.Fatalf("Notify.Distributor.Enabled = true, want false")
}
}
func TestEnabledDistributorNotifyValidation(t *testing.T) {
tests := []struct {
name string
mutate func(*Config)
wantErr string
}{
{
name: "Endpoint",
mutate: func(cfg *Config) {
cfg.Notify.Distributor.Endpoint = "distributor.example.com"
},
wantErr: "notify.distributor.endpoint",
},
{
name: "TokenEnvEmpty",
mutate: func(cfg *Config) {
cfg.Notify.Distributor.TokenEnv = ""
},
wantErr: "notify.distributor.token_env",
},
{
name: "TokenEnvInvalid",
mutate: func(cfg *Config) {
cfg.Notify.Distributor.TokenEnv = "1TOKEN"
},
wantErr: "notify.distributor.token_env",
},
{
name: "Timeout",
mutate: func(cfg *Config) {
cfg.Notify.Distributor.Timeout = 0
},
wantErr: "notify.distributor.timeout",
},
{
name: "FailurePolicy",
mutate: func(cfg *Config) {
cfg.Notify.Distributor.FailurePolicy = "warn"
},
wantErr: "notify.distributor.failure_policy",
},
{
name: "BundleTemplate",
mutate: func(cfg *Config) {
cfg.Notify.Distributor.BundleIDTemplate = "{unknown}"
},
wantErr: "notify.distributor.bundle_id_template",
},
{
name: "IdempotencyTemplate",
mutate: func(cfg *Config) {
cfg.Notify.Distributor.IdempotencyKeyTemplate = "{unknown}"
},
wantErr: "notify.distributor.idempotency_key_template",
},
{
name: "ReportPathTemplateUnknown",
mutate: func(cfg *Config) {
cfg.Notify.Distributor.ReportPathTemplate = "{bundle_id}"
},
wantErr: "notify.distributor.report_path_template",
},
{
name: "ReportPathTemplateInvalidPath",
mutate: func(cfg *Config) {
cfg.Notify.Distributor.ReportPathTemplate = "/{batch_output_name}"
},
wantErr: "notify.distributor.report_path_template",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
cfg := Defaults()
cfg.Notify.Distributor.Enabled = true
tt.mutate(&cfg)
err := Validate(cfg)
if err == nil {
t.Fatal("Validate() error = nil, want error")
}
if !strings.Contains(err.Error(), tt.wantErr) {
t.Fatalf("error = %q, want %q", err.Error(), tt.wantErr)
}
})
}
}
func TestDistributorTemplateRendering(t *testing.T) {
values := DistributorTemplateValues{
LocationID: "home",
ReportID: "daily",
RunID: "20260607T120000Z",
ArtifactGroup: "daily",
BatchOutputName: "daily.md",
BundleID: "weatherreporter.home.daily.20260607T120000Z",
}
bundleID, err := RenderDistributorBundleID("weatherreporter.{location_id}.{report_id}.{run_id}", values)
if err != nil {
t.Fatalf("RenderDistributorBundleID() error = %v", err)
}
if bundleID != "weatherreporter.home.daily.20260607T120000Z" {
t.Fatalf("bundleID = %q, want rendered value", bundleID)
}
idempotencyKey, err := RenderDistributorIdempotencyKey("{bundle_id}", values)
if err != nil {
t.Fatalf("RenderDistributorIdempotencyKey() error = %v", err)
}
if idempotencyKey != "weatherreporter.home.daily.20260607T120000Z" {
t.Fatalf("idempotencyKey = %q, want rendered bundle ID", idempotencyKey)
}
reportPath, err := RenderDistributorReportPath("reports/{batch_output_name}", values)
if err != nil {
t.Fatalf("RenderDistributorReportPath() error = %v", err)
}
if reportPath != "reports/daily.md" {
t.Fatalf("reportPath = %q, want reports/daily.md", reportPath)
}
}
func TestDistributorTemplateRejectsUnknownAndMalformedVariables(t *testing.T) {
tests := []struct {
name string
template string
}{
{name: "Unknown", template: "{unknown}"},
{name: "Unclosed", template: "{location_id"},
{name: "Unopened", template: "location_id}"},
{name: "Empty", template: "{}"},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
_, err := RenderDistributorBundleID(tt.template, DistributorTemplateValues{})
if err == nil {
t.Fatal("RenderDistributorBundleID() error = nil, want error")
}
})
}
}
func TestDistributorReportPathValidation(t *testing.T) {
tests := []struct {
name string
path string
ok bool
}{
{name: "Simple", path: "daily.md", ok: true},
{name: "Nested", path: "reports/daily.md", ok: true},
{name: "Empty", path: "", ok: false},
{name: "Absolute", path: "/reports/daily.md", ok: false},
{name: "WindowsAbsolute", path: "C:/reports/daily.md", ok: false},
{name: "Backslash", path: `reports\daily.md`, ok: false},
{name: "CurrentSegment", path: "reports/./daily.md", ok: false},
{name: "ParentSegment", path: "reports/../daily.md", ok: false},
{name: "EmptySegment", path: "reports//daily.md", ok: false},
{name: "Manifest", path: "reports/manifest.json", ok: false},
{name: "DistributorMetadata", path: "reports/.distributor.json", ok: false},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
err := ValidateDistributorReportPath(tt.path)
if tt.ok && err != nil {
t.Fatalf("ValidateDistributorReportPath() error = %v", err)
}
if !tt.ok && err == nil {
t.Fatal("ValidateDistributorReportPath() error = nil, want error")
}
})
}
}
func TestDistributorReportPathRenderingRejectsInvalidValues(t *testing.T) {
tests := []struct {
name string
batchOutputName string
}{
{name: "Absolute", batchOutputName: "/daily.md"},
{name: "Backslash", batchOutputName: `reports\daily.md`},
{name: "CurrentSegment", batchOutputName: "./daily.md"},
{name: "ParentSegment", batchOutputName: "../daily.md"},
{name: "EmptySegment", batchOutputName: "reports//daily.md"},
{name: "Manifest", batchOutputName: "manifest.json"},
{name: "DistributorMetadata", batchOutputName: ".distributor.json"},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
_, err := RenderDistributorReportPath("{batch_output_name}", DistributorTemplateValues{
BatchOutputName: tt.batchOutputName,
})
if err == nil {
t.Fatal("RenderDistributorReportPath() error = nil, want error")
}
})
}
}
func TestLoadFileLoadsSecretsBeforeReturningNotifyConfig(t *testing.T) {
dir := t.TempDir()
secretsDir := filepath.Join(dir, "secrets")
if err := os.Mkdir(secretsDir, 0o700); err != nil {
t.Fatalf("create secrets directory: %v", err)
}
if err := os.WriteFile(filepath.Join(secretsDir, "DISTRIBUTOR_UPLOAD_TOKEN"), []byte("loaded-token"), 0o600); err != nil {
t.Fatalf("write secret: %v", err)
}
path := filepath.Join(dir, "config.yml")
configYAML := "secrets:\n" +
" directory: " + secretsDir + "\n" +
"notify:\n" +
" distributor:\n" +
" enabled: true\n"
if err := os.WriteFile(path, []byte(configYAML), 0o600); err != nil {
t.Fatalf("write config fixture: %v", err)
}
t.Setenv("DISTRIBUTOR_UPLOAD_TOKEN", "")
cfg, err := LoadFile(path)
if err != nil {
t.Fatalf("LoadFile() error = %v", err)
}
if cfg.Notify.Distributor.TokenEnv != "DISTRIBUTOR_UPLOAD_TOKEN" {
t.Fatalf("TokenEnv = %q, want DISTRIBUTOR_UPLOAD_TOKEN", cfg.Notify.Distributor.TokenEnv)
}
if got := os.Getenv(cfg.Notify.Distributor.TokenEnv); got != "loaded-token" {
t.Fatalf("environment value = %q, want loaded-token", got)
}
}
func TestLoadSecretsDisabledLeavesEnvironmentUnchanged(t *testing.T) { func TestLoadSecretsDisabledLeavesEnvironmentUnchanged(t *testing.T) {
t.Setenv("WEATHERREPORTER_DISABLED_SECRET", "original") t.Setenv("WEATHERREPORTER_DISABLED_SECRET", "original")

View File

@@ -21,6 +21,18 @@ func Defaults() Config {
Secrets: SecretsConfig{ Secrets: SecretsConfig{
Directory: "", Directory: "",
}, },
Notify: NotifyConfig{
Distributor: DistributorNotifyConfig{
Enabled: false,
Endpoint: "https://distributor.example.com",
TokenEnv: "DISTRIBUTOR_UPLOAD_TOKEN",
Timeout: 30 * time.Second,
FailurePolicy: NotifyFailureError,
BundleIDTemplate: "weatherreporter.{location_id}.{report_id}.{run_id}",
IdempotencyKeyTemplate: "{bundle_id}",
ReportPathTemplate: "{batch_output_name}",
},
},
MissingSource: MissingSourceConfig{ MissingSource: MissingSourceConfig{
Default: MissingSourceWarn, Default: MissingSourceWarn,
Sources: map[string]MissingSourcePolicy{}, Sources: map[string]MissingSourcePolicy{},

View File

@@ -0,0 +1,145 @@
package config
import (
"fmt"
"path/filepath"
"strings"
)
type DistributorTemplateValues struct {
LocationID string
ReportID string
RunID string
ArtifactGroup string
BatchOutputName string
BundleID string
}
var distributorTemplateVariables = map[string]struct{}{
"location_id": {},
"report_id": {},
"run_id": {},
"artifact_group": {},
"batch_output_name": {},
}
var distributorIdempotencyTemplateVariables = map[string]struct{}{
"location_id": {},
"report_id": {},
"run_id": {},
"artifact_group": {},
"batch_output_name": {},
"bundle_id": {},
}
func RenderDistributorBundleID(template string, values DistributorTemplateValues) (string, error) {
return renderDistributorTemplate("notify.distributor.bundle_id_template", template, values, distributorTemplateVariables)
}
func RenderDistributorIdempotencyKey(template string, values DistributorTemplateValues) (string, error) {
return renderDistributorTemplate("notify.distributor.idempotency_key_template", template, values, distributorIdempotencyTemplateVariables)
}
func RenderDistributorReportPath(template string, values DistributorTemplateValues) (string, error) {
rendered, err := renderDistributorTemplate("notify.distributor.report_path_template", template, values, distributorTemplateVariables)
if err != nil {
return "", err
}
if err := ValidateDistributorReportPath(rendered); err != nil {
return "", err
}
return rendered, nil
}
func validateDistributorTemplate(name, template string, allowed map[string]struct{}) error {
_, err := renderDistributorTemplate(name, template, DistributorTemplateValues{}, allowed)
return err
}
func renderDistributorTemplate(name, template string, values DistributorTemplateValues, allowed map[string]struct{}) (string, error) {
var rendered strings.Builder
for i := 0; i < len(template); {
switch template[i] {
case '{':
end := strings.IndexByte(template[i+1:], '}')
if end < 0 {
return "", fmt.Errorf("%s contains an unclosed template variable", name)
}
variable := template[i+1 : i+1+end]
if variable == "" {
return "", fmt.Errorf("%s contains an empty template variable", name)
}
if _, ok := allowed[variable]; !ok {
return "", fmt.Errorf("%s contains unknown template variable %q", name, variable)
}
rendered.WriteString(distributorTemplateValue(variable, values))
i += end + 2
case '}':
return "", fmt.Errorf("%s contains an unopened template variable", name)
default:
rendered.WriteByte(template[i])
i++
}
}
return rendered.String(), nil
}
func distributorTemplateValue(variable string, values DistributorTemplateValues) string {
switch variable {
case "location_id":
return values.LocationID
case "report_id":
return values.ReportID
case "run_id":
return values.RunID
case "artifact_group":
return values.ArtifactGroup
case "batch_output_name":
return values.BatchOutputName
case "bundle_id":
return values.BundleID
default:
return ""
}
}
func ValidateDistributorReportPath(path string) error {
if path == "" {
return fmt.Errorf("notify.distributor.report_path_template renders an empty path")
}
if isDistributorAbsolutePath(path) {
return fmt.Errorf("notify.distributor.report_path_template must render a relative path")
}
if strings.Contains(path, "\\") {
return fmt.Errorf("notify.distributor.report_path_template must not render backslashes")
}
segments := strings.Split(path, "/")
for _, segment := range segments {
if segment == "" {
return fmt.Errorf("notify.distributor.report_path_template must not render empty path segments")
}
if segment == "." || segment == ".." {
return fmt.Errorf("notify.distributor.report_path_template must not render . or .. path segments")
}
if segment == "manifest.json" || segment == ".distributor.json" {
return fmt.Errorf("notify.distributor.report_path_template must not render reserved path segment %q", segment)
}
}
return nil
}
func isDistributorAbsolutePath(path string) bool {
if filepath.IsAbs(path) || strings.HasPrefix(path, "/") {
return true
}
if len(path) >= 3 && isASCIIAlpha(path[0]) && path[1] == ':' && (path[2] == '/' || path[2] == '\\') {
return true
}
return false
}
func isASCIIAlpha(ch byte) bool {
return (ch >= 'A' && ch <= 'Z') || (ch >= 'a' && ch <= 'z')
}

View File

@@ -49,6 +49,10 @@ func Validate(cfg Config) error {
} }
} }
if err := validateDistributorNotify(cfg.Notify.Distributor); err != nil {
return err
}
if cfg.Scriptorium.Binary == "" { if cfg.Scriptorium.Binary == "" {
return fmt.Errorf("scriptorium.binary is required") return fmt.Errorf("scriptorium.binary is required")
} }
@@ -75,6 +79,56 @@ func Validate(cfg Config) error {
return nil return nil
} }
func validateDistributorNotify(cfg DistributorNotifyConfig) error {
if !cfg.Enabled {
return nil
}
parsed, err := url.Parse(cfg.Endpoint)
if err != nil || parsed.Scheme == "" || parsed.Host == "" {
return fmt.Errorf("notify.distributor.endpoint must be an absolute URL when enabled")
}
if cfg.TokenEnv == "" {
return fmt.Errorf("notify.distributor.token_env is required when enabled")
}
if !secretNamePattern.MatchString(cfg.TokenEnv) {
return fmt.Errorf("notify.distributor.token_env must be a valid environment variable name")
}
if cfg.Timeout <= 0 {
return fmt.Errorf("notify.distributor.timeout must be greater than zero when enabled")
}
if cfg.FailurePolicy != NotifyFailureError {
return fmt.Errorf("notify.distributor.failure_policy must be error when enabled")
}
if cfg.BundleIDTemplate == "" {
return fmt.Errorf("notify.distributor.bundle_id_template is required when enabled")
}
if err := validateDistributorTemplate("notify.distributor.bundle_id_template", cfg.BundleIDTemplate, distributorTemplateVariables); err != nil {
return err
}
if cfg.IdempotencyKeyTemplate == "" {
return fmt.Errorf("notify.distributor.idempotency_key_template is required when enabled")
}
if err := validateDistributorTemplate("notify.distributor.idempotency_key_template", cfg.IdempotencyKeyTemplate, distributorIdempotencyTemplateVariables); err != nil {
return err
}
if cfg.ReportPathTemplate == "" {
return fmt.Errorf("notify.distributor.report_path_template is required when enabled")
}
values := DistributorTemplateValues{
LocationID: "location",
ReportID: "report",
RunID: "run",
ArtifactGroup: "artifact",
BatchOutputName: "report.md",
}
if _, err := RenderDistributorReportPath(cfg.ReportPathTemplate, values); err != nil {
return err
}
return nil
}
func validatePolicy(name string, policy MissingSourcePolicy) error { func validatePolicy(name string, policy MissingSourcePolicy) error {
switch policy { switch policy {
case MissingSourceError, MissingSourceWarn, MissingSourceNone: case MissingSourceError, MissingSourceWarn, MissingSourceNone: