eric/weatherreporter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

Updated the distributor bundle path template

Browse Source
This commit is contained in:
Eric Rakestraw
2026-06-08 10:29:42 -05:00
parent d71c7e4d28
commit 8577fc29e4
18 changed files with 286 additions and 111 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
internal/config/config.go
View File

@@ -59,7 +59,7 @@ type DistributorNotifyConfig struct {
PipelineIDTemplate string `yaml:"pipeline_id_template"`
BundleIDTemplate string `yaml:"bundle_id_template"`
IdempotencyKeyTemplate string `yaml:"idempotency_key_template"`
ReportPathTemplate string `yaml:"report_path_template"`
ReportPathTemplates []string `yaml:"report_path_templates"`
}
type MissingSourceConfig struct {

60
internal/config/config_test.go
View File

@@ -53,8 +53,12 @@ func TestDefaults(t *testing.T) {
if cfg.Notify.Distributor.IdempotencyKeyTemplate != "{bundle_id}.{run_id}" {
t.Fatalf("Notify.Distributor.IdempotencyKeyTemplate = %q, want default", cfg.Notify.Distributor.IdempotencyKeyTemplate)
}
if cfg.Notify.Distributor.ReportPathTemplate != "{batch_output_name}" {
t.Fatalf("Notify.Distributor.ReportPathTemplate = %q, want default", cfg.Notify.Distributor.ReportPathTemplate)
wantReportPaths := []string{
"{valid_start_date}/{artifact_group}/{valid_start_date}-{artifact_group}-{run_id}.md",
"{valid_start_date}/{artifact_group}/latest.md",
}
if strings.Join(cfg.Notify.Distributor.ReportPathTemplates, "\n") != strings.Join(wantReportPaths, "\n") {
t.Fatalf("Notify.Distributor.ReportPathTemplates = %#v, want %#v", cfg.Notify.Distributor.ReportPathTemplates, wantReportPaths)
}
if cfg.MissingSource.Default != MissingSourceWarn {
t.Fatalf("MissingSource.Default = %q, want warn", cfg.MissingSource.Default)
@@ -82,6 +86,9 @@ func TestLoadExampleConfig(t *testing.T) {
if cfg.Notify.Distributor.PipelineIDTemplate != "weatherreporter.{artifact_group}" {
t.Fatalf("PipelineIDTemplate = %q, want example pipeline template", cfg.Notify.Distributor.PipelineIDTemplate)
}
if len(cfg.Notify.Distributor.ReportPathTemplates) != 2 {
t.Fatalf("ReportPathTemplates = %#v, want example archive and latest paths", cfg.Notify.Distributor.ReportPathTemplates)
}
}
func TestLoadMinimalExampleConfig(t *testing.T) {
@@ -241,19 +248,33 @@ func TestEnabledDistributorNotifyValidation(t *testing.T) {
},
wantErr: "notify.distributor.idempotency_key_template",
},
{
name: "ReportPathTemplatesEmpty",
mutate: func(cfg *Config) {
cfg.Notify.Distributor.ReportPathTemplates = nil
},
wantErr: "notify.distributor.report_path_templates",
},
{
name: "ReportPathTemplateUnknown",
mutate: func(cfg *Config) {
cfg.Notify.Distributor.ReportPathTemplate = "{bundle_id}"
cfg.Notify.Distributor.ReportPathTemplates = []string{"{unknown}"}
},
wantErr: "notify.distributor.report_path_template",
wantErr: "notify.distributor.report_path_templates",
},
{
name: "ReportPathTemplateInvalidPath",
mutate: func(cfg *Config) {
cfg.Notify.Distributor.ReportPathTemplate = "/{batch_output_name}"
cfg.Notify.Distributor.ReportPathTemplates = []string{"/{batch_output_name}"}
},
wantErr: "notify.distributor.report_path_template",
wantErr: "notify.distributor.report_path_templates",
},
{
name: "ReportPathTemplateDuplicatePath",
mutate: func(cfg *Config) {
cfg.Notify.Distributor.ReportPathTemplates = []string{"latest.md", "latest.md"}
},
wantErr: "notify.distributor.report_path_templates",
},
}
@@ -282,6 +303,12 @@ func TestDistributorTemplateRendering(t *testing.T) {
RunID: "20260607T120000Z",
ArtifactGroup: "daily",
BatchOutputName: "daily.md",
ValidStartDate: "2026-06-07",
ValidEndDate: "2026-06-08",
ValidStartTime: "1800",
ValidEndTime: "0600",
ValidStartStamp: "2026-06-07T1800",
ValidEndStamp: "2026-06-08T0600",
BundleID: "weatherreporter.home.daily",
}
@@ -309,12 +336,19 @@ func TestDistributorTemplateRendering(t *testing.T) {
t.Fatalf("idempotencyKey = %q, want rendered run key", idempotencyKey)
}
reportPath, err := RenderDistributorReportPath("reports/{batch_output_name}", values)
reportPaths, err := RenderDistributorReportPaths([]string{
"{valid_start_date}/{artifact_group}/{valid_start_stamp}-{valid_end_stamp}-{run_id}.md",
"{valid_start_date}/{artifact_group}/latest.md",
}, values)
if err != nil {
t.Fatalf("RenderDistributorReportPath() error = %v", err)
t.Fatalf("RenderDistributorReportPaths() error = %v", err)
}
if reportPath != "reports/daily.md" {
t.Fatalf("reportPath = %q, want reports/daily.md", reportPath)
wantPaths := []string{
"2026-06-07/daily/2026-06-07T1800-2026-06-08T0600-20260607T120000Z.md",
"2026-06-07/daily/latest.md",
}
if strings.Join(reportPaths, "\n") != strings.Join(wantPaths, "\n") {
t.Fatalf("reportPaths = %#v, want %#v", reportPaths, wantPaths)
}
}
@@ -360,7 +394,7 @@ func TestDistributorReportPathValidation(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
err := ValidateDistributorReportPath(tt.path)
err := ValidateDistributorReportPath("test.path", tt.path)
if tt.ok && err != nil {
t.Fatalf("ValidateDistributorReportPath() error = %v", err)
}
@@ -387,11 +421,11 @@ func TestDistributorReportPathRenderingRejectsInvalidValues(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
_, err := RenderDistributorReportPath("{batch_output_name}", DistributorTemplateValues{
_, err := RenderDistributorReportPaths([]string{"{batch_output_name}"}, DistributorTemplateValues{
BatchOutputName: tt.batchOutputName,
})
if err == nil {
t.Fatal("RenderDistributorReportPath() error = nil, want error")
t.Fatal("RenderDistributorReportPaths() error = nil, want error")
}
})
}

5
internal/config/defaults.go
View File

@@ -31,7 +31,10 @@ func Defaults() Config {
PipelineIDTemplate: "",
BundleIDTemplate: "weatherreporter.{location_id}.{report_id}",
IdempotencyKeyTemplate: "{bundle_id}.{run_id}",
ReportPathTemplate: "{batch_output_name}",
ReportPathTemplates: []string{
"{valid_start_date}/{artifact_group}/{valid_start_date}-{artifact_group}-{run_id}.md",
"{valid_start_date}/{artifact_group}/latest.md",
},
},
},
MissingSource: MissingSourceConfig{

71
internal/config/notify_templates.go
View File

@@ -12,6 +12,12 @@ type DistributorTemplateValues struct {
RunID string
ArtifactGroup string
BatchOutputName string
ValidStartDate string
ValidEndDate string
ValidStartTime string
ValidEndTime string
ValidStartStamp string
ValidEndStamp string
BundleID string
}
@@ -21,6 +27,12 @@ var distributorTemplateVariables = map[string]struct{}{
"run_id": {},
"artifact_group": {},
"batch_output_name": {},
"valid_start_date": {},
"valid_end_date": {},
"valid_start_time": {},
"valid_end_time": {},
"valid_start_stamp": {},
"valid_end_stamp": {},
}
var distributorIdempotencyTemplateVariables = map[string]struct{}{
@@ -29,6 +41,12 @@ var distributorIdempotencyTemplateVariables = map[string]struct{}{
"run_id": {},
"artifact_group": {},
"batch_output_name": {},
"valid_start_date": {},
"valid_end_date": {},
"valid_start_time": {},
"valid_end_time": {},
"valid_start_stamp": {},
"valid_end_stamp": {},
"bundle_id": {},
}
@@ -53,15 +71,28 @@ func RenderDistributorIdempotencyKey(template string, values DistributorTemplate
return renderDistributorTemplate("notify.distributor.idempotency_key_template", template, values, distributorIdempotencyTemplateVariables)
}
func RenderDistributorReportPath(template string, values DistributorTemplateValues) (string, error) {
rendered, err := renderDistributorTemplate("notify.distributor.report_path_template", template, values, distributorTemplateVariables)
if err != nil {
return "", err
func RenderDistributorReportPaths(templates []string, values DistributorTemplateValues) ([]string, error) {
if len(templates) == 0 {
return nil, fmt.Errorf("notify.distributor.report_path_templates must contain at least one entry")
}
if err := ValidateDistributorReportPath(rendered); err != nil {
return "", err
paths := make([]string, 0, len(templates))
seen := make(map[string]struct{}, len(templates))
for i, template := range templates {
name := fmt.Sprintf("notify.distributor.report_path_templates[%d]", i)
rendered, err := renderDistributorTemplate(name, template, values, distributorTemplateVariables)
if err != nil {
return nil, err
}
if err := ValidateDistributorReportPath(name, rendered); err != nil {
return nil, err
}
if _, ok := seen[rendered]; ok {
return nil, fmt.Errorf("notify.distributor.report_path_templates renders duplicate path %q", rendered)
}
seen[rendered] = struct{}{}
paths = append(paths, rendered)
}
return rendered, nil
return paths, nil
}
func validateDistributorTemplate(name, template string, allowed map[string]struct{}) error {
@@ -109,6 +140,18 @@ func distributorTemplateValue(variable string, values DistributorTemplateValues)
return values.ArtifactGroup
case "batch_output_name":
return values.BatchOutputName
case "valid_start_date":
return values.ValidStartDate
case "valid_end_date":
return values.ValidEndDate
case "valid_start_time":
return values.ValidStartTime
case "valid_end_time":
return values.ValidEndTime
case "valid_start_stamp":
return values.ValidStartStamp
case "valid_end_stamp":
return values.ValidEndStamp
case "bundle_id":
return values.BundleID
default:
@@ -116,27 +159,27 @@ func distributorTemplateValue(variable string, values DistributorTemplateValues)
}
}
func ValidateDistributorReportPath(path string) error {
func ValidateDistributorReportPath(name, path string) error {
if path == "" {
return fmt.Errorf("notify.distributor.report_path_template renders an empty path")
return fmt.Errorf("%s renders an empty path", name)
}
if isDistributorAbsolutePath(path) {
return fmt.Errorf("notify.distributor.report_path_template must render a relative path")
return fmt.Errorf("%s must render a relative path", name)
}
if strings.Contains(path, "\\") {
return fmt.Errorf("notify.distributor.report_path_template must not render backslashes")
return fmt.Errorf("%s must not render backslashes", name)
}
segments := strings.Split(path, "/")
for _, segment := range segments {
if segment == "" {
return fmt.Errorf("notify.distributor.report_path_template must not render empty path segments")
return fmt.Errorf("%s must not render empty path segments", name)
}
if segment == "." || segment == ".." {
return fmt.Errorf("notify.distributor.report_path_template must not render . or .. path segments")
return fmt.Errorf("%s must not render . or .. path segments", name)
}
if segment == "manifest.json" || segment == ".distributor.json" {
return fmt.Errorf("notify.distributor.report_path_template must not render reserved path segment %q", segment)
return fmt.Errorf("%s must not render reserved path segment %q", name, segment)
}
}

12
internal/config/validate.go
View File

@@ -118,8 +118,8 @@ func validateDistributorNotify(cfg DistributorNotifyConfig) error {
if err := validateDistributorTemplate("notify.distributor.idempotency_key_template", cfg.IdempotencyKeyTemplate, distributorIdempotencyTemplateVariables); err != nil {
return err
}
if cfg.ReportPathTemplate == "" {
return fmt.Errorf("notify.distributor.report_path_template is required when enabled")
if len(cfg.ReportPathTemplates) == 0 {
return fmt.Errorf("notify.distributor.report_path_templates must contain at least one entry when enabled")
}
values := DistributorTemplateValues{
LocationID: "location",
@@ -127,6 +127,12 @@ func validateDistributorNotify(cfg DistributorNotifyConfig) error {
RunID: "run",
ArtifactGroup: "artifact",
BatchOutputName: "report.md",
ValidStartDate: "2026-05-29",
ValidEndDate: "2026-05-30",
ValidStartTime: "0000",
ValidEndTime: "0000",
ValidStartStamp: "2026-05-29T0000",
ValidEndStamp: "2026-05-30T0000",
}
bundleID, err := RenderDistributorBundleID(cfg.BundleIDTemplate, values)
if err != nil {
@@ -136,7 +142,7 @@ func validateDistributorNotify(cfg DistributorNotifyConfig) error {
if _, err := RenderDistributorPipelineID(cfg.PipelineIDTemplate, values); err != nil {
return err
}
if _, err := RenderDistributorReportPath(cfg.ReportPathTemplate, values); err != nil {
if _, err := RenderDistributorReportPaths(cfg.ReportPathTemplates, values); err != nil {
return err
}