diff --git a/docs/config.md b/docs/config.md index bb8c458..ecfebb1 100644 --- a/docs/config.md +++ b/docs/config.md @@ -101,15 +101,6 @@ contains all managed Markdown reports from that batch. Default: `weatherreporter.{location_id}.{report_id}`. - `idempotency_key_template`: template for single-report distributor idempotency keys. Default: `{bundle_id}.{run_id}`. -- `report_path_templates`: ordered list of templates for Markdown report paths - inside distributor bundles. For `generate `, each rendered path maps - to the generated report's managed Markdown source. For `run morning` and - `run evening`, the templates are rendered once per included report and each - rendered path maps that report's managed Markdown source into the one batch - bundle. Default: - ```yaml - - "{valid_start_date}/{artifact_group}/{valid_start_date}-{artifact_group}-{run_id}.md" - ``` - `batch.enabled`: whether batch distributor notification config is active when distributor notification is enabled. Default: `true`. - `batch.pipeline_id_template`: template for batch distributor pipeline IDs. diff --git a/docs/internal/app-orchestration.md b/docs/internal/app-orchestration.md index b163f61..0f436db 100644 --- a/docs/internal/app-orchestration.md +++ b/docs/internal/app-orchestration.md @@ -163,10 +163,11 @@ all planned reports finish, app orchestration evaluates batch notification: 3. If any planned report failed, the batch notification result is `skipped` with reason `one or more reports failed`, and distributor is not called. 4. If every report succeeded, app orchestration renders batch pipeline, bundle - ID, and idempotency key templates, renders `report_path_templates` for each - included report, validates every managed source path and bundle path, checks - duplicate bundle paths across the batch, calls the notifier once with a - multi-file request, and saves a batch notification debug artifact. + ID, and idempotency key templates, renders report-specific distributor + paths for each included report, validates every managed source path and + bundle path, checks duplicate bundle paths across the batch, calls the + notifier once with a multi-file request, and saves a batch notification + debug artifact. Batch notification failure records a top-level failed notification, increments the aggregate batch failure count, and returns an aggregate batch error without diff --git a/docs/internal/distributor-adapter.md b/docs/internal/distributor-adapter.md index 31169a7..edb8c6b 100644 --- a/docs/internal/distributor-adapter.md +++ b/docs/internal/distributor-adapter.md @@ -61,12 +61,12 @@ and bundle paths from: - `pipeline_id_template` - `bundle_id_template` - `idempotency_key_template` -- `report_path_templates` +- report-specific path templates For batch uploads, the app layer renders pipeline ID, bundle ID, and -idempotency key from `notify.distributor.batch.*`, renders -`report_path_templates` once per included report, and passes the resulting -multi-file request to this adapter. +idempotency key from `notify.distributor.batch.*`, resolves report-specific +path templates once per included report, and passes the resulting multi-file +request to this adapter. The token value is read from the environment variable named by `token_env` after config loading and `secrets.directory` processing. diff --git a/docs/operations.md b/docs/operations.md index 9968df9..c5b92df 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -256,17 +256,16 @@ top-level `notification` object with fields such as `status`, `reason`, Distributor notification is configured with `notify.distributor` and is disabled by default. For `generate `, weatherreporter uploads the managed Markdown report path recorded in the report result and metadata. That -single source file can be mapped to one or more configured bundle paths. By -default, it is mapped to one dated report path. Extra copies written by `--out` -or `--out-dir` are operator conveniences only. +single source file is mapped to report-specific bundle paths. Extra copies +written by `--out` or `--out-dir` are operator conveniences only. For `run morning` and `run evening`, per-report notification is suppressed. If `notify.distributor.enabled` and `notify.distributor.batch.enabled` are both true, the batch uploads once after all reports finish successfully. The upload contains one file mapping set per included report. Each mapping uses the -managed Markdown report as the source and renders `report_path_templates` using -that report's normal template values. All rendered bundle paths across the -batch must be unique. If any report fails, weatherreporter records a top-level +managed Markdown report as the source and report-specific path templates for +that report. All rendered bundle paths across the batch must be unique. If any +report fails, weatherreporter records a top-level notification status of `skipped` with reason `one or more reports failed` and does not call distributor. If batch notification is disabled, run commands do not fall back to per-report uploads. diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index 262215f..73657a4 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -300,15 +300,15 @@ Relevant docs: [Configuration reference](config.md), Symptom: a batch returns nonzero with duplicate bundle path context before a distributor run ID is accepted. -Likely cause: `notify.distributor.report_path_templates` rendered the same +Likely cause: report-specific distributor path templates rendered the same bundle-relative path for two included reports in the same batch. Diagnostic: inspect the error in stdout or stderr. The validation error includes the duplicate bundle path plus the report IDs, RunIDs, and managed source paths involved. -Safe fix: change `notify.distributor.report_path_templates` so every report in -a batch renders a unique path. Include values such as `{artifact_group}`, +Safe fix: configure a per-report distributor path override so every report in a +batch renders a unique path. Include values such as `{artifact_group}`, `{valid_start_date}`, `{batch_output_name}`, or `{run_id}` when needed. Relevant docs: [Configuration reference](config.md), @@ -411,7 +411,7 @@ Diagnostic: inspect stdout JSON or stderr status lines for `notificationError` or the top-level batch notification `error`. Confirm `notify.distributor.endpoint`, `notify.distributor.pipeline_id_template`, -`notify.distributor.report_path_templates`, and token configuration. Token +report-specific distributor paths, and token configuration. Token values are redacted from weatherreporter errors. If the upload was accepted but destination output did not change, inspect the diff --git a/examples/config.yml b/examples/config.yml index 9cf2438..ec61a10 100644 --- a/examples/config.yml +++ b/examples/config.yml @@ -24,8 +24,6 @@ notify: pipeline_id_template: "weatherreporter.{report_id}" bundle_id_template: "weatherreporter.{location_id}.{report_id}" idempotency_key_template: "{bundle_id}.{run_id}" - report_path_templates: - - "{valid_start_date}/{artifact_group}/{valid_start_date}-{artifact_group}-{run_id}.md" batch: enabled: true pipeline_id_template: "weatherreporter" diff --git a/internal/app/app.go b/internal/app/app.go index 405fec5..117524b 100644 --- a/internal/app/app.go +++ b/internal/app/app.go @@ -971,7 +971,7 @@ func distributorTemplateValuesForReport(cfg config.Config, resolved report.Resol } func renderDistributorReportBundlePaths(cfg config.Config, resolved report.Resolved, runID string, sourcePath string, values config.DistributorTemplateValues) ([]string, error) { - templates, name, err := distributorReportPathTemplates(cfg, resolved.Definition) + templates, name, err := distributorPathTemplatesForReport(cfg, resolved.Definition) if err != nil { return nil, distributorReportPathError(resolved.Definition.ID, runID, sourcePath, err) } @@ -982,7 +982,7 @@ func renderDistributorReportBundlePaths(cfg config.Config, resolved report.Resol return paths, nil } -func distributorReportPathTemplates(cfg config.Config, definition report.Definition) ([]string, string, error) { +func distributorPathTemplatesForReport(cfg config.Config, definition report.Definition) ([]string, string, error) { overrides, err := cfg.ReportDistributorPathOverrides() if err != nil { return nil, "", err diff --git a/internal/config/config.go b/internal/config/config.go index be5177b..9f60bd5 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -66,7 +66,6 @@ type DistributorNotifyConfig struct { PipelineIDTemplate string `yaml:"pipeline_id_template"` BundleIDTemplate string `yaml:"bundle_id_template"` IdempotencyKeyTemplate string `yaml:"idempotency_key_template"` - ReportPathTemplates []string `yaml:"report_path_templates"` Batch DistributorBatchNotifyConfig `yaml:"batch"` } @@ -152,6 +151,57 @@ func (c *ReportConfig) UnmarshalYAML(value *yaml.Node) error { return nil } +func (c *DistributorNotifyConfig) UnmarshalYAML(value *yaml.Node) error { + if value.Kind != yaml.MappingNode { + return fmt.Errorf("notify distributor entry must be a mapping") + } + for i := 0; i < len(value.Content); i += 2 { + key := value.Content[i].Value + node := value.Content[i+1] + switch key { + case "enabled": + if err := node.Decode(&c.Enabled); err != nil { + return err + } + case "endpoint": + if err := node.Decode(&c.Endpoint); err != nil { + return err + } + case "token_env": + if err := node.Decode(&c.TokenEnv); err != nil { + return err + } + case "timeout": + if err := node.Decode(&c.Timeout); err != nil { + return err + } + case "failure_policy": + if err := node.Decode(&c.FailurePolicy); err != nil { + return err + } + case "pipeline_id_template": + if err := node.Decode(&c.PipelineIDTemplate); err != nil { + return err + } + case "bundle_id_template": + if err := node.Decode(&c.BundleIDTemplate); err != nil { + return err + } + case "idempotency_key_template": + if err := node.Decode(&c.IdempotencyKeyTemplate); err != nil { + return err + } + case "batch": + if err := node.Decode(&c.Batch); err != nil { + return err + } + default: + return fmt.Errorf("unknown notify distributor field %q", key) + } + } + return nil +} + func (c *ReportDistributorConfig) UnmarshalYAML(value *yaml.Node) error { if value.Kind != yaml.MappingNode { return fmt.Errorf("report distributor entry must be a mapping") diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 631c84e..8892746 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -58,12 +58,6 @@ func TestDefaults(t *testing.T) { if cfg.Notify.Distributor.IdempotencyKeyTemplate != "{bundle_id}.{run_id}" { t.Fatalf("Notify.Distributor.IdempotencyKeyTemplate = %q, want default", cfg.Notify.Distributor.IdempotencyKeyTemplate) } - wantReportPaths := []string{ - "{valid_start_date}/{artifact_group}/{valid_start_date}-{artifact_group}-{run_id}.md", - } - if strings.Join(cfg.Notify.Distributor.ReportPathTemplates, "\n") != strings.Join(wantReportPaths, "\n") { - t.Fatalf("Notify.Distributor.ReportPathTemplates = %#v, want %#v", cfg.Notify.Distributor.ReportPathTemplates, wantReportPaths) - } if !cfg.Notify.Distributor.Batch.Enabled { t.Fatalf("Notify.Distributor.Batch.Enabled = false, want true") } @@ -102,9 +96,6 @@ func TestLoadExampleConfig(t *testing.T) { if cfg.Notify.Distributor.PipelineIDTemplate != "weatherreporter.{report_id}" { t.Fatalf("PipelineIDTemplate = %q, want example pipeline template", cfg.Notify.Distributor.PipelineIDTemplate) } - if len(cfg.Notify.Distributor.ReportPathTemplates) != 1 { - t.Fatalf("ReportPathTemplates = %#v, want example archive path", cfg.Notify.Distributor.ReportPathTemplates) - } if !cfg.Notify.Distributor.Batch.Enabled { t.Fatalf("Notify.Distributor.Batch.Enabled = false, want true") } @@ -1065,6 +1056,37 @@ func TestDisabledDistributorNotifyAcceptsOmittedFields(t *testing.T) { } } +func TestDistributorNotifyRejectsRemovedGlobalReportPaths(t *testing.T) { + removedField := "report_path" + "_templates" + _, err := LoadFile(writeConfig(t, ` +notify: + distributor: + `+removedField+`: + - index.md +`)) + if err == nil { + t.Fatal("LoadFile() error = nil, want removed global path field error") + } + if !strings.Contains(err.Error(), `unknown notify distributor field "`+removedField+`"`) { + t.Fatalf("error = %q, want removed global path field rejection", err.Error()) + } +} + +func TestDistributorNotifyRejectsUnknownFields(t *testing.T) { + _, err := LoadFile(writeConfig(t, ` +notify: + distributor: + paths: + - index.md +`)) + if err == nil { + t.Fatal("LoadFile() error = nil, want unknown distributor field error") + } + if !strings.Contains(err.Error(), `unknown notify distributor field "paths"`) { + t.Fatalf("error = %q, want unknown field rejection", err.Error()) + } +} + func TestDisabledDistributorNotifyAcceptsMalformedBatchTemplates(t *testing.T) { cfg := Defaults() cfg.Notify.Distributor.Enabled = false @@ -1154,32 +1176,11 @@ func TestEnabledDistributorNotifyValidation(t *testing.T) { wantErr: "notify.distributor.idempotency_key_template", }, { - name: "ReportPathTemplatesEmpty", + name: "BatchTemplate", mutate: func(cfg *Config) { - cfg.Notify.Distributor.ReportPathTemplates = nil + cfg.Notify.Distributor.Batch.BundleIDTemplate = "{run_id}" }, - wantErr: "notify.distributor.report_path_templates", - }, - { - name: "ReportPathTemplateUnknown", - mutate: func(cfg *Config) { - cfg.Notify.Distributor.ReportPathTemplates = []string{"{unknown}"} - }, - wantErr: "notify.distributor.report_path_templates", - }, - { - name: "ReportPathTemplateInvalidPath", - mutate: func(cfg *Config) { - cfg.Notify.Distributor.ReportPathTemplates = []string{"/{batch_output_name}"} - }, - wantErr: "notify.distributor.report_path_templates", - }, - { - name: "ReportPathTemplateDuplicatePath", - mutate: func(cfg *Config) { - cfg.Notify.Distributor.ReportPathTemplates = []string{"latest.md", "latest.md"} - }, - wantErr: "notify.distributor.report_path_templates", + wantErr: "notify.distributor.batch.bundle_id_template", }, } diff --git a/internal/config/defaults.go b/internal/config/defaults.go index 418b3fc..df68a9e 100644 --- a/internal/config/defaults.go +++ b/internal/config/defaults.go @@ -31,9 +31,6 @@ func Defaults() Config { PipelineIDTemplate: "", BundleIDTemplate: "weatherreporter.{location_id}.{report_id}", IdempotencyKeyTemplate: "{bundle_id}.{run_id}", - ReportPathTemplates: []string{ - "{valid_start_date}/{artifact_group}/{valid_start_date}-{artifact_group}-{run_id}.md", - }, Batch: DistributorBatchNotifyConfig{ Enabled: true, PipelineIDTemplate: "weatherreporter", diff --git a/internal/config/validate.go b/internal/config/validate.go index de80589..a880a91 100644 --- a/internal/config/validate.go +++ b/internal/config/validate.go @@ -124,9 +124,6 @@ func validateDistributorNotify(cfg DistributorNotifyConfig) error { if err := validateDistributorTemplate("notify.distributor.idempotency_key_template", cfg.IdempotencyKeyTemplate, distributorIdempotencyTemplateVariables); err != nil { return err } - if len(cfg.ReportPathTemplates) == 0 { - return fmt.Errorf("notify.distributor.report_path_templates must contain at least one entry when enabled") - } values := sampleDistributorTemplateValues() bundleID, err := RenderDistributorBundleID(cfg.BundleIDTemplate, values) if err != nil { @@ -136,9 +133,6 @@ func validateDistributorNotify(cfg DistributorNotifyConfig) error { if _, err := RenderDistributorPipelineID(cfg.PipelineIDTemplate, values); err != nil { return err } - if _, err := RenderDistributorReportPaths("notify.distributor.report_path_templates", cfg.ReportPathTemplates, values); err != nil { - return err - } if err := validateDistributorBatchNotify(cfg.Batch); err != nil { return err }