From 8577fc29e4ae2bf851a05e9bd08a6a7c1a93aa40 Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Mon, 8 Jun 2026 10:29:42 -0500 Subject: [PATCH] Updated the distributor bundle path template --- docs/config.md | 22 ++++-- docs/internal/app-orchestration.md | 9 +-- docs/internal/distributor-adapter.md | 17 +++-- docs/internal/state.md | 3 +- docs/operations.md | 21 +++--- docs/troubleshooting.md | 6 +- examples/config.yml | 4 +- internal/adapters/distributor/client.go | 66 ++++++++++++------ internal/adapters/distributor/client_test.go | 42 ++++++++---- internal/app/app.go | 41 +++++++++-- internal/app/app_test.go | 10 ++- internal/config/config.go | 2 +- internal/config/config_test.go | 60 +++++++++++++---- internal/config/defaults.go | 5 +- internal/config/notify_templates.go | 71 ++++++++++++++++---- internal/config/validate.go | 12 +++- internal/state/filesystem_test.go | 4 +- internal/state/store.go | 2 +- 18 files changed, 286 insertions(+), 111 deletions(-) diff --git a/docs/config.md b/docs/config.md index 1e1a30d..a35b2bb 100644 --- a/docs/config.md +++ b/docs/config.md @@ -98,20 +98,28 @@ weatherreporter uploads one distributor bundle per generated report after `weatherreporter.{location_id}.{report_id}`. - `idempotency_key_template`: template for distributor idempotency keys. Default: `{bundle_id}.{run_id}`. -- `report_path_template`: template for the Markdown report path inside the - distributor bundle. Default: `{batch_output_name}`. +- `report_path_templates`: ordered list of templates for Markdown report paths + inside the distributor bundle. Each rendered path maps to the same managed + Markdown report source. Default: + ```yaml + - "{valid_start_date}/{artifact_group}/{valid_start_date}-{artifact_group}-{run_id}.md" + - "{valid_start_date}/{artifact_group}/latest.md" + ``` Supported template variables are `location_id`, `report_id`, `run_id`, -`artifact_group`, and `batch_output_name`. `pipeline_id_template` and -`idempotency_key_template` may also use `bundle_id`. +`artifact_group`, `batch_output_name`, `valid_start_date`, `valid_end_date`, +`valid_start_time`, `valid_end_time`, `valid_start_stamp`, and +`valid_end_stamp`. Date values use `YYYY-MM-DD`, time values use `HHMM`, and +stamp values use `YYYY-MM-DDTHHMM` in the effective report timezone. +`pipeline_id_template` and `idempotency_key_template` may also use `bundle_id`. The rendered pipeline ID selects the configured distributor `http_upload` workflow. The rendered bundle ID is the stable logical source identity for the report stream. The rendered idempotency key is the per-run retry identity. -Rendered report paths must be relative paths with `/` separators. They must not -contain backslashes, empty path segments, `.`, `..`, `manifest.json`, or -`.distributor.json`. +Rendered report paths must be unique relative paths with `/` separators. They +must not contain backslashes, empty path segments, `.`, `..`, `manifest.json`, +or `.distributor.json`. The upload token is read from the environment variable named by `token_env` after config loading and `secrets.directory` processing. Config files should diff --git a/docs/internal/app-orchestration.md b/docs/internal/app-orchestration.md index b39ddf0..c5e9cab 100644 --- a/docs/internal/app-orchestration.md +++ b/docs/internal/app-orchestration.md @@ -90,8 +90,9 @@ generation returns an error after writing output, the managed report and metadata remain inspectable. Notification is not attempted after Weather API, briefing, prompt input, render, Scriptorium run, or metadata-save failures. When notification is attempted, the debug artifact records request identity, -including rendered pipeline ID, accepted upload fields, distributor status -fields, raw status report JSON when available, and redacted failure context. +including rendered pipeline ID, bundle paths, accepted upload fields, +distributor status fields, raw status report JSON when available, and redacted +failure context. `--out` copies are never used as notification source files. ## Batch Workflow @@ -141,5 +142,5 @@ Inspect: - Render preflight precedes Scriptorium report generation. - Recent Changes are computed from structured briefing snapshots. - Metadata links artifacts produced for a run. -- Distributor notification uses the managed Markdown report path, not extra - output copies. +- Distributor notification maps the managed Markdown report path to configured + bundle paths; extra output copies are not upload sources. diff --git a/docs/internal/distributor-adapter.md b/docs/internal/distributor-adapter.md index 62f38e1..145b36a 100644 --- a/docs/internal/distributor-adapter.md +++ b/docs/internal/distributor-adapter.md @@ -21,8 +21,7 @@ Inputs: - pipeline ID - bundle ID - idempotency key -- source Markdown report path -- bundle-relative Markdown path +- source Markdown report path and bundle-relative path mappings - bundle created timestamp - context for cancellation @@ -55,24 +54,24 @@ The adapter is built from `notify.distributor` config: - `token_env` - `timeout` -The app layer renders pipeline ID, bundle ID, idempotency key, and bundle path +The app layer renders pipeline ID, bundle ID, idempotency key, and bundle paths from: - `pipeline_id_template` - `bundle_id_template` - `idempotency_key_template` -- `report_path_template` +- `report_path_templates` The token value is read from the environment variable named by `token_env` after config loading and `secrets.directory` processing. ## Upload Behavior -The adapter calls distributor `UploadFiles` with exactly one file: +The adapter calls distributor `UploadFiles` with one or more file mappings: - pipeline ID: the rendered distributor workflow selector - source path: the managed Markdown report path selected by app orchestration -- bundle path: the rendered bundle-relative report path +- bundle paths: rendered bundle-relative report paths - created: the report generation timestamp The adapter creates a distributor upload client with the configured endpoint, @@ -91,11 +90,11 @@ with the status report preserved. ## Failure Behavior The adapter validates required endpoint, token env name, token value, pipeline -ID, bundle ID, idempotency key, source path, bundle path, and upload client -inputs before uploading. +ID, bundle ID, idempotency key, upload files, source paths, bundle paths, and +upload client inputs before uploading. Upload failures include endpoint, pipeline ID, bundle ID, idempotency key, -source path, and bundle path context. Token values are redacted from adapter +source paths, and bundle paths context. Token values are redacted from adapter errors. Distributor idempotency conflicts are exposed as a weatherreporter-owned diff --git a/docs/internal/state.md b/docs/internal/state.md index 6903d59..1870788 100644 --- a/docs/internal/state.md +++ b/docs/internal/state.md @@ -94,7 +94,8 @@ prepared by creating their parent directory; Scriptorium writes the report body to the prepared path. Extra Markdown copies are handled by app orchestration. Distributor notification debug artifacts are written atomically when notification is attempted and include rendered distributor pipeline ID, bundle -ID, idempotency key, upload status, latest run status, and redacted errors. +ID, idempotency key, bundle paths, upload status, latest run status, and +redacted errors. Inspection helpers read existing metadata, briefing, and data package files. Missing metadata directories return no inspection records or no prior snapshot diff --git a/docs/operations.md b/docs/operations.md index da8699b..4fe79ef 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -144,10 +144,11 @@ attempted report. Notification fields are `notificationStatus`, ## Distributor Notification -Distributor notification is configured with `notify.distributor` and is disabled -by default. When enabled, weatherreporter uploads exactly one file per -successfully generated report: the managed Markdown report path recorded in the -report result and metadata. Extra copies written by `--out` or `--out-dir` are +Distributor notification is configured with `notify.distributor` and is +disabled by default. When enabled, weatherreporter uploads the managed Markdown +report path recorded in the report result and metadata. That single source file +can be mapped to multiple configured bundle paths, such as a dated archival path +and a `latest.md` path. Extra copies written by `--out` or `--out-dir` are operator conveniences only. The rendered pipeline ID selects the configured distributor `http_upload` @@ -159,9 +160,9 @@ weatherreporter.{location_id}.{report_id} ``` The default idempotency key appends RunID to the rendered bundle ID so each -report generation has a distinct retry identity. The default bundle path for the -Markdown file is the report definition's batch output name, such as `daily.md`, -`tomorrow.md`, `three-day.md`, or `weekend.md`. +report generation has a distinct retry identity. Default bundle paths use the +valid-period start date, artifact group, and RunID, and also publish +`latest.md` under the same dated artifact-group directory. Notification happens after final metadata save. Weather API, briefing, data-package, render preflight, Scriptorium run, and metadata-save failures do @@ -171,7 +172,7 @@ JSON summary, and the batch returns nonzero. Each notification attempt writes a debug artifact under `notifications/`. The artifact records the rendered pipeline ID, bundle ID, idempotency key, managed -source path, bundle-relative path, bundle created timestamp, accepted upload +source path, bundle-relative paths, bundle created timestamp, accepted upload response, and the latest distributor run status response when available. Weatherreporter polls status until distributor reports `succeeded` or `failed`, or until the configured notification timeout expires. The run status includes @@ -182,7 +183,9 @@ actions such as `replace_older`, `skip_same`, `skip_destination_newer`, or Weatherreporter is responsible for selecting the managed Markdown report, constructing a source bundle, and submitting it to the configured distributor HTTP endpoint. Distributor remains responsible for destination routing, -publication, and any downstream Markdown-to-HTML transformation. +publication, and any downstream Markdown-to-HTML transformation. Distributor +leaves destination files alone when they are not tracked by a newly uploaded +bundle, so previously uploaded dated report paths can remain available. ## Inspection diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index 98e3c0a..eea2e8a 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -265,13 +265,13 @@ Symptom: notification fails with distributor upload rejection, HTTP status, or bundle validation context. Likely cause: the distributor endpoint rejected the token, pipeline ID, bundle -ID, idempotency key, source file, or bundle path. +ID, idempotency key, source file, or one of the rendered bundle paths. Diagnostic: inspect stdout JSON or stderr status lines for `notificationError`. Confirm `notify.distributor.endpoint`, `notify.distributor.pipeline_id_template`, -`notify.distributor.report_path_template`, and token configuration. Token values -are redacted from weatherreporter errors. +`notify.distributor.report_path_templates`, and token configuration. Token +values are redacted from weatherreporter errors. If the upload was accepted but destination output did not change, inspect the notification artifact's `runStatus.report`. Distributor actions such as diff --git a/examples/config.yml b/examples/config.yml index 8195bd8..54e28e1 100644 --- a/examples/config.yml +++ b/examples/config.yml @@ -24,7 +24,9 @@ notify: pipeline_id_template: "weatherreporter.{artifact_group}" bundle_id_template: "weatherreporter.{location_id}.{report_id}" idempotency_key_template: "{bundle_id}.{run_id}" - report_path_template: "{batch_output_name}" + report_path_templates: + - "{valid_start_date}/{artifact_group}/{valid_start_date}-{artifact_group}-{run_id}.md" + - "{valid_start_date}/{artifact_group}/latest.md" missing_source: default: warn diff --git a/internal/adapters/distributor/client.go b/internal/adapters/distributor/client.go index c1cdba8..af362f1 100644 --- a/internal/adapters/distributor/client.go +++ b/internal/adapters/distributor/client.go @@ -28,11 +28,15 @@ type UploadRequest struct { PipelineID string BundleID string IdempotencyKey string - SourcePath string - BundlePath string + Files []UploadFile CreatedAt time.Time } +type UploadFile struct { + SourcePath string + BundlePath string +} + type UploadResult struct { RunID string Status string @@ -81,8 +85,7 @@ type uploadFilesOptions struct { PipelineID string BundleID string IdempotencyKey string - SourcePath string - BundlePath string + Files []UploadFile CreatedAt time.Time } @@ -139,11 +142,16 @@ func (c *Client) Upload(ctx context.Context, req UploadRequest) (UploadResult, e if req.IdempotencyKey == "" { return UploadResult{}, fmt.Errorf("distributor idempotency key is required for bundle %q", req.BundleID) } - if req.SourcePath == "" { - return UploadResult{}, fmt.Errorf("distributor source path is required for bundle %q", req.BundleID) + if len(req.Files) == 0 { + return UploadResult{}, fmt.Errorf("distributor upload files are required for bundle %q", req.BundleID) } - if req.BundlePath == "" { - return UploadResult{}, fmt.Errorf("distributor bundle path is required for bundle %q", req.BundleID) + for i, file := range req.Files { + if file.SourcePath == "" { + return UploadResult{}, fmt.Errorf("distributor source path is required for bundle %q file %d", req.BundleID, i) + } + if file.BundlePath == "" { + return UploadResult{}, fmt.Errorf("distributor bundle path is required for bundle %q file %d", req.BundleID, i) + } } if c.newUploadClient == nil { return UploadResult{}, fmt.Errorf("distributor upload client factory is required for endpoint %q", c.Endpoint) @@ -173,8 +181,7 @@ func (c *Client) Upload(ctx context.Context, req UploadRequest) (UploadResult, e PipelineID: req.PipelineID, BundleID: req.BundleID, IdempotencyKey: req.IdempotencyKey, - SourcePath: req.SourcePath, - BundlePath: req.BundlePath, + Files: append([]UploadFile(nil), req.Files...), CreatedAt: req.CreatedAt, }) if err != nil { @@ -183,8 +190,8 @@ func (c *Client) Upload(ctx context.Context, req UploadRequest) (UploadResult, e PipelineID: req.PipelineID, BundleID: req.BundleID, IdempotencyKey: req.IdempotencyKey, - SourcePath: req.SourcePath, - BundlePath: req.BundlePath, + SourcePaths: uploadSourcePaths(req.Files), + BundlePaths: uploadBundlePaths(req.Files), Token: token, }) } @@ -271,14 +278,19 @@ func newDistributorUploadClient(endpoint, token string, timeout time.Duration) ( } func (c distributorUploadClient) UploadFiles(ctx context.Context, opts uploadFilesOptions) (uploadFilesResult, error) { + files := make([]distributorbundle.BundleFile, 0, len(opts.Files)) + for _, file := range opts.Files { + files = append(files, distributorbundle.BundleFile{ + SourcePath: file.SourcePath, + Path: file.BundlePath, + }) + } result, err := c.client.UploadFiles(ctx, distributorupload.UploadFilesOptions{ PipelineID: opts.PipelineID, ID: opts.BundleID, Created: opts.CreatedAt, IdempotencyKey: opts.IdempotencyKey, - Files: []distributorbundle.BundleFile{ - {SourcePath: opts.SourcePath, Path: opts.BundlePath}, - }, + Files: files, }) if err != nil { return uploadFilesResult{}, err @@ -311,8 +323,8 @@ type uploadErrorContext struct { PipelineID string BundleID string IdempotencyKey string - SourcePath string - BundlePath string + SourcePaths []string + BundlePaths []string Token string } @@ -322,10 +334,26 @@ func wrapUploadError(err error, ctx uploadErrorContext) error { err = redactToken(err, ctx.Token) if isConflict { return &IdempotencyConflictError{ - Err: fmt.Errorf("upload distributor bundle %q to pipeline %q at endpoint %q with idempotency key %q from source %q as bundle path %q: idempotency conflict: %w", ctx.BundleID, ctx.PipelineID, ctx.Endpoint, ctx.IdempotencyKey, ctx.SourcePath, ctx.BundlePath, err), + Err: fmt.Errorf("upload distributor bundle %q to pipeline %q at endpoint %q with idempotency key %q from sources %q as bundle paths %q: idempotency conflict: %w", ctx.BundleID, ctx.PipelineID, ctx.Endpoint, ctx.IdempotencyKey, ctx.SourcePaths, ctx.BundlePaths, err), } } - return fmt.Errorf("upload distributor bundle %q to pipeline %q at endpoint %q with idempotency key %q from source %q as bundle path %q: %w", ctx.BundleID, ctx.PipelineID, ctx.Endpoint, ctx.IdempotencyKey, ctx.SourcePath, ctx.BundlePath, err) + return fmt.Errorf("upload distributor bundle %q to pipeline %q at endpoint %q with idempotency key %q from sources %q as bundle paths %q: %w", ctx.BundleID, ctx.PipelineID, ctx.Endpoint, ctx.IdempotencyKey, ctx.SourcePaths, ctx.BundlePaths, err) +} + +func uploadSourcePaths(files []UploadFile) []string { + paths := make([]string, 0, len(files)) + for _, file := range files { + paths = append(paths, file.SourcePath) + } + return paths +} + +func uploadBundlePaths(files []UploadFile) []string { + paths := make([]string, 0, len(files)) + for _, file := range files { + paths = append(paths, file.BundlePath) + } + return paths } func redactToken(err error, token string) error { diff --git a/internal/adapters/distributor/client_test.go b/internal/adapters/distributor/client_test.go index a6cf9dc..d646d0e 100644 --- a/internal/adapters/distributor/client_test.go +++ b/internal/adapters/distributor/client_test.go @@ -14,7 +14,7 @@ import ( "gitea.maximumdirect.net/eric/weatherreporter/internal/config" ) -func TestUploadUsesConfiguredClientAndSingleFile(t *testing.T) { +func TestUploadUsesConfiguredClientAndFiles(t *testing.T) { cfg := config.Defaults().Notify.Distributor cfg.Endpoint = "https://distributor.example.test" cfg.TokenEnv = "DISTRIBUTOR_UPLOAD_TOKEN" @@ -33,9 +33,11 @@ func TestUploadUsesConfiguredClientAndSingleFile(t *testing.T) { PipelineID: "weatherreporter.daily", BundleID: "weatherreporter.home.daily.run", IdempotencyKey: "weatherreporter.home.daily.run", - SourcePath: "/tmp/report.md", - BundlePath: "daily.md", - CreatedAt: time.Date(2026, 6, 7, 12, 0, 0, 123, time.UTC), + Files: []UploadFile{ + {SourcePath: "/tmp/report.md", BundlePath: "2026-06-07/daily/report.md"}, + {SourcePath: "/tmp/report.md", BundlePath: "2026-06-07/daily/latest.md"}, + }, + CreatedAt: time.Date(2026, 6, 7, 12, 0, 0, 123, time.UTC), }) if err != nil { t.Fatalf("Upload() error = %v", err) @@ -65,11 +67,14 @@ func TestUploadUsesConfiguredClientAndSingleFile(t *testing.T) { if got.IdempotencyKey != "weatherreporter.home.daily.run" { t.Fatalf("IdempotencyKey = %q, want weatherreporter.home.daily.run", got.IdempotencyKey) } - if got.SourcePath != "/tmp/report.md" { - t.Fatalf("SourcePath = %q, want /tmp/report.md", got.SourcePath) + if len(got.Files) != 2 { + t.Fatalf("files = %#v, want two mappings", got.Files) } - if got.BundlePath != "daily.md" { - t.Fatalf("BundlePath = %q, want daily.md", got.BundlePath) + if got.Files[0].SourcePath != "/tmp/report.md" || got.Files[0].BundlePath != "2026-06-07/daily/report.md" { + t.Fatalf("first file = %#v, want archive mapping", got.Files[0]) + } + if got.Files[1].SourcePath != "/tmp/report.md" || got.Files[1].BundlePath != "2026-06-07/daily/latest.md" { + t.Fatalf("second file = %#v, want latest mapping", got.Files[1]) } if got.CreatedAt.IsZero() { t.Fatal("CreatedAt is zero, want generated report timestamp") @@ -102,17 +107,24 @@ func TestUploadRejectsMissingInputs(t *testing.T) { }, wantErr: "pipeline id is required", }, + { + name: "Files", + mutate: func(c *Client, req *UploadRequest) { + req.Files = nil + }, + wantErr: "upload files are required", + }, { name: "SourcePath", mutate: func(c *Client, req *UploadRequest) { - req.SourcePath = "" + req.Files[0].SourcePath = "" }, wantErr: "source path is required", }, { name: "BundlePath", mutate: func(c *Client, req *UploadRequest) { - req.BundlePath = "" + req.Files[0].BundlePath = "" }, wantErr: "bundle path is required", }, @@ -181,7 +193,7 @@ func TestUploadWrapsUploadFailureWithContextWithoutToken(t *testing.T) { if err == nil { t.Fatal("Upload() error = nil, want error") } - for _, want := range []string{cfg.Endpoint, req.PipelineID, req.BundleID, req.IdempotencyKey, req.SourcePath, req.BundlePath} { + for _, want := range []string{cfg.Endpoint, req.PipelineID, req.BundleID, req.IdempotencyKey, req.Files[0].SourcePath, req.Files[0].BundlePath, req.Files[1].BundlePath} { if !strings.Contains(err.Error(), want) { t.Fatalf("error = %q, want context %q", err.Error(), want) } @@ -332,9 +344,11 @@ func validUploadRequest() UploadRequest { PipelineID: "weatherreporter.daily", BundleID: "weatherreporter.home.daily.run", IdempotencyKey: "weatherreporter.home.daily.run", - SourcePath: "/tmp/report.md", - BundlePath: "daily.md", - CreatedAt: time.Date(2026, 6, 7, 12, 0, 0, 123, time.UTC), + Files: []UploadFile{ + {SourcePath: "/tmp/report.md", BundlePath: "2026-06-07/daily/report.md"}, + {SourcePath: "/tmp/report.md", BundlePath: "2026-06-07/daily/latest.md"}, + }, + CreatedAt: time.Date(2026, 6, 7, 12, 0, 0, 123, time.UTC), } } diff --git a/internal/app/app.go b/internal/app/app.go index ff2b84b..1d63c33 100644 --- a/internal/app/app.go +++ b/internal/app/app.go @@ -162,7 +162,7 @@ type NotificationRequest struct { BundleID string IdempotencyKey string ReportPath string - BundlePath string + BundlePaths []string CreatedAt time.Time } @@ -637,6 +637,9 @@ func buildNotificationRequest(cfg config.Config, resolved report.Resolved, repor ArtifactGroup: resolved.Definition.ArtifactGroup, BatchOutputName: resolved.Definition.BatchOutputName, } + if err := addDistributorValidPeriodValues(&values, resolved.ValidPeriod, cfg.WeatherAPI.Timezone); err != nil { + return NotificationRequest{}, err + } bundleID, err := config.RenderDistributorBundleID(cfg.Notify.Distributor.BundleIDTemplate, values) if err != nil { return NotificationRequest{}, err @@ -650,7 +653,7 @@ func buildNotificationRequest(cfg config.Config, resolved report.Resolved, repor if err != nil { return NotificationRequest{}, err } - bundlePath, err := config.RenderDistributorReportPath(cfg.Notify.Distributor.ReportPathTemplate, values) + bundlePaths, err := config.RenderDistributorReportPaths(cfg.Notify.Distributor.ReportPathTemplates, values) if err != nil { return NotificationRequest{}, err } @@ -661,11 +664,27 @@ func buildNotificationRequest(cfg config.Config, resolved report.Resolved, repor BundleID: bundleID, IdempotencyKey: idempotencyKey, ReportPath: reportPath, - BundlePath: bundlePath, + BundlePaths: bundlePaths, CreatedAt: metadata.GeneratedAt, }, nil } +func addDistributorValidPeriodValues(values *config.DistributorTemplateValues, period timeutil.Period, timezone string) error { + location, err := timeutil.LoadLocation(timezone) + if err != nil { + return err + } + start := period.Start.In(location) + end := period.End.In(location) + values.ValidStartDate = start.Format(timeutil.DateLayout) + values.ValidEndDate = end.Format(timeutil.DateLayout) + values.ValidStartTime = start.Format("1504") + values.ValidEndTime = end.Format("1504") + values.ValidStartStamp = start.Format("2006-01-02T1504") + values.ValidEndStamp = end.Format("2006-01-02T1504") + return nil +} + func saveNotificationArtifact(ctx context.Context, store state.Store, resolved report.Resolved, cfg config.Config, metadata state.Metadata, req NotificationRequest, result *NotificationResult, notifyErr error) (string, error) { if store == nil { return "", fmt.Errorf("state store is required") @@ -680,7 +699,7 @@ func saveNotificationArtifact(ctx context.Context, store state.Store, resolved r BundleID: req.BundleID, IdempotencyKey: req.IdempotencyKey, SourcePath: req.ReportPath, - BundlePath: req.BundlePath, + BundlePaths: append([]string(nil), req.BundlePaths...), BundleCreated: req.CreatedAt, Status: "attempted", } @@ -729,8 +748,7 @@ func (n distributorNotifier) Notify(ctx context.Context, req NotificationRequest PipelineID: req.PipelineID, BundleID: req.BundleID, IdempotencyKey: req.IdempotencyKey, - SourcePath: req.ReportPath, - BundlePath: req.BundlePath, + Files: distributorUploadFiles(req.ReportPath, req.BundlePaths), CreatedAt: req.CreatedAt, }) notification := &NotificationResult{ @@ -758,6 +776,17 @@ func (n distributorNotifier) Notify(ctx context.Context, req NotificationRequest return notification, nil } +func distributorUploadFiles(sourcePath string, bundlePaths []string) []distributoradapter.UploadFile { + files := make([]distributoradapter.UploadFile, 0, len(bundlePaths)) + for _, bundlePath := range bundlePaths { + files = append(files, distributoradapter.UploadFile{ + SourcePath: sourcePath, + BundlePath: bundlePath, + }) + } + return files +} + func BuildBriefing(req BriefingRequest, bundle *forecast.Bundle) (briefing.Package, error) { location, err := timeutil.LoadLocation(req.Config.WeatherAPI.Timezone) if err != nil { diff --git a/internal/app/app_test.go b/internal/app/app_test.go index 47c01b9..e6ec7ef 100644 --- a/internal/app/app_test.go +++ b/internal/app/app_test.go @@ -335,7 +335,11 @@ func TestGenerateReportNotifiesManagedReportPath(t *testing.T) { if err := json.Unmarshal(notificationData, ¬ificationArtifact); err != nil { t.Fatalf("decode notification artifact: %v", err) } - if notificationArtifact.PipelineID != "weatherreporter.daily" || notificationArtifact.BundleCreated.IsZero() || notificationArtifact.RunStatus == nil || !strings.Contains(string(notificationArtifact.RunStatus.Report), "replace_older") { + wantBundlePaths := []string{ + "2026-05-29/daily/2026-05-29-daily-" + result.Metadata.RunID + ".md", + "2026-05-29/daily/latest.md", + } + if notificationArtifact.PipelineID != "weatherreporter.daily" || strings.Join(notificationArtifact.BundlePaths, "\n") != strings.Join(wantBundlePaths, "\n") || notificationArtifact.BundleCreated.IsZero() || notificationArtifact.RunStatus == nil || !strings.Contains(string(notificationArtifact.RunStatus.Report), "replace_older") { t.Fatalf("notification artifact = %#v, want requested pipeline, status report, and created timestamp", notificationArtifact) } if len(notifier.requests) != 1 { @@ -348,8 +352,8 @@ func TestGenerateReportNotifiesManagedReportPath(t *testing.T) { if req.ReportPath == outputPath { t.Fatalf("notification used output copy %q, want managed report path", outputPath) } - if req.BundlePath != "daily.md" { - t.Fatalf("notification BundlePath = %q, want daily.md", req.BundlePath) + if strings.Join(req.BundlePaths, "\n") != strings.Join(wantBundlePaths, "\n") { + t.Fatalf("notification BundlePaths = %#v, want %#v", req.BundlePaths, wantBundlePaths) } if req.PipelineID != "weatherreporter.daily" { t.Fatalf("notification PipelineID = %q, want rendered pipeline", req.PipelineID) diff --git a/internal/config/config.go b/internal/config/config.go index 093cee0..21b9775 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -59,7 +59,7 @@ type DistributorNotifyConfig struct { PipelineIDTemplate string `yaml:"pipeline_id_template"` BundleIDTemplate string `yaml:"bundle_id_template"` IdempotencyKeyTemplate string `yaml:"idempotency_key_template"` - ReportPathTemplate string `yaml:"report_path_template"` + ReportPathTemplates []string `yaml:"report_path_templates"` } type MissingSourceConfig struct { diff --git a/internal/config/config_test.go b/internal/config/config_test.go index f4f7f80..881440e 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -53,8 +53,12 @@ func TestDefaults(t *testing.T) { if cfg.Notify.Distributor.IdempotencyKeyTemplate != "{bundle_id}.{run_id}" { t.Fatalf("Notify.Distributor.IdempotencyKeyTemplate = %q, want default", cfg.Notify.Distributor.IdempotencyKeyTemplate) } - if cfg.Notify.Distributor.ReportPathTemplate != "{batch_output_name}" { - t.Fatalf("Notify.Distributor.ReportPathTemplate = %q, want default", cfg.Notify.Distributor.ReportPathTemplate) + wantReportPaths := []string{ + "{valid_start_date}/{artifact_group}/{valid_start_date}-{artifact_group}-{run_id}.md", + "{valid_start_date}/{artifact_group}/latest.md", + } + if strings.Join(cfg.Notify.Distributor.ReportPathTemplates, "\n") != strings.Join(wantReportPaths, "\n") { + t.Fatalf("Notify.Distributor.ReportPathTemplates = %#v, want %#v", cfg.Notify.Distributor.ReportPathTemplates, wantReportPaths) } if cfg.MissingSource.Default != MissingSourceWarn { t.Fatalf("MissingSource.Default = %q, want warn", cfg.MissingSource.Default) @@ -82,6 +86,9 @@ func TestLoadExampleConfig(t *testing.T) { if cfg.Notify.Distributor.PipelineIDTemplate != "weatherreporter.{artifact_group}" { t.Fatalf("PipelineIDTemplate = %q, want example pipeline template", cfg.Notify.Distributor.PipelineIDTemplate) } + if len(cfg.Notify.Distributor.ReportPathTemplates) != 2 { + t.Fatalf("ReportPathTemplates = %#v, want example archive and latest paths", cfg.Notify.Distributor.ReportPathTemplates) + } } func TestLoadMinimalExampleConfig(t *testing.T) { @@ -241,19 +248,33 @@ func TestEnabledDistributorNotifyValidation(t *testing.T) { }, wantErr: "notify.distributor.idempotency_key_template", }, + { + name: "ReportPathTemplatesEmpty", + mutate: func(cfg *Config) { + cfg.Notify.Distributor.ReportPathTemplates = nil + }, + wantErr: "notify.distributor.report_path_templates", + }, { name: "ReportPathTemplateUnknown", mutate: func(cfg *Config) { - cfg.Notify.Distributor.ReportPathTemplate = "{bundle_id}" + cfg.Notify.Distributor.ReportPathTemplates = []string{"{unknown}"} }, - wantErr: "notify.distributor.report_path_template", + wantErr: "notify.distributor.report_path_templates", }, { name: "ReportPathTemplateInvalidPath", mutate: func(cfg *Config) { - cfg.Notify.Distributor.ReportPathTemplate = "/{batch_output_name}" + cfg.Notify.Distributor.ReportPathTemplates = []string{"/{batch_output_name}"} }, - wantErr: "notify.distributor.report_path_template", + wantErr: "notify.distributor.report_path_templates", + }, + { + name: "ReportPathTemplateDuplicatePath", + mutate: func(cfg *Config) { + cfg.Notify.Distributor.ReportPathTemplates = []string{"latest.md", "latest.md"} + }, + wantErr: "notify.distributor.report_path_templates", }, } @@ -282,6 +303,12 @@ func TestDistributorTemplateRendering(t *testing.T) { RunID: "20260607T120000Z", ArtifactGroup: "daily", BatchOutputName: "daily.md", + ValidStartDate: "2026-06-07", + ValidEndDate: "2026-06-08", + ValidStartTime: "1800", + ValidEndTime: "0600", + ValidStartStamp: "2026-06-07T1800", + ValidEndStamp: "2026-06-08T0600", BundleID: "weatherreporter.home.daily", } @@ -309,12 +336,19 @@ func TestDistributorTemplateRendering(t *testing.T) { t.Fatalf("idempotencyKey = %q, want rendered run key", idempotencyKey) } - reportPath, err := RenderDistributorReportPath("reports/{batch_output_name}", values) + reportPaths, err := RenderDistributorReportPaths([]string{ + "{valid_start_date}/{artifact_group}/{valid_start_stamp}-{valid_end_stamp}-{run_id}.md", + "{valid_start_date}/{artifact_group}/latest.md", + }, values) if err != nil { - t.Fatalf("RenderDistributorReportPath() error = %v", err) + t.Fatalf("RenderDistributorReportPaths() error = %v", err) } - if reportPath != "reports/daily.md" { - t.Fatalf("reportPath = %q, want reports/daily.md", reportPath) + wantPaths := []string{ + "2026-06-07/daily/2026-06-07T1800-2026-06-08T0600-20260607T120000Z.md", + "2026-06-07/daily/latest.md", + } + if strings.Join(reportPaths, "\n") != strings.Join(wantPaths, "\n") { + t.Fatalf("reportPaths = %#v, want %#v", reportPaths, wantPaths) } } @@ -360,7 +394,7 @@ func TestDistributorReportPathValidation(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - err := ValidateDistributorReportPath(tt.path) + err := ValidateDistributorReportPath("test.path", tt.path) if tt.ok && err != nil { t.Fatalf("ValidateDistributorReportPath() error = %v", err) } @@ -387,11 +421,11 @@ func TestDistributorReportPathRenderingRejectsInvalidValues(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - _, err := RenderDistributorReportPath("{batch_output_name}", DistributorTemplateValues{ + _, err := RenderDistributorReportPaths([]string{"{batch_output_name}"}, DistributorTemplateValues{ BatchOutputName: tt.batchOutputName, }) if err == nil { - t.Fatal("RenderDistributorReportPath() error = nil, want error") + t.Fatal("RenderDistributorReportPaths() error = nil, want error") } }) } diff --git a/internal/config/defaults.go b/internal/config/defaults.go index 30f176e..55f5dd0 100644 --- a/internal/config/defaults.go +++ b/internal/config/defaults.go @@ -31,7 +31,10 @@ func Defaults() Config { PipelineIDTemplate: "", BundleIDTemplate: "weatherreporter.{location_id}.{report_id}", IdempotencyKeyTemplate: "{bundle_id}.{run_id}", - ReportPathTemplate: "{batch_output_name}", + ReportPathTemplates: []string{ + "{valid_start_date}/{artifact_group}/{valid_start_date}-{artifact_group}-{run_id}.md", + "{valid_start_date}/{artifact_group}/latest.md", + }, }, }, MissingSource: MissingSourceConfig{ diff --git a/internal/config/notify_templates.go b/internal/config/notify_templates.go index 3f701d5..3ce3ffd 100644 --- a/internal/config/notify_templates.go +++ b/internal/config/notify_templates.go @@ -12,6 +12,12 @@ type DistributorTemplateValues struct { RunID string ArtifactGroup string BatchOutputName string + ValidStartDate string + ValidEndDate string + ValidStartTime string + ValidEndTime string + ValidStartStamp string + ValidEndStamp string BundleID string } @@ -21,6 +27,12 @@ var distributorTemplateVariables = map[string]struct{}{ "run_id": {}, "artifact_group": {}, "batch_output_name": {}, + "valid_start_date": {}, + "valid_end_date": {}, + "valid_start_time": {}, + "valid_end_time": {}, + "valid_start_stamp": {}, + "valid_end_stamp": {}, } var distributorIdempotencyTemplateVariables = map[string]struct{}{ @@ -29,6 +41,12 @@ var distributorIdempotencyTemplateVariables = map[string]struct{}{ "run_id": {}, "artifact_group": {}, "batch_output_name": {}, + "valid_start_date": {}, + "valid_end_date": {}, + "valid_start_time": {}, + "valid_end_time": {}, + "valid_start_stamp": {}, + "valid_end_stamp": {}, "bundle_id": {}, } @@ -53,15 +71,28 @@ func RenderDistributorIdempotencyKey(template string, values DistributorTemplate return renderDistributorTemplate("notify.distributor.idempotency_key_template", template, values, distributorIdempotencyTemplateVariables) } -func RenderDistributorReportPath(template string, values DistributorTemplateValues) (string, error) { - rendered, err := renderDistributorTemplate("notify.distributor.report_path_template", template, values, distributorTemplateVariables) - if err != nil { - return "", err +func RenderDistributorReportPaths(templates []string, values DistributorTemplateValues) ([]string, error) { + if len(templates) == 0 { + return nil, fmt.Errorf("notify.distributor.report_path_templates must contain at least one entry") } - if err := ValidateDistributorReportPath(rendered); err != nil { - return "", err + paths := make([]string, 0, len(templates)) + seen := make(map[string]struct{}, len(templates)) + for i, template := range templates { + name := fmt.Sprintf("notify.distributor.report_path_templates[%d]", i) + rendered, err := renderDistributorTemplate(name, template, values, distributorTemplateVariables) + if err != nil { + return nil, err + } + if err := ValidateDistributorReportPath(name, rendered); err != nil { + return nil, err + } + if _, ok := seen[rendered]; ok { + return nil, fmt.Errorf("notify.distributor.report_path_templates renders duplicate path %q", rendered) + } + seen[rendered] = struct{}{} + paths = append(paths, rendered) } - return rendered, nil + return paths, nil } func validateDistributorTemplate(name, template string, allowed map[string]struct{}) error { @@ -109,6 +140,18 @@ func distributorTemplateValue(variable string, values DistributorTemplateValues) return values.ArtifactGroup case "batch_output_name": return values.BatchOutputName + case "valid_start_date": + return values.ValidStartDate + case "valid_end_date": + return values.ValidEndDate + case "valid_start_time": + return values.ValidStartTime + case "valid_end_time": + return values.ValidEndTime + case "valid_start_stamp": + return values.ValidStartStamp + case "valid_end_stamp": + return values.ValidEndStamp case "bundle_id": return values.BundleID default: @@ -116,27 +159,27 @@ func distributorTemplateValue(variable string, values DistributorTemplateValues) } } -func ValidateDistributorReportPath(path string) error { +func ValidateDistributorReportPath(name, path string) error { if path == "" { - return fmt.Errorf("notify.distributor.report_path_template renders an empty path") + return fmt.Errorf("%s renders an empty path", name) } if isDistributorAbsolutePath(path) { - return fmt.Errorf("notify.distributor.report_path_template must render a relative path") + return fmt.Errorf("%s must render a relative path", name) } if strings.Contains(path, "\\") { - return fmt.Errorf("notify.distributor.report_path_template must not render backslashes") + return fmt.Errorf("%s must not render backslashes", name) } segments := strings.Split(path, "/") for _, segment := range segments { if segment == "" { - return fmt.Errorf("notify.distributor.report_path_template must not render empty path segments") + return fmt.Errorf("%s must not render empty path segments", name) } if segment == "." || segment == ".." { - return fmt.Errorf("notify.distributor.report_path_template must not render . or .. path segments") + return fmt.Errorf("%s must not render . or .. path segments", name) } if segment == "manifest.json" || segment == ".distributor.json" { - return fmt.Errorf("notify.distributor.report_path_template must not render reserved path segment %q", segment) + return fmt.Errorf("%s must not render reserved path segment %q", name, segment) } } diff --git a/internal/config/validate.go b/internal/config/validate.go index 2e9ba77..fcf3447 100644 --- a/internal/config/validate.go +++ b/internal/config/validate.go @@ -118,8 +118,8 @@ func validateDistributorNotify(cfg DistributorNotifyConfig) error { if err := validateDistributorTemplate("notify.distributor.idempotency_key_template", cfg.IdempotencyKeyTemplate, distributorIdempotencyTemplateVariables); err != nil { return err } - if cfg.ReportPathTemplate == "" { - return fmt.Errorf("notify.distributor.report_path_template is required when enabled") + if len(cfg.ReportPathTemplates) == 0 { + return fmt.Errorf("notify.distributor.report_path_templates must contain at least one entry when enabled") } values := DistributorTemplateValues{ LocationID: "location", @@ -127,6 +127,12 @@ func validateDistributorNotify(cfg DistributorNotifyConfig) error { RunID: "run", ArtifactGroup: "artifact", BatchOutputName: "report.md", + ValidStartDate: "2026-05-29", + ValidEndDate: "2026-05-30", + ValidStartTime: "0000", + ValidEndTime: "0000", + ValidStartStamp: "2026-05-29T0000", + ValidEndStamp: "2026-05-30T0000", } bundleID, err := RenderDistributorBundleID(cfg.BundleIDTemplate, values) if err != nil { @@ -136,7 +142,7 @@ func validateDistributorNotify(cfg DistributorNotifyConfig) error { if _, err := RenderDistributorPipelineID(cfg.PipelineIDTemplate, values); err != nil { return err } - if _, err := RenderDistributorReportPath(cfg.ReportPathTemplate, values); err != nil { + if _, err := RenderDistributorReportPaths(cfg.ReportPathTemplates, values); err != nil { return err } diff --git a/internal/state/filesystem_test.go b/internal/state/filesystem_test.go index 997e796..860033a 100644 --- a/internal/state/filesystem_test.go +++ b/internal/state/filesystem_test.go @@ -69,7 +69,7 @@ func TestSaveArtifactsAndMetadataRoundTrip(t *testing.T) { BundleID: "weatherreporter.home.daily.run", IdempotencyKey: "weatherreporter.home.daily.run", SourcePath: "/tmp/report.md", - BundlePath: "daily.md", + BundlePaths: []string{"2026-05-29/daily/report.md", "2026-05-29/daily/latest.md"}, BundleCreated: resolved.GeneratedAt, Status: "succeeded", RunStatus: &DistributorRunStatus{RunID: "distributor-run", Status: "succeeded"}, @@ -103,7 +103,7 @@ func TestSaveArtifactsAndMetadataRoundTrip(t *testing.T) { if err := json.Unmarshal(notificationData, ¬ification); err != nil { t.Fatalf("decode notification: %v", err) } - if notification.SchemaVersion != DistributorNotificationSchemaVersion || notification.PipelineID != "weatherreporter.daily" || notification.RunStatus == nil || notification.RunStatus.Status != "succeeded" { + if notification.SchemaVersion != DistributorNotificationSchemaVersion || notification.PipelineID != "weatherreporter.daily" || len(notification.BundlePaths) != 2 || notification.RunStatus == nil || notification.RunStatus.Status != "succeeded" { t.Fatalf("notification = %#v, want persisted distributor status", notification) } paths, err := store.Paths(resolved) diff --git a/internal/state/store.go b/internal/state/store.go index e293dc1..5f4843d 100644 --- a/internal/state/store.go +++ b/internal/state/store.go @@ -49,7 +49,7 @@ type DistributorNotificationArtifact struct { BundleID string `json:"bundleId,omitempty"` IdempotencyKey string `json:"idempotencyKey,omitempty"` SourcePath string `json:"sourcePath,omitempty"` - BundlePath string `json:"bundlePath,omitempty"` + BundlePaths []string `json:"bundlePaths,omitempty"` BundleCreated time.Time `json:"bundleCreated,omitempty"` Status string `json:"status"` Upload *DistributorUploadResult `json:"upload,omitempty"`