Implement the distributor v0.5 PipelineID update

This commit is contained in:
2026-06-08 07:04:31 -05:00
parent 9bc8156615
commit d71c7e4d28
24 changed files with 264 additions and 103 deletions

View File

@@ -92,16 +92,22 @@ weatherreporter uploads one distributor bundle per generated report after
- `timeout`: distributor operation timeout. Must be greater than zero when - `timeout`: distributor operation timeout. Must be greater than zero when
enabled. Default: `30s`. enabled. Default: `30s`.
- `failure_policy`: must be `error` when enabled. Default: `error`. - `failure_policy`: must be `error` when enabled. Default: `error`.
- `pipeline_id_template`: template for the distributor pipeline ID. Required
when enabled. Default: empty.
- `bundle_id_template`: template for distributor bundle IDs. Default: - `bundle_id_template`: template for distributor bundle IDs. Default:
`weatherreporter.{location_id}.{report_id}.{run_id}`. `weatherreporter.{location_id}.{report_id}`.
- `idempotency_key_template`: template for distributor idempotency keys. - `idempotency_key_template`: template for distributor idempotency keys.
Default: `{bundle_id}`. Default: `{bundle_id}.{run_id}`.
- `report_path_template`: template for the Markdown report path inside the - `report_path_template`: template for the Markdown report path inside the
distributor bundle. Default: `{batch_output_name}`. distributor bundle. Default: `{batch_output_name}`.
Supported template variables are `location_id`, `report_id`, `run_id`, Supported template variables are `location_id`, `report_id`, `run_id`,
`artifact_group`, and `batch_output_name`. `idempotency_key_template` may also `artifact_group`, and `batch_output_name`. `pipeline_id_template` and
use `bundle_id`. `idempotency_key_template` may also use `bundle_id`.
The rendered pipeline ID selects the configured distributor `http_upload`
workflow. The rendered bundle ID is the stable logical source identity for the
report stream. The rendered idempotency key is the per-run retry identity.
Rendered report paths must be relative paths with `/` separators. They must not Rendered report paths must be relative paths with `/` separators. They must not
contain backslashes, empty path segments, `.`, `..`, `manifest.json`, or contain backslashes, empty path segments, `.`, `..`, `manifest.json`, or

View File

@@ -9,13 +9,16 @@ This document is the copyable implementation guide for submitting producer outpu
The upstream application needs these values from deployment or operator configuration: The upstream application needs these values from deployment or operator configuration:
- distributor endpoint: the HTTP server base URL, such as `https://distributor.example.com`; - distributor endpoint: the HTTP server base URL, such as `https://distributor.example.com`;
- upload token: bearer token for exactly one configured `http_upload` pipeline; - upload token: bearer token that authenticates the producer;
- pipeline id: configured `http_upload` pipeline that should process this upload;
- generated files: regular local files to include in the source bundle; - generated files: regular local files to include in the source bundle;
- bundle id: stable identifier for this producer output; - bundle id: stable identifier for the logical report stream or artifact;
- idempotency key: stable key for retrying the same producer operation. - idempotency key: unique key for one producer run, reused only when retrying that same run.
Do not put destination routing, public URLs, transform settings, or credentials in the source manifest. Those belong in the `distributor` pipeline configuration. Do not put destination routing, public URLs, transform settings, or credentials in the source manifest. Those belong in the `distributor` pipeline configuration.
The token, pipeline id, bundle id, and idempotency key have different jobs. The token authenticates the producer. The pipeline id selects the configured distributor workflow, including destinations and publishing policy. The bundle id tells `distributor` whether a new upload is a newer version of the same source; keep it stable across runs that should replace the same managed destination artifact. The idempotency key tells `distributor` whether an upload request is a retry; change it for each distinct producer run so new content is enqueued.
## Recommended Workflow ## Recommended Workflow
Use `gitea.maximumdirect.net/eric/distributor/pkg/upload`. Use `gitea.maximumdirect.net/eric/distributor/pkg/upload`.
@@ -53,7 +56,9 @@ func SubmitReport(reportPath, summaryPath string) error {
return fmt.Errorf("distributor endpoint and token are required") return fmt.Errorf("distributor endpoint and token are required")
} }
reportID := "weather.hourly.brentwood.2026-06-07T15" pipelineID := "weather-hourly"
reportID := "weather.hourly.brentwood"
runID := time.Now().UTC().Format("20060102T150405.000000000Z")
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel() defer cancel()
@@ -66,8 +71,9 @@ func SubmitReport(reportPath, summaryPath string) error {
} }
result, err := client.UploadFiles(ctx, upload.UploadFilesOptions{ result, err := client.UploadFiles(ctx, upload.UploadFilesOptions{
PipelineID: pipelineID,
ID: reportID, ID: reportID,
IdempotencyKey: reportID, IdempotencyKey: reportID + "." + runID,
Files: []bundle.BundleFile{ Files: []bundle.BundleFile{
{SourcePath: reportPath, Path: "report.md"}, {SourcePath: reportPath, Path: "report.md"},
{SourcePath: summaryPath, Path: "summary.txt"}, {SourcePath: summaryPath, Path: "summary.txt"},
@@ -88,8 +94,11 @@ func SubmitReport(reportPath, summaryPath string) error {
## Producer Responsibilities ## Producer Responsibilities
- Use a stable bundle id for the producer output, such as a report type plus logical timestamp. - Use a stable bundle id for the logical producer output that should replace the same destination artifact, such as `weather.hourly.brentwood`.
- Use a stable idempotency key for cross-process retries of the same producer operation. - Set `PipelineID` to the configured upload pipeline that should process the bundle.
- Do not include per-run timestamps, random values, or job ids in the bundle id unless each run should be treated as a different source.
- Use an idempotency key that changes for every distinct producer run, such as `<bundle-id>.<run-id>`.
- Reuse the same idempotency key only when retrying the exact same producer run with the same source manifest.
- Map each generated file to a clean slash-separated bundle path, such as `report.md` or `assets/chart.png`. - Map each generated file to a clean slash-separated bundle path, such as `report.md` or `assets/chart.png`.
- Include only regular files. Symlinks, directories as files, devices, FIFOs, and sockets are rejected. - Include only regular files. Symlinks, directories as files, devices, FIFOs, and sockets are rejected.
- Keep file contents stable after upload inputs are selected. Bundle digests are calculated from file bytes. - Keep file contents stable after upload inputs are selected. Bundle digests are calculated from file bytes.
@@ -99,9 +108,9 @@ Valid bundle paths are relative slash paths. They must not be empty, absolute, c
## Idempotency And Status ## Idempotency And Status
`pkg/upload` sends `Idempotency-Key` on every upload. If the caller omits one, the package generates a random key for that call and reuses it for in-process retries. That is enough for transient network retry within one process. `pkg/upload` sends `Idempotency-Key` on every upload. If the caller omits one, the package generates a random key for that call and reuses it for in-process retries. That is enough for transient network retry within one process, but it does not give cross-process retry identity.
For producer jobs that may retry after process restart, supply a stable key derived from the producer operation, such as the report id or job id. Reusing the same key with the same normalized source manifest returns the original accepted run. Reusing the same key with different source content returns a conflict. For producer jobs that may retry after process restart, supply a key derived from the producer run, such as `<bundle-id>.<run-id>`. Reusing the same key with the same token, pipeline id, and normalized source manifest returns the original accepted run. Reusing the same key with different source content in that scope returns a conflict. Reusing one key across multiple distinct report generations prevents those generations from being treated as new uploads.
`Status` polls `/runs/<run-id>` while the distributor server retains the in-memory status record. Status values are `accepted`, `queued`, `running`, `succeeded`, and `failed`. Completed records expire according to the server's `server.http.retention` setting, and server restart clears status and idempotency records. `Status` polls `/runs/<run-id>` while the distributor server retains the in-memory status record. Status values are `accepted`, `queued`, `running`, `succeeded`, and `failed`. Completed records expire according to the server's `server.http.retention` setting, and server restart clears status and idempotency records.

View File

@@ -19,7 +19,7 @@ Use `WriteBundle` when producer-generated files live outside the final bundle ro
```go ```go
manifest, err := bundle.WriteBundle(bundle.WriteBundleOptions{ manifest, err := bundle.WriteBundle(bundle.WriteBundleOptions{
Root: "/var/spool/distributor/weather/hourly-2026-06-07T15", Root: "/var/spool/distributor/weather/hourly-2026-06-07T15",
ID: "weather.hourly.brentwood.2026-06-07T15", ID: "weather.hourly.brentwood",
Files: []bundle.BundleFile{ Files: []bundle.BundleFile{
{SourcePath: "/tmp/weather/report.md", Path: "report.md"}, {SourcePath: "/tmp/weather/report.md", Path: "report.md"},
{SourcePath: "/tmp/weather/summary.txt", Path: "summary.txt"}, {SourcePath: "/tmp/weather/summary.txt", Path: "summary.txt"},
@@ -41,7 +41,7 @@ Use `BuildManifest` and `WriteManifest` when files are already staged under the
root := "/var/spool/distributor/weather/hourly-2026-06-07T15" root := "/var/spool/distributor/weather/hourly-2026-06-07T15"
manifest, err := bundle.BuildManifest(bundle.BuildOptions{ manifest, err := bundle.BuildManifest(bundle.BuildOptions{
Root: root, Root: root,
ID: "weather.hourly.brentwood.2026-06-07T15", ID: "weather.hourly.brentwood",
Files: []string{"report.md", "summary.txt"}, Files: []string{"report.md", "summary.txt"},
}) })
if err != nil { if err != nil {
@@ -72,6 +72,8 @@ Invalid paths include:
Explicit file lists preserve caller order. File order is part of the bundle digest, so producers should choose it deliberately and keep it stable. Explicit file lists preserve caller order. File order is part of the bundle digest, so producers should choose it deliberately and keep it stable.
The manifest `ID` is the logical source identity used by `distributor` destination comparison. Keep it stable for runs that should replace the same managed destination artifact. If every run uses a different manifest `ID`, `distributor` treats those runs as different sources and may report a destination conflict instead of replacing older output.
## Validation And Digest Helpers ## Validation And Digest Helpers
Use `ValidateBundle` before handing an existing local bundle to another process. It verifies manifest semantics, file existence, regular-file type, file size, per-file SHA-256 digests, and bundle digest. Use `ValidateBundle` before handing an existing local bundle to another process. It verifies manifest semantics, file existence, regular-file type, file size, per-file SHA-256 digests, and bundle digest.

View File

@@ -8,7 +8,7 @@ Import path:
import "gitea.maximumdirect.net/eric/distributor/pkg/upload" import "gitea.maximumdirect.net/eric/distributor/pkg/upload"
``` ```
`pkg/upload` is the producer-facing HTTP upload client. It builds on `pkg/bundle`, packages valid source bundles as gzip-compressed tar archives, sends bearer authentication, includes idempotency keys, and exposes a status polling helper. `pkg/upload` is the producer-facing HTTP upload client. It builds on `pkg/bundle`, packages valid source bundles as gzip-compressed tar archives, sends bearer authentication, routes uploads to a configured pipeline, includes idempotency keys, and exposes a status polling helper.
`UploadFiles` examples also use: `UploadFiles` examples also use:
@@ -30,7 +30,7 @@ if err != nil {
} }
``` ```
`Endpoint` is the distributor server base URL. The client derives `/upload` and `/runs/<run-id>`. `Token` is required and is sent as `Authorization: Bearer <token>`. Token values are redacted from client errors. `Endpoint` is the distributor server base URL. The client derives `/v1/pipelines/<pipeline-id>/upload` and `/runs/<run-id>`. `Token` is required and is sent as `Authorization: Bearer <token>`. Token values are redacted from client errors.
`HTTPClient` and `Retry` are optional. Defaults use a 30 second HTTP timeout and safe retry settings. `HTTPClient` and `Retry` are optional. Defaults use a 30 second HTTP timeout and safe retry settings.
@@ -40,8 +40,9 @@ Use `UploadFiles` when the producer has generated output files but has not assem
```go ```go
result, err := client.UploadFiles(ctx, upload.UploadFilesOptions{ result, err := client.UploadFiles(ctx, upload.UploadFilesOptions{
ID: "weather.hourly.brentwood.2026-06-07T15", PipelineID: "weather-hourly",
IdempotencyKey: "weather.hourly.brentwood.2026-06-07T15", ID: "weather.hourly.brentwood",
IdempotencyKey: "weather.hourly.brentwood.20260607T150000Z",
Files: []bundle.BundleFile{ Files: []bundle.BundleFile{
{SourcePath: "/tmp/weather/report.md", Path: "report.md"}, {SourcePath: "/tmp/weather/report.md", Path: "report.md"},
{SourcePath: "/tmp/weather/summary.txt", Path: "summary.txt"}, {SourcePath: "/tmp/weather/summary.txt", Path: "summary.txt"},
@@ -53,7 +54,7 @@ if err != nil {
_ = result.RunID _ = result.RunID
``` ```
`UploadFiles` creates a temporary bundle, writes and validates a manifest, uploads the archive, and removes temporary files when the call returns. It does not write into producer source directories. `PipelineID` is required and selects the configured distributor workflow for this upload. `ID` is the source manifest id and identifies the logical artifact inside that workflow. `UploadFiles` creates a temporary bundle, writes and validates a manifest, uploads the archive, and removes temporary files when the call returns. It does not write into producer source directories.
## Upload An Existing Bundle ## Upload An Existing Bundle
@@ -61,8 +62,9 @@ Use `UploadBundle` when the producer already has a complete local bundle root co
```go ```go
result, err := client.UploadBundle(ctx, upload.UploadBundleOptions{ result, err := client.UploadBundle(ctx, upload.UploadBundleOptions{
PipelineID: "weather-hourly",
Root: "/var/spool/weather/hourly-2026-06-07T15", Root: "/var/spool/weather/hourly-2026-06-07T15",
IdempotencyKey: "weather.hourly.brentwood.2026-06-07T15", IdempotencyKey: "weather.hourly.brentwood.20260607T150000Z",
}) })
if err != nil { if err != nil {
return err return err
@@ -70,7 +72,7 @@ if err != nil {
_ = result.RunID _ = result.RunID
``` ```
`UploadBundle` validates the local bundle by default and uploads only `manifest.json` plus manifest-listed files. Unlisted files are not uploaded. `PipelineID` is required for existing bundles too. `UploadBundle` validates the local bundle by default and uploads only `manifest.json` plus manifest-listed files. Unlisted files are not uploaded.
## Result And Status ## Result And Status
@@ -94,7 +96,9 @@ Status values are `accepted`, `queued`, `running`, `succeeded`, and `failed`. Co
Every upload request includes `Idempotency-Key`. Every upload request includes `Idempotency-Key`.
If `IdempotencyKey` is omitted, the client generates a random 128-bit lowercase hexadecimal key for that upload operation and reuses it for retries within the same call. For cross-process retry safety, producers should pass a stable key derived from the producer job or report id. If `IdempotencyKey` is omitted, the client generates a random 128-bit lowercase hexadecimal key for that upload operation and reuses it for retries within the same call. For cross-process retry safety, producers should pass a key derived from the producer run, such as `<bundle-id>.<run-id>`.
Do not reuse the same idempotency key for multiple distinct report generations. Reuse it only when retrying the exact same run with the same token, pipeline id, and source manifest. A repeated key with the same manifest in that scope returns the original accepted run instead of enqueueing another run; a repeated key with different content returns an idempotency conflict.
The client retries only safe cases: The client retries only safe cases:
@@ -102,7 +106,7 @@ The client retries only safe cases:
- temporary network errors; - temporary network errors;
- ambiguous mid-upload failures. - ambiguous mid-upload failures.
It does not retry after `202 Accepted` and does not retry `400`, `401`, `409`, `413`, or `415`. It does not retry after `202 Accepted` and does not retry `400`, `401`, `403`, `404`, `409`, `413`, or `415`.
Detect conflicting key reuse with `errors.As`: Detect conflicting key reuse with `errors.As`:

View File

@@ -90,8 +90,8 @@ generation returns an error after writing output, the managed report and
metadata remain inspectable. Notification is not attempted after Weather API, metadata remain inspectable. Notification is not attempted after Weather API,
briefing, prompt input, render, Scriptorium run, or metadata-save failures. briefing, prompt input, render, Scriptorium run, or metadata-save failures.
When notification is attempted, the debug artifact records request identity, When notification is attempted, the debug artifact records request identity,
accepted upload fields, distributor status fields, raw status report JSON when including rendered pipeline ID, accepted upload fields, distributor status
available, and redacted failure context. fields, raw status report JSON when available, and redacted failure context.
`--out` copies are never used as notification source files. `--out` copies are never used as notification source files.
## Batch Workflow ## Batch Workflow

View File

@@ -18,6 +18,7 @@ Inputs:
- distributor endpoint URL - distributor endpoint URL
- token environment variable name - token environment variable name
- upload timeout - upload timeout
- pipeline ID
- bundle ID - bundle ID
- idempotency key - idempotency key
- source Markdown report path - source Markdown report path
@@ -54,8 +55,10 @@ The adapter is built from `notify.distributor` config:
- `token_env` - `token_env`
- `timeout` - `timeout`
The app layer renders bundle ID, idempotency key, and bundle path from: The app layer renders pipeline ID, bundle ID, idempotency key, and bundle path
from:
- `pipeline_id_template`
- `bundle_id_template` - `bundle_id_template`
- `idempotency_key_template` - `idempotency_key_template`
- `report_path_template` - `report_path_template`
@@ -67,6 +70,7 @@ after config loading and `secrets.directory` processing.
The adapter calls distributor `UploadFiles` with exactly one file: The adapter calls distributor `UploadFiles` with exactly one file:
- pipeline ID: the rendered distributor workflow selector
- source path: the managed Markdown report path selected by app orchestration - source path: the managed Markdown report path selected by app orchestration
- bundle path: the rendered bundle-relative report path - bundle path: the rendered bundle-relative report path
- created: the report generation timestamp - created: the report generation timestamp
@@ -86,12 +90,13 @@ with the status report preserved.
## Failure Behavior ## Failure Behavior
The adapter validates required endpoint, token env name, token value, bundle ID, The adapter validates required endpoint, token env name, token value, pipeline
idempotency key, source path, bundle path, and upload client inputs before ID, bundle ID, idempotency key, source path, bundle path, and upload client
uploading. inputs before uploading.
Upload failures include endpoint, bundle ID, idempotency key, source path, and Upload failures include endpoint, pipeline ID, bundle ID, idempotency key,
bundle path context. Token values are redacted from adapter errors. source path, and bundle path context. Token values are redacted from adapter
errors.
Distributor idempotency conflicts are exposed as a weatherreporter-owned Distributor idempotency conflicts are exposed as a weatherreporter-owned
`IdempotencyConflictError`, so callers do not depend on upstream distributor `IdempotencyConflictError`, so callers do not depend on upstream distributor

View File

@@ -93,7 +93,8 @@ Durable JSON writes use shared atomic file helpers. Managed Markdown reports are
prepared by creating their parent directory; Scriptorium writes the report body prepared by creating their parent directory; Scriptorium writes the report body
to the prepared path. Extra Markdown copies are handled by app orchestration. to the prepared path. Extra Markdown copies are handled by app orchestration.
Distributor notification debug artifacts are written atomically when Distributor notification debug artifacts are written atomically when
notification is attempted. notification is attempted and include rendered distributor pipeline ID, bundle
ID, idempotency key, upload status, latest run status, and redacted errors.
Inspection helpers read existing metadata, briefing, and data package files. Inspection helpers read existing metadata, briefing, and data package files.
Missing metadata directories return no inspection records or no prior snapshot Missing metadata directories return no inspection records or no prior snapshot

View File

@@ -150,16 +150,18 @@ successfully generated report: the managed Markdown report path recorded in the
report result and metadata. Extra copies written by `--out` or `--out-dir` are report result and metadata. Extra copies written by `--out` or `--out-dir` are
operator conveniences only. operator conveniences only.
The default bundle ID is derived from producer name, location ID, report ID, and The rendered pipeline ID selects the configured distributor `http_upload`
RunID: workflow. The default bundle ID is a stable logical source identity derived from
producer name, location ID, and report ID:
```text ```text
weatherreporter.{location_id}.{report_id}.{run_id} weatherreporter.{location_id}.{report_id}
``` ```
The default idempotency key is the rendered bundle ID. The default bundle path The default idempotency key appends RunID to the rendered bundle ID so each
for the Markdown file is the report definition's batch output name, such as report generation has a distinct retry identity. The default bundle path for the
`daily.md`, `tomorrow.md`, `three-day.md`, or `weekend.md`. Markdown file is the report definition's batch output name, such as `daily.md`,
`tomorrow.md`, `three-day.md`, or `weekend.md`.
Notification happens after final metadata save. Weather API, briefing, Notification happens after final metadata save. Weather API, briefing,
data-package, render preflight, Scriptorium run, and metadata-save failures do data-package, render preflight, Scriptorium run, and metadata-save failures do
@@ -168,14 +170,14 @@ other reports continue, the failed report includes notification fields in the
JSON summary, and the batch returns nonzero. JSON summary, and the batch returns nonzero.
Each notification attempt writes a debug artifact under `notifications/`. The Each notification attempt writes a debug artifact under `notifications/`. The
artifact records the rendered bundle ID, idempotency key, managed source path, artifact records the rendered pipeline ID, bundle ID, idempotency key, managed
bundle-relative path, bundle created timestamp, accepted upload response, and source path, bundle-relative path, bundle created timestamp, accepted upload
the latest distributor run status response when available. Weatherreporter polls response, and the latest distributor run status response when available.
status until distributor reports `succeeded` or `failed`, or until the configured Weatherreporter polls status until distributor reports `succeeded` or `failed`,
notification timeout expires. The run status includes the distributor status, or until the configured notification timeout expires. The run status includes
error text, and raw run report JSON, which can show actions such as the distributor status, error text, and raw run report JSON, which can show
`replace_older`, `skip_same`, `skip_destination_newer`, or `failed`. Token values actions such as `replace_older`, `skip_same`, `skip_destination_newer`, or
are not written. `failed`. Token values are not written.
Weatherreporter is responsible for selecting the managed Markdown report, Weatherreporter is responsible for selecting the managed Markdown report,
constructing a source bundle, and submitting it to the configured distributor constructing a source bundle, and submitting it to the configured distributor

View File

@@ -240,17 +240,17 @@ Relevant docs: [Configuration reference](config.md),
Symptom: notification fails with idempotency conflict context. Symptom: notification fails with idempotency conflict context.
Likely cause: the same idempotency key was reused for different bundle content. Likely cause: the same idempotency key was reused for different bundle content
By default the key derives from bundle ID, which includes location ID, report within the same distributor token and pipeline. By default the bundle ID is a
ID, and RunID. stable report-stream identity and the idempotency key appends RunID.
Diagnostic: inspect the failed batch JSON or stderr line for bundle and Diagnostic: inspect the failed batch JSON or stderr line for pipeline, bundle,
idempotency context. Compare the configured templates with the report RunID and and idempotency context. Compare the configured templates with the report RunID
report path. and report path.
Also inspect the notification artifact linked from metadata. It records the Also inspect the notification artifact linked from metadata. It records the
rendered bundle ID, idempotency key, upload result, distributor run status, rendered pipeline ID, bundle ID, idempotency key, upload result, distributor run
status error, and raw run report JSON when available. status, status error, and raw run report JSON when available.
Safe fix: keep idempotency templates stable for retries of the same generated Safe fix: keep idempotency templates stable for retries of the same generated
report, but do not reuse the same rendered key for different generated report report, but do not reuse the same rendered key for different generated report
@@ -264,11 +264,12 @@ Relevant docs: [Operations guide](operations.md),
Symptom: notification fails with distributor upload rejection, HTTP status, or Symptom: notification fails with distributor upload rejection, HTTP status, or
bundle validation context. bundle validation context.
Likely cause: the distributor endpoint rejected the token, bundle ID, Likely cause: the distributor endpoint rejected the token, pipeline ID, bundle
idempotency key, source file, or bundle path. ID, idempotency key, source file, or bundle path.
Diagnostic: inspect stdout JSON or stderr status lines for Diagnostic: inspect stdout JSON or stderr status lines for
`notificationError`. Confirm `notify.distributor.endpoint`, `notificationError`. Confirm `notify.distributor.endpoint`,
`notify.distributor.pipeline_id_template`,
`notify.distributor.report_path_template`, and token configuration. Token values `notify.distributor.report_path_template`, and token configuration. Token values
are redacted from weatherreporter errors. are redacted from weatherreporter errors.

View File

@@ -21,6 +21,7 @@ notify:
token_env: DISTRIBUTOR_UPLOAD_TOKEN token_env: DISTRIBUTOR_UPLOAD_TOKEN
timeout: 30s timeout: 30s
failure_policy: error failure_policy: error
pipeline_id_template: "weatherreporter.{artifact_group}"
bundle_id_template: "weatherreporter.{location_id}.{report_id}" bundle_id_template: "weatherreporter.{location_id}.{report_id}"
idempotency_key_template: "{bundle_id}.{run_id}" idempotency_key_template: "{bundle_id}.{run_id}"
report_path_template: "{batch_output_name}" report_path_template: "{batch_output_name}"

2
go.mod
View File

@@ -4,4 +4,4 @@ go 1.26
require gopkg.in/yaml.v3 v3.0.1 require gopkg.in/yaml.v3 v3.0.1
require gitea.maximumdirect.net/eric/distributor v0.4.0 require gitea.maximumdirect.net/eric/distributor v0.5.0

4
go.sum
View File

@@ -1,5 +1,5 @@
gitea.maximumdirect.net/eric/distributor v0.4.0 h1:SRrTFjVLMv4wFZmMLwnaYtJaQlQ/wsB4OzcaSEEb524= gitea.maximumdirect.net/eric/distributor v0.5.0 h1:+al7Bw+kMv6V35a3Sm5rUtCTQhwOn5b9x3RsclPMKJk=
gitea.maximumdirect.net/eric/distributor v0.4.0/go.mod h1:G03FCFZPHpsUKC6SeMgTdbfNRpPQBdyTtDUj04e1Tu8= gitea.maximumdirect.net/eric/distributor v0.5.0/go.mod h1:G03FCFZPHpsUKC6SeMgTdbfNRpPQBdyTtDUj04e1Tu8=
github.com/aws/aws-sdk-go-v2 v1.41.9 h1:/rYeyO2+HrMztAmxAq9++XJtFMqSIpSsNA0yDGALYq4= github.com/aws/aws-sdk-go-v2 v1.41.9 h1:/rYeyO2+HrMztAmxAq9++XJtFMqSIpSsNA0yDGALYq4=
github.com/aws/aws-sdk-go-v2 v1.41.9/go.mod h1:+HsoOEX80qAVUitj1A2DhCNTjmb3edVyuDypb6LNEeo= github.com/aws/aws-sdk-go-v2 v1.41.9/go.mod h1:+HsoOEX80qAVUitj1A2DhCNTjmb3edVyuDypb6LNEeo=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.11 h1:h5+3VT69KUBK24grGuuA5saDJTj2IIjLb9au668Fo5I= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.11 h1:h5+3VT69KUBK24grGuuA5saDJTj2IIjLb9au668Fo5I=

View File

@@ -25,6 +25,7 @@ type Client struct {
} }
type UploadRequest struct { type UploadRequest struct {
PipelineID string
BundleID string BundleID string
IdempotencyKey string IdempotencyKey string
SourcePath string SourcePath string
@@ -77,6 +78,7 @@ type uploadClient interface {
} }
type uploadFilesOptions struct { type uploadFilesOptions struct {
PipelineID string
BundleID string BundleID string
IdempotencyKey string IdempotencyKey string
SourcePath string SourcePath string
@@ -128,6 +130,9 @@ func (c *Client) Upload(ctx context.Context, req UploadRequest) (UploadResult, e
if c.TokenEnv == "" { if c.TokenEnv == "" {
return UploadResult{}, fmt.Errorf("distributor token environment variable is required") return UploadResult{}, fmt.Errorf("distributor token environment variable is required")
} }
if req.PipelineID == "" {
return UploadResult{}, fmt.Errorf("distributor pipeline id is required")
}
if req.BundleID == "" { if req.BundleID == "" {
return UploadResult{}, fmt.Errorf("distributor bundle id is required") return UploadResult{}, fmt.Errorf("distributor bundle id is required")
} }
@@ -165,6 +170,7 @@ func (c *Client) Upload(ctx context.Context, req UploadRequest) (UploadResult, e
defer cancel() defer cancel()
result, err := uploadClient.UploadFiles(runCtx, uploadFilesOptions{ result, err := uploadClient.UploadFiles(runCtx, uploadFilesOptions{
PipelineID: req.PipelineID,
BundleID: req.BundleID, BundleID: req.BundleID,
IdempotencyKey: req.IdempotencyKey, IdempotencyKey: req.IdempotencyKey,
SourcePath: req.SourcePath, SourcePath: req.SourcePath,
@@ -174,6 +180,7 @@ func (c *Client) Upload(ctx context.Context, req UploadRequest) (UploadResult, e
if err != nil { if err != nil {
return UploadResult{}, wrapUploadError(err, uploadErrorContext{ return UploadResult{}, wrapUploadError(err, uploadErrorContext{
Endpoint: c.Endpoint, Endpoint: c.Endpoint,
PipelineID: req.PipelineID,
BundleID: req.BundleID, BundleID: req.BundleID,
IdempotencyKey: req.IdempotencyKey, IdempotencyKey: req.IdempotencyKey,
SourcePath: req.SourcePath, SourcePath: req.SourcePath,
@@ -265,6 +272,7 @@ func newDistributorUploadClient(endpoint, token string, timeout time.Duration) (
func (c distributorUploadClient) UploadFiles(ctx context.Context, opts uploadFilesOptions) (uploadFilesResult, error) { func (c distributorUploadClient) UploadFiles(ctx context.Context, opts uploadFilesOptions) (uploadFilesResult, error) {
result, err := c.client.UploadFiles(ctx, distributorupload.UploadFilesOptions{ result, err := c.client.UploadFiles(ctx, distributorupload.UploadFilesOptions{
PipelineID: opts.PipelineID,
ID: opts.BundleID, ID: opts.BundleID,
Created: opts.CreatedAt, Created: opts.CreatedAt,
IdempotencyKey: opts.IdempotencyKey, IdempotencyKey: opts.IdempotencyKey,
@@ -300,6 +308,7 @@ func (c distributorUploadClient) Status(ctx context.Context, runID string) (runS
type uploadErrorContext struct { type uploadErrorContext struct {
Endpoint string Endpoint string
PipelineID string
BundleID string BundleID string
IdempotencyKey string IdempotencyKey string
SourcePath string SourcePath string
@@ -313,10 +322,10 @@ func wrapUploadError(err error, ctx uploadErrorContext) error {
err = redactToken(err, ctx.Token) err = redactToken(err, ctx.Token)
if isConflict { if isConflict {
return &IdempotencyConflictError{ return &IdempotencyConflictError{
Err: fmt.Errorf("upload distributor bundle %q to endpoint %q with idempotency key %q from source %q as bundle path %q: idempotency conflict: %w", ctx.BundleID, ctx.Endpoint, ctx.IdempotencyKey, ctx.SourcePath, ctx.BundlePath, err), Err: fmt.Errorf("upload distributor bundle %q to pipeline %q at endpoint %q with idempotency key %q from source %q as bundle path %q: idempotency conflict: %w", ctx.BundleID, ctx.PipelineID, ctx.Endpoint, ctx.IdempotencyKey, ctx.SourcePath, ctx.BundlePath, err),
} }
} }
return fmt.Errorf("upload distributor bundle %q to endpoint %q with idempotency key %q from source %q as bundle path %q: %w", ctx.BundleID, ctx.Endpoint, ctx.IdempotencyKey, ctx.SourcePath, ctx.BundlePath, err) return fmt.Errorf("upload distributor bundle %q to pipeline %q at endpoint %q with idempotency key %q from source %q as bundle path %q: %w", ctx.BundleID, ctx.PipelineID, ctx.Endpoint, ctx.IdempotencyKey, ctx.SourcePath, ctx.BundlePath, err)
} }
func redactToken(err error, token string) error { func redactToken(err error, token string) error {

View File

@@ -30,6 +30,7 @@ func TestUploadUsesConfiguredClientAndSingleFile(t *testing.T) {
client := newClient(cfg, factory.newClient) client := newClient(cfg, factory.newClient)
result, err := client.Upload(context.Background(), UploadRequest{ result, err := client.Upload(context.Background(), UploadRequest{
PipelineID: "weatherreporter.daily",
BundleID: "weatherreporter.home.daily.run", BundleID: "weatherreporter.home.daily.run",
IdempotencyKey: "weatherreporter.home.daily.run", IdempotencyKey: "weatherreporter.home.daily.run",
SourcePath: "/tmp/report.md", SourcePath: "/tmp/report.md",
@@ -55,6 +56,9 @@ func TestUploadUsesConfiguredClientAndSingleFile(t *testing.T) {
t.Fatalf("factory timeout = %s, want 15s", factory.timeout) t.Fatalf("factory timeout = %s, want 15s", factory.timeout)
} }
got := factory.client.opts got := factory.client.opts
if got.PipelineID != "weatherreporter.daily" {
t.Fatalf("PipelineID = %q, want weatherreporter.daily", got.PipelineID)
}
if got.BundleID != "weatherreporter.home.daily.run" { if got.BundleID != "weatherreporter.home.daily.run" {
t.Fatalf("BundleID = %q, want weatherreporter.home.daily.run", got.BundleID) t.Fatalf("BundleID = %q, want weatherreporter.home.daily.run", got.BundleID)
} }
@@ -91,6 +95,13 @@ func TestUploadRejectsMissingInputs(t *testing.T) {
}, },
wantErr: "token environment variable", wantErr: "token environment variable",
}, },
{
name: "PipelineID",
mutate: func(c *Client, req *UploadRequest) {
req.PipelineID = ""
},
wantErr: "pipeline id is required",
},
{ {
name: "SourcePath", name: "SourcePath",
mutate: func(c *Client, req *UploadRequest) { mutate: func(c *Client, req *UploadRequest) {
@@ -170,7 +181,7 @@ func TestUploadWrapsUploadFailureWithContextWithoutToken(t *testing.T) {
if err == nil { if err == nil {
t.Fatal("Upload() error = nil, want error") t.Fatal("Upload() error = nil, want error")
} }
for _, want := range []string{cfg.Endpoint, req.BundleID, req.IdempotencyKey, req.SourcePath, req.BundlePath} { for _, want := range []string{cfg.Endpoint, req.PipelineID, req.BundleID, req.IdempotencyKey, req.SourcePath, req.BundlePath} {
if !strings.Contains(err.Error(), want) { if !strings.Contains(err.Error(), want) {
t.Fatalf("error = %q, want context %q", err.Error(), want) t.Fatalf("error = %q, want context %q", err.Error(), want)
} }
@@ -318,6 +329,7 @@ func TestUploadPreservesIdempotencyConflictDiagnosis(t *testing.T) {
func validUploadRequest() UploadRequest { func validUploadRequest() UploadRequest {
return UploadRequest{ return UploadRequest{
PipelineID: "weatherreporter.daily",
BundleID: "weatherreporter.home.daily.run", BundleID: "weatherreporter.home.daily.run",
IdempotencyKey: "weatherreporter.home.daily.run", IdempotencyKey: "weatherreporter.home.daily.run",
SourcePath: "/tmp/report.md", SourcePath: "/tmp/report.md",

View File

@@ -114,24 +114,25 @@ type BatchResult struct {
} }
type BatchReportResult struct { type BatchReportResult struct {
ReportID report.ID `json:"reportId"` ReportID report.ID `json:"reportId"`
ReportName string `json:"reportName"` ReportName string `json:"reportName"`
PromptID string `json:"promptId"` PromptID string `json:"promptId"`
RunID string `json:"runId"` RunID string `json:"runId"`
Status string `json:"status"` Status string `json:"status"`
Error string `json:"error,omitempty"` Error string `json:"error,omitempty"`
NotificationStatus string `json:"notificationStatus,omitempty"` NotificationStatus string `json:"notificationStatus,omitempty"`
NotificationRunID string `json:"notificationRunId,omitempty"` NotificationRunID string `json:"notificationRunId,omitempty"`
NotificationError string `json:"notificationError,omitempty"` NotificationPipelineID string `json:"notificationPipelineId,omitempty"`
NotificationPath string `json:"notificationPath,omitempty"` NotificationError string `json:"notificationError,omitempty"`
GeneratedAt time.Time `json:"generatedAt"` NotificationPath string `json:"notificationPath,omitempty"`
ValidPeriod timeutil.Period `json:"validPeriod"` GeneratedAt time.Time `json:"generatedAt"`
BriefingPath string `json:"briefingPath,omitempty"` ValidPeriod timeutil.Period `json:"validPeriod"`
DataPackagePath string `json:"dataPackagePath,omitempty"` BriefingPath string `json:"briefingPath,omitempty"`
PreflightPath string `json:"preflightPath,omitempty"` DataPackagePath string `json:"dataPackagePath,omitempty"`
ReportPath string `json:"reportPath,omitempty"` PreflightPath string `json:"preflightPath,omitempty"`
OutputPath string `json:"outputPath,omitempty"` ReportPath string `json:"reportPath,omitempty"`
MetadataPath string `json:"metadataPath,omitempty"` OutputPath string `json:"outputPath,omitempty"`
MetadataPath string `json:"metadataPath,omitempty"`
} }
type BatchError struct { type BatchError struct {
@@ -157,6 +158,7 @@ type Notifier interface {
type NotificationRequest struct { type NotificationRequest struct {
ReportID report.ID ReportID report.ID
RunID string RunID string
PipelineID string
BundleID string BundleID string
IdempotencyKey string IdempotencyKey string
ReportPath string ReportPath string
@@ -280,6 +282,7 @@ func RunBatchDetailed(ctx context.Context, req BatchRequest) (*BatchResult, erro
if errors.As(err, &notificationErr) { if errors.As(err, &notificationErr) {
item.NotificationStatus = "failed" item.NotificationStatus = "failed"
item.NotificationError = notificationErr.Error() item.NotificationError = notificationErr.Error()
item.NotificationPipelineID = notificationErr.Request.PipelineID
if paths, pathErr := store.Paths(resolved); pathErr == nil { if paths, pathErr := store.Paths(resolved); pathErr == nil {
item.NotificationPath = paths.Notification item.NotificationPath = paths.Notification
} }
@@ -297,6 +300,7 @@ func RunBatchDetailed(ctx context.Context, req BatchRequest) (*BatchResult, erro
if reportResult.Notification != nil { if reportResult.Notification != nil {
item.NotificationStatus = reportResult.Notification.Status item.NotificationStatus = reportResult.Notification.Status
item.NotificationRunID = reportResult.Notification.RunID item.NotificationRunID = reportResult.Notification.RunID
item.NotificationPipelineID = reportResult.Notification.PipelineID
} }
result.Succeeded++ result.Succeeded++
} }
@@ -638,6 +642,10 @@ func buildNotificationRequest(cfg config.Config, resolved report.Resolved, repor
return NotificationRequest{}, err return NotificationRequest{}, err
} }
values.BundleID = bundleID values.BundleID = bundleID
pipelineID, err := config.RenderDistributorPipelineID(cfg.Notify.Distributor.PipelineIDTemplate, values)
if err != nil {
return NotificationRequest{}, err
}
idempotencyKey, err := config.RenderDistributorIdempotencyKey(cfg.Notify.Distributor.IdempotencyKeyTemplate, values) idempotencyKey, err := config.RenderDistributorIdempotencyKey(cfg.Notify.Distributor.IdempotencyKeyTemplate, values)
if err != nil { if err != nil {
return NotificationRequest{}, err return NotificationRequest{}, err
@@ -649,6 +657,7 @@ func buildNotificationRequest(cfg config.Config, resolved report.Resolved, repor
return NotificationRequest{ return NotificationRequest{
ReportID: resolved.Definition.ID, ReportID: resolved.Definition.ID,
RunID: metadata.RunID, RunID: metadata.RunID,
PipelineID: pipelineID,
BundleID: bundleID, BundleID: bundleID,
IdempotencyKey: idempotencyKey, IdempotencyKey: idempotencyKey,
ReportPath: reportPath, ReportPath: reportPath,
@@ -667,6 +676,7 @@ func saveNotificationArtifact(ctx context.Context, store state.Store, resolved r
ReportID: resolved.Definition.ID, ReportID: resolved.Definition.ID,
AttemptedAt: time.Now(), AttemptedAt: time.Now(),
Endpoint: cfg.Notify.Distributor.Endpoint, Endpoint: cfg.Notify.Distributor.Endpoint,
PipelineID: req.PipelineID,
BundleID: req.BundleID, BundleID: req.BundleID,
IdempotencyKey: req.IdempotencyKey, IdempotencyKey: req.IdempotencyKey,
SourcePath: req.ReportPath, SourcePath: req.ReportPath,
@@ -716,6 +726,7 @@ type distributorNotifier struct {
func (n distributorNotifier) Notify(ctx context.Context, req NotificationRequest) (*NotificationResult, error) { func (n distributorNotifier) Notify(ctx context.Context, req NotificationRequest) (*NotificationResult, error) {
result, err := n.client.Upload(ctx, distributoradapter.UploadRequest{ result, err := n.client.Upload(ctx, distributoradapter.UploadRequest{
PipelineID: req.PipelineID,
BundleID: req.BundleID, BundleID: req.BundleID,
IdempotencyKey: req.IdempotencyKey, IdempotencyKey: req.IdempotencyKey,
SourcePath: req.ReportPath, SourcePath: req.ReportPath,
@@ -723,6 +734,7 @@ func (n distributorNotifier) Notify(ctx context.Context, req NotificationRequest
CreatedAt: req.CreatedAt, CreatedAt: req.CreatedAt,
}) })
notification := &NotificationResult{ notification := &NotificationResult{
PipelineID: req.PipelineID,
BundleID: req.BundleID, BundleID: req.BundleID,
IdempotencyKey: req.IdempotencyKey, IdempotencyKey: req.IdempotencyKey,
RunID: result.RunID, RunID: result.RunID,
@@ -731,7 +743,9 @@ func (n distributorNotifier) Notify(ctx context.Context, req NotificationRequest
StatusError: result.StatusError, StatusError: result.StatusError,
} }
if result.RunStatus != nil { if result.RunStatus != nil {
notification.PipelineID = result.RunStatus.PipelineID if result.RunStatus.PipelineID != "" {
notification.PipelineID = result.RunStatus.PipelineID
}
notification.AcceptedAt = result.RunStatus.AcceptedAt notification.AcceptedAt = result.RunStatus.AcceptedAt
notification.StartedAt = result.RunStatus.StartedAt notification.StartedAt = result.RunStatus.StartedAt
notification.FinishedAt = result.RunStatus.FinishedAt notification.FinishedAt = result.RunStatus.FinishedAt

View File

@@ -288,6 +288,7 @@ func TestGenerateReportNotifiesManagedReportPath(t *testing.T) {
cfg := dailyTestConfig(t, server) cfg := dailyTestConfig(t, server)
cfg.Workspace.Root = t.TempDir() cfg.Workspace.Root = t.TempDir()
cfg.Notify.Distributor.Enabled = true cfg.Notify.Distributor.Enabled = true
cfg.Notify.Distributor.PipelineIDTemplate = "weatherreporter.{artifact_group}"
resolved, err := ResolveGenerate(GenerateRequest{ resolved, err := ResolveGenerate(GenerateRequest{
Config: cfg, Config: cfg,
Report: ReportDaily, Report: ReportDaily,
@@ -330,8 +331,12 @@ func TestGenerateReportNotifiesManagedReportPath(t *testing.T) {
if err != nil { if err != nil {
t.Fatalf("read notification artifact: %v", err) t.Fatalf("read notification artifact: %v", err)
} }
if !strings.Contains(string(notificationData), `"replace_older"`) || !strings.Contains(string(notificationData), `"bundleCreated"`) { var notificationArtifact state.DistributorNotificationArtifact
t.Fatalf("notification artifact missing status report or created timestamp:\n%s", string(notificationData)) if err := json.Unmarshal(notificationData, &notificationArtifact); err != nil {
t.Fatalf("decode notification artifact: %v", err)
}
if notificationArtifact.PipelineID != "weatherreporter.daily" || notificationArtifact.BundleCreated.IsZero() || notificationArtifact.RunStatus == nil || !strings.Contains(string(notificationArtifact.RunStatus.Report), "replace_older") {
t.Fatalf("notification artifact = %#v, want requested pipeline, status report, and created timestamp", notificationArtifact)
} }
if len(notifier.requests) != 1 { if len(notifier.requests) != 1 {
t.Fatalf("notification requests = %d, want 1", len(notifier.requests)) t.Fatalf("notification requests = %d, want 1", len(notifier.requests))
@@ -346,11 +351,14 @@ func TestGenerateReportNotifiesManagedReportPath(t *testing.T) {
if req.BundlePath != "daily.md" { if req.BundlePath != "daily.md" {
t.Fatalf("notification BundlePath = %q, want daily.md", req.BundlePath) t.Fatalf("notification BundlePath = %q, want daily.md", req.BundlePath)
} }
if req.BundleID != "weatherreporter.home.daily_today."+result.Metadata.RunID { if req.PipelineID != "weatherreporter.daily" {
t.Fatalf("notification PipelineID = %q, want rendered pipeline", req.PipelineID)
}
if req.BundleID != "weatherreporter.home.daily_today" {
t.Fatalf("notification BundleID = %q, want default template", req.BundleID) t.Fatalf("notification BundleID = %q, want default template", req.BundleID)
} }
if req.IdempotencyKey != req.BundleID { if req.IdempotencyKey != req.BundleID+"."+result.Metadata.RunID {
t.Fatalf("IdempotencyKey = %q, want bundle id %q", req.IdempotencyKey, req.BundleID) t.Fatalf("IdempotencyKey = %q, want per-run key", req.IdempotencyKey)
} }
if req.RunID != result.Metadata.RunID { if req.RunID != result.Metadata.RunID {
t.Fatalf("notification RunID = %q, want report run id %q", req.RunID, result.Metadata.RunID) t.Fatalf("notification RunID = %q, want report run id %q", req.RunID, result.Metadata.RunID)
@@ -365,6 +373,7 @@ func TestGenerateReportNotificationFailureFailsReport(t *testing.T) {
cfg := dailyTestConfig(t, server) cfg := dailyTestConfig(t, server)
cfg.Workspace.Root = t.TempDir() cfg.Workspace.Root = t.TempDir()
cfg.Notify.Distributor.Enabled = true cfg.Notify.Distributor.Enabled = true
cfg.Notify.Distributor.PipelineIDTemplate = "weatherreporter.{artifact_group}"
resolved, err := ResolveGenerate(GenerateRequest{ resolved, err := ResolveGenerate(GenerateRequest{
Config: cfg, Config: cfg,
Report: ReportDaily, Report: ReportDaily,
@@ -441,6 +450,7 @@ func TestGenerateReportDoesNotNotifyAfterRenderOrRunFailure(t *testing.T) {
cfg := dailyTestConfig(t, server) cfg := dailyTestConfig(t, server)
cfg.Workspace.Root = t.TempDir() cfg.Workspace.Root = t.TempDir()
cfg.Notify.Distributor.Enabled = true cfg.Notify.Distributor.Enabled = true
cfg.Notify.Distributor.PipelineIDTemplate = "weatherreporter.{artifact_group}"
resolved, err := ResolveGenerate(GenerateRequest{ resolved, err := ResolveGenerate(GenerateRequest{
Config: cfg, Config: cfg,
Report: ReportDaily, Report: ReportDaily,
@@ -473,6 +483,7 @@ func TestGenerateReportDoesNotNotifyAfterFetchFailure(t *testing.T) {
cfg.WeatherAPI.Timezone = "America/Chicago" cfg.WeatherAPI.Timezone = "America/Chicago"
cfg.Workspace.Root = t.TempDir() cfg.Workspace.Root = t.TempDir()
cfg.Notify.Distributor.Enabled = true cfg.Notify.Distributor.Enabled = true
cfg.Notify.Distributor.PipelineIDTemplate = "weatherreporter.{artifact_group}"
resolved, err := ResolveGenerate(GenerateRequest{ resolved, err := ResolveGenerate(GenerateRequest{
Config: cfg, Config: cfg,
Report: ReportDaily, Report: ReportDaily,
@@ -1257,6 +1268,7 @@ func TestRunBatchContinuesAfterNotificationFailure(t *testing.T) {
cfg.WeatherAPI.Timezone = "America/Chicago" cfg.WeatherAPI.Timezone = "America/Chicago"
cfg.Workspace.Root = t.TempDir() cfg.Workspace.Root = t.TempDir()
cfg.Notify.Distributor.Enabled = true cfg.Notify.Distributor.Enabled = true
cfg.Notify.Distributor.PipelineIDTemplate = "weatherreporter.{artifact_group}"
notifier := &recordingNotifier{ notifier := &recordingNotifier{
errByReport: map[report.ID]error{ errByReport: map[report.ID]error{
report.ThreeDay: errors.New("distributor unavailable"), report.ThreeDay: errors.New("distributor unavailable"),
@@ -1289,6 +1301,9 @@ func TestRunBatchContinuesAfterNotificationFailure(t *testing.T) {
if item.NotificationStatus != "failed" { if item.NotificationStatus != "failed" {
t.Fatalf("3-day notification status = %q, want failed", item.NotificationStatus) t.Fatalf("3-day notification status = %q, want failed", item.NotificationStatus)
} }
if item.NotificationPipelineID != "weatherreporter.three-day" {
t.Fatalf("3-day notification pipeline = %q, want weatherreporter.three-day", item.NotificationPipelineID)
}
if !strings.Contains(item.NotificationError, "distributor unavailable") { if !strings.Contains(item.NotificationError, "distributor unavailable") {
t.Fatalf("3-day notification error = %q, want distributor unavailable", item.NotificationError) t.Fatalf("3-day notification error = %q, want distributor unavailable", item.NotificationError)
} }
@@ -1300,6 +1315,9 @@ func TestRunBatchContinuesAfterNotificationFailure(t *testing.T) {
if item.NotificationStatus != "accepted" { if item.NotificationStatus != "accepted" {
t.Fatalf("report %s notification status = %q, want accepted", item.ReportID, item.NotificationStatus) t.Fatalf("report %s notification status = %q, want accepted", item.ReportID, item.NotificationStatus)
} }
if item.NotificationPipelineID == "" {
t.Fatalf("report %s notification pipeline is empty", item.ReportID)
}
} }
if !failedThreeDay { if !failedThreeDay {
t.Fatalf("reports = %#v, want notification failure on 3-day item", result.Reports) t.Fatalf("reports = %#v, want notification failure on 3-day item", result.Reports)
@@ -1500,9 +1518,13 @@ func (n *recordingNotifier) Notify(_ context.Context, req NotificationRequest) (
if result.IdempotencyKey == "" { if result.IdempotencyKey == "" {
result.IdempotencyKey = req.IdempotencyKey result.IdempotencyKey = req.IdempotencyKey
} }
if result.PipelineID == "" {
result.PipelineID = req.PipelineID
}
return &result, nil return &result, nil
} }
return &NotificationResult{ return &NotificationResult{
PipelineID: req.PipelineID,
BundleID: req.BundleID, BundleID: req.BundleID,
IdempotencyKey: req.IdempotencyKey, IdempotencyKey: req.IdempotencyKey,
Status: "accepted", Status: "accepted",

View File

@@ -469,7 +469,12 @@ func TestRunEveningUsesOutputDirectoryAndSummary(t *testing.T) {
func TestRunEveningReportsNotificationSuccess(t *testing.T) { func TestRunEveningReportsNotificationSuccess(t *testing.T) {
server := dailyServer(t) server := dailyServer(t)
distributorServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { distributorServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/upload" { if r.URL.Path == "/runs/distributor-run-1" {
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{"run_id":"distributor-run-1","pipeline_id":"weatherreporter.daily","status":"succeeded","report":{"actions":[{"action":"replace_older"}]}}`))
return
}
if r.URL.Path != "/v1/pipelines/weatherreporter.daily/upload" {
http.NotFound(w, r) http.NotFound(w, r)
return return
} }
@@ -481,7 +486,7 @@ func TestRunEveningReportsNotificationSuccess(t *testing.T) {
scriptoriumPath := writeFakeScriptorium(t, tempDir) scriptoriumPath := writeFakeScriptorium(t, tempDir)
configPath := filepath.Join(tempDir, "config.yml") configPath := filepath.Join(tempDir, "config.yml")
workspaceRoot := filepath.Join(tempDir, "workspace") workspaceRoot := filepath.Join(tempDir, "workspace")
configBody := "weather_api:\n base_url: " + server.URL + "/\n timezone: America/Chicago\nscriptorium:\n binary: " + scriptoriumPath + "\nworkspace:\n root: " + workspaceRoot + "\nnotify:\n distributor:\n enabled: true\n endpoint: " + distributorServer.URL + "\n token_env: CLI_DISTRIBUTOR_TOKEN\n" configBody := "weather_api:\n base_url: " + server.URL + "/\n timezone: America/Chicago\nscriptorium:\n binary: " + scriptoriumPath + "\nworkspace:\n root: " + workspaceRoot + "\nnotify:\n distributor:\n enabled: true\n endpoint: " + distributorServer.URL + "\n token_env: CLI_DISTRIBUTOR_TOKEN\n pipeline_id_template: weatherreporter.{artifact_group}\n"
if err := os.WriteFile(configPath, []byte(configBody), 0o600); err != nil { if err := os.WriteFile(configPath, []byte(configBody), 0o600); err != nil {
t.Fatalf("write config: %v", err) t.Fatalf("write config: %v", err)
} }
@@ -505,10 +510,10 @@ func TestRunEveningReportsNotificationSuccess(t *testing.T) {
if len(summary.Reports) != 1 { if len(summary.Reports) != 1 {
t.Fatalf("reports = %#v, want one report", summary.Reports) t.Fatalf("reports = %#v, want one report", summary.Reports)
} }
if summary.Reports[0].NotificationStatus != "accepted" || summary.Reports[0].NotificationRunID != "distributor-run-1" { if summary.Reports[0].NotificationStatus != "succeeded" || summary.Reports[0].NotificationRunID != "distributor-run-1" || summary.Reports[0].NotificationPipelineID != "weatherreporter.daily" {
t.Fatalf("notification fields = %#v", summary.Reports[0]) t.Fatalf("notification fields = %#v", summary.Reports[0])
} }
if !strings.Contains(stderr.String(), `notificationStatus="accepted"`) || !strings.Contains(stderr.String(), `notificationRunId="distributor-run-1"`) { if !strings.Contains(stderr.String(), `notificationStatus="succeeded"`) || !strings.Contains(stderr.String(), `notificationRunId="distributor-run-1"`) {
t.Fatalf("stderr missing notification fields:\n%s", stderr.String()) t.Fatalf("stderr missing notification fields:\n%s", stderr.String())
} }
if strings.Contains(stdout.String(), "cli-secret-token") || strings.Contains(stderr.String(), "cli-secret-token") { if strings.Contains(stdout.String(), "cli-secret-token") || strings.Contains(stderr.String(), "cli-secret-token") {
@@ -527,7 +532,7 @@ func TestRunEveningReportsNotificationFailureWithoutToken(t *testing.T) {
scriptoriumPath := writeFakeScriptorium(t, tempDir) scriptoriumPath := writeFakeScriptorium(t, tempDir)
configPath := filepath.Join(tempDir, "config.yml") configPath := filepath.Join(tempDir, "config.yml")
workspaceRoot := filepath.Join(tempDir, "workspace") workspaceRoot := filepath.Join(tempDir, "workspace")
configBody := "weather_api:\n base_url: " + server.URL + "/\n timezone: America/Chicago\nscriptorium:\n binary: " + scriptoriumPath + "\nworkspace:\n root: " + workspaceRoot + "\nnotify:\n distributor:\n enabled: true\n endpoint: " + distributorServer.URL + "\n token_env: CLI_DISTRIBUTOR_TOKEN\n" configBody := "weather_api:\n base_url: " + server.URL + "/\n timezone: America/Chicago\nscriptorium:\n binary: " + scriptoriumPath + "\nworkspace:\n root: " + workspaceRoot + "\nnotify:\n distributor:\n enabled: true\n endpoint: " + distributorServer.URL + "\n token_env: CLI_DISTRIBUTOR_TOKEN\n pipeline_id_template: weatherreporter.{artifact_group}\n"
if err := os.WriteFile(configPath, []byte(configBody), 0o600); err != nil { if err := os.WriteFile(configPath, []byte(configBody), 0o600); err != nil {
t.Fatalf("write config: %v", err) t.Fatalf("write config: %v", err)
} }

View File

@@ -56,6 +56,7 @@ type DistributorNotifyConfig struct {
TokenEnv string `yaml:"token_env"` TokenEnv string `yaml:"token_env"`
Timeout time.Duration `yaml:"timeout"` Timeout time.Duration `yaml:"timeout"`
FailurePolicy NotifyFailurePolicy `yaml:"failure_policy"` FailurePolicy NotifyFailurePolicy `yaml:"failure_policy"`
PipelineIDTemplate string `yaml:"pipeline_id_template"`
BundleIDTemplate string `yaml:"bundle_id_template"` BundleIDTemplate string `yaml:"bundle_id_template"`
IdempotencyKeyTemplate string `yaml:"idempotency_key_template"` IdempotencyKeyTemplate string `yaml:"idempotency_key_template"`
ReportPathTemplate string `yaml:"report_path_template"` ReportPathTemplate string `yaml:"report_path_template"`

View File

@@ -44,10 +44,13 @@ func TestDefaults(t *testing.T) {
if cfg.Notify.Distributor.FailurePolicy != NotifyFailureError { if cfg.Notify.Distributor.FailurePolicy != NotifyFailureError {
t.Fatalf("Notify.Distributor.FailurePolicy = %q, want error", cfg.Notify.Distributor.FailurePolicy) t.Fatalf("Notify.Distributor.FailurePolicy = %q, want error", cfg.Notify.Distributor.FailurePolicy)
} }
if cfg.Notify.Distributor.BundleIDTemplate != "weatherreporter.{location_id}.{report_id}.{run_id}" { if cfg.Notify.Distributor.PipelineIDTemplate != "" {
t.Fatalf("Notify.Distributor.PipelineIDTemplate = %q, want empty", cfg.Notify.Distributor.PipelineIDTemplate)
}
if cfg.Notify.Distributor.BundleIDTemplate != "weatherreporter.{location_id}.{report_id}" {
t.Fatalf("Notify.Distributor.BundleIDTemplate = %q, want default", cfg.Notify.Distributor.BundleIDTemplate) t.Fatalf("Notify.Distributor.BundleIDTemplate = %q, want default", cfg.Notify.Distributor.BundleIDTemplate)
} }
if cfg.Notify.Distributor.IdempotencyKeyTemplate != "{bundle_id}" { if cfg.Notify.Distributor.IdempotencyKeyTemplate != "{bundle_id}.{run_id}" {
t.Fatalf("Notify.Distributor.IdempotencyKeyTemplate = %q, want default", cfg.Notify.Distributor.IdempotencyKeyTemplate) t.Fatalf("Notify.Distributor.IdempotencyKeyTemplate = %q, want default", cfg.Notify.Distributor.IdempotencyKeyTemplate)
} }
if cfg.Notify.Distributor.ReportPathTemplate != "{batch_output_name}" { if cfg.Notify.Distributor.ReportPathTemplate != "{batch_output_name}" {
@@ -76,6 +79,9 @@ func TestLoadExampleConfig(t *testing.T) {
if cfg.Location.ID != "home" || cfg.Location.Name != "Brentwood" || cfg.Location.Region != "St. Louis Metro" { if cfg.Location.ID != "home" || cfg.Location.Name != "Brentwood" || cfg.Location.Region != "St. Louis Metro" {
t.Fatalf("Location = %#v, want example location", cfg.Location) t.Fatalf("Location = %#v, want example location", cfg.Location)
} }
if cfg.Notify.Distributor.PipelineIDTemplate != "weatherreporter.{artifact_group}" {
t.Fatalf("PipelineIDTemplate = %q, want example pipeline template", cfg.Notify.Distributor.PipelineIDTemplate)
}
} }
func TestLoadMinimalExampleConfig(t *testing.T) { func TestLoadMinimalExampleConfig(t *testing.T) {
@@ -200,6 +206,27 @@ func TestEnabledDistributorNotifyValidation(t *testing.T) {
}, },
wantErr: "notify.distributor.failure_policy", wantErr: "notify.distributor.failure_policy",
}, },
{
name: "PipelineTemplateEmpty",
mutate: func(cfg *Config) {
cfg.Notify.Distributor.PipelineIDTemplate = ""
},
wantErr: "notify.distributor.pipeline_id_template",
},
{
name: "PipelineTemplateUnknown",
mutate: func(cfg *Config) {
cfg.Notify.Distributor.PipelineIDTemplate = "{unknown}"
},
wantErr: "notify.distributor.pipeline_id_template",
},
{
name: "PipelineTemplateRenderedEmpty",
mutate: func(cfg *Config) {
cfg.Notify.Distributor.PipelineIDTemplate = " "
},
wantErr: "notify.distributor.pipeline_id_template",
},
{ {
name: "BundleTemplate", name: "BundleTemplate",
mutate: func(cfg *Config) { mutate: func(cfg *Config) {
@@ -234,6 +261,7 @@ func TestEnabledDistributorNotifyValidation(t *testing.T) {
t.Run(tt.name, func(t *testing.T) { t.Run(tt.name, func(t *testing.T) {
cfg := Defaults() cfg := Defaults()
cfg.Notify.Distributor.Enabled = true cfg.Notify.Distributor.Enabled = true
cfg.Notify.Distributor.PipelineIDTemplate = "weatherreporter.{artifact_group}"
tt.mutate(&cfg) tt.mutate(&cfg)
err := Validate(cfg) err := Validate(cfg)
@@ -254,23 +282,31 @@ func TestDistributorTemplateRendering(t *testing.T) {
RunID: "20260607T120000Z", RunID: "20260607T120000Z",
ArtifactGroup: "daily", ArtifactGroup: "daily",
BatchOutputName: "daily.md", BatchOutputName: "daily.md",
BundleID: "weatherreporter.home.daily.20260607T120000Z", BundleID: "weatherreporter.home.daily",
} }
bundleID, err := RenderDistributorBundleID("weatherreporter.{location_id}.{report_id}.{run_id}", values) bundleID, err := RenderDistributorBundleID("weatherreporter.{location_id}.{report_id}", values)
if err != nil { if err != nil {
t.Fatalf("RenderDistributorBundleID() error = %v", err) t.Fatalf("RenderDistributorBundleID() error = %v", err)
} }
if bundleID != "weatherreporter.home.daily.20260607T120000Z" { if bundleID != "weatherreporter.home.daily" {
t.Fatalf("bundleID = %q, want rendered value", bundleID) t.Fatalf("bundleID = %q, want rendered value", bundleID)
} }
idempotencyKey, err := RenderDistributorIdempotencyKey("{bundle_id}", values) pipelineID, err := RenderDistributorPipelineID("weatherreporter.{artifact_group}.{bundle_id}", values)
if err != nil {
t.Fatalf("RenderDistributorPipelineID() error = %v", err)
}
if pipelineID != "weatherreporter.daily.weatherreporter.home.daily" {
t.Fatalf("pipelineID = %q, want rendered pipeline ID", pipelineID)
}
idempotencyKey, err := RenderDistributorIdempotencyKey("{bundle_id}.{run_id}", values)
if err != nil { if err != nil {
t.Fatalf("RenderDistributorIdempotencyKey() error = %v", err) t.Fatalf("RenderDistributorIdempotencyKey() error = %v", err)
} }
if idempotencyKey != "weatherreporter.home.daily.20260607T120000Z" { if idempotencyKey != "weatherreporter.home.daily.20260607T120000Z" {
t.Fatalf("idempotencyKey = %q, want rendered bundle ID", idempotencyKey) t.Fatalf("idempotencyKey = %q, want rendered run key", idempotencyKey)
} }
reportPath, err := RenderDistributorReportPath("reports/{batch_output_name}", values) reportPath, err := RenderDistributorReportPath("reports/{batch_output_name}", values)
@@ -375,7 +411,8 @@ func TestLoadFileLoadsSecretsBeforeReturningNotifyConfig(t *testing.T) {
" directory: " + secretsDir + "\n" + " directory: " + secretsDir + "\n" +
"notify:\n" + "notify:\n" +
" distributor:\n" + " distributor:\n" +
" enabled: true\n" " enabled: true\n" +
" pipeline_id_template: weatherreporter.{artifact_group}\n"
if err := os.WriteFile(path, []byte(configYAML), 0o600); err != nil { if err := os.WriteFile(path, []byte(configYAML), 0o600); err != nil {
t.Fatalf("write config fixture: %v", err) t.Fatalf("write config fixture: %v", err)
} }

View File

@@ -28,8 +28,9 @@ func Defaults() Config {
TokenEnv: "DISTRIBUTOR_UPLOAD_TOKEN", TokenEnv: "DISTRIBUTOR_UPLOAD_TOKEN",
Timeout: 30 * time.Second, Timeout: 30 * time.Second,
FailurePolicy: NotifyFailureError, FailurePolicy: NotifyFailureError,
BundleIDTemplate: "weatherreporter.{location_id}.{report_id}.{run_id}", PipelineIDTemplate: "",
IdempotencyKeyTemplate: "{bundle_id}", BundleIDTemplate: "weatherreporter.{location_id}.{report_id}",
IdempotencyKeyTemplate: "{bundle_id}.{run_id}",
ReportPathTemplate: "{batch_output_name}", ReportPathTemplate: "{batch_output_name}",
}, },
}, },

View File

@@ -32,10 +32,23 @@ var distributorIdempotencyTemplateVariables = map[string]struct{}{
"bundle_id": {}, "bundle_id": {},
} }
var distributorPipelineTemplateVariables = distributorIdempotencyTemplateVariables
func RenderDistributorBundleID(template string, values DistributorTemplateValues) (string, error) { func RenderDistributorBundleID(template string, values DistributorTemplateValues) (string, error) {
return renderDistributorTemplate("notify.distributor.bundle_id_template", template, values, distributorTemplateVariables) return renderDistributorTemplate("notify.distributor.bundle_id_template", template, values, distributorTemplateVariables)
} }
func RenderDistributorPipelineID(template string, values DistributorTemplateValues) (string, error) {
rendered, err := renderDistributorTemplate("notify.distributor.pipeline_id_template", template, values, distributorPipelineTemplateVariables)
if err != nil {
return "", err
}
if strings.TrimSpace(rendered) == "" {
return "", fmt.Errorf("notify.distributor.pipeline_id_template renders an empty pipeline id")
}
return rendered, nil
}
func RenderDistributorIdempotencyKey(template string, values DistributorTemplateValues) (string, error) { func RenderDistributorIdempotencyKey(template string, values DistributorTemplateValues) (string, error) {
return renderDistributorTemplate("notify.distributor.idempotency_key_template", template, values, distributorIdempotencyTemplateVariables) return renderDistributorTemplate("notify.distributor.idempotency_key_template", template, values, distributorIdempotencyTemplateVariables)
} }

View File

@@ -100,6 +100,12 @@ func validateDistributorNotify(cfg DistributorNotifyConfig) error {
if cfg.FailurePolicy != NotifyFailureError { if cfg.FailurePolicy != NotifyFailureError {
return fmt.Errorf("notify.distributor.failure_policy must be error when enabled") return fmt.Errorf("notify.distributor.failure_policy must be error when enabled")
} }
if cfg.PipelineIDTemplate == "" {
return fmt.Errorf("notify.distributor.pipeline_id_template is required when enabled")
}
if err := validateDistributorTemplate("notify.distributor.pipeline_id_template", cfg.PipelineIDTemplate, distributorPipelineTemplateVariables); err != nil {
return err
}
if cfg.BundleIDTemplate == "" { if cfg.BundleIDTemplate == "" {
return fmt.Errorf("notify.distributor.bundle_id_template is required when enabled") return fmt.Errorf("notify.distributor.bundle_id_template is required when enabled")
} }
@@ -122,6 +128,14 @@ func validateDistributorNotify(cfg DistributorNotifyConfig) error {
ArtifactGroup: "artifact", ArtifactGroup: "artifact",
BatchOutputName: "report.md", BatchOutputName: "report.md",
} }
bundleID, err := RenderDistributorBundleID(cfg.BundleIDTemplate, values)
if err != nil {
return err
}
values.BundleID = bundleID
if _, err := RenderDistributorPipelineID(cfg.PipelineIDTemplate, values); err != nil {
return err
}
if _, err := RenderDistributorReportPath(cfg.ReportPathTemplate, values); err != nil { if _, err := RenderDistributorReportPath(cfg.ReportPathTemplate, values); err != nil {
return err return err
} }

View File

@@ -65,6 +65,7 @@ func TestSaveArtifactsAndMetadataRoundTrip(t *testing.T) {
ReportID: resolved.Definition.ID, ReportID: resolved.Definition.ID,
AttemptedAt: resolved.GeneratedAt, AttemptedAt: resolved.GeneratedAt,
Endpoint: "https://distributor.example.test", Endpoint: "https://distributor.example.test",
PipelineID: "weatherreporter.daily",
BundleID: "weatherreporter.home.daily.run", BundleID: "weatherreporter.home.daily.run",
IdempotencyKey: "weatherreporter.home.daily.run", IdempotencyKey: "weatherreporter.home.daily.run",
SourcePath: "/tmp/report.md", SourcePath: "/tmp/report.md",
@@ -102,7 +103,7 @@ func TestSaveArtifactsAndMetadataRoundTrip(t *testing.T) {
if err := json.Unmarshal(notificationData, &notification); err != nil { if err := json.Unmarshal(notificationData, &notification); err != nil {
t.Fatalf("decode notification: %v", err) t.Fatalf("decode notification: %v", err)
} }
if notification.SchemaVersion != DistributorNotificationSchemaVersion || notification.RunStatus == nil || notification.RunStatus.Status != "succeeded" { if notification.SchemaVersion != DistributorNotificationSchemaVersion || notification.PipelineID != "weatherreporter.daily" || notification.RunStatus == nil || notification.RunStatus.Status != "succeeded" {
t.Fatalf("notification = %#v, want persisted distributor status", notification) t.Fatalf("notification = %#v, want persisted distributor status", notification)
} }
paths, err := store.Paths(resolved) paths, err := store.Paths(resolved)

View File

@@ -45,6 +45,7 @@ type DistributorNotificationArtifact struct {
ReportID report.ID `json:"reportId"` ReportID report.ID `json:"reportId"`
AttemptedAt time.Time `json:"attemptedAt"` AttemptedAt time.Time `json:"attemptedAt"`
Endpoint string `json:"endpoint"` Endpoint string `json:"endpoint"`
PipelineID string `json:"pipelineId,omitempty"`
BundleID string `json:"bundleId,omitempty"` BundleID string `json:"bundleId,omitempty"`
IdempotencyKey string `json:"idempotencyKey,omitempty"` IdempotencyKey string `json:"idempotencyKey,omitempty"`
SourcePath string `json:"sourcePath,omitempty"` SourcePath string `json:"sourcePath,omitempty"`