Implement the distributor v0.5 PipelineID update
This commit is contained in:
@@ -92,16 +92,22 @@ weatherreporter uploads one distributor bundle per generated report after
|
|||||||
- `timeout`: distributor operation timeout. Must be greater than zero when
|
- `timeout`: distributor operation timeout. Must be greater than zero when
|
||||||
enabled. Default: `30s`.
|
enabled. Default: `30s`.
|
||||||
- `failure_policy`: must be `error` when enabled. Default: `error`.
|
- `failure_policy`: must be `error` when enabled. Default: `error`.
|
||||||
|
- `pipeline_id_template`: template for the distributor pipeline ID. Required
|
||||||
|
when enabled. Default: empty.
|
||||||
- `bundle_id_template`: template for distributor bundle IDs. Default:
|
- `bundle_id_template`: template for distributor bundle IDs. Default:
|
||||||
`weatherreporter.{location_id}.{report_id}.{run_id}`.
|
`weatherreporter.{location_id}.{report_id}`.
|
||||||
- `idempotency_key_template`: template for distributor idempotency keys.
|
- `idempotency_key_template`: template for distributor idempotency keys.
|
||||||
Default: `{bundle_id}`.
|
Default: `{bundle_id}.{run_id}`.
|
||||||
- `report_path_template`: template for the Markdown report path inside the
|
- `report_path_template`: template for the Markdown report path inside the
|
||||||
distributor bundle. Default: `{batch_output_name}`.
|
distributor bundle. Default: `{batch_output_name}`.
|
||||||
|
|
||||||
Supported template variables are `location_id`, `report_id`, `run_id`,
|
Supported template variables are `location_id`, `report_id`, `run_id`,
|
||||||
`artifact_group`, and `batch_output_name`. `idempotency_key_template` may also
|
`artifact_group`, and `batch_output_name`. `pipeline_id_template` and
|
||||||
use `bundle_id`.
|
`idempotency_key_template` may also use `bundle_id`.
|
||||||
|
|
||||||
|
The rendered pipeline ID selects the configured distributor `http_upload`
|
||||||
|
workflow. The rendered bundle ID is the stable logical source identity for the
|
||||||
|
report stream. The rendered idempotency key is the per-run retry identity.
|
||||||
|
|
||||||
Rendered report paths must be relative paths with `/` separators. They must not
|
Rendered report paths must be relative paths with `/` separators. They must not
|
||||||
contain backslashes, empty path segments, `.`, `..`, `manifest.json`, or
|
contain backslashes, empty path segments, `.`, `..`, `manifest.json`, or
|
||||||
|
|||||||
@@ -9,13 +9,16 @@ This document is the copyable implementation guide for submitting producer outpu
|
|||||||
The upstream application needs these values from deployment or operator configuration:
|
The upstream application needs these values from deployment or operator configuration:
|
||||||
|
|
||||||
- distributor endpoint: the HTTP server base URL, such as `https://distributor.example.com`;
|
- distributor endpoint: the HTTP server base URL, such as `https://distributor.example.com`;
|
||||||
- upload token: bearer token for exactly one configured `http_upload` pipeline;
|
- upload token: bearer token that authenticates the producer;
|
||||||
|
- pipeline id: configured `http_upload` pipeline that should process this upload;
|
||||||
- generated files: regular local files to include in the source bundle;
|
- generated files: regular local files to include in the source bundle;
|
||||||
- bundle id: stable identifier for this producer output;
|
- bundle id: stable identifier for the logical report stream or artifact;
|
||||||
- idempotency key: stable key for retrying the same producer operation.
|
- idempotency key: unique key for one producer run, reused only when retrying that same run.
|
||||||
|
|
||||||
Do not put destination routing, public URLs, transform settings, or credentials in the source manifest. Those belong in the `distributor` pipeline configuration.
|
Do not put destination routing, public URLs, transform settings, or credentials in the source manifest. Those belong in the `distributor` pipeline configuration.
|
||||||
|
|
||||||
|
The token, pipeline id, bundle id, and idempotency key have different jobs. The token authenticates the producer. The pipeline id selects the configured distributor workflow, including destinations and publishing policy. The bundle id tells `distributor` whether a new upload is a newer version of the same source; keep it stable across runs that should replace the same managed destination artifact. The idempotency key tells `distributor` whether an upload request is a retry; change it for each distinct producer run so new content is enqueued.
|
||||||
|
|
||||||
## Recommended Workflow
|
## Recommended Workflow
|
||||||
|
|
||||||
Use `gitea.maximumdirect.net/eric/distributor/pkg/upload`.
|
Use `gitea.maximumdirect.net/eric/distributor/pkg/upload`.
|
||||||
@@ -53,7 +56,9 @@ func SubmitReport(reportPath, summaryPath string) error {
|
|||||||
return fmt.Errorf("distributor endpoint and token are required")
|
return fmt.Errorf("distributor endpoint and token are required")
|
||||||
}
|
}
|
||||||
|
|
||||||
reportID := "weather.hourly.brentwood.2026-06-07T15"
|
pipelineID := "weather-hourly"
|
||||||
|
reportID := "weather.hourly.brentwood"
|
||||||
|
runID := time.Now().UTC().Format("20060102T150405.000000000Z")
|
||||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||||
defer cancel()
|
defer cancel()
|
||||||
|
|
||||||
@@ -66,8 +71,9 @@ func SubmitReport(reportPath, summaryPath string) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
result, err := client.UploadFiles(ctx, upload.UploadFilesOptions{
|
result, err := client.UploadFiles(ctx, upload.UploadFilesOptions{
|
||||||
|
PipelineID: pipelineID,
|
||||||
ID: reportID,
|
ID: reportID,
|
||||||
IdempotencyKey: reportID,
|
IdempotencyKey: reportID + "." + runID,
|
||||||
Files: []bundle.BundleFile{
|
Files: []bundle.BundleFile{
|
||||||
{SourcePath: reportPath, Path: "report.md"},
|
{SourcePath: reportPath, Path: "report.md"},
|
||||||
{SourcePath: summaryPath, Path: "summary.txt"},
|
{SourcePath: summaryPath, Path: "summary.txt"},
|
||||||
@@ -88,8 +94,11 @@ func SubmitReport(reportPath, summaryPath string) error {
|
|||||||
|
|
||||||
## Producer Responsibilities
|
## Producer Responsibilities
|
||||||
|
|
||||||
- Use a stable bundle id for the producer output, such as a report type plus logical timestamp.
|
- Use a stable bundle id for the logical producer output that should replace the same destination artifact, such as `weather.hourly.brentwood`.
|
||||||
- Use a stable idempotency key for cross-process retries of the same producer operation.
|
- Set `PipelineID` to the configured upload pipeline that should process the bundle.
|
||||||
|
- Do not include per-run timestamps, random values, or job ids in the bundle id unless each run should be treated as a different source.
|
||||||
|
- Use an idempotency key that changes for every distinct producer run, such as `<bundle-id>.<run-id>`.
|
||||||
|
- Reuse the same idempotency key only when retrying the exact same producer run with the same source manifest.
|
||||||
- Map each generated file to a clean slash-separated bundle path, such as `report.md` or `assets/chart.png`.
|
- Map each generated file to a clean slash-separated bundle path, such as `report.md` or `assets/chart.png`.
|
||||||
- Include only regular files. Symlinks, directories as files, devices, FIFOs, and sockets are rejected.
|
- Include only regular files. Symlinks, directories as files, devices, FIFOs, and sockets are rejected.
|
||||||
- Keep file contents stable after upload inputs are selected. Bundle digests are calculated from file bytes.
|
- Keep file contents stable after upload inputs are selected. Bundle digests are calculated from file bytes.
|
||||||
@@ -99,9 +108,9 @@ Valid bundle paths are relative slash paths. They must not be empty, absolute, c
|
|||||||
|
|
||||||
## Idempotency And Status
|
## Idempotency And Status
|
||||||
|
|
||||||
`pkg/upload` sends `Idempotency-Key` on every upload. If the caller omits one, the package generates a random key for that call and reuses it for in-process retries. That is enough for transient network retry within one process.
|
`pkg/upload` sends `Idempotency-Key` on every upload. If the caller omits one, the package generates a random key for that call and reuses it for in-process retries. That is enough for transient network retry within one process, but it does not give cross-process retry identity.
|
||||||
|
|
||||||
For producer jobs that may retry after process restart, supply a stable key derived from the producer operation, such as the report id or job id. Reusing the same key with the same normalized source manifest returns the original accepted run. Reusing the same key with different source content returns a conflict.
|
For producer jobs that may retry after process restart, supply a key derived from the producer run, such as `<bundle-id>.<run-id>`. Reusing the same key with the same token, pipeline id, and normalized source manifest returns the original accepted run. Reusing the same key with different source content in that scope returns a conflict. Reusing one key across multiple distinct report generations prevents those generations from being treated as new uploads.
|
||||||
|
|
||||||
`Status` polls `/runs/<run-id>` while the distributor server retains the in-memory status record. Status values are `accepted`, `queued`, `running`, `succeeded`, and `failed`. Completed records expire according to the server's `server.http.retention` setting, and server restart clears status and idempotency records.
|
`Status` polls `/runs/<run-id>` while the distributor server retains the in-memory status record. Status values are `accepted`, `queued`, `running`, `succeeded`, and `failed`. Completed records expire according to the server's `server.http.retention` setting, and server restart clears status and idempotency records.
|
||||||
|
|
||||||
|
|||||||
@@ -19,7 +19,7 @@ Use `WriteBundle` when producer-generated files live outside the final bundle ro
|
|||||||
```go
|
```go
|
||||||
manifest, err := bundle.WriteBundle(bundle.WriteBundleOptions{
|
manifest, err := bundle.WriteBundle(bundle.WriteBundleOptions{
|
||||||
Root: "/var/spool/distributor/weather/hourly-2026-06-07T15",
|
Root: "/var/spool/distributor/weather/hourly-2026-06-07T15",
|
||||||
ID: "weather.hourly.brentwood.2026-06-07T15",
|
ID: "weather.hourly.brentwood",
|
||||||
Files: []bundle.BundleFile{
|
Files: []bundle.BundleFile{
|
||||||
{SourcePath: "/tmp/weather/report.md", Path: "report.md"},
|
{SourcePath: "/tmp/weather/report.md", Path: "report.md"},
|
||||||
{SourcePath: "/tmp/weather/summary.txt", Path: "summary.txt"},
|
{SourcePath: "/tmp/weather/summary.txt", Path: "summary.txt"},
|
||||||
@@ -41,7 +41,7 @@ Use `BuildManifest` and `WriteManifest` when files are already staged under the
|
|||||||
root := "/var/spool/distributor/weather/hourly-2026-06-07T15"
|
root := "/var/spool/distributor/weather/hourly-2026-06-07T15"
|
||||||
manifest, err := bundle.BuildManifest(bundle.BuildOptions{
|
manifest, err := bundle.BuildManifest(bundle.BuildOptions{
|
||||||
Root: root,
|
Root: root,
|
||||||
ID: "weather.hourly.brentwood.2026-06-07T15",
|
ID: "weather.hourly.brentwood",
|
||||||
Files: []string{"report.md", "summary.txt"},
|
Files: []string{"report.md", "summary.txt"},
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -72,6 +72,8 @@ Invalid paths include:
|
|||||||
|
|
||||||
Explicit file lists preserve caller order. File order is part of the bundle digest, so producers should choose it deliberately and keep it stable.
|
Explicit file lists preserve caller order. File order is part of the bundle digest, so producers should choose it deliberately and keep it stable.
|
||||||
|
|
||||||
|
The manifest `ID` is the logical source identity used by `distributor` destination comparison. Keep it stable for runs that should replace the same managed destination artifact. If every run uses a different manifest `ID`, `distributor` treats those runs as different sources and may report a destination conflict instead of replacing older output.
|
||||||
|
|
||||||
## Validation And Digest Helpers
|
## Validation And Digest Helpers
|
||||||
|
|
||||||
Use `ValidateBundle` before handing an existing local bundle to another process. It verifies manifest semantics, file existence, regular-file type, file size, per-file SHA-256 digests, and bundle digest.
|
Use `ValidateBundle` before handing an existing local bundle to another process. It verifies manifest semantics, file existence, regular-file type, file size, per-file SHA-256 digests, and bundle digest.
|
||||||
|
|||||||
@@ -8,7 +8,7 @@ Import path:
|
|||||||
import "gitea.maximumdirect.net/eric/distributor/pkg/upload"
|
import "gitea.maximumdirect.net/eric/distributor/pkg/upload"
|
||||||
```
|
```
|
||||||
|
|
||||||
`pkg/upload` is the producer-facing HTTP upload client. It builds on `pkg/bundle`, packages valid source bundles as gzip-compressed tar archives, sends bearer authentication, includes idempotency keys, and exposes a status polling helper.
|
`pkg/upload` is the producer-facing HTTP upload client. It builds on `pkg/bundle`, packages valid source bundles as gzip-compressed tar archives, sends bearer authentication, routes uploads to a configured pipeline, includes idempotency keys, and exposes a status polling helper.
|
||||||
|
|
||||||
`UploadFiles` examples also use:
|
`UploadFiles` examples also use:
|
||||||
|
|
||||||
@@ -30,7 +30,7 @@ if err != nil {
|
|||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
`Endpoint` is the distributor server base URL. The client derives `/upload` and `/runs/<run-id>`. `Token` is required and is sent as `Authorization: Bearer <token>`. Token values are redacted from client errors.
|
`Endpoint` is the distributor server base URL. The client derives `/v1/pipelines/<pipeline-id>/upload` and `/runs/<run-id>`. `Token` is required and is sent as `Authorization: Bearer <token>`. Token values are redacted from client errors.
|
||||||
|
|
||||||
`HTTPClient` and `Retry` are optional. Defaults use a 30 second HTTP timeout and safe retry settings.
|
`HTTPClient` and `Retry` are optional. Defaults use a 30 second HTTP timeout and safe retry settings.
|
||||||
|
|
||||||
@@ -40,8 +40,9 @@ Use `UploadFiles` when the producer has generated output files but has not assem
|
|||||||
|
|
||||||
```go
|
```go
|
||||||
result, err := client.UploadFiles(ctx, upload.UploadFilesOptions{
|
result, err := client.UploadFiles(ctx, upload.UploadFilesOptions{
|
||||||
ID: "weather.hourly.brentwood.2026-06-07T15",
|
PipelineID: "weather-hourly",
|
||||||
IdempotencyKey: "weather.hourly.brentwood.2026-06-07T15",
|
ID: "weather.hourly.brentwood",
|
||||||
|
IdempotencyKey: "weather.hourly.brentwood.20260607T150000Z",
|
||||||
Files: []bundle.BundleFile{
|
Files: []bundle.BundleFile{
|
||||||
{SourcePath: "/tmp/weather/report.md", Path: "report.md"},
|
{SourcePath: "/tmp/weather/report.md", Path: "report.md"},
|
||||||
{SourcePath: "/tmp/weather/summary.txt", Path: "summary.txt"},
|
{SourcePath: "/tmp/weather/summary.txt", Path: "summary.txt"},
|
||||||
@@ -53,7 +54,7 @@ if err != nil {
|
|||||||
_ = result.RunID
|
_ = result.RunID
|
||||||
```
|
```
|
||||||
|
|
||||||
`UploadFiles` creates a temporary bundle, writes and validates a manifest, uploads the archive, and removes temporary files when the call returns. It does not write into producer source directories.
|
`PipelineID` is required and selects the configured distributor workflow for this upload. `ID` is the source manifest id and identifies the logical artifact inside that workflow. `UploadFiles` creates a temporary bundle, writes and validates a manifest, uploads the archive, and removes temporary files when the call returns. It does not write into producer source directories.
|
||||||
|
|
||||||
## Upload An Existing Bundle
|
## Upload An Existing Bundle
|
||||||
|
|
||||||
@@ -61,8 +62,9 @@ Use `UploadBundle` when the producer already has a complete local bundle root co
|
|||||||
|
|
||||||
```go
|
```go
|
||||||
result, err := client.UploadBundle(ctx, upload.UploadBundleOptions{
|
result, err := client.UploadBundle(ctx, upload.UploadBundleOptions{
|
||||||
|
PipelineID: "weather-hourly",
|
||||||
Root: "/var/spool/weather/hourly-2026-06-07T15",
|
Root: "/var/spool/weather/hourly-2026-06-07T15",
|
||||||
IdempotencyKey: "weather.hourly.brentwood.2026-06-07T15",
|
IdempotencyKey: "weather.hourly.brentwood.20260607T150000Z",
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
@@ -70,7 +72,7 @@ if err != nil {
|
|||||||
_ = result.RunID
|
_ = result.RunID
|
||||||
```
|
```
|
||||||
|
|
||||||
`UploadBundle` validates the local bundle by default and uploads only `manifest.json` plus manifest-listed files. Unlisted files are not uploaded.
|
`PipelineID` is required for existing bundles too. `UploadBundle` validates the local bundle by default and uploads only `manifest.json` plus manifest-listed files. Unlisted files are not uploaded.
|
||||||
|
|
||||||
## Result And Status
|
## Result And Status
|
||||||
|
|
||||||
@@ -94,7 +96,9 @@ Status values are `accepted`, `queued`, `running`, `succeeded`, and `failed`. Co
|
|||||||
|
|
||||||
Every upload request includes `Idempotency-Key`.
|
Every upload request includes `Idempotency-Key`.
|
||||||
|
|
||||||
If `IdempotencyKey` is omitted, the client generates a random 128-bit lowercase hexadecimal key for that upload operation and reuses it for retries within the same call. For cross-process retry safety, producers should pass a stable key derived from the producer job or report id.
|
If `IdempotencyKey` is omitted, the client generates a random 128-bit lowercase hexadecimal key for that upload operation and reuses it for retries within the same call. For cross-process retry safety, producers should pass a key derived from the producer run, such as `<bundle-id>.<run-id>`.
|
||||||
|
|
||||||
|
Do not reuse the same idempotency key for multiple distinct report generations. Reuse it only when retrying the exact same run with the same token, pipeline id, and source manifest. A repeated key with the same manifest in that scope returns the original accepted run instead of enqueueing another run; a repeated key with different content returns an idempotency conflict.
|
||||||
|
|
||||||
The client retries only safe cases:
|
The client retries only safe cases:
|
||||||
|
|
||||||
@@ -102,7 +106,7 @@ The client retries only safe cases:
|
|||||||
- temporary network errors;
|
- temporary network errors;
|
||||||
- ambiguous mid-upload failures.
|
- ambiguous mid-upload failures.
|
||||||
|
|
||||||
It does not retry after `202 Accepted` and does not retry `400`, `401`, `409`, `413`, or `415`.
|
It does not retry after `202 Accepted` and does not retry `400`, `401`, `403`, `404`, `409`, `413`, or `415`.
|
||||||
|
|
||||||
Detect conflicting key reuse with `errors.As`:
|
Detect conflicting key reuse with `errors.As`:
|
||||||
|
|
||||||
|
|||||||
@@ -90,8 +90,8 @@ generation returns an error after writing output, the managed report and
|
|||||||
metadata remain inspectable. Notification is not attempted after Weather API,
|
metadata remain inspectable. Notification is not attempted after Weather API,
|
||||||
briefing, prompt input, render, Scriptorium run, or metadata-save failures.
|
briefing, prompt input, render, Scriptorium run, or metadata-save failures.
|
||||||
When notification is attempted, the debug artifact records request identity,
|
When notification is attempted, the debug artifact records request identity,
|
||||||
accepted upload fields, distributor status fields, raw status report JSON when
|
including rendered pipeline ID, accepted upload fields, distributor status
|
||||||
available, and redacted failure context.
|
fields, raw status report JSON when available, and redacted failure context.
|
||||||
`--out` copies are never used as notification source files.
|
`--out` copies are never used as notification source files.
|
||||||
|
|
||||||
## Batch Workflow
|
## Batch Workflow
|
||||||
|
|||||||
@@ -18,6 +18,7 @@ Inputs:
|
|||||||
- distributor endpoint URL
|
- distributor endpoint URL
|
||||||
- token environment variable name
|
- token environment variable name
|
||||||
- upload timeout
|
- upload timeout
|
||||||
|
- pipeline ID
|
||||||
- bundle ID
|
- bundle ID
|
||||||
- idempotency key
|
- idempotency key
|
||||||
- source Markdown report path
|
- source Markdown report path
|
||||||
@@ -54,8 +55,10 @@ The adapter is built from `notify.distributor` config:
|
|||||||
- `token_env`
|
- `token_env`
|
||||||
- `timeout`
|
- `timeout`
|
||||||
|
|
||||||
The app layer renders bundle ID, idempotency key, and bundle path from:
|
The app layer renders pipeline ID, bundle ID, idempotency key, and bundle path
|
||||||
|
from:
|
||||||
|
|
||||||
|
- `pipeline_id_template`
|
||||||
- `bundle_id_template`
|
- `bundle_id_template`
|
||||||
- `idempotency_key_template`
|
- `idempotency_key_template`
|
||||||
- `report_path_template`
|
- `report_path_template`
|
||||||
@@ -67,6 +70,7 @@ after config loading and `secrets.directory` processing.
|
|||||||
|
|
||||||
The adapter calls distributor `UploadFiles` with exactly one file:
|
The adapter calls distributor `UploadFiles` with exactly one file:
|
||||||
|
|
||||||
|
- pipeline ID: the rendered distributor workflow selector
|
||||||
- source path: the managed Markdown report path selected by app orchestration
|
- source path: the managed Markdown report path selected by app orchestration
|
||||||
- bundle path: the rendered bundle-relative report path
|
- bundle path: the rendered bundle-relative report path
|
||||||
- created: the report generation timestamp
|
- created: the report generation timestamp
|
||||||
@@ -86,12 +90,13 @@ with the status report preserved.
|
|||||||
|
|
||||||
## Failure Behavior
|
## Failure Behavior
|
||||||
|
|
||||||
The adapter validates required endpoint, token env name, token value, bundle ID,
|
The adapter validates required endpoint, token env name, token value, pipeline
|
||||||
idempotency key, source path, bundle path, and upload client inputs before
|
ID, bundle ID, idempotency key, source path, bundle path, and upload client
|
||||||
uploading.
|
inputs before uploading.
|
||||||
|
|
||||||
Upload failures include endpoint, bundle ID, idempotency key, source path, and
|
Upload failures include endpoint, pipeline ID, bundle ID, idempotency key,
|
||||||
bundle path context. Token values are redacted from adapter errors.
|
source path, and bundle path context. Token values are redacted from adapter
|
||||||
|
errors.
|
||||||
|
|
||||||
Distributor idempotency conflicts are exposed as a weatherreporter-owned
|
Distributor idempotency conflicts are exposed as a weatherreporter-owned
|
||||||
`IdempotencyConflictError`, so callers do not depend on upstream distributor
|
`IdempotencyConflictError`, so callers do not depend on upstream distributor
|
||||||
|
|||||||
@@ -93,7 +93,8 @@ Durable JSON writes use shared atomic file helpers. Managed Markdown reports are
|
|||||||
prepared by creating their parent directory; Scriptorium writes the report body
|
prepared by creating their parent directory; Scriptorium writes the report body
|
||||||
to the prepared path. Extra Markdown copies are handled by app orchestration.
|
to the prepared path. Extra Markdown copies are handled by app orchestration.
|
||||||
Distributor notification debug artifacts are written atomically when
|
Distributor notification debug artifacts are written atomically when
|
||||||
notification is attempted.
|
notification is attempted and include rendered distributor pipeline ID, bundle
|
||||||
|
ID, idempotency key, upload status, latest run status, and redacted errors.
|
||||||
|
|
||||||
Inspection helpers read existing metadata, briefing, and data package files.
|
Inspection helpers read existing metadata, briefing, and data package files.
|
||||||
Missing metadata directories return no inspection records or no prior snapshot
|
Missing metadata directories return no inspection records or no prior snapshot
|
||||||
|
|||||||
@@ -150,16 +150,18 @@ successfully generated report: the managed Markdown report path recorded in the
|
|||||||
report result and metadata. Extra copies written by `--out` or `--out-dir` are
|
report result and metadata. Extra copies written by `--out` or `--out-dir` are
|
||||||
operator conveniences only.
|
operator conveniences only.
|
||||||
|
|
||||||
The default bundle ID is derived from producer name, location ID, report ID, and
|
The rendered pipeline ID selects the configured distributor `http_upload`
|
||||||
RunID:
|
workflow. The default bundle ID is a stable logical source identity derived from
|
||||||
|
producer name, location ID, and report ID:
|
||||||
|
|
||||||
```text
|
```text
|
||||||
weatherreporter.{location_id}.{report_id}.{run_id}
|
weatherreporter.{location_id}.{report_id}
|
||||||
```
|
```
|
||||||
|
|
||||||
The default idempotency key is the rendered bundle ID. The default bundle path
|
The default idempotency key appends RunID to the rendered bundle ID so each
|
||||||
for the Markdown file is the report definition's batch output name, such as
|
report generation has a distinct retry identity. The default bundle path for the
|
||||||
`daily.md`, `tomorrow.md`, `three-day.md`, or `weekend.md`.
|
Markdown file is the report definition's batch output name, such as `daily.md`,
|
||||||
|
`tomorrow.md`, `three-day.md`, or `weekend.md`.
|
||||||
|
|
||||||
Notification happens after final metadata save. Weather API, briefing,
|
Notification happens after final metadata save. Weather API, briefing,
|
||||||
data-package, render preflight, Scriptorium run, and metadata-save failures do
|
data-package, render preflight, Scriptorium run, and metadata-save failures do
|
||||||
@@ -168,14 +170,14 @@ other reports continue, the failed report includes notification fields in the
|
|||||||
JSON summary, and the batch returns nonzero.
|
JSON summary, and the batch returns nonzero.
|
||||||
|
|
||||||
Each notification attempt writes a debug artifact under `notifications/`. The
|
Each notification attempt writes a debug artifact under `notifications/`. The
|
||||||
artifact records the rendered bundle ID, idempotency key, managed source path,
|
artifact records the rendered pipeline ID, bundle ID, idempotency key, managed
|
||||||
bundle-relative path, bundle created timestamp, accepted upload response, and
|
source path, bundle-relative path, bundle created timestamp, accepted upload
|
||||||
the latest distributor run status response when available. Weatherreporter polls
|
response, and the latest distributor run status response when available.
|
||||||
status until distributor reports `succeeded` or `failed`, or until the configured
|
Weatherreporter polls status until distributor reports `succeeded` or `failed`,
|
||||||
notification timeout expires. The run status includes the distributor status,
|
or until the configured notification timeout expires. The run status includes
|
||||||
error text, and raw run report JSON, which can show actions such as
|
the distributor status, error text, and raw run report JSON, which can show
|
||||||
`replace_older`, `skip_same`, `skip_destination_newer`, or `failed`. Token values
|
actions such as `replace_older`, `skip_same`, `skip_destination_newer`, or
|
||||||
are not written.
|
`failed`. Token values are not written.
|
||||||
|
|
||||||
Weatherreporter is responsible for selecting the managed Markdown report,
|
Weatherreporter is responsible for selecting the managed Markdown report,
|
||||||
constructing a source bundle, and submitting it to the configured distributor
|
constructing a source bundle, and submitting it to the configured distributor
|
||||||
|
|||||||
@@ -240,17 +240,17 @@ Relevant docs: [Configuration reference](config.md),
|
|||||||
|
|
||||||
Symptom: notification fails with idempotency conflict context.
|
Symptom: notification fails with idempotency conflict context.
|
||||||
|
|
||||||
Likely cause: the same idempotency key was reused for different bundle content.
|
Likely cause: the same idempotency key was reused for different bundle content
|
||||||
By default the key derives from bundle ID, which includes location ID, report
|
within the same distributor token and pipeline. By default the bundle ID is a
|
||||||
ID, and RunID.
|
stable report-stream identity and the idempotency key appends RunID.
|
||||||
|
|
||||||
Diagnostic: inspect the failed batch JSON or stderr line for bundle and
|
Diagnostic: inspect the failed batch JSON or stderr line for pipeline, bundle,
|
||||||
idempotency context. Compare the configured templates with the report RunID and
|
and idempotency context. Compare the configured templates with the report RunID
|
||||||
report path.
|
and report path.
|
||||||
|
|
||||||
Also inspect the notification artifact linked from metadata. It records the
|
Also inspect the notification artifact linked from metadata. It records the
|
||||||
rendered bundle ID, idempotency key, upload result, distributor run status,
|
rendered pipeline ID, bundle ID, idempotency key, upload result, distributor run
|
||||||
status error, and raw run report JSON when available.
|
status, status error, and raw run report JSON when available.
|
||||||
|
|
||||||
Safe fix: keep idempotency templates stable for retries of the same generated
|
Safe fix: keep idempotency templates stable for retries of the same generated
|
||||||
report, but do not reuse the same rendered key for different generated report
|
report, but do not reuse the same rendered key for different generated report
|
||||||
@@ -264,11 +264,12 @@ Relevant docs: [Operations guide](operations.md),
|
|||||||
Symptom: notification fails with distributor upload rejection, HTTP status, or
|
Symptom: notification fails with distributor upload rejection, HTTP status, or
|
||||||
bundle validation context.
|
bundle validation context.
|
||||||
|
|
||||||
Likely cause: the distributor endpoint rejected the token, bundle ID,
|
Likely cause: the distributor endpoint rejected the token, pipeline ID, bundle
|
||||||
idempotency key, source file, or bundle path.
|
ID, idempotency key, source file, or bundle path.
|
||||||
|
|
||||||
Diagnostic: inspect stdout JSON or stderr status lines for
|
Diagnostic: inspect stdout JSON or stderr status lines for
|
||||||
`notificationError`. Confirm `notify.distributor.endpoint`,
|
`notificationError`. Confirm `notify.distributor.endpoint`,
|
||||||
|
`notify.distributor.pipeline_id_template`,
|
||||||
`notify.distributor.report_path_template`, and token configuration. Token values
|
`notify.distributor.report_path_template`, and token configuration. Token values
|
||||||
are redacted from weatherreporter errors.
|
are redacted from weatherreporter errors.
|
||||||
|
|
||||||
|
|||||||
@@ -21,6 +21,7 @@ notify:
|
|||||||
token_env: DISTRIBUTOR_UPLOAD_TOKEN
|
token_env: DISTRIBUTOR_UPLOAD_TOKEN
|
||||||
timeout: 30s
|
timeout: 30s
|
||||||
failure_policy: error
|
failure_policy: error
|
||||||
|
pipeline_id_template: "weatherreporter.{artifact_group}"
|
||||||
bundle_id_template: "weatherreporter.{location_id}.{report_id}"
|
bundle_id_template: "weatherreporter.{location_id}.{report_id}"
|
||||||
idempotency_key_template: "{bundle_id}.{run_id}"
|
idempotency_key_template: "{bundle_id}.{run_id}"
|
||||||
report_path_template: "{batch_output_name}"
|
report_path_template: "{batch_output_name}"
|
||||||
|
|||||||
2
go.mod
2
go.mod
@@ -4,4 +4,4 @@ go 1.26
|
|||||||
|
|
||||||
require gopkg.in/yaml.v3 v3.0.1
|
require gopkg.in/yaml.v3 v3.0.1
|
||||||
|
|
||||||
require gitea.maximumdirect.net/eric/distributor v0.4.0
|
require gitea.maximumdirect.net/eric/distributor v0.5.0
|
||||||
|
|||||||
4
go.sum
4
go.sum
@@ -1,5 +1,5 @@
|
|||||||
gitea.maximumdirect.net/eric/distributor v0.4.0 h1:SRrTFjVLMv4wFZmMLwnaYtJaQlQ/wsB4OzcaSEEb524=
|
gitea.maximumdirect.net/eric/distributor v0.5.0 h1:+al7Bw+kMv6V35a3Sm5rUtCTQhwOn5b9x3RsclPMKJk=
|
||||||
gitea.maximumdirect.net/eric/distributor v0.4.0/go.mod h1:G03FCFZPHpsUKC6SeMgTdbfNRpPQBdyTtDUj04e1Tu8=
|
gitea.maximumdirect.net/eric/distributor v0.5.0/go.mod h1:G03FCFZPHpsUKC6SeMgTdbfNRpPQBdyTtDUj04e1Tu8=
|
||||||
github.com/aws/aws-sdk-go-v2 v1.41.9 h1:/rYeyO2+HrMztAmxAq9++XJtFMqSIpSsNA0yDGALYq4=
|
github.com/aws/aws-sdk-go-v2 v1.41.9 h1:/rYeyO2+HrMztAmxAq9++XJtFMqSIpSsNA0yDGALYq4=
|
||||||
github.com/aws/aws-sdk-go-v2 v1.41.9/go.mod h1:+HsoOEX80qAVUitj1A2DhCNTjmb3edVyuDypb6LNEeo=
|
github.com/aws/aws-sdk-go-v2 v1.41.9/go.mod h1:+HsoOEX80qAVUitj1A2DhCNTjmb3edVyuDypb6LNEeo=
|
||||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.11 h1:h5+3VT69KUBK24grGuuA5saDJTj2IIjLb9au668Fo5I=
|
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.11 h1:h5+3VT69KUBK24grGuuA5saDJTj2IIjLb9au668Fo5I=
|
||||||
|
|||||||
@@ -25,6 +25,7 @@ type Client struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type UploadRequest struct {
|
type UploadRequest struct {
|
||||||
|
PipelineID string
|
||||||
BundleID string
|
BundleID string
|
||||||
IdempotencyKey string
|
IdempotencyKey string
|
||||||
SourcePath string
|
SourcePath string
|
||||||
@@ -77,6 +78,7 @@ type uploadClient interface {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type uploadFilesOptions struct {
|
type uploadFilesOptions struct {
|
||||||
|
PipelineID string
|
||||||
BundleID string
|
BundleID string
|
||||||
IdempotencyKey string
|
IdempotencyKey string
|
||||||
SourcePath string
|
SourcePath string
|
||||||
@@ -128,6 +130,9 @@ func (c *Client) Upload(ctx context.Context, req UploadRequest) (UploadResult, e
|
|||||||
if c.TokenEnv == "" {
|
if c.TokenEnv == "" {
|
||||||
return UploadResult{}, fmt.Errorf("distributor token environment variable is required")
|
return UploadResult{}, fmt.Errorf("distributor token environment variable is required")
|
||||||
}
|
}
|
||||||
|
if req.PipelineID == "" {
|
||||||
|
return UploadResult{}, fmt.Errorf("distributor pipeline id is required")
|
||||||
|
}
|
||||||
if req.BundleID == "" {
|
if req.BundleID == "" {
|
||||||
return UploadResult{}, fmt.Errorf("distributor bundle id is required")
|
return UploadResult{}, fmt.Errorf("distributor bundle id is required")
|
||||||
}
|
}
|
||||||
@@ -165,6 +170,7 @@ func (c *Client) Upload(ctx context.Context, req UploadRequest) (UploadResult, e
|
|||||||
defer cancel()
|
defer cancel()
|
||||||
|
|
||||||
result, err := uploadClient.UploadFiles(runCtx, uploadFilesOptions{
|
result, err := uploadClient.UploadFiles(runCtx, uploadFilesOptions{
|
||||||
|
PipelineID: req.PipelineID,
|
||||||
BundleID: req.BundleID,
|
BundleID: req.BundleID,
|
||||||
IdempotencyKey: req.IdempotencyKey,
|
IdempotencyKey: req.IdempotencyKey,
|
||||||
SourcePath: req.SourcePath,
|
SourcePath: req.SourcePath,
|
||||||
@@ -174,6 +180,7 @@ func (c *Client) Upload(ctx context.Context, req UploadRequest) (UploadResult, e
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return UploadResult{}, wrapUploadError(err, uploadErrorContext{
|
return UploadResult{}, wrapUploadError(err, uploadErrorContext{
|
||||||
Endpoint: c.Endpoint,
|
Endpoint: c.Endpoint,
|
||||||
|
PipelineID: req.PipelineID,
|
||||||
BundleID: req.BundleID,
|
BundleID: req.BundleID,
|
||||||
IdempotencyKey: req.IdempotencyKey,
|
IdempotencyKey: req.IdempotencyKey,
|
||||||
SourcePath: req.SourcePath,
|
SourcePath: req.SourcePath,
|
||||||
@@ -265,6 +272,7 @@ func newDistributorUploadClient(endpoint, token string, timeout time.Duration) (
|
|||||||
|
|
||||||
func (c distributorUploadClient) UploadFiles(ctx context.Context, opts uploadFilesOptions) (uploadFilesResult, error) {
|
func (c distributorUploadClient) UploadFiles(ctx context.Context, opts uploadFilesOptions) (uploadFilesResult, error) {
|
||||||
result, err := c.client.UploadFiles(ctx, distributorupload.UploadFilesOptions{
|
result, err := c.client.UploadFiles(ctx, distributorupload.UploadFilesOptions{
|
||||||
|
PipelineID: opts.PipelineID,
|
||||||
ID: opts.BundleID,
|
ID: opts.BundleID,
|
||||||
Created: opts.CreatedAt,
|
Created: opts.CreatedAt,
|
||||||
IdempotencyKey: opts.IdempotencyKey,
|
IdempotencyKey: opts.IdempotencyKey,
|
||||||
@@ -300,6 +308,7 @@ func (c distributorUploadClient) Status(ctx context.Context, runID string) (runS
|
|||||||
|
|
||||||
type uploadErrorContext struct {
|
type uploadErrorContext struct {
|
||||||
Endpoint string
|
Endpoint string
|
||||||
|
PipelineID string
|
||||||
BundleID string
|
BundleID string
|
||||||
IdempotencyKey string
|
IdempotencyKey string
|
||||||
SourcePath string
|
SourcePath string
|
||||||
@@ -313,10 +322,10 @@ func wrapUploadError(err error, ctx uploadErrorContext) error {
|
|||||||
err = redactToken(err, ctx.Token)
|
err = redactToken(err, ctx.Token)
|
||||||
if isConflict {
|
if isConflict {
|
||||||
return &IdempotencyConflictError{
|
return &IdempotencyConflictError{
|
||||||
Err: fmt.Errorf("upload distributor bundle %q to endpoint %q with idempotency key %q from source %q as bundle path %q: idempotency conflict: %w", ctx.BundleID, ctx.Endpoint, ctx.IdempotencyKey, ctx.SourcePath, ctx.BundlePath, err),
|
Err: fmt.Errorf("upload distributor bundle %q to pipeline %q at endpoint %q with idempotency key %q from source %q as bundle path %q: idempotency conflict: %w", ctx.BundleID, ctx.PipelineID, ctx.Endpoint, ctx.IdempotencyKey, ctx.SourcePath, ctx.BundlePath, err),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return fmt.Errorf("upload distributor bundle %q to endpoint %q with idempotency key %q from source %q as bundle path %q: %w", ctx.BundleID, ctx.Endpoint, ctx.IdempotencyKey, ctx.SourcePath, ctx.BundlePath, err)
|
return fmt.Errorf("upload distributor bundle %q to pipeline %q at endpoint %q with idempotency key %q from source %q as bundle path %q: %w", ctx.BundleID, ctx.PipelineID, ctx.Endpoint, ctx.IdempotencyKey, ctx.SourcePath, ctx.BundlePath, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
func redactToken(err error, token string) error {
|
func redactToken(err error, token string) error {
|
||||||
|
|||||||
@@ -30,6 +30,7 @@ func TestUploadUsesConfiguredClientAndSingleFile(t *testing.T) {
|
|||||||
client := newClient(cfg, factory.newClient)
|
client := newClient(cfg, factory.newClient)
|
||||||
|
|
||||||
result, err := client.Upload(context.Background(), UploadRequest{
|
result, err := client.Upload(context.Background(), UploadRequest{
|
||||||
|
PipelineID: "weatherreporter.daily",
|
||||||
BundleID: "weatherreporter.home.daily.run",
|
BundleID: "weatherreporter.home.daily.run",
|
||||||
IdempotencyKey: "weatherreporter.home.daily.run",
|
IdempotencyKey: "weatherreporter.home.daily.run",
|
||||||
SourcePath: "/tmp/report.md",
|
SourcePath: "/tmp/report.md",
|
||||||
@@ -55,6 +56,9 @@ func TestUploadUsesConfiguredClientAndSingleFile(t *testing.T) {
|
|||||||
t.Fatalf("factory timeout = %s, want 15s", factory.timeout)
|
t.Fatalf("factory timeout = %s, want 15s", factory.timeout)
|
||||||
}
|
}
|
||||||
got := factory.client.opts
|
got := factory.client.opts
|
||||||
|
if got.PipelineID != "weatherreporter.daily" {
|
||||||
|
t.Fatalf("PipelineID = %q, want weatherreporter.daily", got.PipelineID)
|
||||||
|
}
|
||||||
if got.BundleID != "weatherreporter.home.daily.run" {
|
if got.BundleID != "weatherreporter.home.daily.run" {
|
||||||
t.Fatalf("BundleID = %q, want weatherreporter.home.daily.run", got.BundleID)
|
t.Fatalf("BundleID = %q, want weatherreporter.home.daily.run", got.BundleID)
|
||||||
}
|
}
|
||||||
@@ -91,6 +95,13 @@ func TestUploadRejectsMissingInputs(t *testing.T) {
|
|||||||
},
|
},
|
||||||
wantErr: "token environment variable",
|
wantErr: "token environment variable",
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
name: "PipelineID",
|
||||||
|
mutate: func(c *Client, req *UploadRequest) {
|
||||||
|
req.PipelineID = ""
|
||||||
|
},
|
||||||
|
wantErr: "pipeline id is required",
|
||||||
|
},
|
||||||
{
|
{
|
||||||
name: "SourcePath",
|
name: "SourcePath",
|
||||||
mutate: func(c *Client, req *UploadRequest) {
|
mutate: func(c *Client, req *UploadRequest) {
|
||||||
@@ -170,7 +181,7 @@ func TestUploadWrapsUploadFailureWithContextWithoutToken(t *testing.T) {
|
|||||||
if err == nil {
|
if err == nil {
|
||||||
t.Fatal("Upload() error = nil, want error")
|
t.Fatal("Upload() error = nil, want error")
|
||||||
}
|
}
|
||||||
for _, want := range []string{cfg.Endpoint, req.BundleID, req.IdempotencyKey, req.SourcePath, req.BundlePath} {
|
for _, want := range []string{cfg.Endpoint, req.PipelineID, req.BundleID, req.IdempotencyKey, req.SourcePath, req.BundlePath} {
|
||||||
if !strings.Contains(err.Error(), want) {
|
if !strings.Contains(err.Error(), want) {
|
||||||
t.Fatalf("error = %q, want context %q", err.Error(), want)
|
t.Fatalf("error = %q, want context %q", err.Error(), want)
|
||||||
}
|
}
|
||||||
@@ -318,6 +329,7 @@ func TestUploadPreservesIdempotencyConflictDiagnosis(t *testing.T) {
|
|||||||
|
|
||||||
func validUploadRequest() UploadRequest {
|
func validUploadRequest() UploadRequest {
|
||||||
return UploadRequest{
|
return UploadRequest{
|
||||||
|
PipelineID: "weatherreporter.daily",
|
||||||
BundleID: "weatherreporter.home.daily.run",
|
BundleID: "weatherreporter.home.daily.run",
|
||||||
IdempotencyKey: "weatherreporter.home.daily.run",
|
IdempotencyKey: "weatherreporter.home.daily.run",
|
||||||
SourcePath: "/tmp/report.md",
|
SourcePath: "/tmp/report.md",
|
||||||
|
|||||||
@@ -114,24 +114,25 @@ type BatchResult struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type BatchReportResult struct {
|
type BatchReportResult struct {
|
||||||
ReportID report.ID `json:"reportId"`
|
ReportID report.ID `json:"reportId"`
|
||||||
ReportName string `json:"reportName"`
|
ReportName string `json:"reportName"`
|
||||||
PromptID string `json:"promptId"`
|
PromptID string `json:"promptId"`
|
||||||
RunID string `json:"runId"`
|
RunID string `json:"runId"`
|
||||||
Status string `json:"status"`
|
Status string `json:"status"`
|
||||||
Error string `json:"error,omitempty"`
|
Error string `json:"error,omitempty"`
|
||||||
NotificationStatus string `json:"notificationStatus,omitempty"`
|
NotificationStatus string `json:"notificationStatus,omitempty"`
|
||||||
NotificationRunID string `json:"notificationRunId,omitempty"`
|
NotificationRunID string `json:"notificationRunId,omitempty"`
|
||||||
NotificationError string `json:"notificationError,omitempty"`
|
NotificationPipelineID string `json:"notificationPipelineId,omitempty"`
|
||||||
NotificationPath string `json:"notificationPath,omitempty"`
|
NotificationError string `json:"notificationError,omitempty"`
|
||||||
GeneratedAt time.Time `json:"generatedAt"`
|
NotificationPath string `json:"notificationPath,omitempty"`
|
||||||
ValidPeriod timeutil.Period `json:"validPeriod"`
|
GeneratedAt time.Time `json:"generatedAt"`
|
||||||
BriefingPath string `json:"briefingPath,omitempty"`
|
ValidPeriod timeutil.Period `json:"validPeriod"`
|
||||||
DataPackagePath string `json:"dataPackagePath,omitempty"`
|
BriefingPath string `json:"briefingPath,omitempty"`
|
||||||
PreflightPath string `json:"preflightPath,omitempty"`
|
DataPackagePath string `json:"dataPackagePath,omitempty"`
|
||||||
ReportPath string `json:"reportPath,omitempty"`
|
PreflightPath string `json:"preflightPath,omitempty"`
|
||||||
OutputPath string `json:"outputPath,omitempty"`
|
ReportPath string `json:"reportPath,omitempty"`
|
||||||
MetadataPath string `json:"metadataPath,omitempty"`
|
OutputPath string `json:"outputPath,omitempty"`
|
||||||
|
MetadataPath string `json:"metadataPath,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type BatchError struct {
|
type BatchError struct {
|
||||||
@@ -157,6 +158,7 @@ type Notifier interface {
|
|||||||
type NotificationRequest struct {
|
type NotificationRequest struct {
|
||||||
ReportID report.ID
|
ReportID report.ID
|
||||||
RunID string
|
RunID string
|
||||||
|
PipelineID string
|
||||||
BundleID string
|
BundleID string
|
||||||
IdempotencyKey string
|
IdempotencyKey string
|
||||||
ReportPath string
|
ReportPath string
|
||||||
@@ -280,6 +282,7 @@ func RunBatchDetailed(ctx context.Context, req BatchRequest) (*BatchResult, erro
|
|||||||
if errors.As(err, ¬ificationErr) {
|
if errors.As(err, ¬ificationErr) {
|
||||||
item.NotificationStatus = "failed"
|
item.NotificationStatus = "failed"
|
||||||
item.NotificationError = notificationErr.Error()
|
item.NotificationError = notificationErr.Error()
|
||||||
|
item.NotificationPipelineID = notificationErr.Request.PipelineID
|
||||||
if paths, pathErr := store.Paths(resolved); pathErr == nil {
|
if paths, pathErr := store.Paths(resolved); pathErr == nil {
|
||||||
item.NotificationPath = paths.Notification
|
item.NotificationPath = paths.Notification
|
||||||
}
|
}
|
||||||
@@ -297,6 +300,7 @@ func RunBatchDetailed(ctx context.Context, req BatchRequest) (*BatchResult, erro
|
|||||||
if reportResult.Notification != nil {
|
if reportResult.Notification != nil {
|
||||||
item.NotificationStatus = reportResult.Notification.Status
|
item.NotificationStatus = reportResult.Notification.Status
|
||||||
item.NotificationRunID = reportResult.Notification.RunID
|
item.NotificationRunID = reportResult.Notification.RunID
|
||||||
|
item.NotificationPipelineID = reportResult.Notification.PipelineID
|
||||||
}
|
}
|
||||||
result.Succeeded++
|
result.Succeeded++
|
||||||
}
|
}
|
||||||
@@ -638,6 +642,10 @@ func buildNotificationRequest(cfg config.Config, resolved report.Resolved, repor
|
|||||||
return NotificationRequest{}, err
|
return NotificationRequest{}, err
|
||||||
}
|
}
|
||||||
values.BundleID = bundleID
|
values.BundleID = bundleID
|
||||||
|
pipelineID, err := config.RenderDistributorPipelineID(cfg.Notify.Distributor.PipelineIDTemplate, values)
|
||||||
|
if err != nil {
|
||||||
|
return NotificationRequest{}, err
|
||||||
|
}
|
||||||
idempotencyKey, err := config.RenderDistributorIdempotencyKey(cfg.Notify.Distributor.IdempotencyKeyTemplate, values)
|
idempotencyKey, err := config.RenderDistributorIdempotencyKey(cfg.Notify.Distributor.IdempotencyKeyTemplate, values)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return NotificationRequest{}, err
|
return NotificationRequest{}, err
|
||||||
@@ -649,6 +657,7 @@ func buildNotificationRequest(cfg config.Config, resolved report.Resolved, repor
|
|||||||
return NotificationRequest{
|
return NotificationRequest{
|
||||||
ReportID: resolved.Definition.ID,
|
ReportID: resolved.Definition.ID,
|
||||||
RunID: metadata.RunID,
|
RunID: metadata.RunID,
|
||||||
|
PipelineID: pipelineID,
|
||||||
BundleID: bundleID,
|
BundleID: bundleID,
|
||||||
IdempotencyKey: idempotencyKey,
|
IdempotencyKey: idempotencyKey,
|
||||||
ReportPath: reportPath,
|
ReportPath: reportPath,
|
||||||
@@ -667,6 +676,7 @@ func saveNotificationArtifact(ctx context.Context, store state.Store, resolved r
|
|||||||
ReportID: resolved.Definition.ID,
|
ReportID: resolved.Definition.ID,
|
||||||
AttemptedAt: time.Now(),
|
AttemptedAt: time.Now(),
|
||||||
Endpoint: cfg.Notify.Distributor.Endpoint,
|
Endpoint: cfg.Notify.Distributor.Endpoint,
|
||||||
|
PipelineID: req.PipelineID,
|
||||||
BundleID: req.BundleID,
|
BundleID: req.BundleID,
|
||||||
IdempotencyKey: req.IdempotencyKey,
|
IdempotencyKey: req.IdempotencyKey,
|
||||||
SourcePath: req.ReportPath,
|
SourcePath: req.ReportPath,
|
||||||
@@ -716,6 +726,7 @@ type distributorNotifier struct {
|
|||||||
|
|
||||||
func (n distributorNotifier) Notify(ctx context.Context, req NotificationRequest) (*NotificationResult, error) {
|
func (n distributorNotifier) Notify(ctx context.Context, req NotificationRequest) (*NotificationResult, error) {
|
||||||
result, err := n.client.Upload(ctx, distributoradapter.UploadRequest{
|
result, err := n.client.Upload(ctx, distributoradapter.UploadRequest{
|
||||||
|
PipelineID: req.PipelineID,
|
||||||
BundleID: req.BundleID,
|
BundleID: req.BundleID,
|
||||||
IdempotencyKey: req.IdempotencyKey,
|
IdempotencyKey: req.IdempotencyKey,
|
||||||
SourcePath: req.ReportPath,
|
SourcePath: req.ReportPath,
|
||||||
@@ -723,6 +734,7 @@ func (n distributorNotifier) Notify(ctx context.Context, req NotificationRequest
|
|||||||
CreatedAt: req.CreatedAt,
|
CreatedAt: req.CreatedAt,
|
||||||
})
|
})
|
||||||
notification := &NotificationResult{
|
notification := &NotificationResult{
|
||||||
|
PipelineID: req.PipelineID,
|
||||||
BundleID: req.BundleID,
|
BundleID: req.BundleID,
|
||||||
IdempotencyKey: req.IdempotencyKey,
|
IdempotencyKey: req.IdempotencyKey,
|
||||||
RunID: result.RunID,
|
RunID: result.RunID,
|
||||||
@@ -731,7 +743,9 @@ func (n distributorNotifier) Notify(ctx context.Context, req NotificationRequest
|
|||||||
StatusError: result.StatusError,
|
StatusError: result.StatusError,
|
||||||
}
|
}
|
||||||
if result.RunStatus != nil {
|
if result.RunStatus != nil {
|
||||||
notification.PipelineID = result.RunStatus.PipelineID
|
if result.RunStatus.PipelineID != "" {
|
||||||
|
notification.PipelineID = result.RunStatus.PipelineID
|
||||||
|
}
|
||||||
notification.AcceptedAt = result.RunStatus.AcceptedAt
|
notification.AcceptedAt = result.RunStatus.AcceptedAt
|
||||||
notification.StartedAt = result.RunStatus.StartedAt
|
notification.StartedAt = result.RunStatus.StartedAt
|
||||||
notification.FinishedAt = result.RunStatus.FinishedAt
|
notification.FinishedAt = result.RunStatus.FinishedAt
|
||||||
|
|||||||
@@ -288,6 +288,7 @@ func TestGenerateReportNotifiesManagedReportPath(t *testing.T) {
|
|||||||
cfg := dailyTestConfig(t, server)
|
cfg := dailyTestConfig(t, server)
|
||||||
cfg.Workspace.Root = t.TempDir()
|
cfg.Workspace.Root = t.TempDir()
|
||||||
cfg.Notify.Distributor.Enabled = true
|
cfg.Notify.Distributor.Enabled = true
|
||||||
|
cfg.Notify.Distributor.PipelineIDTemplate = "weatherreporter.{artifact_group}"
|
||||||
resolved, err := ResolveGenerate(GenerateRequest{
|
resolved, err := ResolveGenerate(GenerateRequest{
|
||||||
Config: cfg,
|
Config: cfg,
|
||||||
Report: ReportDaily,
|
Report: ReportDaily,
|
||||||
@@ -330,8 +331,12 @@ func TestGenerateReportNotifiesManagedReportPath(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("read notification artifact: %v", err)
|
t.Fatalf("read notification artifact: %v", err)
|
||||||
}
|
}
|
||||||
if !strings.Contains(string(notificationData), `"replace_older"`) || !strings.Contains(string(notificationData), `"bundleCreated"`) {
|
var notificationArtifact state.DistributorNotificationArtifact
|
||||||
t.Fatalf("notification artifact missing status report or created timestamp:\n%s", string(notificationData))
|
if err := json.Unmarshal(notificationData, ¬ificationArtifact); err != nil {
|
||||||
|
t.Fatalf("decode notification artifact: %v", err)
|
||||||
|
}
|
||||||
|
if notificationArtifact.PipelineID != "weatherreporter.daily" || notificationArtifact.BundleCreated.IsZero() || notificationArtifact.RunStatus == nil || !strings.Contains(string(notificationArtifact.RunStatus.Report), "replace_older") {
|
||||||
|
t.Fatalf("notification artifact = %#v, want requested pipeline, status report, and created timestamp", notificationArtifact)
|
||||||
}
|
}
|
||||||
if len(notifier.requests) != 1 {
|
if len(notifier.requests) != 1 {
|
||||||
t.Fatalf("notification requests = %d, want 1", len(notifier.requests))
|
t.Fatalf("notification requests = %d, want 1", len(notifier.requests))
|
||||||
@@ -346,11 +351,14 @@ func TestGenerateReportNotifiesManagedReportPath(t *testing.T) {
|
|||||||
if req.BundlePath != "daily.md" {
|
if req.BundlePath != "daily.md" {
|
||||||
t.Fatalf("notification BundlePath = %q, want daily.md", req.BundlePath)
|
t.Fatalf("notification BundlePath = %q, want daily.md", req.BundlePath)
|
||||||
}
|
}
|
||||||
if req.BundleID != "weatherreporter.home.daily_today."+result.Metadata.RunID {
|
if req.PipelineID != "weatherreporter.daily" {
|
||||||
|
t.Fatalf("notification PipelineID = %q, want rendered pipeline", req.PipelineID)
|
||||||
|
}
|
||||||
|
if req.BundleID != "weatherreporter.home.daily_today" {
|
||||||
t.Fatalf("notification BundleID = %q, want default template", req.BundleID)
|
t.Fatalf("notification BundleID = %q, want default template", req.BundleID)
|
||||||
}
|
}
|
||||||
if req.IdempotencyKey != req.BundleID {
|
if req.IdempotencyKey != req.BundleID+"."+result.Metadata.RunID {
|
||||||
t.Fatalf("IdempotencyKey = %q, want bundle id %q", req.IdempotencyKey, req.BundleID)
|
t.Fatalf("IdempotencyKey = %q, want per-run key", req.IdempotencyKey)
|
||||||
}
|
}
|
||||||
if req.RunID != result.Metadata.RunID {
|
if req.RunID != result.Metadata.RunID {
|
||||||
t.Fatalf("notification RunID = %q, want report run id %q", req.RunID, result.Metadata.RunID)
|
t.Fatalf("notification RunID = %q, want report run id %q", req.RunID, result.Metadata.RunID)
|
||||||
@@ -365,6 +373,7 @@ func TestGenerateReportNotificationFailureFailsReport(t *testing.T) {
|
|||||||
cfg := dailyTestConfig(t, server)
|
cfg := dailyTestConfig(t, server)
|
||||||
cfg.Workspace.Root = t.TempDir()
|
cfg.Workspace.Root = t.TempDir()
|
||||||
cfg.Notify.Distributor.Enabled = true
|
cfg.Notify.Distributor.Enabled = true
|
||||||
|
cfg.Notify.Distributor.PipelineIDTemplate = "weatherreporter.{artifact_group}"
|
||||||
resolved, err := ResolveGenerate(GenerateRequest{
|
resolved, err := ResolveGenerate(GenerateRequest{
|
||||||
Config: cfg,
|
Config: cfg,
|
||||||
Report: ReportDaily,
|
Report: ReportDaily,
|
||||||
@@ -441,6 +450,7 @@ func TestGenerateReportDoesNotNotifyAfterRenderOrRunFailure(t *testing.T) {
|
|||||||
cfg := dailyTestConfig(t, server)
|
cfg := dailyTestConfig(t, server)
|
||||||
cfg.Workspace.Root = t.TempDir()
|
cfg.Workspace.Root = t.TempDir()
|
||||||
cfg.Notify.Distributor.Enabled = true
|
cfg.Notify.Distributor.Enabled = true
|
||||||
|
cfg.Notify.Distributor.PipelineIDTemplate = "weatherreporter.{artifact_group}"
|
||||||
resolved, err := ResolveGenerate(GenerateRequest{
|
resolved, err := ResolveGenerate(GenerateRequest{
|
||||||
Config: cfg,
|
Config: cfg,
|
||||||
Report: ReportDaily,
|
Report: ReportDaily,
|
||||||
@@ -473,6 +483,7 @@ func TestGenerateReportDoesNotNotifyAfterFetchFailure(t *testing.T) {
|
|||||||
cfg.WeatherAPI.Timezone = "America/Chicago"
|
cfg.WeatherAPI.Timezone = "America/Chicago"
|
||||||
cfg.Workspace.Root = t.TempDir()
|
cfg.Workspace.Root = t.TempDir()
|
||||||
cfg.Notify.Distributor.Enabled = true
|
cfg.Notify.Distributor.Enabled = true
|
||||||
|
cfg.Notify.Distributor.PipelineIDTemplate = "weatherreporter.{artifact_group}"
|
||||||
resolved, err := ResolveGenerate(GenerateRequest{
|
resolved, err := ResolveGenerate(GenerateRequest{
|
||||||
Config: cfg,
|
Config: cfg,
|
||||||
Report: ReportDaily,
|
Report: ReportDaily,
|
||||||
@@ -1257,6 +1268,7 @@ func TestRunBatchContinuesAfterNotificationFailure(t *testing.T) {
|
|||||||
cfg.WeatherAPI.Timezone = "America/Chicago"
|
cfg.WeatherAPI.Timezone = "America/Chicago"
|
||||||
cfg.Workspace.Root = t.TempDir()
|
cfg.Workspace.Root = t.TempDir()
|
||||||
cfg.Notify.Distributor.Enabled = true
|
cfg.Notify.Distributor.Enabled = true
|
||||||
|
cfg.Notify.Distributor.PipelineIDTemplate = "weatherreporter.{artifact_group}"
|
||||||
notifier := &recordingNotifier{
|
notifier := &recordingNotifier{
|
||||||
errByReport: map[report.ID]error{
|
errByReport: map[report.ID]error{
|
||||||
report.ThreeDay: errors.New("distributor unavailable"),
|
report.ThreeDay: errors.New("distributor unavailable"),
|
||||||
@@ -1289,6 +1301,9 @@ func TestRunBatchContinuesAfterNotificationFailure(t *testing.T) {
|
|||||||
if item.NotificationStatus != "failed" {
|
if item.NotificationStatus != "failed" {
|
||||||
t.Fatalf("3-day notification status = %q, want failed", item.NotificationStatus)
|
t.Fatalf("3-day notification status = %q, want failed", item.NotificationStatus)
|
||||||
}
|
}
|
||||||
|
if item.NotificationPipelineID != "weatherreporter.three-day" {
|
||||||
|
t.Fatalf("3-day notification pipeline = %q, want weatherreporter.three-day", item.NotificationPipelineID)
|
||||||
|
}
|
||||||
if !strings.Contains(item.NotificationError, "distributor unavailable") {
|
if !strings.Contains(item.NotificationError, "distributor unavailable") {
|
||||||
t.Fatalf("3-day notification error = %q, want distributor unavailable", item.NotificationError)
|
t.Fatalf("3-day notification error = %q, want distributor unavailable", item.NotificationError)
|
||||||
}
|
}
|
||||||
@@ -1300,6 +1315,9 @@ func TestRunBatchContinuesAfterNotificationFailure(t *testing.T) {
|
|||||||
if item.NotificationStatus != "accepted" {
|
if item.NotificationStatus != "accepted" {
|
||||||
t.Fatalf("report %s notification status = %q, want accepted", item.ReportID, item.NotificationStatus)
|
t.Fatalf("report %s notification status = %q, want accepted", item.ReportID, item.NotificationStatus)
|
||||||
}
|
}
|
||||||
|
if item.NotificationPipelineID == "" {
|
||||||
|
t.Fatalf("report %s notification pipeline is empty", item.ReportID)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
if !failedThreeDay {
|
if !failedThreeDay {
|
||||||
t.Fatalf("reports = %#v, want notification failure on 3-day item", result.Reports)
|
t.Fatalf("reports = %#v, want notification failure on 3-day item", result.Reports)
|
||||||
@@ -1500,9 +1518,13 @@ func (n *recordingNotifier) Notify(_ context.Context, req NotificationRequest) (
|
|||||||
if result.IdempotencyKey == "" {
|
if result.IdempotencyKey == "" {
|
||||||
result.IdempotencyKey = req.IdempotencyKey
|
result.IdempotencyKey = req.IdempotencyKey
|
||||||
}
|
}
|
||||||
|
if result.PipelineID == "" {
|
||||||
|
result.PipelineID = req.PipelineID
|
||||||
|
}
|
||||||
return &result, nil
|
return &result, nil
|
||||||
}
|
}
|
||||||
return &NotificationResult{
|
return &NotificationResult{
|
||||||
|
PipelineID: req.PipelineID,
|
||||||
BundleID: req.BundleID,
|
BundleID: req.BundleID,
|
||||||
IdempotencyKey: req.IdempotencyKey,
|
IdempotencyKey: req.IdempotencyKey,
|
||||||
Status: "accepted",
|
Status: "accepted",
|
||||||
|
|||||||
@@ -469,7 +469,12 @@ func TestRunEveningUsesOutputDirectoryAndSummary(t *testing.T) {
|
|||||||
func TestRunEveningReportsNotificationSuccess(t *testing.T) {
|
func TestRunEveningReportsNotificationSuccess(t *testing.T) {
|
||||||
server := dailyServer(t)
|
server := dailyServer(t)
|
||||||
distributorServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
distributorServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
if r.URL.Path != "/upload" {
|
if r.URL.Path == "/runs/distributor-run-1" {
|
||||||
|
w.Header().Set("Content-Type", "application/json")
|
||||||
|
_, _ = w.Write([]byte(`{"run_id":"distributor-run-1","pipeline_id":"weatherreporter.daily","status":"succeeded","report":{"actions":[{"action":"replace_older"}]}}`))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if r.URL.Path != "/v1/pipelines/weatherreporter.daily/upload" {
|
||||||
http.NotFound(w, r)
|
http.NotFound(w, r)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -481,7 +486,7 @@ func TestRunEveningReportsNotificationSuccess(t *testing.T) {
|
|||||||
scriptoriumPath := writeFakeScriptorium(t, tempDir)
|
scriptoriumPath := writeFakeScriptorium(t, tempDir)
|
||||||
configPath := filepath.Join(tempDir, "config.yml")
|
configPath := filepath.Join(tempDir, "config.yml")
|
||||||
workspaceRoot := filepath.Join(tempDir, "workspace")
|
workspaceRoot := filepath.Join(tempDir, "workspace")
|
||||||
configBody := "weather_api:\n base_url: " + server.URL + "/\n timezone: America/Chicago\nscriptorium:\n binary: " + scriptoriumPath + "\nworkspace:\n root: " + workspaceRoot + "\nnotify:\n distributor:\n enabled: true\n endpoint: " + distributorServer.URL + "\n token_env: CLI_DISTRIBUTOR_TOKEN\n"
|
configBody := "weather_api:\n base_url: " + server.URL + "/\n timezone: America/Chicago\nscriptorium:\n binary: " + scriptoriumPath + "\nworkspace:\n root: " + workspaceRoot + "\nnotify:\n distributor:\n enabled: true\n endpoint: " + distributorServer.URL + "\n token_env: CLI_DISTRIBUTOR_TOKEN\n pipeline_id_template: weatherreporter.{artifact_group}\n"
|
||||||
if err := os.WriteFile(configPath, []byte(configBody), 0o600); err != nil {
|
if err := os.WriteFile(configPath, []byte(configBody), 0o600); err != nil {
|
||||||
t.Fatalf("write config: %v", err)
|
t.Fatalf("write config: %v", err)
|
||||||
}
|
}
|
||||||
@@ -505,10 +510,10 @@ func TestRunEveningReportsNotificationSuccess(t *testing.T) {
|
|||||||
if len(summary.Reports) != 1 {
|
if len(summary.Reports) != 1 {
|
||||||
t.Fatalf("reports = %#v, want one report", summary.Reports)
|
t.Fatalf("reports = %#v, want one report", summary.Reports)
|
||||||
}
|
}
|
||||||
if summary.Reports[0].NotificationStatus != "accepted" || summary.Reports[0].NotificationRunID != "distributor-run-1" {
|
if summary.Reports[0].NotificationStatus != "succeeded" || summary.Reports[0].NotificationRunID != "distributor-run-1" || summary.Reports[0].NotificationPipelineID != "weatherreporter.daily" {
|
||||||
t.Fatalf("notification fields = %#v", summary.Reports[0])
|
t.Fatalf("notification fields = %#v", summary.Reports[0])
|
||||||
}
|
}
|
||||||
if !strings.Contains(stderr.String(), `notificationStatus="accepted"`) || !strings.Contains(stderr.String(), `notificationRunId="distributor-run-1"`) {
|
if !strings.Contains(stderr.String(), `notificationStatus="succeeded"`) || !strings.Contains(stderr.String(), `notificationRunId="distributor-run-1"`) {
|
||||||
t.Fatalf("stderr missing notification fields:\n%s", stderr.String())
|
t.Fatalf("stderr missing notification fields:\n%s", stderr.String())
|
||||||
}
|
}
|
||||||
if strings.Contains(stdout.String(), "cli-secret-token") || strings.Contains(stderr.String(), "cli-secret-token") {
|
if strings.Contains(stdout.String(), "cli-secret-token") || strings.Contains(stderr.String(), "cli-secret-token") {
|
||||||
@@ -527,7 +532,7 @@ func TestRunEveningReportsNotificationFailureWithoutToken(t *testing.T) {
|
|||||||
scriptoriumPath := writeFakeScriptorium(t, tempDir)
|
scriptoriumPath := writeFakeScriptorium(t, tempDir)
|
||||||
configPath := filepath.Join(tempDir, "config.yml")
|
configPath := filepath.Join(tempDir, "config.yml")
|
||||||
workspaceRoot := filepath.Join(tempDir, "workspace")
|
workspaceRoot := filepath.Join(tempDir, "workspace")
|
||||||
configBody := "weather_api:\n base_url: " + server.URL + "/\n timezone: America/Chicago\nscriptorium:\n binary: " + scriptoriumPath + "\nworkspace:\n root: " + workspaceRoot + "\nnotify:\n distributor:\n enabled: true\n endpoint: " + distributorServer.URL + "\n token_env: CLI_DISTRIBUTOR_TOKEN\n"
|
configBody := "weather_api:\n base_url: " + server.URL + "/\n timezone: America/Chicago\nscriptorium:\n binary: " + scriptoriumPath + "\nworkspace:\n root: " + workspaceRoot + "\nnotify:\n distributor:\n enabled: true\n endpoint: " + distributorServer.URL + "\n token_env: CLI_DISTRIBUTOR_TOKEN\n pipeline_id_template: weatherreporter.{artifact_group}\n"
|
||||||
if err := os.WriteFile(configPath, []byte(configBody), 0o600); err != nil {
|
if err := os.WriteFile(configPath, []byte(configBody), 0o600); err != nil {
|
||||||
t.Fatalf("write config: %v", err)
|
t.Fatalf("write config: %v", err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -56,6 +56,7 @@ type DistributorNotifyConfig struct {
|
|||||||
TokenEnv string `yaml:"token_env"`
|
TokenEnv string `yaml:"token_env"`
|
||||||
Timeout time.Duration `yaml:"timeout"`
|
Timeout time.Duration `yaml:"timeout"`
|
||||||
FailurePolicy NotifyFailurePolicy `yaml:"failure_policy"`
|
FailurePolicy NotifyFailurePolicy `yaml:"failure_policy"`
|
||||||
|
PipelineIDTemplate string `yaml:"pipeline_id_template"`
|
||||||
BundleIDTemplate string `yaml:"bundle_id_template"`
|
BundleIDTemplate string `yaml:"bundle_id_template"`
|
||||||
IdempotencyKeyTemplate string `yaml:"idempotency_key_template"`
|
IdempotencyKeyTemplate string `yaml:"idempotency_key_template"`
|
||||||
ReportPathTemplate string `yaml:"report_path_template"`
|
ReportPathTemplate string `yaml:"report_path_template"`
|
||||||
|
|||||||
@@ -44,10 +44,13 @@ func TestDefaults(t *testing.T) {
|
|||||||
if cfg.Notify.Distributor.FailurePolicy != NotifyFailureError {
|
if cfg.Notify.Distributor.FailurePolicy != NotifyFailureError {
|
||||||
t.Fatalf("Notify.Distributor.FailurePolicy = %q, want error", cfg.Notify.Distributor.FailurePolicy)
|
t.Fatalf("Notify.Distributor.FailurePolicy = %q, want error", cfg.Notify.Distributor.FailurePolicy)
|
||||||
}
|
}
|
||||||
if cfg.Notify.Distributor.BundleIDTemplate != "weatherreporter.{location_id}.{report_id}.{run_id}" {
|
if cfg.Notify.Distributor.PipelineIDTemplate != "" {
|
||||||
|
t.Fatalf("Notify.Distributor.PipelineIDTemplate = %q, want empty", cfg.Notify.Distributor.PipelineIDTemplate)
|
||||||
|
}
|
||||||
|
if cfg.Notify.Distributor.BundleIDTemplate != "weatherreporter.{location_id}.{report_id}" {
|
||||||
t.Fatalf("Notify.Distributor.BundleIDTemplate = %q, want default", cfg.Notify.Distributor.BundleIDTemplate)
|
t.Fatalf("Notify.Distributor.BundleIDTemplate = %q, want default", cfg.Notify.Distributor.BundleIDTemplate)
|
||||||
}
|
}
|
||||||
if cfg.Notify.Distributor.IdempotencyKeyTemplate != "{bundle_id}" {
|
if cfg.Notify.Distributor.IdempotencyKeyTemplate != "{bundle_id}.{run_id}" {
|
||||||
t.Fatalf("Notify.Distributor.IdempotencyKeyTemplate = %q, want default", cfg.Notify.Distributor.IdempotencyKeyTemplate)
|
t.Fatalf("Notify.Distributor.IdempotencyKeyTemplate = %q, want default", cfg.Notify.Distributor.IdempotencyKeyTemplate)
|
||||||
}
|
}
|
||||||
if cfg.Notify.Distributor.ReportPathTemplate != "{batch_output_name}" {
|
if cfg.Notify.Distributor.ReportPathTemplate != "{batch_output_name}" {
|
||||||
@@ -76,6 +79,9 @@ func TestLoadExampleConfig(t *testing.T) {
|
|||||||
if cfg.Location.ID != "home" || cfg.Location.Name != "Brentwood" || cfg.Location.Region != "St. Louis Metro" {
|
if cfg.Location.ID != "home" || cfg.Location.Name != "Brentwood" || cfg.Location.Region != "St. Louis Metro" {
|
||||||
t.Fatalf("Location = %#v, want example location", cfg.Location)
|
t.Fatalf("Location = %#v, want example location", cfg.Location)
|
||||||
}
|
}
|
||||||
|
if cfg.Notify.Distributor.PipelineIDTemplate != "weatherreporter.{artifact_group}" {
|
||||||
|
t.Fatalf("PipelineIDTemplate = %q, want example pipeline template", cfg.Notify.Distributor.PipelineIDTemplate)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestLoadMinimalExampleConfig(t *testing.T) {
|
func TestLoadMinimalExampleConfig(t *testing.T) {
|
||||||
@@ -200,6 +206,27 @@ func TestEnabledDistributorNotifyValidation(t *testing.T) {
|
|||||||
},
|
},
|
||||||
wantErr: "notify.distributor.failure_policy",
|
wantErr: "notify.distributor.failure_policy",
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
name: "PipelineTemplateEmpty",
|
||||||
|
mutate: func(cfg *Config) {
|
||||||
|
cfg.Notify.Distributor.PipelineIDTemplate = ""
|
||||||
|
},
|
||||||
|
wantErr: "notify.distributor.pipeline_id_template",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "PipelineTemplateUnknown",
|
||||||
|
mutate: func(cfg *Config) {
|
||||||
|
cfg.Notify.Distributor.PipelineIDTemplate = "{unknown}"
|
||||||
|
},
|
||||||
|
wantErr: "notify.distributor.pipeline_id_template",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "PipelineTemplateRenderedEmpty",
|
||||||
|
mutate: func(cfg *Config) {
|
||||||
|
cfg.Notify.Distributor.PipelineIDTemplate = " "
|
||||||
|
},
|
||||||
|
wantErr: "notify.distributor.pipeline_id_template",
|
||||||
|
},
|
||||||
{
|
{
|
||||||
name: "BundleTemplate",
|
name: "BundleTemplate",
|
||||||
mutate: func(cfg *Config) {
|
mutate: func(cfg *Config) {
|
||||||
@@ -234,6 +261,7 @@ func TestEnabledDistributorNotifyValidation(t *testing.T) {
|
|||||||
t.Run(tt.name, func(t *testing.T) {
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
cfg := Defaults()
|
cfg := Defaults()
|
||||||
cfg.Notify.Distributor.Enabled = true
|
cfg.Notify.Distributor.Enabled = true
|
||||||
|
cfg.Notify.Distributor.PipelineIDTemplate = "weatherreporter.{artifact_group}"
|
||||||
tt.mutate(&cfg)
|
tt.mutate(&cfg)
|
||||||
|
|
||||||
err := Validate(cfg)
|
err := Validate(cfg)
|
||||||
@@ -254,23 +282,31 @@ func TestDistributorTemplateRendering(t *testing.T) {
|
|||||||
RunID: "20260607T120000Z",
|
RunID: "20260607T120000Z",
|
||||||
ArtifactGroup: "daily",
|
ArtifactGroup: "daily",
|
||||||
BatchOutputName: "daily.md",
|
BatchOutputName: "daily.md",
|
||||||
BundleID: "weatherreporter.home.daily.20260607T120000Z",
|
BundleID: "weatherreporter.home.daily",
|
||||||
}
|
}
|
||||||
|
|
||||||
bundleID, err := RenderDistributorBundleID("weatherreporter.{location_id}.{report_id}.{run_id}", values)
|
bundleID, err := RenderDistributorBundleID("weatherreporter.{location_id}.{report_id}", values)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("RenderDistributorBundleID() error = %v", err)
|
t.Fatalf("RenderDistributorBundleID() error = %v", err)
|
||||||
}
|
}
|
||||||
if bundleID != "weatherreporter.home.daily.20260607T120000Z" {
|
if bundleID != "weatherreporter.home.daily" {
|
||||||
t.Fatalf("bundleID = %q, want rendered value", bundleID)
|
t.Fatalf("bundleID = %q, want rendered value", bundleID)
|
||||||
}
|
}
|
||||||
|
|
||||||
idempotencyKey, err := RenderDistributorIdempotencyKey("{bundle_id}", values)
|
pipelineID, err := RenderDistributorPipelineID("weatherreporter.{artifact_group}.{bundle_id}", values)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("RenderDistributorPipelineID() error = %v", err)
|
||||||
|
}
|
||||||
|
if pipelineID != "weatherreporter.daily.weatherreporter.home.daily" {
|
||||||
|
t.Fatalf("pipelineID = %q, want rendered pipeline ID", pipelineID)
|
||||||
|
}
|
||||||
|
|
||||||
|
idempotencyKey, err := RenderDistributorIdempotencyKey("{bundle_id}.{run_id}", values)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("RenderDistributorIdempotencyKey() error = %v", err)
|
t.Fatalf("RenderDistributorIdempotencyKey() error = %v", err)
|
||||||
}
|
}
|
||||||
if idempotencyKey != "weatherreporter.home.daily.20260607T120000Z" {
|
if idempotencyKey != "weatherreporter.home.daily.20260607T120000Z" {
|
||||||
t.Fatalf("idempotencyKey = %q, want rendered bundle ID", idempotencyKey)
|
t.Fatalf("idempotencyKey = %q, want rendered run key", idempotencyKey)
|
||||||
}
|
}
|
||||||
|
|
||||||
reportPath, err := RenderDistributorReportPath("reports/{batch_output_name}", values)
|
reportPath, err := RenderDistributorReportPath("reports/{batch_output_name}", values)
|
||||||
@@ -375,7 +411,8 @@ func TestLoadFileLoadsSecretsBeforeReturningNotifyConfig(t *testing.T) {
|
|||||||
" directory: " + secretsDir + "\n" +
|
" directory: " + secretsDir + "\n" +
|
||||||
"notify:\n" +
|
"notify:\n" +
|
||||||
" distributor:\n" +
|
" distributor:\n" +
|
||||||
" enabled: true\n"
|
" enabled: true\n" +
|
||||||
|
" pipeline_id_template: weatherreporter.{artifact_group}\n"
|
||||||
if err := os.WriteFile(path, []byte(configYAML), 0o600); err != nil {
|
if err := os.WriteFile(path, []byte(configYAML), 0o600); err != nil {
|
||||||
t.Fatalf("write config fixture: %v", err)
|
t.Fatalf("write config fixture: %v", err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -28,8 +28,9 @@ func Defaults() Config {
|
|||||||
TokenEnv: "DISTRIBUTOR_UPLOAD_TOKEN",
|
TokenEnv: "DISTRIBUTOR_UPLOAD_TOKEN",
|
||||||
Timeout: 30 * time.Second,
|
Timeout: 30 * time.Second,
|
||||||
FailurePolicy: NotifyFailureError,
|
FailurePolicy: NotifyFailureError,
|
||||||
BundleIDTemplate: "weatherreporter.{location_id}.{report_id}.{run_id}",
|
PipelineIDTemplate: "",
|
||||||
IdempotencyKeyTemplate: "{bundle_id}",
|
BundleIDTemplate: "weatherreporter.{location_id}.{report_id}",
|
||||||
|
IdempotencyKeyTemplate: "{bundle_id}.{run_id}",
|
||||||
ReportPathTemplate: "{batch_output_name}",
|
ReportPathTemplate: "{batch_output_name}",
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
|
|||||||
@@ -32,10 +32,23 @@ var distributorIdempotencyTemplateVariables = map[string]struct{}{
|
|||||||
"bundle_id": {},
|
"bundle_id": {},
|
||||||
}
|
}
|
||||||
|
|
||||||
|
var distributorPipelineTemplateVariables = distributorIdempotencyTemplateVariables
|
||||||
|
|
||||||
func RenderDistributorBundleID(template string, values DistributorTemplateValues) (string, error) {
|
func RenderDistributorBundleID(template string, values DistributorTemplateValues) (string, error) {
|
||||||
return renderDistributorTemplate("notify.distributor.bundle_id_template", template, values, distributorTemplateVariables)
|
return renderDistributorTemplate("notify.distributor.bundle_id_template", template, values, distributorTemplateVariables)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func RenderDistributorPipelineID(template string, values DistributorTemplateValues) (string, error) {
|
||||||
|
rendered, err := renderDistributorTemplate("notify.distributor.pipeline_id_template", template, values, distributorPipelineTemplateVariables)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(rendered) == "" {
|
||||||
|
return "", fmt.Errorf("notify.distributor.pipeline_id_template renders an empty pipeline id")
|
||||||
|
}
|
||||||
|
return rendered, nil
|
||||||
|
}
|
||||||
|
|
||||||
func RenderDistributorIdempotencyKey(template string, values DistributorTemplateValues) (string, error) {
|
func RenderDistributorIdempotencyKey(template string, values DistributorTemplateValues) (string, error) {
|
||||||
return renderDistributorTemplate("notify.distributor.idempotency_key_template", template, values, distributorIdempotencyTemplateVariables)
|
return renderDistributorTemplate("notify.distributor.idempotency_key_template", template, values, distributorIdempotencyTemplateVariables)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -100,6 +100,12 @@ func validateDistributorNotify(cfg DistributorNotifyConfig) error {
|
|||||||
if cfg.FailurePolicy != NotifyFailureError {
|
if cfg.FailurePolicy != NotifyFailureError {
|
||||||
return fmt.Errorf("notify.distributor.failure_policy must be error when enabled")
|
return fmt.Errorf("notify.distributor.failure_policy must be error when enabled")
|
||||||
}
|
}
|
||||||
|
if cfg.PipelineIDTemplate == "" {
|
||||||
|
return fmt.Errorf("notify.distributor.pipeline_id_template is required when enabled")
|
||||||
|
}
|
||||||
|
if err := validateDistributorTemplate("notify.distributor.pipeline_id_template", cfg.PipelineIDTemplate, distributorPipelineTemplateVariables); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
if cfg.BundleIDTemplate == "" {
|
if cfg.BundleIDTemplate == "" {
|
||||||
return fmt.Errorf("notify.distributor.bundle_id_template is required when enabled")
|
return fmt.Errorf("notify.distributor.bundle_id_template is required when enabled")
|
||||||
}
|
}
|
||||||
@@ -122,6 +128,14 @@ func validateDistributorNotify(cfg DistributorNotifyConfig) error {
|
|||||||
ArtifactGroup: "artifact",
|
ArtifactGroup: "artifact",
|
||||||
BatchOutputName: "report.md",
|
BatchOutputName: "report.md",
|
||||||
}
|
}
|
||||||
|
bundleID, err := RenderDistributorBundleID(cfg.BundleIDTemplate, values)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
values.BundleID = bundleID
|
||||||
|
if _, err := RenderDistributorPipelineID(cfg.PipelineIDTemplate, values); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
if _, err := RenderDistributorReportPath(cfg.ReportPathTemplate, values); err != nil {
|
if _, err := RenderDistributorReportPath(cfg.ReportPathTemplate, values); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -65,6 +65,7 @@ func TestSaveArtifactsAndMetadataRoundTrip(t *testing.T) {
|
|||||||
ReportID: resolved.Definition.ID,
|
ReportID: resolved.Definition.ID,
|
||||||
AttemptedAt: resolved.GeneratedAt,
|
AttemptedAt: resolved.GeneratedAt,
|
||||||
Endpoint: "https://distributor.example.test",
|
Endpoint: "https://distributor.example.test",
|
||||||
|
PipelineID: "weatherreporter.daily",
|
||||||
BundleID: "weatherreporter.home.daily.run",
|
BundleID: "weatherreporter.home.daily.run",
|
||||||
IdempotencyKey: "weatherreporter.home.daily.run",
|
IdempotencyKey: "weatherreporter.home.daily.run",
|
||||||
SourcePath: "/tmp/report.md",
|
SourcePath: "/tmp/report.md",
|
||||||
@@ -102,7 +103,7 @@ func TestSaveArtifactsAndMetadataRoundTrip(t *testing.T) {
|
|||||||
if err := json.Unmarshal(notificationData, ¬ification); err != nil {
|
if err := json.Unmarshal(notificationData, ¬ification); err != nil {
|
||||||
t.Fatalf("decode notification: %v", err)
|
t.Fatalf("decode notification: %v", err)
|
||||||
}
|
}
|
||||||
if notification.SchemaVersion != DistributorNotificationSchemaVersion || notification.RunStatus == nil || notification.RunStatus.Status != "succeeded" {
|
if notification.SchemaVersion != DistributorNotificationSchemaVersion || notification.PipelineID != "weatherreporter.daily" || notification.RunStatus == nil || notification.RunStatus.Status != "succeeded" {
|
||||||
t.Fatalf("notification = %#v, want persisted distributor status", notification)
|
t.Fatalf("notification = %#v, want persisted distributor status", notification)
|
||||||
}
|
}
|
||||||
paths, err := store.Paths(resolved)
|
paths, err := store.Paths(resolved)
|
||||||
|
|||||||
@@ -45,6 +45,7 @@ type DistributorNotificationArtifact struct {
|
|||||||
ReportID report.ID `json:"reportId"`
|
ReportID report.ID `json:"reportId"`
|
||||||
AttemptedAt time.Time `json:"attemptedAt"`
|
AttemptedAt time.Time `json:"attemptedAt"`
|
||||||
Endpoint string `json:"endpoint"`
|
Endpoint string `json:"endpoint"`
|
||||||
|
PipelineID string `json:"pipelineId,omitempty"`
|
||||||
BundleID string `json:"bundleId,omitempty"`
|
BundleID string `json:"bundleId,omitempty"`
|
||||||
IdempotencyKey string `json:"idempotencyKey,omitempty"`
|
IdempotencyKey string `json:"idempotencyKey,omitempty"`
|
||||||
SourcePath string `json:"sourcePath,omitempty"`
|
SourcePath string `json:"sourcePath,omitempty"`
|
||||||
|
|||||||
Reference in New Issue
Block a user