58 Commits

Author SHA1 Message Date
2b1fb26e7d Revise the daily report prompt text to include further detail regarding geographic scope 2026-08-05 09:31:30 -05:00
3c7383e2ce Revise the daily report prompt text 2026-08-03 08:29:39 -05:00
eed47b4f68 Finish profile comparison follow-up fixes 2026-08-02 14:24:24 +00:00
6c185b8d0e Finalize profile comparison implementation 2026-08-02 13:35:31 +00:00
faf547e4a8 Complete comparison failure summaries 2026-08-02 13:28:16 +00:00
acb476a142 Report committed comparison cleanup failures 2026-08-02 13:23:18 +00:00
606b4423f1 Authorize comparison replacement at commit time 2026-08-02 13:17:50 +00:00
1716702c99 Make prompt debug creation concurrency safe 2026-08-02 13:11:41 +00:00
e0229d9c90 Document profile comparison workflow 2026-08-02 06:10:15 +00:00
ccf6b66880 Complete comparison command output 2026-08-02 06:02:24 +00:00
b489c56a48 Add comparison command request parsing 2026-08-02 05:56:31 +00:00
d39e42de30 Assemble comparison application workflow 2026-08-02 05:50:29 +00:00
d642791c10 Add concurrent comparison profile execution 2026-08-02 05:39:51 +00:00
236e3d16c4 Separate report execution from publication 2026-08-02 05:35:59 +00:00
4fa873983d Extract immutable report preparation 2026-08-02 05:30:52 +00:00
de1ae896b3 Add comparison profile preflight 2026-08-02 05:25:03 +00:00
6173e50d25 Publish comparison bundles safely 2026-08-02 05:21:04 +00:00
3bca2f41f7 Define comparison artifact contracts 2026-08-02 05:12:36 +00:00
af9cb0c0dc Document Weatherreporter v0.11.0
All checks were successful
ci/woodpecker/tag/release Pipeline was successful
2026-08-02 02:09:47 +00:00
20c82776dc Finish output directory follow-up work 2026-08-02 02:08:30 +00:00
f364ce773d Complete configurable output directory implementation 2026-08-02 01:42:58 +00:00
0dc6a06cd3 Document configured output directories 2026-08-02 01:41:03 +00:00
2af6a5cfd2 Apply configured output directories 2026-08-02 01:38:06 +00:00
0c4c575eea Add output directory configuration contract 2026-08-02 01:34:23 +00:00
114f7f5f85 Make release validation portable
All checks were successful
ci/woodpecker/tag/release Pipeline was successful
2026-08-02 00:36:54 +00:00
328c7a5693 Document Weatherreporter v0.10.0
Some checks failed
ci/woodpecker/tag/release Pipeline failed
2026-08-02 00:29:36 +00:00
fe176a2abc Finish stateless execution cleanup 2026-08-02 00:15:41 +00:00
ab9218b124 Complete stateless execution remediation 2026-08-01 22:01:28 +00:00
8d6ab0eb56 Remove per-report batch notification state 2026-08-01 21:56:25 +00:00
76cd399c76 Keep batch notification failures out of report counts 2026-08-01 21:54:53 +00:00
bf1746a756 Preflight batch output destinations 2026-08-01 21:51:42 +00:00
28bdc04fba Prevent output publication after cancellation 2026-08-01 21:49:18 +00:00
b67fae886e Complete stateless execution exit gate 2026-08-01 20:18:54 +00:00
71a2eae87b Reconcile internal stateless documentation 2026-08-01 20:16:47 +00:00
bd34ec57f8 Document stateless output operations 2026-08-01 20:11:57 +00:00
97215ddb9b Expand stateless workflow test coverage 2026-08-01 20:06:40 +00:00
dd7881acfb Remove workspace state subsystem 2026-08-01 20:00:54 +00:00
ece31567b8 Remove historical inspection commands 2026-08-01 19:58:11 +00:00
7ffc3dc603 Run report generation without workspace state 2026-08-01 19:52:22 +00:00
4bdba6f2b7 Stop persisting notification receipts 2026-08-01 19:40:51 +00:00
b184ca7cbd Move prompt debug capture out of state 2026-08-01 19:35:12 +00:00
62a12dd661 Write reports to operator-selected outputs 2026-08-01 19:33:12 +00:00
ac8d618111 Remove dormant forecast comparison policy 2026-08-01 19:24:34 +00:00
5ddd3ee19c Remove recent changes from prompt execution 2026-08-01 19:22:11 +00:00
8be9b020d4 Record stateless execution architecture decision 2026-08-01 19:18:45 +00:00
7f5a9c0357 Plan the stateless execution refactor 2026-08-01 19:16:44 +00:00
7d591487e4 Clean up roadmap and troubleshooting documentation 2026-08-01 18:16:01 +00:00
1250247986 Correct profile test boundaries and fallback coverage 2026-08-01 17:24:21 +00:00
117c5336ba Finalize domain prompt profile roadmap 2026-08-01 14:27:39 +00:00
c5ec4f83b2 Document logical prompt profile configuration 2026-08-01 14:25:09 +00:00
39c097a710 Verify profile selection in application workflows 2026-08-01 14:20:57 +00:00
993120a9f2 Adopt logical prompt profile defaults 2026-08-01 14:16:55 +00:00
c20e285d5f Wire embedded profile fallbacks 2026-08-01 14:14:36 +00:00
acbe22dcad Add embedded weather profile catalog 2026-08-01 14:13:11 +00:00
cc97ae186c Plan domain profiles and ephemeral state 2026-08-01 14:07:23 +00:00
51c35f7c22 Upgrade Promptkit to version 0.5.0 2026-08-01 13:38:18 +00:00
f014a078ee Plan domain-specific prompt profiles 2026-08-01 02:15:45 +00:00
8c19ad763b Require precipitation timing in generated text 2026-08-01 01:25:57 +00:00
144 changed files with 8499 additions and 8079 deletions

3
.gitignore vendored
View File

@@ -1,6 +1,5 @@
# Compiled application binary and testing workspace
# Compiled application binary
/weatherreporter
/workspace
# ---> Go
# If you prefer the allow list template instead of the deny list, see community template:

View File

@@ -1,25 +1,31 @@
# weatherreporter
Weatherreporter is a Go CLI that turns normalized weather data into managed,
Weatherreporter is a Go CLI that turns normalized weather data into
human-facing Markdown reports.
It provides repeatable reports with inspectable local artifacts, so operators
can review what was collected and generated for every run.
It produces a Markdown report at an operator-owned destination and can upload
the completed output through Distributor. It can also compare explicitly
selected Promptkit profiles against one shared prepared report and publish a
local comparison bundle.
## Quickstart
```sh
weatherreporter generate today --out ./today.md
weatherreporter generate today
```
Configure a Weather API endpoint first; see the
[configuration reference](docs/config.md).
[configuration reference](docs/config.md). The report is written to
`today.md` in the current directory when `output.directory` is not configured.
Set that configuration value for an ordinary publication directory, or use
`--out` for one command. See the [CLI reference](docs/cli.md) and [operations
guide](docs/operations.md) for command and operating details.
## Documentation
- [CLI reference](docs/cli.md)
- [Configuration reference](docs/config.md)
- [Operations guide](docs/operations.md)
- [Troubleshooting](docs/troubleshooting.md)
- [Comparison bundle contract](docs/integrations/comparison-bundle.md)
- [Development guide](docs/development.md)
- [Architecture policy](docs/policy/architecture.md)

View File

@@ -0,0 +1,100 @@
# 0001: Make Weatherreporter Execution Stateless
Status: Accepted
Date: 2026-08-01
## Context
Weather reports are ephemeral products. Forecasts and current conditions change
continuously, so the useful response to an old, failed, or superseded report is
normally a new generation rather than replaying or inspecting a prior run.
The existing run-addressed workspace retains module snapshots, prompt inputs,
execution receipts, generated text, rendered reports, metadata, and
notification receipts. That provenance store accumulates operational history
whose recovery and compatibility obligations are disproportionate to the value
of an ephemeral weather report. It also exists solely to support local Recent
Changes comparison for a rarely used report section.
The temporary roadmap that defined the feature scope and implementation plan
has been retired under the repository's documentation lifecycle. The
[architecture policy](../policy/architecture.md) defines the resulting system
invariants; this decision records their durable rationale.
## Decision
Weatherreporter will operate as a stateless transformation pipeline:
```text
Weather API input
-> deterministic facts and modules
-> Promptkit data package and generated text
-> repository-owned Markdown rendering
-> operator-owned report output
-> optional Distributor upload
```
Ordinary invocations will retain intermediate values only for the active
process and will publish one operator-owned Markdown output atomically. A
failed or canceled generation must not truncate or partially replace an
existing selected output. Single-report Distributor notification follows
successful publication; batch notification follows successful publication of
every planned report.
Weatherreporter will remove local Recent Changes comparison instead of
retaining application state to support it. It will remove run-addressed
workspace artifacts, historical inspection, and backward-compatible workspace
decoding. RunIDs may remain active correlation and Distributor idempotency
values, but will not identify retained application history.
Explicit `--llm-debug-dir` capture remains the sole diagnostic-file exception.
The operator selects and manages that secure location; ordinary execution does
not create an implicit debug location or a general logging store, and debug
capture must continue to exclude credentials.
Any future forecast comparison must use a structured product supplied by the
Weather API rather than local Weatherreporter history. The proposed
[Upstream Forecast Change Product](../roadmap/future.md#upstream-forecast-change-product)
defines the required upstream direction. A future integration must not add a
local snapshot fallback.
## Alternatives Considered
### Retain The Bounded Current-State Design
Retaining a managed workspace with current metadata, receipts, and snapshots
would preserve inspection and local comparison, but keeps an application-owned
history subsystem, artifact compatibility burden, and recovery surface that do
not match the report lifecycle.
### Time-Based Retention
Expiring workspace material after a fixed period reduces accumulation but still
requires retention policy, cleanup behavior, failure handling, and historical
format support. It does not remove the mismatch between retained provenance and
ephemeral report products.
### Bounded Run History
Keeping only a fixed number of prior runs limits storage volume but still makes
Weatherreporter responsible for run selection, comparison, inspection, and
state migration. It also creates arbitrary history gaps without establishing an
authoritative forecast baseline.
## Consequences
The CLI, configuration, prompt-input, workspace, and inspection contracts will
change together. Legacy workspace material will not be migrated, decoded, or
automatically deleted; operators remain responsible for any desired cleanup.
Current action results will carry active identity, selected profile, safe
effective model information, output location, notification result, and safe
errors instead of historical artifact paths. Tests will protect atomic output,
batch and notification ordering, explicit secure debug capture, and the
absence of ordinary application-managed state.
This decision deliberately leaves the Weather API responsible for any future
forecast-history comparison. It avoids a cache, archive, retention engine,
manifest, resume mechanism, or replacement inspection surface in
Weatherreporter.

View File

@@ -1,17 +1,19 @@
# Weatherreporter CLI
`weatherreporter` generates weather reports, runs report batches, and inspects
artifacts already stored in its workspace.
`weatherreporter` generates Markdown weather reports, runs report batches, and
compares explicitly selected Promptkit profiles against one prepared report. It
has no command for inspecting prior runs or application-owned state.
## Shortest Useful Command
```sh
weatherreporter generate today --out ./today.md
weatherreporter generate today
```
The command uses the configured Weather API and writes an extra Markdown copy
at `./today.md`. See the [configuration reference](config.md) to supply the
required Weather API endpoint.
The command uses the configured Weather API and atomically writes `today.md`.
With no configured output directory, it writes in the current directory. See
the [configuration reference](config.md) to supply the required Weather API
endpoint and choose an ordinary output directory.
## Commands And Usage
@@ -24,12 +26,7 @@ weatherreporter generate tomorrow [--config PATH] [--units VALUE] [--tz NAME] [-
weatherreporter generate hourly [--config PATH] [--units VALUE] [--tz NAME] [--out PATH] [--llm-debug-dir PATH] [--quiet]
weatherreporter run morning [--config PATH] [--units VALUE] [--tz NAME] [--out-dir PATH] [--llm-debug-dir PATH] [--quiet]
weatherreporter run evening [--config PATH] [--units VALUE] [--tz NAME] [--out-dir PATH] [--llm-debug-dir PATH] [--quiet]
weatherreporter inspect reports [--config PATH] [--limit N]
weatherreporter inspect metadata [--config PATH] RUN_ID
weatherreporter inspect modules [--config PATH] RUN_ID
weatherreporter inspect data-package [--config PATH] RUN_ID
weatherreporter inspect prior [--config PATH] RUN_ID
weatherreporter inspect sources [--config PATH] RUN_ID
weatherreporter compare REPORT --profile PROFILE --profile PROFILE [--config PATH] [--units VALUE] [--tz NAME] [--date YYYY-MM-DD] [--out-dir PATH] [--replace] [--llm-debug-dir PATH] [--quiet]
```
`weatherreporter --version` prints the version embedded in the executable.
@@ -38,86 +35,140 @@ builds report `development`.
| Command | Contract |
| --- | --- |
| `generate daily` | Requires `--date YYYY-MM-DD`; the date is interpreted in the effective report timezone. |
| `generate today` | Accepts an optional `--date YYYY-MM-DD`; without it, the current local date in the effective report timezone is used. |
| `generate tomorrow` | Uses the next local civil day and accepts the common generate flags. |
| `generate hourly` | Covers the next six hours in the effective report timezone. It does not accept `--date`, `--hours`, or `--duration`. |
| `run morning` and `run evening` | Run their defined report batches. `--out-dir` writes extra Markdown copies; `--out` is not accepted. |
| `generate daily` | Requires `--date YYYY-MM-DD`; the date is interpreted in the effective report timezone. Its default filename is `daily-YYYY-MM-DD.md`. |
| `generate today` | Accepts an optional `--date YYYY-MM-DD`; without it, the current local date in the effective report timezone is used. Its default filename is `today.md`. |
| `generate tomorrow` | Uses the next local civil day and writes `tomorrow.md` by default. |
| `generate hourly` | Covers the next six hours in the effective report timezone and writes `hourly.md` by default. It does not accept `--date`, `--hours`, or `--duration`. |
| `run morning` and `run evening` | Run their defined report batches beneath the configured output directory, or the current directory when none is configured. `--out-dir` selects another directory. `--out` is not accepted. |
| `compare REPORT` | Accepts `daily`, `today`, `tomorrow`, or `hourly`. It requires at least two distinct, nonblank `--profile` values in their supplied order. Daily requires `--date`; Today accepts it optionally; Tomorrow and Hourly do not accept it. |
`generate` accepts the four report command names shown above. `run` accepts
only `morning` and `evening`. Batch membership, workspace artifacts, and
notification sequencing are described in the [operations guide](operations.md).
only `morning` and `evening`. `compare` always requires explicit profile
selection: `promptkit.profile` is not used as a comparison default. Batch
membership and notification ordering are described in the
[operations guide](operations.md).
## Output, Errors, And Quiet Mode
Action commands (`generate` and `run`) write a JSON summary to stdout unless
`--quiet` is set. `run` also writes compact per-report and batch status lines
to stderr. A pre-run error, such as an invalid flag, missing required argument,
or configuration-load failure, produces no partial JSON summary. When an action
fails after it has produced a result, its summary has `"status": "failed"` and
an `error` field.
For `generate`, the report's default filename is placed beneath
`output.directory` when configured, otherwise the current directory. `--out
PATH` selects one complete output file instead. A relative path is resolved
from the current directory; an absolute path is used as given. For a batch,
the configured directory has the same role and `--out-dir PATH` selects its
output directory instead. For `compare`, `--out-dir PATH` selects one exact
bundle directory; otherwise the report-derived comparison directory is placed
beneath the configured directory or current directory. `--replace` is required
to replace an existing nonempty recognized comparison bundle. See the
[configuration reference](config.md) for the field's validation and path rules
and the [comparison bundle contract](integrations/comparison-bundle.md) for the
bundle format.
Outputs are written atomically. A generation, rendering, write, or cancellation
failure before publication leaves an existing destination unchanged. A
notification failure occurs after publication, so the newly written output
remains available.
Action commands (`generate`, `run`, and `compare`) write a JSON summary to
stdout unless `--quiet` is set. `run` also writes compact per-report and batch
status lines to stderr. A pre-run error, such as an invalid flag, missing
required argument, or configuration-load failure, produces no partial JSON
summary. When an action fails after it has produced a result, its summary has
`"status": "failed"` and an `error` field.
`--quiet` is supported by action commands only. It suppresses action summaries
and routine batch status output; it does not suppress command errors.
Inspection commands always write their requested JSON value to stdout and do
not accept `--quiet`.
### Generate Summary
A generate summary always identifies the command, report, run, generation
time, valid period, and status:
A generate summary identifies the command, report, run, generation time, valid
period, prompt version, timezone, and status. Successful output has an absolute
`outputPath`:
```json
{
"command": "generate",
"reportId": "today",
"reportName": "Today Report",
"promptId": "weather.today_generated_text",
"promptVersion": "2.0.0",
"runId": "20260529T120000.000000000Z_today",
"status": "succeeded",
"generatedAt": "2026-05-29T12:00:00Z",
"validPeriod": {
"start": "2026-05-29T00:00:00-05:00",
"end": "2026-05-30T00:00:00-05:00"
}
"timezone": "America/Chicago",
"outputPath": "/srv/weather/today.md"
}
```
When available, the summary also includes `reportPath`, `metadataPath`,
`dataPackagePath`, `preparationPath`, `executionPath`, `generatedTextRawPath`,
`generatedTextPath`, `renderContextPath`, and `llmDebugPath`. `outputPath` is included only
when `--out` wrote an extra copy. Distributor notification, when attempted,
adds `notificationPath` and may add a compact `notification` object.
When available, the summary also includes the effective `profileId`,
`backendId`, `modelName`, `sourceWarnings`, `validationStatus`, requested
`llmDebugPath`, and compact Distributor `notification` result. It does not
include historical or transient artifact paths such as metadata, prompt input,
raw generated text, render context, or notification receipts.
### Run Summary And Stderr
A run summary contains `command`, `batch`, `status`, `startedAt`, `finishedAt`,
`total`, `succeeded`, `failed`, and a `reports` array. It may also contain a
top-level `notification` object and `error`. Batch status is `failed` if any
report or the batch notification fails.
`total`, `succeeded`, `failed`, and a `reports` array. Each report item includes
its identity, status, effective profile and model details when available,
source warnings, validation status, and absolute `outputPath` after publication.
The top-level summary may also contain a batch `notification` object and
`error`. Batch status is `failed` if any report or the batch notification fails.
The `total`, `succeeded`, and `failed` counters describe report items only, so
a failed batch notification can leave `failed` at `0` while the top-level
notification and action status are `failed`.
Without `--quiet`, batch status lines use this form:
```text
report=today status=succeeded output="reports/today.md"
report=today status=succeeded output="/srv/weather/reports/today.md"
batch=morning total=2 succeeded=2 failed=0
```
### Compare Summary
A comparison summary contains these fields in this order: `command`,
`comparisonId`, `reportId`, `reportName`, `promptId`, `promptVersion`,
`promptHash`, `status`, `startedAt`, `finishedAt`, `timezone`, `validPeriod`,
`outputDirectory`, `manifestPath`, `dataPackagePath`, `total`, `succeeded`,
`failed`, `results`, and optional `error`. Published artifact paths and each
successful `results[].reportPath` are absolute. `results` preserves the
supplied profile order and each item contains `position`, `profileId`, optional
`backendId`, `modelName`, `status`, optional `validationStatus`, optional
`reportPath`, optional `llmDebugPath`, and optional safe `error`.
The comparison status is `succeeded` only when every selected profile succeeds
and the bundle is published. Individual profile failures still publish a
complete partial bundle and return a failed command result. Cancellation or a
failure before publication omits the artifact paths and returns a safe
top-level error; the resolved `outputDirectory` and finalized timestamp remain
when available. The safe error includes only a category and message: aggregate
and unclassified application failures use `application`; cancellation uses
`canceled`; deadlines use `deadline_exceeded`; prompt execution uses its
published Promptkit category; destination failures use `destination_<kind>`;
and committed cleanup failures use `publication_cleanup`. It does not expose
provider diagnostics, filesystem causes, or recovery paths. See the
[comparison bundle contract](integrations/comparison-bundle.md) for durable
artifact fields and failure invariants.
If the bundle is published but cleanup of its replaced prior bundle fails, the
summary still includes the published artifact paths and has status `failed`.
Its JSON error is `publication_cleanup` with the message `comparison published
but cleanup did not complete`; the returned command error identifies the
retained backup path for operator recovery.
## Flag Reference
| Flag | Accepted by | Meaning |
| --- | --- | --- |
| `-h`, `--help` | top level | Show help. |
| `-h`, `--help` | top level, `compare` | Show help without loading configuration or contacting a provider. |
| `--config PATH` | all commands | Load `PATH` instead of `/usr/local/etc/weatherreporter/config.yml`. |
| `--units VALUE` | `generate`, `run` | Override `weather_api.units` for this command. |
| `--tz NAME` | `generate`, `run` | Override `weather_api.timezone` for this command. |
| `--out PATH` | every `generate` command | Write an extra Markdown report copy. |
| `--llm-debug-dir PATH` | every `generate` and `run` command | Write requested sensitive prompt diagnostics outside the managed workspace. The path must be absolute. |
| `--out-dir PATH` | `run morning`, `run evening` | Write extra Markdown report copies in `PATH`. |
| `--quiet` | `generate`, `run` | Suppress action summaries and routine batch status output. |
| `--date YYYY-MM-DD` | `generate daily`, `generate today` | Required for Daily; optional for Today. |
| `--limit N` | `inspect reports` | Maximum runs to list. Defaults to `20`; `0` means no limit. |
| `--units VALUE` | `generate`, `run`, `compare` | Override `weather_api.units` for this command. |
| `--tz NAME` | `generate`, `run`, `compare` | Override `weather_api.timezone` for this command. |
| `--out PATH` | every `generate` command | Write the report to this complete file destination instead of the configured or current-directory default. |
| `--llm-debug-dir PATH` | every `generate`, `run`, and `compare` command | Write requested sensitive prompt diagnostics under this absolute path. |
| `--profile PROFILE` | `compare` | Select one explicit profile. Repeat at least twice with distinct, nonblank IDs. |
| `--out-dir PATH` | `run morning`, `run evening`, `compare` | Write batch reports beneath this directory, or select the exact comparison directory. |
| `--replace` | `compare` | Authorize replacement of a recognized nonempty comparison bundle. |
| `--quiet` | `generate`, `run`, `compare` | Suppress successful action output and routine batch status output. |
| `--date YYYY-MM-DD` | `generate daily`, `generate today`, `compare daily`, `compare today` | Required for Daily; optional for Today. |
Distributor notification is configured through `notify.distributor`; there are
no Distributor-specific CLI flags. See the [configuration reference](config.md).
@@ -125,33 +176,10 @@ no Distributor-specific CLI flags. See the [configuration reference](config.md).
## Invocation Examples
```sh
weatherreporter generate daily --date 2026-05-29 --out ./daily.md
weatherreporter generate today --date 2026-05-29 --out ./today.md
weatherreporter generate hourly --out ./hourly.md
weatherreporter generate daily --date 2026-05-29
weatherreporter generate today --out ./reports/today.md
weatherreporter generate hourly --out /srv/weather/hourly.md
weatherreporter generate today --llm-debug-dir /var/tmp/weatherreporter-debug
weatherreporter run morning --out-dir ./reports --llm-debug-dir /var/tmp/weatherreporter-debug
weatherreporter compare daily --date 2026-05-29 --profile weather-light --profile weather-balanced --out-dir ./comparison-daily-2026-05-29
```
## Inspection Commands
```sh
weatherreporter inspect reports --limit 10
weatherreporter inspect metadata 20260529T100000.000000000Z_today
weatherreporter inspect modules 20260529T100000.000000000Z_today
weatherreporter inspect data-package 20260529T100000.000000000Z_today
weatherreporter inspect prior 20260529T100000.000000000Z_today
weatherreporter inspect sources 20260529T100000.000000000Z_today
```
| Command | JSON returned |
| --- | --- |
| `inspect reports` | Recent generated runs, including artifact paths and source-warning counts. |
| `inspect metadata RUN_ID` | Persisted metadata for the run. |
| `inspect modules RUN_ID` | The run's persisted ordered module snapshot. |
| `inspect data-package RUN_ID` | The run's persisted prompt data package. |
| `inspect prior RUN_ID` | Prior comparable snapshot metadata, or `null` when none exists. |
| `inspect sources RUN_ID` | Source provenance and source warnings without full weather payloads. |
Inspection is read-only: it does not collect weather data or invoke Promptkit.
See the [operations guide](operations.md) for artifact lifecycle
and recovery.

View File

@@ -13,8 +13,8 @@ explicit `--config PATH` must exist. Values are applied in this order:
2. the configuration file, when present; and
3. the `--units` and `--tz` command-line overrides.
Environment variables do not override configuration fields. Output flags write
extra report copies for a command and do not change configuration.
Environment variables do not override configuration fields. Output flags select
operator-owned destinations for one command and do not change configuration.
## Maintained Examples
@@ -22,8 +22,12 @@ extra report copies for a command and do not change configuration.
collection and generation configuration.
- [config.yml](../examples/config.yml) is a representative production-oriented
configuration using synthetic endpoints and no credentials.
- [weather-light-local-profile.yml](../examples/weather-light-local-profile.yml)
is a complete endpoint-only override for the embedded `weather-light`
profile.
Both files are loaded by the configuration test suite.
The configuration examples are loaded by the configuration test suite. The
profile example is inspected through the Promptkit adapter test suite.
## Minimal Configuration
@@ -77,6 +81,32 @@ Missing directories, unreadable files, subdirectories, symlinks, non-regular
files, and invalid names fail configuration loading. Put only secret values in
this directory, never in the YAML file.
### `output`
`output.directory` selects the ordinary operator-owned publication directory
for individual reports, batches, and the default parent of comparison bundles.
| Field | Default | Rules |
| --- | --- | --- |
| `directory` | empty | An omitted or empty value uses the invocation working directory. A nonempty value must contain at least one non-whitespace character. |
The configured value is preserved while configuration loads: it is not cleaned,
made absolute, inspected, created, or expanded through environment variables or
a home-directory shortcut. At execution, an absolute directory is used as
given; a relative directory resolves from the invocation working directory, not
from the configuration file's location. A missing directory is created when a
report is successfully published. An existing non-directory or an uninspectable
path fails output preflight before prompt inspection, weather collection, or
publication.
For one `generate` command, `--out` is a complete file destination and takes
precedence over `output.directory`. For `run`, `--out-dir` takes precedence.
For `compare`, `--out-dir` selects its exact bundle directory; without it, the
comparison's report-derived directory is placed beneath `output.directory`.
Those explicit flags do not inspect or rebase beneath the configured directory.
See the [CLI reference](cli.md) for command selection and the [operations
guide](operations.md) for publication and failure handling.
### `notify.distributor`
Distributor notification is disabled by default. Its fields are:
@@ -125,7 +155,7 @@ The default paths are:
| `tomorrow` | `daily/{valid_start_date}/{run_id}.md`, `daily/{valid_start_date}/index.md`, `tomorrow/index.md` |
See the [operations guide](operations.md) for notification timing, uploaded
artifact selection, and failure handling.
output selection, and failure handling.
### `missing_source`
@@ -138,35 +168,42 @@ source keys are `observations`, `current`, `narrative`, `alerts`, `discussion`,
### `promptkit`
Promptkit configuration selects the executor and prompt/profile checks for
every `generate` and `run` command. A top-level `scriptorium:` configuration
every `generate`, `run`, and `compare` command. A top-level `scriptorium:` configuration
key is rejected with a migration error; it is not translated or ignored.
Prompt debug capture has no YAML setting. Use `--llm-debug-dir PATH` on an
individual `generate` or `run` command when explicitly needed.
individual `generate`, `run`, or `compare` command when explicitly needed.
| Field | Default | Rules |
| --- | --- | --- |
| `profile` | empty | Optional explicit execution profile. Otherwise the prompt's declared default is used. |
| `profile_file` | empty | Optional external profile file. Cannot be combined with `profile_dir`. |
| `profile_dir` | empty | Optional external profile directory. Cannot be combined with `profile_file`. |
| `profile` | empty | Optional global profile selection for every report in one command. When empty, each exact prompt version selects its declared default. |
| `profile_file` | empty | Optional external Promptkit profile file. It cannot be combined with `profile_dir`. A same-ID profile completely replaces Weatherreporter's embedded definition. |
| `profile_dir` | empty | Optional external Promptkit profile directory. It cannot be combined with `profile_file`. A same-ID profile completely replaces Weatherreporter's embedded definition. |
| `timeout` | `2m` | Must be greater than zero. |
| `local.endpoint` | empty | Optional absolute URL for the conventional local backend. A blank endpoint leaves it unregistered. |
| `local.concurrency_limit` | `1` | Maximum local backend concurrency. `0` is unlimited; negative values are invalid. |
### `workspace`
`profile` selects an ID; `profile_file` and `profile_dir` supply definitions.
They are separate decisions. An explicit `profile` applies to every selected
report. Otherwise Hourly selects `weather-light`, while Daily, Today, and
Tomorrow select `weather-balanced` through their exact `2.0.0` prompt
definitions.
| Field | Default |
| --- | --- |
| `root` | `workspace` |
| `snapshots_dir` | `snapshots` |
| `reports_dir` | `reports` |
| `data_packages_dir` | `data-packages` |
| `preflight_dir` | `preflight` |
| `notifications_dir` | `notifications` |
Promptkit resolves a selected profile definition from a test or embedding
consumer's explicit in-memory profile, then the configured `profile_file` or
`profile_dir`, then Weatherreporter's embedded catalog, and finally Promptkit's
built-in catalog. Sources provide complete definitions; fields are never
merged. A matching malformed external profile fails rather than using the
embedded definition. The [Promptkit integration guide](integrations/promptkit.md)
owns the catalog and precedence details.
`workspace.root` is required. Each workspace subdirectory must be a relative
path that stays within the root. See the [operations guide](operations.md) for
the managed workspace layout and lifecycle.
To replace the default Hourly definition with a local OpenAI-compatible
endpoint, set `profile_file` to a copy of
[weather-light-local-profile.yml](../examples/weather-light-local-profile.yml).
The example has no credential and should be edited for the local endpoint and
model before use. An alternative profile may use `backend: local`; in that
case `promptkit.local.endpoint` supplies the conventional local backend
endpoint.
### `dayparts`
@@ -176,18 +213,6 @@ derivation. Every item needs `name`, `start`, and `end`; start and end use
(`06:00``10:00`), `midday` (`10:00``15:00`), `afternoon`
(`15:00``17:00`), and `evening` (`17:00``24:00`).
### `recent_change`
| Field | Default |
| --- | --- |
| `temperature_degrees` | `5` |
| `precip_probability_points` | `20` |
| `wind_gust_miles_per_hour` | `10` |
| `precip_timing_shift_minutes` | `120` |
These thresholds control when Recent Changes are included in prompt input for a
prior comparable module snapshot.
### `reports`
`reports` optionally overrides a report's ordered deterministic modules and

View File

@@ -6,8 +6,8 @@ kind of change to its canonical documentation.
Weatherreporter is a Go CLI that collects normalized weather data, derives
deterministic report facts and module snapshots, executes Promptkit for
single-report generated text, renders managed Markdown reports, and can upload completed
reports through Distributor. Start with the [README](../README.md) for product
single-report generated text, renders Markdown reports, and can upload completed
operator-owned outputs through Distributor. Start with the [README](../README.md) for product
context and the [architecture policy](policy/architecture.md) for system
boundaries and invariants.
@@ -21,16 +21,16 @@ boundaries and invariants.
| Adding, changing, reviewing, or deleting tests | [Testing policy](policy/testing.md) and focused package tests | The policy defines risk-based sufficiency, durable test boundaries, doubles, and test-maintenance criteria. |
| CLI commands, flags, output, quiet mode, or command wiring | [CLI reference](cli.md) and [CLI internals](internal/cli.md) | The reference owns the user contract; the internal guide owns command composition and output flow. |
| Configuration fields, defaults, loading, overrides, validation, or secrets | [Configuration reference](config.md), [architecture policy](policy/architecture.md), and tests under `internal/config` | These separate the user-visible contract, architectural rules, and executable behavior. |
| Top-level generation, batch, collection, inspection, or notification workflow | [App orchestration internals](internal/app-orchestration.md) | It owns workflow ordering, persistence points, failure propagation, and orchestration invariants. |
| Top-level generation, batch, comparison, collection, output publication, or notification workflow | [App orchestration internals](internal/app-orchestration.md), [comparison execution internals](internal/comparison-execution.md), and [comparison publication internals](internal/comparison-publication.md) | They own workflow ordering, concurrent profile execution, output publication, failure propagation, and orchestration invariants. |
| Weather API transport, source envelopes, source warnings, or collection | [Weather API integration](integrations/weatherapi.md), [weather-data internals](internal/weather-data.md), and [collection internals](internal/collect.md) | These separate the external contract, normalized source facts, and app-facing collection behavior. |
| Forecast periods, weather derivation, collected facts, or derived facts | [Forecast derivation internals](internal/forecast-derivation.md) and [fact contracts](internal/facts.md) | They own deterministic derivation and the fact boundaries used by reports. |
| Report definitions, valid periods, report IDs, output naming, or batch composition | [Report registry internals](internal/report-registry.md) and [app orchestration internals](internal/app-orchestration.md) | Report definitions own selection and period rules; orchestration owns execution. |
| Module IDs, module composition, briefing values, or prompt-facing exports | [Module contract internals](internal/module.md), [module builder internals](internal/briefing.md), and [prompt-input internals](internal/prompt-input.md) | These own module contracts, value construction, and the curated prompt-package boundary. |
| Recent Changes comparison | [Changes internals](internal/changes.md) and [operations guide](operations.md) | The internal guide owns structured comparison; operations owns user-visible artifact behavior. |
| Prompt execution, profiles, prompt inputs, or result handling | `internal/promptexec`, the Promptkit adapter, and [prompt-input internals](internal/prompt-input.md) | These separate the executor contract and input construction. |
| Prompt execution, profiles, prepared report inputs, or result handling | `internal/promptexec`, the Promptkit adapter, [prepared report internals](internal/prepared-report.md), and [prompt-input internals](internal/prompt-input.md) | These separate the executor contract, immutable preparation, and input construction. |
| Durable comparison bundles or their compatibility | [Comparison bundle contract](integrations/comparison-bundle.md) and [comparison publication internals](internal/comparison-publication.md) | The integration document owns the external schema; internals own how it is published. |
| Generated-text schemas, validation, render contexts, templates, or Markdown rendering | [Generated-text internals](internal/generatedtext.md), [report-template internals](internal/reporttemplate.md), and [report template guide](templates.md) | These own structured text, renderer implementation, and the maintainer-facing template surface. |
| Workspace paths, metadata, atomic persistence, lookup, inspection, or recovery | [State internals](internal/state.md), [operations guide](operations.md), and [troubleshooting guide](troubleshooting.md) | These separate implementation, operator workflows, and symptom-based recovery. |
| Distributor bundles, uploads, notification artifacts, or failures | [Distributor adapter internals](internal/distributor-adapter.md), [Distributor integration contracts](integrations/distributor/), and [operations guide](operations.md) | These separate adapter behavior, external contracts, and operational lifecycle. |
| Output destinations, atomic publication, prompt diagnosis, or legacy cleanup | [Operations guide](operations.md), [App orchestration internals](internal/app-orchestration.md), and [comparison publication internals](internal/comparison-publication.md) | Operations owns operator workflows; internals own implementation boundaries. |
| Distributor bundles, uploads, notification results, or failures | [Distributor adapter internals](internal/distributor-adapter.md), [Distributor integration contracts](integrations/distributor/), and [operations guide](operations.md) | These separate adapter behavior, external contracts, and operational lifecycle. |
| Maintained example configuration | [Configuration reference](config.md) and files under `examples/` | The reference owns field meaning; examples own complete copyable files. |
| Release preparation, tagging, publication, or verification | [Release procedure](release.md) | It owns version selection, release-note preparation, candidate validation, tag publication, CI behavior, and post-publication checks. |
| Proposed, deferred, or unimplemented work | Documents under `docs/roadmap/` | Future behavior and implementation status belong only in roadmaps until implemented. |
@@ -45,13 +45,14 @@ present before introducing a new package or abstraction.
| --- | --- |
| `cmd/weatherreporter` | Binary entry point. |
| `internal/cli` | Command parsing, flags, help, output, and command wiring. |
| `internal/app` | Generation, batches, collection coordination, notification, and inspection orchestration. |
| `internal/app` | Stateless generation, batches, comparisons, collection coordination, output publication, and notification. |
| `internal/comparison` | Comparison identities, logical bundles, guarded destinations, and atomic bundle publication. |
| `internal/config` | Configuration defaults, loading, precedence, secrets, and validation. |
| `internal/adapters` | Weather API, Promptkit, and Distributor boundaries. |
| `internal/weatherdata`, `internal/forecast`, `internal/facts` | Normalized source facts and deterministic derivation. |
| `internal/report`, `internal/module`, `internal/briefing`, `internal/changes` | Report registry, module contracts and values, and structured comparison. |
| `internal/report`, `internal/module`, `internal/briefing` | Report registry plus module and briefing contracts. |
| `internal/promptinput`, `internal/generatedtext`, `internal/reporttemplate` | Prompt packages, generated-text validation, render contexts, and Markdown templates. |
| `internal/state`, `internal/fileutil`, `internal/timeutil` | Durable artifacts, atomic file operations, clocks, dates, timezones, and periods. |
| `internal/fileutil`, `internal/timeutil` | Atomic output operations, clocks, dates, timezones, and periods. |
| `docs` | User, operator, integration, internal, policy, and roadmap documentation. |
| `examples` | Maintained copyable configuration. |

View File

@@ -0,0 +1,92 @@
# Comparison Bundle Contract
A comparison bundle is the durable, flat artifact produced when one report is
executed with multiple explicit Promptkit profiles. This document is the
canonical contract for consumers of those bundles. Command invocation and JSON
action summaries belong to the [CLI reference](../cli.md); destination handling
and retention belong to the [operations guide](../operations.md).
## Version And Layout
The current and only supported manifest schema version is
`weatherreporter.comparison.v1`. A bundle directory contains exactly these
regular, non-symlinked files:
```text
comparison.json
data-package.yml
NN-profile-slug.md
```
`comparison.json` is the manifest and `data-package.yml` is the exact YAML
input supplied to every selected profile. There is one Markdown file for each
successful result and none for failed results. `NN` is the one-based selected
profile position, zero padded to at least two digits (and widened only when
needed for 100 or more profiles). The profile slug preserves ASCII letters,
digits, `-`, and `_`; each run of other characters becomes one `-`; edge `-`
and `_` characters are removed; the value is capped at 64 bytes; and an empty
slug becomes `profile`. Logical profile IDs remain authoritative in the
manifest.
All manifest paths are basenames relative to the bundle root. They never use
path separators, `.` or `..`. The CLI reports absolute paths only after a
bundle has been published.
## Manifest Schema
The manifest is UTF-8 JSON, encoded as two-space-indented JSON with one
trailing newline. Its fields appear in this order:
```text
schemaVersion, comparisonId, startedAt, finishedAt, reportId, validPeriod,
timezone, promptId, promptVersion, promptHash, dataPackage, total, succeeded,
failed, results
```
`validPeriod` contains `start` and `end`; it is a nonempty half-open period.
`dataPackage` contains `path` (always `data-package.yml`) and `sha256` (the
lowercase, 64-character SHA-256 digest of that file's exact bytes). `results`
is in the explicit profile-selection order. Its result-object fields appear in
this order:
```text
position, profileId, backendId, modelName, status, validationStatus,
reportPath, error
```
`startedAt` and `finishedAt` are nonzero UTC timestamps, and the latter is not
earlier than the former. `validPeriod` retains its resolved time offset.
`reportId`, `timezone`, prompt identity, model name, and comparison ID are
nonblank. `promptHash` and `dataPackage.sha256` are lowercase SHA-256 digests.
## Result Invariants
`total` is at least two and equals the number of results. Positions are
contiguous from one, profile IDs are distinct and nonblank, and
`succeeded + failed == total`.
A successful result has `status: "succeeded"`, `validationStatus: "passed"`,
a unique Markdown `reportPath`, and no `error`. A failed result has
`status: "failed"`, no `reportPath`, and an `error` object with nonblank
`category` and `message`. Its validation status is absent, `failed`, or
`skipped`. Error messages are valid UTF-8 and no longer than 1,024 bytes.
`backendId` and `validationStatus` are omitted when unavailable.
Every successful Markdown file is declared by exactly one successful result.
The directory contains no extra entries. Consumers can therefore verify the
data-package digest and the full manifest-to-file mapping without scanning a
larger workspace.
## Compatibility And Sensitivity
Weatherreporter recognizes a replaceable bundle only when it exactly satisfies
the current version, schema, file set, file types, relative-path rules, and
data-package digest. It rejects unknown manifest fields, multiple JSON values,
extra entries, symlinks, and future or otherwise unsupported versions. Treat a
bundle that fails recognition as an ordinary directory, not as a compatible
bundle.
The manifest contains safe operational provenance, but `data-package.yml` and
the generated Markdown can contain sensitive weather or location context. Do
not assume these artifacts are safe for public distribution. Handle retention,
access, and deletion according to the [operations guide](../operations.md).

View File

@@ -58,8 +58,8 @@ application to record.
Run and idempotency records are in-memory. Completed records expire according
to Distributor's `server.http.retention`, and a Distributor restart removes
retained status and idempotency state. Status polling decisions and persistence
of notification artifacts are internal orchestration behavior; see the
retained status and idempotency state. Status polling decisions are internal
orchestration behavior; see the
[Distributor adapter](../../internal/distributor-adapter.md) and
[application orchestration](../../internal/app-orchestration.md).

View File

@@ -8,15 +8,15 @@ the upload call returns.
## File Mappings
Every mapping pairs a managed Markdown report source with one bundle-relative
path. A single-report notification maps its one managed report to each rendered
Every mapping pairs an operator-owned Markdown output with one bundle-relative
path. A single-report notification maps its published output to each rendered
path configured for that report. A batch notification combines mappings for
every included managed report and rejects duplicate bundle paths.
every included published output and rejects duplicate bundle paths.
The report source is never an `--out` copy or an arbitrary workspace scan. The
application selects it and renders notification paths; see the [operations guide](../../operations.md)
for the managed-upload rule and the [Distributor adapter](../../internal/distributor-adapter.md)
for the adapter boundary.
The report source is the output selected for that command; the application does
not scan local directories. It renders notification paths after publication;
see the [operations guide](../../operations.md) and the
[Distributor adapter](../../internal/distributor-adapter.md) for the boundary.
Bundle paths must be clean, relative, slash-separated paths. They cannot be
empty or absolute, contain backslashes, empty segments, `.` or `..`, or use

View File

@@ -16,7 +16,7 @@ For each notification, Weatherreporter calls `UploadFiles` with:
- the rendered pipeline ID;
- the rendered bundle ID as the source manifest ID;
- the report or batch generation time as `Created`;
- the managed-report-to-bundle-path mappings described in the
- the published-output-to-bundle-path mappings described in the
[bundle mapping contract](pkg-bundle.md); and
- a rendered idempotency key.
@@ -38,8 +38,8 @@ adapter translates it to its own conflict error without exposing the token.
The adapter then calls `Status` for the accepted run. A terminal `failed`
status is a notification failure. A status lookup failure or a timeout before a
terminal status remains attached to the otherwise accepted upload as diagnostic
status information. Polling cadence, final failure handling, redaction, and
notification artifact persistence are internal behavior documented in the
status information. Polling cadence, final failure handling, and redaction are
internal behavior documented in the
[Distributor adapter](../../internal/distributor-adapter.md) and
[application orchestration](../../internal/app-orchestration.md).

View File

@@ -1,22 +1,50 @@
# Promptkit Integration
Weatherreporter uses Promptkit for all generated-text reports. The four logical prompts are
`weather.daily_generated_text`, `weather.today_generated_text`,
`weather.tomorrow_generated_text`, and `weather.hourly_generated_text`, each at version
`1.0.0`. Their prompt assets and generated-text JSON Schemas are embedded by
`internal/promptassets`.
Weatherreporter uses Promptkit for all generated-text reports. The four logical prompts are `weather.daily_generated_text`, `weather.today_generated_text`, `weather.tomorrow_generated_text`, and `weather.hourly_generated_text`, each at version `2.0.0`. Their prompt assets, generated-text JSON Schemas, and Weatherreporter profile catalog are embedded by `internal/promptassets`.
Before collection, Weatherreporter inspects the exact prompt version, requires one required
`data_package` input with content type `application/yaml`, and requires the report's JSON
Schema output contract. It selects `promptkit.profile` when configured, otherwise the
prompt's declared default profile. Profiles that require a direct API key are unsupported; a
profile that reports `APIKeyEnv` requires a nonblank value in that environment variable.
## Logical Profile Catalog
Execution receives the already-persisted YAML package, prepares it once, and returns structured
JSON that Weatherreporter validates before rendering its own Markdown template. Preparation and
execution receipts are project-owned, safe provenance records. Content-rich diagnostics are
opt-in through `--llm-debug-dir`; see [operations](../operations.md) for retention and permissions.
Prompt definitions select a stable Weatherreporter profile ID. The embedded definitions currently use Promptkit's `openrouter` backend:
Prompt/profile configuration is owned by the [configuration reference](../config.md). Adapter
construction and mapping are documented in the [Promptkit adapter internals](../internal/promptkit-adapter.md).
Durable metadata compatibility is described in [state internals](../internal/state.md).
| Profile ID | Model | Reasoning effort | Timeout | Service tier | Default reports |
| --- | --- | --- | --- | --- | --- |
| `weather-light` | `deepseek/deepseek-v4-flash` | Provider default | 180 seconds | `flex` | Hourly |
| `weather-balanced` | `~google/gemini-flash-latest` | `high` | 240 seconds | `flex` | Daily, Today, Tomorrow |
| `weather-deep` | `~anthropic/claude-sonnet-latest` | `high` | 240 seconds | `flex` | None |
The `~` prefix is part of each OpenRouter rolling-alias model ID. The embedded profiles intentionally omit endpoints, credentials, temperature, `top_p`, and output-token limits.
## Selection And Active Execution
Before weather collection, Weatherreporter validates the exact prompt version, output contract, and selected profile. A nonblank `promptkit.profile` selects one profile ID for every report in the command; otherwise the prompt's declared default selects it. Promptkit resolves the selected definition in this order:
1. explicit in-memory profiles used by an embedding consumer or test;
2. the configured `profile_file` or `profile_dir`;
3. Weatherreporter's embedded fallback profiles; and
4. Promptkit's built-in catalog.
A source falls through only when the selected ID is absent. Each source supplies a complete definition, so profile fields are not merged. A malformed matching operator definition is an error and does not fall back.
Profiles that require a direct API key are unsupported; a profile that reports `APIKeyEnv` requires a nonblank value in that environment variable. Active results retain the selected logical profile ID and resolved backend and model. Ordinary errors, summaries, logs, and outputs exclude endpoints, credentials, rendered messages, schemas, request bodies, response bodies, and complete parameter maps.
Promptkit receives the YAML data package as an inline input and returns structured JSON that Weatherreporter validates before rendering its own Markdown template. Safe active provenance remains in memory. Content-rich diagnostics are opt-in through `--llm-debug-dir`; see [operations](../operations.md) for retention and permissions.
## Comparison Execution
For `compare`, Weatherreporter validates one exact prompt and every explicitly
selected profile before weather collection. It prepares one deterministic YAML
data package, retains immutable copies of the report inputs, and executes every
profile against the same exact data-package bytes. Each profile remains an
independent Promptkit execution: one provider or validation failure does not
stop its peers, while caller cancellation applies to every in-flight execution.
Weatherreporter starts selected profile executions concurrently and does not
add an application-level concurrency limit. Promptkit owns backend capacity and
any profile or backend concurrency policy. The durable comparison output and
its compatibility rules are defined by the
[comparison bundle contract](comparison-bundle.md); the user-facing command
contract is in the [CLI reference](../cli.md).
The generated-text schemas require `summary`, `forecast_discussion`, and `precipitation_timing`, and reject additional properties. Prompts return an empty string for `precipitation_timing` when the deterministic package contains no precipitation windows.
Prompt/profile configuration and the maintained local override example are owned by the [configuration reference](../config.md). Adapter construction and mapping are documented in the [Promptkit adapter internals](../internal/promptkit-adapter.md).

View File

@@ -1,45 +1,57 @@
# Application Orchestration Internals
`internal/app` owns top-level generation, batch, collection, inspection, and
notification ordering after the CLI has parsed arguments and loaded configuration.
`internal/app` owns stateless report generation, batch execution, comparison
orchestration, atomic output publication, and notification coordination after
`internal/cli` has parsed arguments and loaded configuration. The user contract
is owned by the [CLI reference](../cli.md) and [operations guide](../operations.md).
## Generation
## Single-Report Flow
`GenerateDetailed` resolves one of the four report definitions, initializes an
optional debug root, and inspects the exact Promptkit prompt/profile before it
collects weather or writes managed state. It then builds facts and modules,
saves the YAML data package, persists preparation metadata from the executor
callback, executes the prepared prompt, saves execution provenance and raw
output, validates generated text, renders Markdown, and optionally copies or
notifies from the managed report.
`GenerateDetailed` resolves the requested report and output destination before initializing an optional explicit debug writer. An explicit output file wins; otherwise the configured output directory is used, falling back to the captured working directory. It validates the exact Promptkit prompt and selected profile before collecting weather data. The resolved profile, backend, and model are carried in the active result.
After a completed prompt run, each successfully written downstream artifact is
atomically added to the execution record before the corresponding metadata
rewrite. Later failures therefore leave the original Promptkit outcome and its
last durable set of reached paths inspectable.
The workflow builds facts, a module snapshot, briefing metadata, and the YAML prompt package in memory. It executes Promptkit, validates the returned generated text, builds a render context, and renders Markdown. `fileutil` atomically writes the completed Markdown to the selected output path. Only after that write succeeds does single-report notification run.
Failure results retain all safe paths reached so far. Validation rejection
persists raw output and execution provenance but does not render a report.
Failures return an active partial result with safe identity, profile, warning, validation, debug, and output information when available. After rendering and immediately before publication, the workflow checks for cancellation or deadline expiry. Any failure before publication leaves an existing destination unchanged. A notification failure retains the newly published output.
## Batches
`RunBatchDetailed` constructs a single debug writer and uses the request's
single executor. Before collection it inspects Today, Tomorrow, and Daily for
morning, or Tomorrow and Daily for evening, deduplicating effective profile
inspection. It then collects once, plans eligible Daily dates, and calls the
same prompt-generation core sequentially for each planned report. Per-report
notification is suppressed; a failed report does not stop later reports.
`RunBatchDetailed` selects an explicit output directory first, otherwise the configured directory and then the captured working directory. It does this before creating at most one explicit debug writer or validating prompt and profile candidates for the selected batch. It collects once, calculates the data-dependent plan, then validates and retains the final output path for every planned report before invoking the same generation core sequentially.
Batch notification is skipped when disabled or when any report failed.
Successful notification uses the completed managed report paths only. Batch
items retain preparation, execution, and optional debug paths when reached.
Each item has an independent result. A failed item does not stop later items; successful items retain their published output paths. Per-report notification is suppressed during a batch. Batch notification runs only after every planned report has published successfully. It is skipped when any item failed. Batch result counters count report items only; a batch notification failure is represented by the top-level notification result and still produces a failed batch outcome.
## Inspection And Boundaries
## Comparisons
Inspection loads persisted state only. It does not collect weather, invoke
Promptkit, or upload reports. The app coordinates project-owned contracts but
does not parse flags, load YAML, implement transport, construct provider SDKs,
or define report-period policy.
`CompareDetailed` validates ordered explicit profile IDs, resolves the report,
and preflights the exact bundle destination before initializing optional prompt
debugging, prompt inspection, or collection. It then inspects the one prompt
and every selected profile, collects once, and delegates shared report
construction to the prepared-report flow. It does not accept a notifier.
Once the destination is resolved, the partial result retains its absolute
output directory even when later preflight, debug initialization, inspection,
collection, or preparation fails. Every initialized result is finalized with a
finished timestamp. If prompt inspection succeeds before a later profile
inspection fails, the partial result retains the resolved prompt ID, version,
and hash. Artifact paths are added only after publication commits.
The comparison execution core starts each inspected profile independently,
keeps results in selection order, and waits for all started work. Independent
profile failures are recorded and do not stop peers. Context cancellation marks
unfinished work and prevents publication. Details of prepared values, execution
and debugging, and publication are documented in [prepared report
internals](prepared-report.md), [comparison execution
internals](comparison-execution.md), and [comparison publication
internals](comparison-publication.md).
When publication has committed its new bundle, application results contain the
absolute manifest, data-package, and successful report paths even if removal of
the previous sibling backup then fails. That cleanup failure is still returned
as an operational error rather than treating the new bundle as unpublished;
the returned error retains the recovery path and underlying filesystem cause.
## Boundaries And Verification
The package does not parse flags, load YAML, implement transport, construct provider SDKs, or define report-period policy. Prompt, profile, weather, and Distributor implementations remain behind project-owned contracts.
Focused checks:

View File

@@ -66,4 +66,4 @@ go test ./internal/briefing
```
Builders emit structured facts, never report prose. The app collects their
outputs into a module snapshot, and state persists that snapshot.
outputs into an in-memory module snapshot for prompt input and rendering.

View File

@@ -1,56 +0,0 @@
# Changes Internals
`internal/changes` deterministically compares a compatible prior module
snapshot with the current snapshot. It returns compact structured changes for
prompt input; it never reads state, finds a prior report, renders Markdown, or
compares generated text. Snapshot construction belongs to
[module internals](module.md), and prior-snapshot discovery belongs to
[state internals](state.md).
## Comparison inputs and output
Each comparator receives a prior snapshot, a current snapshot, and
`Thresholds`. A `Change` has a stable type and message plus previous and
current values where useful. Changes are sorted by type and then message, so
the same inputs always yield the same order.
Threshold values are supplied by application orchestration from the
[Recent Changes configuration](../config.md#recent_change); this package does
not load configuration or choose defaults. Numeric changes are emitted when
the absolute difference meets the configured threshold. Precipitation also
requires a change between its low, possible, likely, and high categories.
## Strategies
| Comparator | Required snapshot data | Compared values |
| --- | --- | --- |
| `CompareDaily` | `derived_daily_summary`, `derived_daypart_summaries` | Low and high temperature, daily precipitation probability and timing, peak gust, alerts, and aggregate indicators |
For daily comparison, `alert_digest` and `precip_timing` are optional: alerts
are compared when present, and timing is compared only when both snapshots
contain it.
The application selects a comparator only after state lookup establishes a
compatible prior snapshot. Daily, Today, and Tomorrow use the daily comparator.
Hourly reports do not produce a Recent Changes list.
## Missing data and failures
Required stanzas that are absent or cannot be decoded return an error with the
snapshot and stanza context. Optional stanzas may be absent. A snapshot with no
eligible predecessor is not a comparison failure: the caller supplies an empty
change list without invoking this package.
The package has no filesystem, transport, CLI, renderer, or persistence
behavior. It does not decide report compatibility or retain snapshots.
## Verification and invariants
Focused tests cover the daily strategy, threshold boundaries, indicator and
alert changes, and missing required stanzas:
```sh
go test ./internal/changes
```
Recent Changes always compare structured snapshot values, never report prose.

View File

@@ -1,26 +1,14 @@
# CLI Internals
`internal/cli` parses terminal arguments, loads configuration, constructs app
requests, and translates app results to bounded JSON summaries. The user
contract belongs in the [CLI reference](../cli.md).
`internal/cli` parses terminal arguments, loads configuration, constructs app requests, and translates app results to bounded JSON summaries. The public contract belongs in the [CLI reference](../cli.md).
The root `--version` flag reports the build version supplied by
`internal/buildinfo`. Tagged release builds replace its development default at
link time.
The root `--version` flag reports the build version supplied by `internal/buildinfo`. Tagged release builds replace its development default at link time.
For each `generate` or `run` action, `Runner` constructs one project-owned
Promptkit executor after configuration loads. It passes the executor and any
`--llm-debug-dir` request into the app. `run` accepts the debug flag as well
as `generate`; the app, not the CLI, secures and initializes the debug root.
For each `generate` or `run` action, `Runner` constructs one project-owned Promptkit executor after configuration loads. It captures an absolute working directory, resolves only a relative explicit output override against it, and passes the working directory, loaded configuration, resolved override, and any `--llm-debug-dir` request to the app. The raw configured fallback remains in the configuration for app-owned destination selection. `run` uses the same explicit-resolution rule for `--out-dir`.
Summaries include identity, status, safe artifact paths, and notification
provenance. They intentionally exclude module values, YAML package bodies, raw
generated text, rendered prompts, schemas, endpoints, credentials, and full
Distributor payloads. A failed action with a partial result still emits its
safe summary before its error is returned.
The CLI dispatches only generation and batch actions. It has no persisted-run or inspection dispatch. Summaries include report identity, status, output path, effective profile/backend/model, source warnings, validation, requested debug path, and notification result when available. They intentionally exclude prompt input, raw generated text, render context, endpoints, credentials, and full Distributor payloads. A failed action with a partial result still emits its safe summary before its error is returned.
CLI code owns no report policy, weather collection, persistence, provider
execution, or notification policy. Focused checks:
CLI code owns no report policy, weather collection, output publication, provider execution, or notification policy. Focused checks:
```sh
go test ./internal/cli

View File

@@ -0,0 +1,27 @@
# Comparison Execution Internals
The comparison execution core receives an already prepared report and an
already inspected, ordered profile list. It initializes an outcome for every
selected profile, launches each started profile in its own goroutine, and
waits for every started goroutine before returning. Results retain the supplied
selection order even though execution completes in an arbitrary order.
Every profile uses the exact inspected prompt identity and a private copy of
the same prepared data package. Provider, generated-text validation, rendering,
or debug-write failure becomes that profile's safe failed outcome and does not
cancel its peers. The application deliberately imposes no additional semaphore:
Promptkit owns backend capacity. Cancellation or a deadline marks unfinished
outcomes as skipped or failed, joins work, and prevents bundle publication.
When debugging is enabled, each execution receives a deterministic reference
derived from the comparison identity, ordered profile position, and safe
profile slug. This keeps concurrent captures separate. The debug writer itself
owns secure-root validation and file permissions. It safely creates shared
missing ancestors during concurrent writes, then rejects symlink and non-
directory components. Operational retention and sensitivity are documented in
the [operations guide](../operations.md).
The output result and its safe errors are converted into the durable contract
only by comparison publication. See [comparison publication
internals](comparison-publication.md) and the external [comparison bundle
contract](../integrations/comparison-bundle.md).

View File

@@ -0,0 +1,30 @@
# Comparison Publication Internals
`internal/comparison` separates the logical bundle from filesystem mechanics.
The application builds a validated manifest, exact shared data-package bytes,
and only the Markdown files for successful profiles. The durable layout,
schema, and compatibility rules are owned by the [comparison bundle
contract](../integrations/comparison-bundle.md).
Destination planning is read-only. It requires an exact absolute target that
is neither the filesystem root nor the working directory, rejects unsafe
symlinks and non-directories, accepts a missing or empty directory, and permits
replacement only for a recognized current bundle. Publication rechecks that
authorization immediately before it writes a private sibling staging directory.
For replacement, it moves the prior bundle to a private sibling backup,
reauthorizes that moved entry, and restores it if installing the new bundle
fails.
The new bundle is committed only after the staged directory has been installed
at the target. From that point its artifact paths are authoritative: a failure
to remove the retained sibling backup does not roll back the new bundle.
Publication returns an inspectable cleanup error with the absolute backup path
and underlying filesystem cause so an operator can recover or remove that
backup manually.
The application preflights before prompt inspection and collection, then
preflights again before publication. A cancellation or any failure before the
commit leaves the prior destination untouched. Completed bundles include
partial profile results; comparison publication never coordinates Distributor
notification. Operator-facing lifecycle and cleanup are in the
[operations guide](../operations.md).

View File

@@ -47,7 +47,7 @@ status, and `RunStatus`, including pipeline ID, lifecycle timestamps, report,
and remote error details.
Status lookup or polling errors are preserved in `UploadResult.StatusError` so
the caller can record an accepted-but-unconfirmed delivery. A terminal failed
the caller can report an accepted-but-unconfirmed delivery. A terminal failed
run returns that result and an error. Upload failures return no result. Upstream
idempotency conflicts become the local `IdempotencyConflictError`, which adds
endpoint, pipeline, bundle, idempotency, and file-path context while redacting

View File

@@ -60,5 +60,5 @@ go test ./internal/facts
```
Facts are derived once for a resolved report from already collected data.
They remain reusable structured values: prompt wording, state persistence,
prior-report comparison, and template presentation are owned elsewhere.
They remain reusable structured values for prompt input and template
presentation, which are owned elsewhere.

View File

@@ -46,9 +46,7 @@ UTC when these APIs are called directly. Optional narrative, discussion, and
alerts remain absent when their normalized products are absent.
Forecast thresholds used for brief indicators and precipitation timing are
implementation rules. User-configurable Recent Changes thresholds are applied
by [changes internals](changes.md), whose defaults are documented in
[configuration](../config.md).
implementation rules.
## Verification and invariants

View File

@@ -17,8 +17,9 @@ value and canonical normalized JSON, loads its canonical schema through
Daily, Today, and Tomorrow use a day-style value with required trimmed summary
and one or more nonblank discussion paragraphs. Hourly requires trimmed summary
and a single trimmed discussion string. Each form permits optional trimmed
precipitation-timing and confidence prose. Typed decoding rejects unknown JSON
and a single trimmed discussion string. Every form also requires the
`precipitation_timing` field; an empty string means there is no supported timing
prose to render. Typed decoding rejects missing required fields and unknown JSON
fields; no general-purpose JSON Schema engine is used at runtime.
## Render contexts
@@ -33,8 +34,8 @@ template iteration rather than maps.
Optional source stanzas become nil or fallback context fields. Missing required
stanzas, type-decoding failures, invalid metadata, or a generated-text type
that does not match the chosen handler fail before template execution. Prompt
packages, raw Promptkit output, state persistence, and template asset lookup
remain outside this package.
packages, raw Promptkit output handling, and template asset lookup remain
outside this package.
## Verification and invariants

View File

@@ -1,7 +1,7 @@
# Module Contract Internals
`internal/module` defines the stable envelope between report composition,
module builders, snapshots, comparisons, templates, and prompt packages. It
module builders, in-memory snapshots, templates, and prompt packages. It
does not define a report, execute a builder, or choose prompt-export policy;
those responsibilities belong to [report registry](report-registry.md) and
[briefing](briefing.md).
@@ -11,10 +11,10 @@ those responsibilities belong to [report registry](report-registry.md) and
Each `Output` has a module ID, stanza name, rich `Value`, and runtime-only
`PromptValue`. `DataPackageValue` returns the prompt value when present and
otherwise the rich value. This permits custom prompt exports without shrinking
the template and inspection value.
the template value.
`NewSnapshot` builds the ordered `weatherreporter.modules.v1` snapshot and
validates it. Snapshot JSON persists IDs, stanza names, and rich values only;
validates it. Its JSON representation contains IDs, stanza names, and rich values only;
`PromptValue` is deliberately excluded. `StanzaValue` decodes a named rich
stanza into a caller-supplied type, reporting a missing stanza separately from
a decoding error.
@@ -47,7 +47,7 @@ are validated by the briefing registry.
## Rich and prompt-facing values
Rich values remain available to snapshots, comparisons, and render contexts.
Rich values remain available to module snapshots and render contexts.
Briefing attaches custom prompt exports only for current conditions, hourly
forecast, and derived daypart summaries; all other current builders use
pass-through values. The prompt package owns how exported stanzas are grouped

View File

@@ -0,0 +1,20 @@
# Prepared Report Internals
`internal/app` builds a `preparedReport` after collection and before profile
execution. This is the immutable boundary shared by ordinary report generation
and profile comparison; it is not a durable artifact.
Preparation builds report facts, the configured module snapshot, briefing
metadata, the curated prompt-input package, serialized YAML, and the
generated-text definition. It deep-copies mutable facts, snapshots, metadata,
and data-package bytes before returning them. Consumers receive independent
copies so one execution cannot change another's input or rendering context.
Single-report generation executes one prepared profile and publishes its
Markdown. Comparison prepares once, gives every selected profile the same YAML
bytes, and only then assembles the resulting logical bundle. The prompt-input
shape is owned by [prompt-input internals](prompt-input.md); profile execution
semantics are owned by [Promptkit integration](../integrations/promptkit.md).
Preparation failure has no publication side effects. Tests for this boundary
cover mutation isolation, byte equality, and reuse by both execution paths.

View File

@@ -1,29 +1,16 @@
# Prompt Input Internals
`internal/promptinput` converts report metadata, an ordered module snapshot,
Recent Changes, and source warnings into the YAML `data_package` consumed by
Promptkit. It owns this package's schema, grouping, serialization, loading,
and validation—not weather collection, module construction, path choice, or
provider execution.
`internal/promptinput` converts report metadata, an ordered module snapshot, and source warnings into the YAML `data_package` supplied inline to Promptkit. It owns the package schema, grouping, serialization, loading, and validation; it does not choose an output destination, collect weather, execute a provider, or retain packages after a command ends.
## Package construction
## Package Construction
`Build` produces `weatherreporter.data_package.v3`. It copies the run ID;
report ID, variant, prompt ID, generation time, timezone, local current date,
and valid period; ordered briefing stanzas; Recent Changes; and source
warnings. A nil Recent Changes slice becomes an empty `items` list.
`Build` produces `weatherreporter.data_package.v4`. It copies the run ID; report ID, variant, prompt ID, generation time, timezone, local current date, and valid period; ordered briefing stanzas; and source warnings. Prompt input contains no historical comparison section.
Briefing starts as a flat snapshot order and stanza-value map. `Build` uses
each output's `DataPackageValue`, so runtime prompt exports take precedence and
rich values are used only as a fallback. Prompt exports are selected by the
[briefing registry](briefing.md), while the rich-versus-prompt contract is in
[module internals](module.md).
Briefing is a flat ordered set of stanza values. `Build` uses each output's `DataPackageValue`, so curated prompt exports take precedence and rich values are used only as a fallback. Prompt exports are selected by the [briefing registry](briefing.md), while the rich-versus-prompt contract is in [module internals](module.md).
## YAML ordering and grouping
## YAML Ordering And Validation
Serialization keeps `metadata` directly under `briefing`. Every other known
stanza is placed in exactly one category, emitted in category order and in its
original snapshot order within that category:
Serialization keeps `metadata` directly under `briefing`. Every other known stanza is placed in one category and emitted in category order while preserving its original module order:
| Category | Current stanzas |
| --- | --- |
@@ -32,30 +19,10 @@ original snapshot order within that category:
| `narrative_products` | narrative forecast, discussions, and weather story |
| `raw_data` | current conditions and hourly forecast |
This YAML presentation does not alter the flat snapshot model. `LoadYAML`
accepts the same category layout and reconstructs flat `Order` and `Values`,
rejecting misplaced, duplicate, unknown, or uncategorized stanzas.
`LoadYAML` accepts this layout and reconstructs the flat order and values. It rejects misplaced, duplicate, unknown, or uncategorized stanzas. `Validate` requires the v4 schema version, report identity and period fields, and at least one ordered briefing stanza. `MarshalYAML` and `LoadYAML` validate their result. `Save` remains a reusable atomic-file helper for callers that explicitly need one; normal application execution passes marshalled YAML directly to Promptkit.
## Validation and persistence
`Validate` requires the current schema version, run and report identifiers,
prompt ID, generation timestamp, timezone, current local date, valid period,
and at least one ordered briefing stanza. It rejects duplicate stanza names,
missing values, and a missing category for every non-metadata stanza.
`MarshalYAML` and `LoadYAML` validate their result. `Save` writes the serialized
YAML atomically; managed workspace paths are owned by [state internals](state.md).
Generated-text artifacts and template render contexts are later workflow
artifacts, not members of this package.
## Verification and invariants
Focused tests cover construction, curated exports, category ordering, YAML
round trips, invalid layout, validation, and atomic saves:
Focused tests cover construction, curated exports, category ordering, YAML round trips, invalid layout, validation, and atomic saves:
```sh
go test ./internal/promptinput
```
The package is narrower than a template render context and never infers changes
from report prose.

View File

@@ -1,18 +1,12 @@
# Promptkit Adapter Internals
`internal/adapters/promptkit` maps Weatherreporter's project-owned executor contract to Promptkit.
The CLI maps `promptkit` configuration to a `PromptExecutorConfig` and constructs one executor
per action. Promptkit dependency types do not escape the adapter.
`internal/adapters/promptkit` maps Weatherreporter's project-owned executor contract to Promptkit. The CLI maps `promptkit` configuration to a `PromptExecutorConfig` and constructs one executor per action. Promptkit dependency types do not escape the adapter.
The adapter exposes exact prompt and profile inspection plus prepared execution. It maps Promptkit
inspection values to project-owned prompt input, output-contract, profile, preparation, execution,
validation, and optional debug values. It classifies adapter failures without copying provider secrets
or unbounded response bodies into application errors or normal state.
The adapter supplies Weatherreporter's embedded prompt, schema, and fallback profile filesystems to each engine. Promptkit resolves configured operator profile sources, the embedded fallback catalog, and its built-in catalog; the adapter does not parse profile YAML, merge sources, or probe endpoints.
The app calls the executor's preparation callback before provider execution to persist safe preparation
provenance. Completed executions are then persisted as safe execution provenance and raw generated text
is validated by `internal/generatedtext`. The adapter does not write workspace state, render Markdown,
choose report definitions, or send Distributor notifications.
The adapter exposes exact prompt and profile validation plus prepared execution. It maps safe prompt identity, logical profile, effective backend/model, preparation, execution, validation, and optional debug values into `promptexec`. `Execute` passes the YAML package as an inline Promptkit input; it does not construct a filesystem URI or write a package file.
The application uses the preparation callback to record active safe provenance in memory and optionally writes content-rich diagnostics only through an explicit debug writer. The adapter returns raw output for application validation and rendering. It does not retain application state, render Markdown, choose report definitions, or send Distributor notifications.
Focused tests:
@@ -20,5 +14,4 @@ Focused tests:
go test ./internal/adapters/promptkit ./internal/cli ./internal/app
```
The public logical prompt/profile/schema contract is owned by the
[Promptkit integration guide](../integrations/promptkit.md).
The public logical prompt/profile/schema contract is owned by the [Promptkit integration guide](../integrations/promptkit.md).

View File

@@ -1,69 +1,30 @@
# Report Registry Internals
`internal/report` owns the registry of report identities and the data declared
for each one: resolution, prompt identity and version, comparison policy,
artifact group, output-copy name, default module composition, and Distributor
path declarations. The public command syntax is owned by the
[CLI reference](../cli.md); configuration aliases and overrides are owned by
the [configuration reference](../config.md).
`internal/report` owns report identities, valid-period resolution, exact prompt identity and version, output names, default module composition, and Distributor path declarations. Public command syntax belongs in the [CLI reference](../cli.md); configuration aliases and overrides belong in the [configuration reference](../config.md).
## Definitions and resolution
## Definitions And Resolution
Each `Definition` declares a stable ID and display name, prompt ID, generation
version, template and generated-text schema IDs, valid-period resolver,
comparison strategy, artifact group, batch-copy filename, Distributor path
templates, generation eligibility, compatible prior IDs, default modules, and
batch eligibility flags. `Resolved` combines that definition with the valid
period and run metadata for one invocation.
Each `Definition` declares a stable ID and display name, prompt ID and version, template and generated-text schema IDs, valid-period resolver, default output name, Distributor path templates, module list, and fixed batch eligibility. `Resolved` combines a definition with one valid period and run identity.
| Report ID | Prompt version | Period policy | Comparison | Registry batch flag | Output copy |
| Report ID | Prompt version | Default profile | Period policy | Fixed batch flag | Default output |
| --- | --- | --- | --- | --- | --- |
| `daily` | `1.0.0` | Explicit local civil day | Same valid date | Dynamic Daily inclusion is app-owned | `daily.md` |
| `today` | `1.0.0` | Selected or current local civil day | Same valid date | Morning | `today.md` |
| `tomorrow` | `1.0.0` | Next local civil day | Same valid date | Evening | `tomorrow.md` |
| `hourly` | `1.0.0` | Rolling six-hour interval | Rolling window | — | `hourly.md` |
| `daily` | `2.0.0` | `weather-balanced` | Explicit local civil day | Dynamic Daily inclusion is app-owned | `daily-YYYY-MM-DD.md` |
| `today` | `2.0.0` | `weather-balanced` | Selected or current local civil day | Morning | `today.md` |
| `tomorrow` | `2.0.0` | `weather-balanced` | Next local civil day | Evening | `tomorrow.md` |
| `hourly` | `2.0.0` | `weather-light` | Rolling six-hour interval | — | `hourly.md` |
Each report pairs its ID and prompt version with matching template and schema
IDs. Exact template fields and schema assets belong to [report templates](../templates.md)
and [generated-text internals](generatedtext.md).
Daily derives its filename from the resolved valid-period start in the effective timezone, so multiple Daily items have distinct destinations. Exact template fields and schema assets belong to [report templates](../templates.md) and [generated-text internals](generatedtext.md). Prompt assets own default profile selection; the registry stores no provider setting.
All valid periods are half-open.
## Collaborators And Boundaries
## Registry collaborators
`DefaultRegistry`, `Lookup`, `Resolve`, and report-name helpers prevent callers from duplicating report identity rules. Registry overrides clone a recognized definition and replace its module list. `DistributorPathTemplates` are consumed by app orchestration; their rendered external bundle-path contract is documented in the [Distributor bundle guide](../integrations/distributor/pkg-bundle.md).
`DefaultRegistry` is the only source of the four report definitions.
`Lookup`, `Resolve`, and report-name helpers prevent callers from duplicating
report identity rules. Registry overrides clone a definition and replace its
module list only after the report ID is recognized.
`morning` and `evening` are registry-owned batch names. Fixed flags declare Today and Tomorrow eligibility; app orchestration determines data-dependent Daily membership and the actual batch plan.
The definition's `DistributorPathTemplates` are internal declarations consumed
by app orchestration. Their rendered external bundle paths and compatibility
contract are documented in the [Distributor bundle guide](../integrations/distributor/pkg-bundle.md), not repeated here.
The registry never collects weather data, parses CLI flags, writes output, executes Promptkit, or delivers a report.
`morning` and `evening` are registry-owned batch names. Registry flags declare
fixed report eligibility; app orchestration determines data-dependent Daily
membership and produces the actual batch plan.
## Module composition and failures
Each definition supplies an ordered `[]module.ConfigItem`; the complete
report-to-module mapping is maintained in [module internals](module.md).
`ArtifactGroup`, `BatchOutputName`, and comparison compatibility
are likewise consumed by state and orchestration rather than recomputed there.
Unknown report IDs or batch names return errors. The registry never collects
weather data, builds modules, parses CLI flags, writes state, executes
Promptkit, or delivers a report.
## Verification and invariants
Focused tests cover definition completeness, command and alias lookup, period
resolution, run IDs, path declarations, composition defaults, and override
validation:
Focused tests cover definition completeness, command and alias lookup, period resolution, run IDs, output names, composition defaults, and override validation:
```sh
go test ./internal/report
```
All report selection goes through the registry, and the registry is the source
of truth for report identity—not rendered report text or app-local constants.

View File

@@ -1,56 +0,0 @@
# State Internals
`internal/state` owns safe workspace paths, atomic artifact writes, metadata,
prior-snapshot lookup, and read-only inspection. Operators should use the
[operations guide](../operations.md) for lifecycle and retention.
## Artifact Paths
For each run, paths are grouped by artifact group and valid start date:
| Artifact | Location |
| --- | --- |
| Module snapshot | `snapshots/<group>/<date>/modules.<run-id>.json` |
| Metadata | `snapshots/<group>/<date>/metadata.<run-id>.json` |
| Data package | `data-packages/<group>/<date>/data_package.<run-id>.yaml` |
| Prompt preparation | `preflight/<group>/<date>/prompt_preparation.<run-id>.json` |
| Prompt execution | `snapshots/<group>/<date>/prompt_execution.<run-id>.json` |
| Raw generated text | `snapshots/<group>/<date>/generated_text_raw.<run-id>.json` |
| Validated generated text | `snapshots/<group>/<date>/generated_text.<run-id>.json` |
| Render context | `snapshots/<group>/<date>/render_context.<run-id>.json` |
| Managed report | `reports/<group>/<date>/report.<run-id>.md` |
| Notification | `notifications/<group>/<date>/distributor.<run-id>.json` |
Batch notification records are `notifications/batches/<batch>/<local-date>/distributor.<batch-run-id>.json`.
## Metadata And Debug Storage
New metadata is `weatherreporter.metadata.v2` and gains preparation and
execution paths only after those artifacts are saved. Legacy V1 records remain
readable; their historic preflight and generated-text-result fields are mapped
to the corresponding preparation and execution views during inspection. New
runs never write V1 records.
Prompt preparation and execution records are validated on both save and load.
They require exact report/prompt identity, complete timing, internally
consistent provenance, and status-appropriate validation or bounded classified
errors. Completed execution provenance keeps Promptkit's run identity distinct
from the Weatherreporter run identity.
For a completed prompt run, the execution record is atomically replaced after
each downstream artifact is saved. Its path set therefore records the raw and
normalized generated text, render context, managed report, requested output
copy, and notification artifact actually reached without changing the original
Promptkit outcome.
`PromptDebugWriter` is separate from workspace state. An empty root disables
it. An enabled absolute root is checked for safe directories and symlinks, then
stores `preparation.json` and `execution.json` beneath
`<root>/<report-id>/<valid-date>/<run-id>/`. Directories are `0700`; files are
atomic `0600`. Normal state discovery does not read this root.
Focused checks:
```sh
go test ./internal/state
```

View File

@@ -1,7 +1,7 @@
# Weather Data Internals
`internal/weatherdata` owns the normalized, wire-independent weather bundle
that passes from collection through rendering and persistence. The Weather API
that passes from collection through rendering. The Weather API
adapter translates provider responses into these types; its request, response,
and availability contract is documented in the
[Weather API integration guide](../integrations/weatherapi.md).
@@ -54,8 +54,7 @@ local provenance and whole-run consumers see it. A policy that treats a missing
source as an error returns no partial bundle.
Warnings describe data completeness, not rendering or delivery failures.
Those failures are recorded by the application and state layers; see
[application orchestration](app-orchestration.md) and [state internals](state.md).
Those failures are reported by [application orchestration](app-orchestration.md).
## Boundaries and verification

View File

@@ -1,29 +1,124 @@
# Weatherreporter Operations
This guide covers normal operation, managed workspace state, inspection,
recovery, and operational caveats. See the [CLI reference](cli.md) for complete
command syntax and the [configuration reference](config.md) for fields,
defaults, and notification templates. For symptom-based diagnosis, see
[Troubleshooting](troubleshooting.md).
This guide covers normal output handling, Distributor notification, secure
prompt diagnostics, and cleanup of legacy application state. See the [CLI
reference](cli.md) for command syntax and the [configuration reference](config.md)
for fields, defaults, and notification templates.
## Normal Operation
After configuring a Weather API endpoint, generate one report:
```sh
weatherreporter generate today --out ./today.md
weatherreporter generate today
```
A generation collects weather data, resolves the report period, builds and
persists the module snapshot and prompt data package, records Promptkit
preparation provenance before provider execution, then persists raw output and
execution provenance, validates the structured generated text, and renders the
managed Markdown report from the validated text and deterministic values.
With no configured output directory, the command writes `today.md` in the
current directory. Set `output.directory` to use one ordinary publication
directory for reports, or choose a one-command operator-owned file with
`--out`; a relative path is resolved from the current directory and an absolute
path is used directly. The explicit flag takes precedence over the configured
directory. Weatherreporter renders in memory and atomically replaces the
selected destination only after generation and rendering succeed. It does not
create a default workspace, metadata, receipts, or intermediate output files.
The managed report and its final metadata are saved before single-report
Distributor notification is attempted. `--out` writes an extra operator copy;
it never changes the managed report or upload source. A successful generate
command prints its summary to stdout unless `--quiet` is used.
A missing configured directory is created only as part of successful report
publication. If its existing path is not a directory or cannot be inspected,
the command stops before prompt inspection or weather collection, leaving any
existing report unchanged. See the [configuration reference](config.md) for the
field definition and validation rules.
Before a destination is published, provider, validation, rendering, write, and
cancellation failures leave an existing report unchanged. A notification
failure happens after publication, so retain and use the completed Markdown
file while resolving the delivery error. The JSON result identifies the
absolute output path and active profile, backend, model, warnings, validation,
debug, and notification information; see the [CLI reference](cli.md) for its
exact fields.
## Batch Outputs And Distributor Notification
Run a scheduled batch with an explicit output directory when appropriate:
```sh
weatherreporter run morning --out-dir ./reports
```
Without `--out-dir`, batch reports are written beneath `output.directory` when
configured, otherwise the current directory. The explicit directory applies
only to that command and takes precedence over the configured fallback.
Morning runs Today, Tomorrow, and every eligible dated Daily Report; evening
runs Tomorrow and the same eligible Daily Reports. Eligible Daily dates begin
after tomorrow and require complete hourly coverage for their local civil day.
A batch collects once, determines the complete report set, and validates every
final output destination before executing its first report prompt. A destination
collision, such as a directory named `tomorrow.md`, stops the batch before any
report output is created or replaced. After successful validation, each selected
report processes independently and successful outputs remain available if
another report fails.
When `notify.distributor.enabled` and batch notification are enabled,
Weatherreporter sends one Distributor upload only after every selected output
exists. If an item fails, the batch notification is skipped and successful
files remain at their selected destinations. A batch notification failure also
leaves all successfully published report files in place. Distributor source
files are those operator-owned Markdown outputs; rendered bundle paths and
delivery status appear in the result, not in a local notification receipt.
Report counters count report items only. A batch notification failure therefore
returns a failed batch status even when all report counters show success; the
top-level notification result contains the delivery diagnostic.
For a single report, Distributor notification follows the atomic output write.
See the [configuration reference](config.md) for pipeline, bundle,
idempotency-key, and per-report path templates.
## Comparison Bundles
Use `compare` when an operator needs to evaluate explicit Promptkit profiles
against the same report input. The command writes one flat, operator-owned
bundle directory and never sends a Distributor notification. Command syntax,
profile validation, JSON output, and exit behavior belong to the
[CLI reference](cli.md); the durable file contract belongs to the
[comparison bundle contract](integrations/comparison-bundle.md).
The output destination follows the normal `output.directory` fallback. An
explicit `--out-dir` takes precedence and names the exact bundle directory,
not a parent to be combined with another name. The standard names are derived
from the report output name, such as `comparison-today` and
`comparison-daily-2026-05-29`; see the [configuration reference](config.md)
for output-directory resolution.
A comparison bundle contains the shared data package, a manifest, and one
Markdown file for every successful profile. Treat all of these files as
potentially sensitive: the data package and generated reports can contain
location or forecast context. Weatherreporter creates no application-owned
history, retention store, or cleanup job. Retain, archive, or remove only the
specific bundle directories your operating policy permits.
The destination is preflighted before prompt inspection and collection, then
rechecked immediately before an atomic publish. A missing or empty directory
is usable. A nonempty directory can be replaced only when `--replace` is given
and it is recognized as a current Weatherreporter comparison bundle; ordinary
directories, symlinks, and unsafe destinations are rejected. Cancellation and
all failures before publication preserve an existing bundle. Profile failures
are different: the command publishes a complete partial bundle, with failed
profiles represented in the manifest and no Markdown file for those profiles.
## Local Prompt Profile Override
Hourly normally selects the embedded `weather-light` profile. To use a local
OpenAI-compatible model without changing prompts or application code, copy
[weather-light-local-profile.yml](../examples/weather-light-local-profile.yml),
set its `endpoint` and `model` for the local server, and configure the copy as
`promptkit.profile_file`. The profile file's `weather-light` definition
completely replaces the embedded definition; it does not affect a report that
selects another profile ID.
Prompt and profile validation occurs before weather collection. A malformed
profile file, missing required credential, or unsupported selected backend
stops the command before collection. A reachable profile can still fail later
if its local model endpoint is unavailable; Weatherreporter does not switch to
a remote profile.
## Optional Prompt Debug Capture
@@ -33,146 +128,55 @@ Use `--llm-debug-dir` only when content-rich prompt diagnostics are required:
weatherreporter generate today --llm-debug-dir /var/tmp/weatherreporter-debug
```
The directory must be absolute and is initialized before prompt inspection or
weather collection. Capture files are stored outside the managed workspace,
with restrictive permissions, under the report ID, valid date, and RunID.
They can contain rendered prompts and generated output, so the normal metadata,
CLI summary, and routine logs contain only the optional directory path—not
their content. A capture-write failure stops that run before later work can
continue.
The directory must be absolute. Requested captures are written with restrictive
permissions beneath the supplied directory, organized by report and run. They
can contain rendered prompts and generated output, so limit access to trusted
operators and remove the captures when they are no longer needed. Comparison
captures additionally identify each selected profile so concurrent executions
remain distinct. Normal output, summaries, and routine logs omit that sensitive
content. Debug capture is never created for an ordinary command without
`--llm-debug-dir`.
Run a scheduled batch with the same configured collection:
If capture creation or writing fails, the affected run fails rather than
silently continuing without the requested diagnostics.
## Diagnosing Failures
Start with the command error and JSON summary. For a report generation failure,
the selected destination was not replaced; for a notification failure, inspect
the completed destination and the notification result. For a batch failure,
use the per-report statuses and retain successful output files. For a comparison
failure, inspect the published manifest when its path is present: individual
profile failures retain their safe result and successful Markdown files, while
cancellation and pre-publication errors leave the prior destination unchanged.
If a replacement commits but cleanup of its prior sibling backup fails, the new
bundle remains valid and its artifact paths appear in the failed command
summary. The summary records a safe `publication_cleanup` error, while the
returned command error reports the retained backup path. Preserve that backup
until it has been inspected and cleaned up manually; do not remove the new
bundle to retry that cleanup.
Enable explicit debug capture only when content-rich Promptkit diagnostics are
necessary.
Weatherreporter does not retain runs for later inspection, resume failed work,
or provide automatic cleanup, archival, remote state, daemon operation, or
automatic storm monitoring.
## Manual Cleanup Of Legacy Workspaces
Older installations may have a directory named `workspace` containing reports,
snapshots, prompt inputs, or notification records from previous versions.
Current commands neither read nor update it. After confirming that no separate
retention requirement applies, remove that specific legacy directory manually;
do not use a broad cleanup command that could remove current operator outputs.
For example, from the directory that contains the old directory:
```sh
weatherreporter run morning --out-dir ./reports --llm-debug-dir /var/tmp/weatherreporter-debug
rm -rf ./workspace
```
Each batch validates its configured prompt/profile candidates, then collects once before it plans reports. Morning runs Today, Tomorrow,
and every eligible dated Daily Report; evening runs Tomorrow and the same
eligible Daily Reports. Eligible Daily dates begin after tomorrow and require
complete hourly coverage for their entire local civil day. A batch continues
after an individual report fails and returns an aggregate failure when any
report or batch notification fails.
`--out-dir` writes extra copies such as `today.md`, `tomorrow.md`, and
`daily-YYYY-MM-DD.md`. These copies are never upload sources. Batch report
copies and notification behavior are summarized in the CLI result; use the
[CLI reference](cli.md) for its exact JSON and stderr contract.
## Managed Workspace
The default workspace root is `workspace`. Artifact paths use the report
definition's artifact group, the valid-period start date in the effective
timezone, and the RunID:
```text
workspace/
reports/<artifact_group>/<YYYY-MM-DD>/report.<run_id>.md
snapshots/<artifact_group>/<YYYY-MM-DD>/modules.<run_id>.json
snapshots/<artifact_group>/<YYYY-MM-DD>/metadata.<run_id>.json
snapshots/<artifact_group>/<YYYY-MM-DD>/generated_text_raw.<run_id>.json
snapshots/<artifact_group>/<YYYY-MM-DD>/prompt_execution.<run_id>.json
snapshots/<artifact_group>/<YYYY-MM-DD>/generated_text.<run_id>.json
snapshots/<artifact_group>/<YYYY-MM-DD>/render_context.<run_id>.json
data-packages/<artifact_group>/<YYYY-MM-DD>/data_package.<run_id>.yaml
preflight/<artifact_group>/<YYYY-MM-DD>/prompt_preparation.<run_id>.json
notifications/<artifact_group>/<YYYY-MM-DD>/distributor.<run_id>.json
notifications/batches/<batch>/<YYYY-MM-DD>/distributor.<batch_run_id>.json
```
The generated-text and render-context artifacts are written for every completed
single-report generation.
A report's metadata links the module snapshot, data package, preparation and
execution receipts, managed report, generated-text artifacts, and any available single-report
notification artifact. Batch notification artifacts are separate batch-level
records under `notifications/batches`.
RunIDs begin with the UTC generation timestamp and report ID. A Daily RunID
also contains its local valid date so multiple Daily reports in one batch have
different managed paths. Batch notification RunIDs contain the UTC batch start
timestamp and batch name.
## Distributor Notification
When `notify.distributor.enabled` is enabled, a successful `generate`
uploads only the managed Markdown report after final metadata has been saved.
The extra copy from `--out` is never uploaded. A notification attempt writes
a redacted debug artifact at
`notifications/<artifact_group>/<YYYY-MM-DD>/distributor.<run_id>.json`; its
path is then recorded in report metadata.
Batches suppress per-report notification. When both Distributor and its batch
notification are enabled, Weatherreporter submits one multi-report upload after
every planned report succeeds. If any report fails, it records a top-level
`skipped` notification with reason `one or more reports failed` and does not
call Distributor. If batch notification is disabled, a batch does not fall back
to individual uploads.
A batch notification attempt writes
`notifications/batches/<batch>/<YYYY-MM-DD>/distributor.<batch_run_id>.json`.
A notification failure makes the batch fail but does not change successful
individual report items into failed items. The debug artifacts contain rendered
identifiers, managed source and bundle paths, upload and status results, and
redacted errors; they do not contain tokens.
## Inspecting Stored Runs
Inspection is read-only: it neither collects weather data nor invokes
Promptkit or Distributor. Start by finding a RunID:
```sh
weatherreporter inspect reports --limit 10
weatherreporter inspect metadata RUN_ID
```
| Command | Reads |
| --- | --- |
| `inspect reports` | Metadata files under the workspace snapshots tree. |
| `inspect metadata RUN_ID` | Metadata located by RunID. |
| `inspect modules RUN_ID` | The module snapshot path recorded in metadata. |
| `inspect data-package RUN_ID` | The data-package path recorded in metadata. |
| `inspect prior RUN_ID` | The run metadata, then compatible earlier metadata for its comparison policy. |
| `inspect sources RUN_ID` | Source provenance and warnings in the run metadata. |
A missing snapshots directory produces no listed reports. An unknown or empty
RunID is an error; use `inspect reports` to obtain a valid value.
New runs write `weatherreporter.metadata.v2`, including preparation and
execution references once those receipts exist. `inspect metadata` also reads
historic V1 records; their legacy preflight and generated-text-result fields
remain visible for compatibility, but Weatherreporter does not write them for
new runs.
## Recovery
Keep the workspace when a run fails: artifacts reached before the failure
remain available where they can be safely persisted.
- A preparation failure can leave its classified receipt and metadata.
- A report-generation failure can leave the managed report, module snapshot,
data package, and metadata.
- A completed prompt validation rejection leaves raw text, an execution receipt,
and metadata. Later generated-text failures can also leave validated text and
a render-context artifact, depending on where they stopped.
- A single-report notification failure preserves the report and final metadata,
including its notification artifact when it was written.
- A batch notification failure preserves each report's artifacts and adds the
top-level batch notification artifact.
Use the RunID from the action summary with the inspection commands above. For
a batch failure, inspect the summary first, then inspect the affected report
RunIDs or the batch notification path. Do not remove the whole workspace as a
first response; retain it until the failure is understood.
## Operational Caveats
- Workspace files and generated reports can contain
sensitive operational context. Set appropriate filesystem permissions and do
not publish them unintentionally.
- Weatherreporter uses one configured Weather API endpoint and local workspace
state.
- It does not provide automatic resume, cleanup, archival, remote state, daemon
operation, or automatic storm monitoring.
This removal cannot be recovered by Weatherreporter. Keep or archive any
historical files that are still needed before deleting them.

View File

@@ -10,29 +10,36 @@ package inventory; focused documents in `docs/internal/` own implementation deta
Weatherreporter is a deterministic weather-report CLI. It collects normalized
weather data, derives facts and modules, builds a curated YAML data package,
compares prior snapshots, executes exact-version Promptkit prompts, validates
structured generated prose, and renders repository-owned Markdown. Completed
managed Markdown may be uploaded through Distributor.
executes exact-version Promptkit prompts, validates structured generated prose,
and renders repository-owned Markdown in memory. Completed Markdown is
atomically published to an operator-owned output destination and may then be
uploaded through Distributor.
An explicit profile comparison prepares one report input once, executes the
same exact prompt and data package across selected profiles concurrently, and
atomically publishes one operator-owned comparison bundle. It remains local:
it does not create application state or send a Distributor notification.
The supported report products are Daily, Today, Tomorrow, and Hourly. A batch
collects once, validates its complete candidate prompt/profile set before
collection, then executes planned reports sequentially with one executor. It
continues after independent report failures and sends a batch notification only
after every planned report succeeds.
collection, then determines and validates every planned output destination
before executing reports sequentially with one executor. It continues after
independent report failures and sends a batch notification only after every
planned report succeeds.
## Ownership And Boundaries
- `internal/cli` owns command parsing, help, summaries, and one executor
construction per action.
- `internal/config` owns defaults, loading, validation, and secret loading.
- `internal/app` owns workflow order, partial results, and notification
coordination through project-owned contracts.
- `internal/app` owns in-memory workflow order, partial results, atomic output
publication, and notification coordination through project-owned contracts.
- `internal/comparison` owns comparison identity, durable logical bundle
validation, safe destination recognition, and atomic bundle publication.
- Deterministic domain packages own weather derivation, report periods, modules,
generated-text validation, and template contexts.
- `internal/adapters/weatherapi`, `internal/adapters/promptkit`, and
`internal/adapters/distributor` own their external dependency mechanics.
- `internal/state` owns workspace paths, V2 metadata, atomic persistence, and
read-only inspection.
Dependency-specific Promptkit types remain inside its adapter. The application
does not parse flags, construct provider clients, or render provider output
@@ -41,28 +48,42 @@ directly.
## Prompt Execution Invariants
- Prompts receive curated module packages, never unbounded raw weather payloads.
- Every execution inspects the exact prompt version and output contract before
- Every execution validates the exact prompt version and output contract before
collection. The selected profile is configured explicitly or declared by the
prompt; unsupported direct-key profiles and missing reported credentials fail
before collection.
- Prepared execution persists safe preparation provenance before provider work.
Completed execution persists safe execution provenance; raw output is
validated before template rendering.
- Prompt and profile validation completes before weather collection. Raw output
is validated before template rendering.
- Comparison validates every explicit profile before collection, prepares one
immutable report input, and delegates backend capacity to Promptkit rather
than adding an application-wide execution limit.
- Generated text fills defined prose slots only. Deterministic facts remain
authoritative and repository-owned templates produce all managed Markdown.
authoritative and repository-owned templates produce all Markdown output.
- Sensitive rendered prompts, schemas, input bodies, provider endpoints, and
credentials never enter normal metadata, summaries, logs, or workspace
artifacts. They are written only to an explicit secure debug root when
requested.
credentials never enter normal summaries or logs. They are written only to
an explicit secure debug root when requested.
## State, Notification, And Testing Invariants
## Output, Notification, And Testing Invariants
- Managed writes are atomic where practical and stay beneath the configured
workspace root. Reached artifacts remain inspectable after later failures.
- New records use `weatherreporter.metadata.v2`; V1 records remain readable for
inspection compatibility.
- Distributor uploads use only the managed Markdown report, never output copies
or workspace scans. Notification follows report and final metadata success.
- Normal execution is stateless: it keeps weather data, prompt input, generated
text, and render context in memory and creates no application-owned durable
state.
- Markdown writes are atomic at an operator-selected destination. A
pre-publication failure, including cancellation observed immediately before
publication, does not replace an existing destination; a notification failure
does not remove a newly published output.
- Configuration or explicit CLI input selects that operator-owned destination;
it does not create an application-owned state boundary.
- Comparison bundles are flat, versioned operator outputs. Their guarded
replacement accepts only a recognized current bundle; cancellation and every
pre-publication failure preserve a prior bundle, while individual profile
failures can publish a complete partial bundle.
- Distributor uploads use only the published Markdown output, never a scan of
local files. Single notification follows publication; batch notification
follows publication of every selected report. Batch counters describe report
outcomes only; a failed batch notification is represented separately at the
batch level.
- Comparison never invokes Distributor notification.
- Default tests are deterministic, offline, and use Promptkit/provider fakes
rather than live provider calls. See the [testing policy](testing.md).

View File

@@ -85,9 +85,8 @@ mechanisms, not secret values.
| Release procedure | `docs/release.md` | Version policy, release preparation, validation, tagging, automated publication, verification, failure handling, and release ordering. | General contributor workflow, product contracts, release-specific change summaries, and implementation history. |
| Release notes | `docs/releases/` | One versioned, changelog-style summary for each release, including compatibility and operator action. The file at the tagged commit supplies the corresponding Gitea release body. | Current CLI, configuration, operations, integration, architecture, and internal contracts; release procedure; implementation plans. |
| CLI contract | `docs/cli.md` | Commands, arguments, flags, invocation semantics, stdout and stderr behavior, summaries, and exit behavior. | Configuration field definitions, complete operating procedures, runtime filesystem layout, and command implementation. |
| Configuration contract | `docs/config.md` | Discovery and precedence, fields, defaults, secrets, validation rules, and user-selectable values. | Complete example files, CLI syntax, runtime state lifecycle, and loading implementation. |
| Operations | `docs/operations.md` | Normal workflows, physical workspace layout, artifacts and metadata, inspection, notification behavior, recovery, cleanup, permissions, and operational caveats. | Complete CLI syntax, configuration field definitions, logical external contracts, and implementation mechanics. |
| Troubleshooting | `docs/troubleshooting.md` | Recurring symptoms, likely causes, diagnostic steps, safe fixes, and links to normal-operation references. | Complete command and configuration references, routine operating procedures, and implementation detail. |
| Configuration contract | `docs/config.md` | Discovery and precedence, fields, defaults, secrets, validation rules, and user-selectable values. | Complete example files, CLI syntax, output lifecycle, and loading implementation. |
| Operations | `docs/operations.md` | Normal output handling, atomic replacement, notification behavior, diagnosis, explicit debug capture, manual legacy-workspace cleanup, permissions, and operational caveats. | Complete CLI syntax, configuration field definitions, logical external contracts, and implementation mechanics. |
| Report template surface | `docs/templates.md` | Implemented template files and partials, render-context fields, editing rules, and maintainer-facing template examples. | Weather derivation, module implementation, generated-text validation internals, and operator procedures. |
| External and durable integration contracts | `docs/integrations/` | Weather API, Promptkit, Distributor, external formats and protocols, durable logical paths and schemas, compatibility behavior, and upstream or downstream responsibilities. | Physical runtime placement and lifecycle, internal transformations, CLI syntax, and configuration defaults. |
| Internal subsystem behavior | `docs/internal/` | Implementation flow, internal collaborators and state transitions, package-local guarantees and failures, and relevant tests. | Global architecture invariants, user-facing contracts, external schemas, operator procedures, and future package plans. |
@@ -110,13 +109,12 @@ structure and invariants. Focused internal documents own implementation
behavior. These documents may link to one another but must not maintain
parallel package or behavior references.
### Commands, Configuration, Operations, And Troubleshooting
### Commands, Configuration, And Operations
CLI documentation answers how to invoke Weatherreporter and what its command
interface does. Configuration documentation answers what settings mean.
Operations answers what happens to runtime state and how to operate or recover
the application. Troubleshooting starts from a symptom and leads to diagnosis
and a safe fix.
Operations answers how to handle operator-owned outputs and runtime failures,
including diagnosis, explicit debug capture, and safe legacy cleanup.
When a workflow crosses these topics, place the complete procedure with the
document that owns the task and link to the other contracts. Do not duplicate

View File

@@ -96,8 +96,8 @@ Use each test type where it protects a distinct risk:
- Integration tests use real deterministic collaborators when correctness
depends on their interaction, while replacing live or nondeterministic
external boundaries.
- App and CLI tests protect representative assembled generation, batch,
inspection, persistence, and notification workflows.
- App and CLI tests protect representative assembled generation, batch, atomic
output, and notification workflows.
- Fixtures must be minimal, synthetic, versioned with the behavior they
exercise, and free of credentials or private data.
- Golden files are appropriate only when the complete output is intentionally
@@ -199,9 +199,9 @@ Each behavior should have a clear test owner:
- Config tests own loading, precedence, defaults, secrets, and validation.
- Domain tests own weather transformations and invariants.
- Adapter tests own HTTP, Promptkit/provider, and upload boundaries.
- Orchestrator tests own workflow ordering, persistence, partial success, and
failure propagation.
- State tests own path derivation, atomic artifacts, lookup, and round trips.
- Orchestrator tests own workflow ordering, output publication, partial success,
and failure propagation.
- Filesystem tests own atomic writes and destination-preservation behavior.
- Template and generated-text tests own schemas, render contexts, and rendered
output contracts.
@@ -219,8 +219,8 @@ observation:
3. Use stubs when a dependency only needs controlled responses.
4. Use mocks when the interaction itself is contractual.
Mocks are appropriate for requirements such as uploading exactly once, saving
metadata before notification, propagating cancellation to Promptkit, or
Mocks are appropriate for requirements such as uploading exactly once,
notifying only after output publication, propagating cancellation to Promptkit, or
avoiding an external call after an earlier workflow failure. Do not use mocks
merely to isolate every object or reproduce the implementation's call graph.

View File

@@ -123,8 +123,8 @@ git diff --cached --check
Follow every added or changed Markdown link and confirm that its local target
exists. Review the candidate for generated binaries, test output, credentials,
temporary files, workspace files, replacements, vendored dependencies, and
other files that do not belong in source control.
temporary files, replacements, vendored dependencies, and other files that do
not belong in source control.
## Publish The Candidate Commit

140
docs/releases/v0.10.0.md Normal file
View File

@@ -0,0 +1,140 @@
# Weatherreporter v0.10.0
Weatherreporter `v0.10.0` makes report execution stateless, adds stable
weather-specific Promptkit profiles, and turns every successful generation
into one atomic operator-owned Markdown output.
## Summary
- Ordinary generation no longer creates or depends on a managed workspace,
historical run artifacts, metadata, receipts, or prior snapshots.
- `generate` and `run` now publish directly to operator-selected paths, with
useful current-directory defaults when output flags are omitted.
- Local Recent Changes comparison and the historical `inspect` command family
have been removed.
- Promptkit `v0.5.0` and three embedded logical profiles provide a stable model
ladder with complete file- or directory-based overrides.
- Prompt input and generated-text contracts have been tightened, and output,
cancellation, batch preflight, notification, and partial-failure behavior
have focused offline coverage.
## Compatibility
This pre-`v1` minor release intentionally breaks CLI, configuration,
prompt-input, action-summary, and workspace contracts from `v0.9.0`.
- The `workspace:` and `recent_change:` configuration sections are no longer
supported. Strict configuration loading rejects them.
- The `inspect reports`, `inspect metadata`, `inspect modules`,
`inspect data-package`, `inspect prior`, and `inspect sources` commands have
been removed. Weatherreporter no longer reads V1 or V2 run metadata or other
historical workspace artifacts.
- Every successful `generate` writes exactly one Markdown file. Without
`--out`, Daily writes `daily-YYYY-MM-DD.md` and Today, Tomorrow, and Hourly
write `today.md`, `tomorrow.md`, and `hourly.md` in the invocation's current
directory. `--out` selects that file rather than creating an extra copy of a
separately managed report.
- `run` writes selected outputs beneath the current directory unless
`--out-dir` selects another directory. Successful items remain available
when another batch item fails.
- Action summaries no longer expose managed report, metadata, snapshot, data
package, prompt preparation, prompt execution, generated-text, render-context,
or notification-receipt paths. They retain the final `outputPath`, optional
`llmDebugPath`, safe effective profile/backend/model details, validation,
warnings, notification status, and safe errors.
- Batch report items no longer contain per-report notification fields. Batch
notification is represented once at the top level. The `total`, `succeeded`,
and `failed` counters describe reports only, so notification failure can
produce a failed action while `failed` remains `0`.
- The prompt data package advances from `weatherreporter.data_package.v3` to
`weatherreporter.data_package.v4` and removes `recent_changes`. All four
embedded prompts advance from `1.1.0` to `2.0.0`.
- Generated-text schemas now require string-valued `precipitation_timing`; the
model returns an empty string when there is no timing text. The unused
`confidence` field has been removed and is rejected as an unknown field.
Existing operator-owned Markdown files remain valid. Existing workspace trees
are ignored rather than migrated or deleted. Distributor continues to receive
the completed Markdown report, but its source is now the selected operator
output rather than a managed report copy.
## Upgrade
Before replacing `v0.9.0`:
1. Remove `workspace:` and `recent_change:` from configuration files.
2. Give scheduled commands a predictable working directory or explicit
`--out` or `--out-dir` destination. Confirm that these selected files may be
atomically replaced on later successful runs.
3. Remove historical `inspect` invocations and update action-summary consumers
to use `outputPath` and the remaining active-workflow fields.
4. Decide whether old workspace contents have any external retention value.
Weatherreporter no longer reads them; after review, they may be removed
manually using the narrowly scoped procedure in the operations guide.
5. Review Promptkit profile selection and credentials. Hourly defaults to
`weather-light`; Daily, Today, and Tomorrow default to `weather-balanced`.
A configured `promptkit.profile` still overrides every report in one action.
The embedded logical profiles are:
| Profile | OpenRouter model | Default use |
| --- | --- | --- |
| `weather-light` | `deepseek/deepseek-v4-flash` | Hourly |
| `weather-balanced` | `~google/gemini-flash-latest` | Daily, Today, Tomorrow |
| `weather-deep` | `~anthropic/claude-sonnet-latest` | Explicit selection |
Override a complete same-ID definition through `promptkit.profile_file` or
`promptkit.profile_dir` to use different models or a local OpenAI-compatible
endpoint. Definitions are replaced rather than field-merged, and a malformed
matching override fails instead of silently falling back.
See the [CLI reference](../cli.md), [configuration
reference](../config.md), [operations guide](../operations.md), and [Promptkit
integration](../integrations/promptkit.md) for the exact current contracts.
## Changes
### Stateless Execution And Operator-Owned Outputs
- Removed local forecast-change comparison, prior-snapshot selection, durable
module and prompt artifacts, managed reports, metadata compatibility, run
discovery, notification receipts, and the complete `internal/state`
subsystem.
- Added an Accepted architecture decision recording the stateless
transformation pipeline and operator-owned output boundary.
- Kept weather, facts, modules, prompt input, generated text, and render context
in memory during ordinary execution.
- Made output publication atomic and ensured cancellation or deadline expiry
observed before publication leaves an existing destination unchanged.
- Added complete batch-destination preflight before the first report prompt,
so a structural collision cannot leave an unreported partial batch.
- Preserved successful outputs after report or Distributor failure. Batch
notification runs only after every selected report succeeds.
### Promptkit Profiles And Prompt Contracts
- Upgraded Promptkit from `v0.4.0` to `v0.5.0`.
- Added embedded `weather-light`, `weather-balanced`, and `weather-deep`
profiles and mapped each exact prompt to its logical default.
- Added embedded-profile fallback after configured `profile_file` or
`profile_dir` lookup, allowing operators to replace a logical profile without
changing report definitions.
- Added a maintained local-endpoint example for replacing `weather-light`.
- Advanced the four prompt definitions to `2.0.0` and the curated data package
to v4 after removing Recent Changes.
- Required `precipitation_timing`, normalized whitespace-only timing to an
empty string, and removed the unused confidence value.
### CLI, Reliability, Documentation, And Testing
- Simplified action summaries to active workflow identity, output, model,
validation, warning, debug, notification, and safe error information.
- Made batch counters report-only while retaining failed action status and
non-zero exit behavior for batch notification failure.
- Kept prompt and profile inspection ahead of weather collection and validated
every batch candidate before collecting once.
- Replaced state-oriented workflow fixtures with focused generation, batch,
output, cancellation, profile-resolution, Distributor, and CLI coverage.
- Reconciled user, operator, integration, internal, policy, and ADR
documentation around the implemented stateless architecture and removed
completed temporary roadmaps.

33
docs/releases/v0.10.1.md Normal file
View File

@@ -0,0 +1,33 @@
# Weatherreporter v0.10.1
This release repairs release validation after the `v0.10.0` pipeline failed in
its privileged build container. Application behavior is unchanged from
`v0.10.0`.
## Summary
The unreadable-secret configuration test now verifies that its process is
actually subject to file permission bits before asserting that a mode-`000`
file cannot be read. This keeps the test meaningful for ordinary users while
allowing the release suite to run correctly in privileged containers.
## Compatibility
This patch release makes no changes to Weatherreporter's CLI, configuration,
report output, integrations, prompts, profiles, or operating behavior. It is
fully compatible with `v0.10.0`.
## Upgrade
No special operator action is required. Use `v0.10.1` in place of `v0.10.0`;
the `v0.10.0` tag remains immutable, but its failed pipeline did not publish
release binaries.
## Changes
- Made the unreadable-secret test capability-aware when the test process can
bypass filesystem permission bits.
- Preserved the production contract that genuinely unreadable secret files
fail configuration loading.
- Restored portable release validation in Woodpecker's privileged Go
container.

37
docs/releases/v0.11.0.md Normal file
View File

@@ -0,0 +1,37 @@
# Weatherreporter v0.11.0
This release adds a configurable default publication directory for generated
weather reports.
## Summary
Operators can now set `output.directory` once for both individual reports and
scheduled batches. Explicit `--out` and `--out-dir` destinations continue to
take precedence, while installations that omit the setting retain the existing
current-directory behavior.
## Compatibility
This release is additive and compatible with `v0.10.1`. Existing configuration
files, commands, report filenames, Promptkit behavior, and Distributor
notification behavior remain valid and unchanged.
## Upgrade
No special action is required. To use the new default destination, configure
`output.directory` as described in the [configuration
reference](../config.md). Existing deployments may continue using the current
working directory or explicit CLI output flags.
## Changes
- Added strict configuration loading and validation for the optional
`output.directory` field.
- Applied the configured directory consistently to `generate` and `run`, with
explicit CLI destinations retaining highest precedence.
- Preserved relative-path handling, absolute result paths, atomic publication,
cancellation safety, and Distributor notification ordering.
- Strengthened output preflight so existing non-directory paths, uninspectable
paths, and dangling symlink components fail before expensive report work.
- Updated the [CLI reference](../cli.md) and [operations
guide](../operations.md) for the new destination-selection behavior.

View File

@@ -3,6 +3,40 @@
This roadmap contains future work only. Each section identifies its planning
status; current behavior is documented outside `docs/roadmap/`.
## Upstream Forecast Change Product
Status: Proposed upstream feature request; unimplemented.
Weatherreporter's local Recent Changes feature was removed by the accepted
[stateless execution decision](../adr/0001-stateless-execution.md). Forecast
version history and comparison are better owned by the Weather API, where the
underlying forecast issuances can be retained and compared consistently for
all consumers.
A future Weather API feature should expose a structured change product with:
- explicit current and baseline forecast issuance timestamps or identifiers;
- documented baseline selection, such as a requested comparison timestamp,
preceding issuance, or fixed rolling period;
- location, timezone, and half-open valid-period identity;
- typed changed values with previous and current values and units;
- stable change categories for temperature, precipitation probability and
timing, wind gusts, alerts, and aggregate hazards;
- an API-owned significance classification or enough structured information
for a stateless consumer to apply a documented presentation threshold; and
- deterministic ordering, missing-baseline behavior, and source metadata.
The API should compare forecast versions, not track a Weatherreporter client's
"previous run." It should not require consumer identity, mutable cursors, or
Weatherreporter-managed history. A missing baseline should be a normal empty
result rather than an error.
Once a stable upstream contract exists, a separate Weatherreporter roadmap may
reintroduce change commentary by collecting that product and mapping it into a
curated prompt-facing module. There must be no local snapshot fallback. The
ordinary Weatherreporter process must remain stateless, and the upstream
feature should have deterministic fixtures before adoption.
## Automatic Storm Monitoring
Status: Proposed and unimplemented.
@@ -13,8 +47,10 @@ Possible direction:
1. Detect candidate storm events from alerts, forecast discussion, weather
story context, hourly thresholds, and material forecast changes.
2. Evaluate candidates through Scriptorium or another narrow evaluator adapter.
3. Persist storm lifecycle state.
2. Evaluate candidates through Promptkit or another narrow evaluator adapter.
3. Keep any required storm lifecycle state in the upstream service or another
explicitly designed external owner rather than silently reintroducing a
Weatherreporter workspace.
4. Generate or update a storm report only when a meaningful event is present.
5. Suppress ordinary low-impact thunder or rain chances.
@@ -54,8 +90,7 @@ Status: Proposed and unimplemented.
Possible future modules:
- `hourly_table` for compact valid-period hourly facts
- `forecast_delta` if a separate stanza is useful beyond current Recent
Changes
- `forecast_delta` after an upstream forecast-change product exists
- `weekend_planning` if weekend-specific planning guidance needs a dedicated
deterministic stanza
- `storm_window_summary` if manual or automatic storm reports need a dedicated
@@ -79,7 +114,7 @@ contracts](../internal/facts.md), [module internals](../internal/module.md), and
- keep broad reusable calculations in `DerivedFacts`
- keep prompt-facing field shape inside module builders
- use typed options for configurable module behavior
- keep module snapshots structured and deterministic for Recent Changes
- keep module output structured and deterministic
## Distributor Notification Enhancements
@@ -92,10 +127,8 @@ behavior is documented in the [Distributor adapter guide](../internal/distributo
unimplemented:
- `failure_policy: warn`
- uploading metadata, module snapshots, data packages, or preflight artifacts
- durable upload retry queues
- distributor-specific CLI flags
- distributor workspace scanning
- destination routing, Markdown-to-HTML transformation, public URLs, or nginx
layout inside weatherreporter
@@ -138,6 +171,7 @@ maintenance costs make the added abstraction worthwhile:
- global test helper package
- logging subsystem
Any future implementation should preserve the existing public CLI, artifact
paths, report identities, module boundaries, and adapter boundaries unless a
separate roadmap explicitly changes them.
Any future implementation should preserve the public CLI, report-output
contract, report identities, module boundaries, and adapter boundaries in
effect when that work begins unless a separate roadmap explicitly changes
them.

View File

@@ -1,555 +1,398 @@
# Promptkit Migration Implementation Plan
# LLM Profile Comparison Implementation Plan
Status: Completed; Stages 119 passed their exit gates.
Status: Complete.
## Purpose And Authority
This document records the completed implementation of the
[Promptkit migration roadmap](promptkit.md) and its post-implementation audit
remediation. The feature roadmap records scope, user intent, policy choices,
and the implemented end state. This plan records implementation sequence,
tests, and completion gates.
This document is the ordered implementation plan for the accepted [LLM
Profile Comparison Roadmap](profile-comparison.md). The roadmap owns the
feature purpose, policy, scope, and desired end state. This plan records the
completed implementation and records the corrective work completed during
post-implementation review.
Stages 1219 were completed in order. They fixed additional defects exposed by
their required tests only when those defects were within the same stated
contract; they did not add new product behavior or reinterpret roadmap
decisions.
All implementation work in this plan is complete. The recorded work leaves the
repository compiling, tested, documented to its implemented boundary, and
internally coherent.
This plan follows the repository's
[architecture](../policy/architecture.md),
[documentation](../policy/documentation.md), and
[testing](../policy/testing.md) policies.
## Implementation Rules
## Continuing Invariants
The following rules governed every implementation stage:
- Keep `gitea.maximumdirect.net/eric/promptkit` pinned at exactly `v0.4.0`.
- Keep Promptkit types inside `internal/adapters/promptkit`, its tests, and the
external prompt-asset contract test.
- Preserve one Promptkit engine per `generate` or `run` invocation and one
shared engine for every sequential report in a batch.
- Preserve exact prompt version `1.0.0`, the exact persisted YAML data-package
bytes, prepared execution, and preparation persistence before provider work.
- Do not add retries, repair attempts, concurrent batch generation, direct
Markdown generation, arbitrary backend registration, or live-provider
tests.
- Keep ordinary artifacts, errors, logs, and summaries free of credentials,
rendered messages, schemas, input bodies, generated bodies, endpoints, and
full effective parameter maps.
- Keep sensitive debug artifacts opt-in, outside normal state, owner-only,
atomic, and free of credentials.
- Treat an artifact path as reached only after the corresponding write or copy
succeeds. Never persist or summarize a merely derivable future path.
- Preserve every safe reached path in partial app and CLI results even when a
later persistence, validation, rendering, copy, or notification step fails.
- Keep v1 metadata read compatibility and write only v2 metadata for new runs.
- Keep the default test suite deterministic, offline, and credential-free.
- Run `git diff --check` before completing every stage. Run the full repository
gate in Stage 19.
- Read `docs/development.md`, the task-specific documents it identifies, all
files under `docs/policy/`, and the feature roadmap before changing code.
- Preserve the existing `generate`, `run`, and `compare` command contracts
except for the explicit comparison corrections defined below.
- Keep Promptkit types and calls behind `internal/adapters/promptkit` and the
dependency-neutral `internal/promptexec` interface.
- Keep comparison artifacts operator-owned and explicit. They are not durable
application state and must never be discovered or consumed implicitly by a
later invocation.
- Preserve comparison's prepare-once, execute-concurrently, order-results-by-
selection, publish-on-profile-failure, and never-notify invariants.
- Do not add a Weatherreporter concurrency limit. Promptkit owns backend
capacity.
- Never expose provider bodies, prompts, schemas, model output, endpoints,
credentials, or arbitrary wrapped error text in normal JSON summaries or
manifests.
- Use deterministic, offline, credential-free tests. Test filesystem safety,
concurrency, and recovery through the narrowest stable behavioral boundary;
do not rely on timing-only sleeps or host permission behavior.
- Run `gofmt` on changed Go files and `git diff --check` in every stage. Run
focused tests while developing and `GOWORK=off go test -count=1 ./...` before
completing each stage. Stages involving concurrency or filesystem mutation
must also run affected packages with `-race`.
- Do not commit, tag, push, or prepare a release unless the implementing prompt
separately requests it.
## Completed Migration Summary
## Completed Stages
Stages 111 are implemented and committed. They remain summarized here to
preserve the history and dependencies of the follow-up work.
### Stage 1: Comparison Artifact And Naming Contracts
| Stage | Completed outcome |
| --- | --- |
| 1 | Removed the unfinished three-day, weekend, and storm product surfaces and retained Daily, Today, Tomorrow, and Hourly with exact prompt version `1.0.0`. |
| 2 | Promoted the four operational prompts and canonical schemas into the embedded `internal/promptassets` source used by Promptkit and generated-text validation. |
| 3 | Added the project-owned `internal/promptexec` inspection, preparation, execution, validation, debug, and error contract. |
| 4 | Added the Promptkit v0.4.0 adapter with prepared execution, explicit value mapping, safe error classification, and offline model-client tests. |
| 5 | Added Promptkit-era preparation and execution artifacts, metadata v2, new paths, and v1 decoding support. |
| 6 | Added explicitly rooted, permission-restricted, atomic LLM debug persistence. |
| 7 | Added Promptkit configuration, executor composition, and pre-collection prompt/profile/credential inspection. |
| 8 | Cut single-report generation over to prepared Promptkit execution and v2 persistence. |
| 9 | Added `--llm-debug-dir` and Promptkit-era single-report summary fields. |
| 10 | Cut morning and evening batches over to one shared Promptkit executor and removed Scriptorium code, configuration, and dependency metadata. |
| 11 | Updated canonical Promptkit documentation, removed the temporary Scriptorium corpus, and ran the available repository checks. |
Added the dependency-neutral comparison model, schema version, manifest
validation and encoding, safe errors, deterministic profile filenames,
comparison identities, default directory names, and content hashing.
The post-implementation audit confirmed the principal dependency and package
boundaries, but found incorrect reached-path bookkeeping, incomplete execution
artifact updates, insufficient artifact validation, extensive loss of
behavioral tests during the final cutover, and roadmap lifecycle text that was
not finalized. The completed remediation addressed those findings without
changing the intended feature scope.
### Stage 2: Destination Recognition And Transactional Publication
| Stage | Completed outcome |
| --- | --- |
| 12 | Corrected reached-path bookkeeping across metadata, app results, batch items, and CLI summaries. |
| 13 | Hardened Promptkit-era durable state validation and restored v1/v2 state coverage. |
| 14 | Recorded every downstream path reached after completed prompt execution. |
| 15 | Restored assembled single-report behavioral and failure coverage. |
| 16 | Simplified prompt-generation orchestration while preserving behavior. |
| 17 | Restored assembled batch, planning, artifact, and notification coverage. |
| 18 | Restored supported CLI, summary, safety, and historical inspection coverage. |
| 19 | Reconciled canonical documentation and passed the complete repository verification gate. |
Added read-only destination planning, strict recognition of current comparison
bundles, private sibling staging, guarded replacement, rollback, and atomic
directory publication.
## Stage 12: Correct Reached-Artifact Bookkeeping
### Stage 3: Ordered Multi-Profile Preflight
Status: Completed.
Added exact prompt inspection followed by sequential profile inspection before
weather collection, including effective backend, model, and credential checks.
### Stage 4: Immutable Shared Report Preparation
Extracted one immutable prepared-report value so comparison collection,
derivation, module construction, and data-package serialization happen once.
### Stage 5: Profile Execution And In-Memory Rendering
Separated profile-specific Promptkit execution, generated-text validation, and
Markdown rendering from output publication while preserving ordinary report
generation behavior.
### Stage 6: Concurrent Ordered Profile Execution
Added one goroutine per selected profile using one shared executor and one
prepared input, deterministic debug identities, isolated profile failures,
joined cancellation, and selection-ordered results.
### Stage 7: Application-Level Comparison
Added `app.CompareDetailed`, coherent complete and partial bundle construction,
aggregate profile-failure behavior, absolute published paths, and the
application-level guarantee that comparison never notifies Distributor.
### Stage 8: Compare Command Parsing
Added the `compare` command request path, repeatable ordered `--profile`, exact
`--out-dir`, guarded `--replace`, applicable common flags, validation, and one
executor construction per invocation.
### Stage 9: CLI Results And Exit Behavior
Added structured success and failure summaries, quiet-mode suppression,
ordered per-profile results, safe bounded errors, and nonzero exit behavior for
partial or command-level failure.
### Stage 10: Canonical Documentation And Initial Validation
Documented the implemented CLI, operations, Promptkit integration, comparison
bundle, application orchestration, execution, publication, architecture, and
development contracts, then passed the original repository-wide validation
gate.
## Stage 11: Make Concurrent Prompt Debug Creation Race-Safe
### Goal
Make metadata, app results, batch items, and CLI summaries truthful at every
failure boundary: a nonblank path means that artifact was successfully
created.
Ensure concurrent comparison profiles can create their distinct debug runs
under one new report/date directory without spuriously failing or leaving a
test goroutine blocked.
### Work
1. Change `state.BuildPromptMetadataFromBriefingMetadata` so it initializes
identity, schema, metadata destination, and only artifacts already saved at
the call site. It must not prepopulate raw-output, normalized-text,
render-context, managed-report, preparation, execution, notification, or
output-copy paths.
2. In `generatePromptReport`, assign each metadata and `ReportResult` path
immediately after that artifact write succeeds and before attempting the
next write. In particular:
1. Update `internal/promptdebug.ensureSecureDirectory` so concurrent creation
of the same missing directory is idempotent. If `os.Mkdir` reports that the
path already exists, inspect the path with `Lstat` and accept it only when it
is the expected real directory. Continue to reject symlinks, non-directories,
unsafe modes, and every unrelated filesystem error.
2. Preserve the existing absolute-path, containment, `0700` directory, `0600`
file, and no-symlink guarantees. Do not weaken debug-root validation or make
all `EEXIST` errors successful.
3. Add a focused prompt-debug concurrency regression that starts multiple
writers beneath a shared missing ancestor, joins every goroutine, and
verifies every expected artifact and permission invariant.
4. Make comparison execution test barriers time-bounded and failure-aware. A
callback failure before executor entry must fail the test promptly rather
than leave `waitForProfileStarts` waiting forever.
5. Retain distinct deterministic debug references and profile-local debug
failure behavior.
- do not initialize `ReportResult.ReportPath` from `Store.Paths`;
- record a saved failed-preparation receipt in the result before saving
metadata;
- record a saved failed or completed execution receipt before saving
metadata;
- retain raw, normalized, context, report, copy, and notification paths
when a later step fails; and
- keep `MetadataPath` unchanged when a metadata rewrite fails, because the
prior successfully written metadata record remains the reached version.
### Tests And Exit Criteria
3. Remove batch-item prepopulation from derived `Store.Paths` values.
`BatchReportResult` receives paths only from the returned `ReportResult` or
from a write that the batch itself successfully completed.
4. Preserve current CLI field names and omission behavior. Human and JSON
summaries must omit every unreached path.
5. Do not change artifact locations, filenames, schemas, report output, or
notification policy in this stage.
### Tests
- Add focused app tests for one representative report using real temporary
state plus a narrow failure-injecting store wrapper.
- Fail the next persistence step immediately after a successful preparation
receipt, execution receipt, raw output, normalized output, render context,
managed report, output copy, and notification artifact; assert that the
returned result contains every reached path and no future path.
- Include one preparation failure, one operational execution failure, and one
completed validation rejection to cover the three execution outcome shapes.
- Add batch and CLI summary assertions proving unreached paths are omitted.
- Run:
- The focused prompt-debug concurrency test passes repeatedly and with the race
detector.
- `TestExecuteComparisonProfilesUsesDistinctDeterministicDebugReferences`
cannot hang when a profile fails before reaching the fake executor.
- Run, at minimum:
```sh
go test ./internal/state ./internal/app ./internal/cli
GOWORK=off go test -count=100 ./internal/promptdebug
GOWORK=off go test -count=100 -run TestExecuteComparisonProfilesUsesDistinctDeterministicDebugReferences ./internal/app
GOWORK=off go test -race -count=1 ./internal/promptdebug ./internal/app
GOWORK=off go test -count=1 ./...
```
## Stage 12: Make Replacement Authorization Commit-Safe
### Goal
Prevent a destination changed after the final read-only preflight from being
treated as the previously authorized empty directory or recognized bundle and
then deleted during replacement.
### Work
1. Replace `DestinationPlan.Exists` as the publication decision with an
explicit destination-state classification: absent, empty real directory, or
recognized current-schema bundle. Keep `Replace` in the plan so publication
can apply the same authorization policy at commit time.
2. Continue to call `PlanDestination` immediately before publication. For an
absent target, install staging with one rename; a concurrently created
target must cause that rename to fail without modifying the new target.
3. For an existing target, rename that exact filesystem entry to the unique
sibling backup before deleting or installing anything. Classify the moved
backup while it is at its stable backup path and authorize it under the
original replacement policy:
- an empty real directory is allowed with or without `Replace`;
- a recognized current-schema comparison bundle is allowed only with
`Replace`; and
- a file, symlink, unrecognized/nonempty directory, unreadable entry, or
other classification failure is not allowed.
4. Treat this post-move classification as the destructive-action
authorization point. If it fails, restore the moved entry to the target and
return an error without installing staging. If the target has concurrently
reappeared or restoration otherwise fails, retain the backup and return an
actionable joined or typed error that identifies its recovery path; never
delete either entry to force restoration.
5. Install staging only after the moved target has passed authorization. Never
remove a backup that did not pass post-move authorization.
6. Preserve the existing final cancellation linearization rule: cancellation
observed before the rename transaction prevents replacement; after the
transaction starts, finish commit or rollback rather than abandoning it.
7. Add a package-private filesystem-operation seam only if needed for
deterministic tests. Keep the public destination and publication APIs free
of test-only hooks.
### Tests And Exit Criteria
- Deterministically replace an initially accepted destination after final
preflight but before its move with each consequential unauthorized type:
unrelated nonempty directory, regular file, and symlink. Publication must
fail, staging must not become the target, and the moved entry must be restored
or retained at a reported recovery path.
- Cover an initially empty directory whose contents change before its move and
a recognized bundle swapped for an unrecognized directory.
- Retain coverage for absent targets, empty directories, recognized bundle
replacement, cancellation before commit, install failure, successful
rollback, failed rollback, and cleanup of ordinary staging failures.
- Run `GOWORK=off go test -race -count=1 ./internal/comparison` and the
repository-wide standard test command.
## Stage 13: Represent Committed Publication Cleanup Failures Accurately
### Goal
Keep application and CLI results truthful when the new comparison bundle has
been committed but removal of the old sibling backup fails.
### Work
1. Change comparison publication to return a dependency-neutral result as well
as an error:
```go
type PublicationResult struct {
Committed bool
RetainedBackupPath string
}
func Publish(
ctx context.Context,
plan DestinationPlan,
bundle LogicalBundle,
) (PublicationResult, error)
```
2. Define `Committed` as meaning the complete staged bundle is now installed
at the target. Pre-commit, staging, authorization, install, and successful-
rollback failures return `Committed == false`. A successful install returns
`Committed == true` even if later backup cleanup fails.
3. Add a typed post-commit cleanup error that unwraps its filesystem cause and
records the retained backup path for operator recovery. On this error,
return `Committed == true` and the absolute retained backup path. Do not
roll back or remove the newly committed valid bundle merely because old
backup cleanup failed.
4. In `app.CompareDetailed`, populate `ManifestPath`, `DataPackagePath`, and
successful profile `ReportPath` values whenever publication reports
`Committed == true`, before returning any cleanup error.
5. Treat post-commit cleanup failure as a command-level operational failure:
return the non-nil structured result plus an error, produce status `failed`,
and exit nonzero even though the published artifact paths are present. The
ordinary safe JSON error must not contain the raw filesystem cause or backup
path; the wrapped diagnostic returned on stderr may identify the retained
backup for recovery.
6. Keep `RetainedBackupPath` out of the versioned comparison manifest. It
describes an incomplete local transaction cleanup, not the logical bundle.
### Tests And Exit Criteria
- Inject a deterministic backup-removal failure after successful installation
and assert the target is the new recognized bundle, the old bundle remains
at the reported backup, `Committed` is true, and the error is inspectable by
type.
- At the application boundary, assert all committed artifact paths are
absolute and populated while the method still returns an error.
- At the CLI boundary, assert status `failed`, nonzero return, present artifact
paths, and a bounded generic safe error with no raw filesystem detail.
- Retain tests showing every pre-commit or rolled-back failure omits published
artifact paths.
- Run comparison, application, and CLI tests with `-race`, then the
repository-wide standard test command.
## Stage 14: Complete Structured Failure Metadata And Classification
### Goal
Make every non-nil comparison result a reliable description of the attempted
run and preserve useful safe error categories in the top-level CLI summary.
### Work
1. In `app.CompareDetailed`, assign the absolute resolved
`OutputDirectory` immediately after output-directory resolution and before
destination preflight. Do not wait for `PlanDestination` to succeed.
2. Once the initial `ComparisonResult` exists, guarantee that every return path
sets a nonzero UTC `FinishedAt` that is not before `StartedAt`. Use one
centralized finalization path or a defer; do not scatter timestamp writes
across individual failures.
3. Continue to omit manifest, data-package, and report paths until publication
commits. Preserve whatever prompt identity fields have actually been
resolved; never invent a hash or profile result for a phase that did not
run.
4. Update `safeComparisonSummaryError` to use this stable mapping, always
passing messages through `comparison.NewSafeError`:
| Error | Category | Safe message |
| --- | --- | --- |
| aggregate profile failure | `application` | existing bounded aggregate message |
| `context.Canceled` | `canceled` | `comparison canceled` |
| `context.DeadlineExceeded` | `deadline_exceeded` | `comparison deadline exceeded` |
| categorized `promptexec` error | exact `promptexec.CategoryOf` value | `comparison prompt operation failed` |
| `comparison.DestinationError` | `destination_<kind>` | `comparison destination preflight failed` |
| post-commit cleanup error | `publication_cleanup` | `comparison published but cleanup did not complete` |
| any unknown error | `application` | `comparison did not complete` |
5. Apply the most specific mapping before a more general wrapped match. In
particular, detect the post-commit cleanup and destination types before
falling back to a nested filesystem or context cause.
6. Do not copy `DestinationError.Target`, wrapped causes, or arbitrary
`error.Error()` text into normal JSON. Detailed returned errors remain
available on stderr and through Go error inspection.
### Tests And Exit Criteria
- Add application tests for destination-preflight, debug initialization,
prompt-preflight, collection, and preparation failures. Whenever a non-nil
result is returned, assert an absolute output directory, nonzero ordered UTC
timestamps, and omission of unpublished artifact paths.
- Add table-driven CLI tests for every mapping row, including wrapped errors,
and assert that unsafe sentinel text cannot enter serialized output.
- Preserve existing ordered profile-level categories and safe messages.
- Run application and CLI tests with `-race`, then the repository-wide standard
test command.
## Stage 15: Remove Temporary Seams, Reconcile Documentation, And Validate
### Goal
Remove review-discovered maintenance debt, document the corrected implemented
behavior in its canonical owners, and complete the release-equivalent gate.
### Work
1. Remove the unused `Runner.resolveComparison` wrapper. Remove the
test-oriented `Runner.executeComparison` seam if it has no production
caller, and rewrite its remaining coverage through `Runner.Run`,
`resolveComparisonAction`, or another stable behavioral boundary.
2. Remove `comparisonProfileOutcome.err` if production code still does not use
it. Keep raw failures in returned/wrapped errors or explicit internal error
types; do not retain an otherwise dead field solely for private test
assertions.
3. Correct the `internal/comparison` package comment so it describes the
package's actual ownership of both logical comparison contracts and
filesystem destination/publication behavior.
4. Update only the canonical current-state documents affected by Stages 11
through 14:
- `docs/internal/comparison-publication.md` owns post-move authorization,
commit state, rollback, retained backups, and cleanup mechanics;
- `docs/internal/comparison-execution.md` owns concurrency and debug-write
behavior;
- `docs/internal/app-orchestration.md` owns partial results and committed
publication error handling;
- `docs/cli.md` owns structured status, safe category, path, and exit
behavior; and
- `docs/operations.md` owns operator recovery for a retained sibling backup.
Link rather than duplicating complete contracts, and update architecture or
integration documentation only if its existing invariant is inaccurate.
5. Mark this plan `Complete` and restore the feature roadmap's implemented
status after every exit criterion below passes. Retain or remove the two
roadmap documents only according to a later maintainer-directed roadmap
cleanup; do not archive them as a second current-state reference in this
stage.
### Tests And Exit Criteria
- Confirm no production-only helper or field remains solely to support tests,
and no test loses meaningful behavioral coverage during cleanup.
- Verify changed relative links and fenced examples. Search current-state docs
for stale claims about comparison publication, debug behavior, results, or
recovery.
- Run the release-equivalent local gate:
```sh
set -eu
test -z "$(git ls-files go.work go.work.sum)"
test ! -e vendor
if grep -Eq '^[[:space:]]*replace([[:space:]]|\()' go.mod; then
printf '%s\n' 'go.mod contains a replacement' >&2
exit 1
fi
GOWORK=off go test -count=1 ./...
GOWORK=off go test -race -count=1 ./...
GOWORK=off go vet ./...
GOWORK=off go build ./...
GOWORK=off go mod tidy -diff
unformatted="$(git ls-files '*.go' | while IFS= read -r file; do gofmt -l "$file"; done)"
test -z "$unformatted"
git diff --check
```
### Exit Gate
Every nonblank path in newly written metadata, app results, batch items, and
CLI summaries names an artifact that exists. Every safe artifact successfully
written before a later failure remains discoverable from the returned partial
result.
## Stage 13: Harden Durable State Contracts And Restore State Coverage
Status: Completed.
### Goal
Make the v1/v2 wire boundary and Promptkit-era artifact validation explicit,
strict, and durably tested.
### Work
1. Strengthen `PromptPreparationArtifact.Validate`:
- require report ID, Weatherreporter RunID, prompt ID, exact prompt version,
data-package path, nonzero start/end times, nonnegative duration, and an
end not earlier than the start;
- for success, require preparation provenance, prohibit an error, and
require its prompt ID/version and data-package path to match the top-level
artifact;
- for failure, require a classified bounded error and prohibit fabricated
preparation provenance.
2. Strengthen `PromptExecutionArtifact.Validate`:
- require report ID, Weatherreporter RunID, prompt ID, exact prompt version,
nonzero start/end times, nonnegative duration, and an end not earlier than
the start;
- for success and validation rejection, require provenance and completed
validation, prohibit an operational error, and require the provenance
prompt ID/version to match the artifact;
- do not compare the provenance RunID with the Weatherreporter RunID because
the provenance value is Promptkit's run identity;
- for operational failure, require a classified bounded error and prohibit
invented provenance or completed validation.
3. Validate required provenance fields for completed executions, including
Promptkit RunID, prompt and rendered hashes, selected profile/backend/model,
and data-package path. Permit usage counters and generated hash to be zero
when the provider legitimately reports no value.
4. Restore focused filesystem and metadata tests for:
- exact v2 paths and filenames;
- preparation/execution round trips and required fields;
- metadata v2 round trips without legacy aliases;
- v1 decoding, normalized internal aliases, and v1-preserving re-marshaling;
- unknown schema rejection;
- report listing, RunID lookup, source/module/data-package inspection, and
retained v1 behavior for historical report IDs;
- atomic writes and unsafe workspace/path rejection; and
- prior-snapshot behavior for the four supported report IDs.
5. Adapt useful tests from the deleted filesystem suite rather than recreating
redundant low-value cases. Do not restore Scriptorium writes or retired
report behavior.
### Tests
Run:
```sh
go test ./internal/state ./internal/app
git diff --check
```
### Exit Gate
The state package rejects incomplete or contradictory Promptkit-era artifacts,
reads historical v1 records, writes only valid v2 records, and has focused
offline coverage for its durable compatibility and filesystem contracts.
## Stage 14: Complete Execution-Artifact Path Tracking
Status: Completed.
### Goal
Make `PromptExecutionArtifact.Paths` accurately record every downstream
artifact reached after a completed Promptkit run.
### Work
1. Treat the execution artifact as an atomically updated durable record of the
completed Promptkit execution and subsequent artifact destinations. Its
status, provenance, validation, usage, and timing remain the provider-run
outcome; later application failures do not change a successful Promptkit
status into an execution failure.
2. Save the initial execution artifact after raw output is persisted, with
`RawOutputPath` populated.
3. After each later successful write, update and atomically resave the same
execution artifact with the corresponding reached path:
- normalized generated text;
- render context;
- managed Markdown report;
- an explicitly requested extra output copy, only after the copy succeeds;
and
- a Distributor notification artifact, including a persisted failure or
status artifact when notification produced one.
4. Keep metadata and execution-artifact path values consistent after every
successful checkpoint. Save the execution artifact before metadata so a
metadata failure does not erase knowledge of a reached downstream artifact.
Failure to update the execution artifact is terminal and returns a partial
result containing the downstream artifact that was already written.
5. Refactor finalization return values only as needed to tell the orchestration
layer which copy and notification paths were actually written. Distributor
must continue uploading only the managed Markdown report.
6. A validation-rejected execution ends after raw output and therefore records
only the raw-output path. An operational execution failure has no completed
provenance and records only safe paths reached before that failure.
### Tests
- Add table-driven execution-artifact lifecycle tests for success and every
downstream failure point.
- Load the persisted execution artifact after normalized-text, context,
template, copy, metadata, and notification failures and assert its status and
exact reached paths.
- Assert that execution artifacts never contain generated bodies, rendered
prompts, schemas, endpoints, parameters, or credentials.
- Run:
```sh
go test ./internal/state ./internal/app
git diff --check
```
### Exit Gate
For every completed Promptkit run, its execution artifact contains exactly the
safe downstream paths reached by the workflow and remains semantically correct
when a later application stage fails.
## Stage 15: Restore Single-Report Behavioral Coverage
Status: Completed.
### Goal
Restore the risk-based application coverage removed during final cutover and
prove the complete single-report Promptkit workflow through project-owned
boundaries.
### Work
1. Reintroduce a focused app test harness using real state, prompt-input,
generated-text validation, render contexts, and templates with deterministic
collector, executor, notifier, clock, and filesystem boundaries.
2. Add representative successful workflows for Daily, Today, Tomorrow, and
Hourly. Verify report identity, exact prompt version, one collection, exact
persisted YAML bytes passed to the executor, expected template output,
optional copy behavior, and managed-report notification source.
3. Cover the required failure matrix:
- inspection and missing credentials before collection;
- preparation failure and callback persistence failure before provider work;
- execution-time credential disappearance;
- capacity rejection without retry;
- cancellation and deadline;
- generation and operational-validation failure;
- completed Promptkit schema rejection with retained raw output;
- generated-text decode/domain rejection;
- render-context and template failure;
- output-copy failure; and
- notification failure.
4. Verify preparation persistence precedes provider execution, debug-write
failure prevents provider execution, and execution-debug failure preserves
previously reached normal and debug artifacts.
5. Verify Recent Changes, prior-snapshot selection, output naming, and
Distributor template values for all four retained reports.
6. Adapt useful tests from the deleted app suite. Omit Scriptorium mechanics,
subprocess interaction assertions, and retired report products.
7. Fix defects exposed by these tests only when the expected behavior is
already decided by the roadmap or canonical policy. Record any new product
question instead of silently choosing it.
### Tests
Run:
```sh
go test ./internal/app
go test -race ./internal/app ./internal/adapters/promptkit
git diff --check
```
### Exit Gate
The single-report workflow has deterministic behavioral coverage for all four
reports, all consequential failure stages, artifact ordering, partial results,
debug isolation, output copying, and notification behavior.
## Stage 16: Refactor Prompt Generation Orchestration
Status: Completed.
### Goal
Reduce the complexity and duplicated persistence logic in
`generatePromptReport` without changing observable behavior.
### Work
1. Use the Stage 1215 tests as the refactoring safety boundary. Do not weaken
assertions to accommodate structural changes.
2. Split the current orchestration into small app-owned operations with clear
inputs and outcomes for:
- deterministic input and initial state construction;
- preparation callback persistence;
- preparation-failure persistence;
- operational-execution-failure persistence;
- completed execution and raw-output persistence;
- normalized text and render-context persistence;
- managed report, optional copy, metadata, and notification finalization;
and
- reached-path updates shared by success and failure paths.
3. Keep workflow order visible in one coordinator. Do not introduce a generic
workflow engine, hidden retry loop, provider-specific app type, or mutable
global state.
4. Centralize the repeated rule that a successful artifact write updates the
result before any following write can fail.
5. Preserve error identities, safe text, atomic writes, exact bytes, debug
ordering, partial results, and notification behavior.
### Tests
Run:
```sh
gofmt -w internal/app/*.go
go test ./internal/app ./internal/state ./internal/cli
go test -race ./internal/app
git diff --check
```
### Exit Gate
The top-level coordinator communicates the workflow order without containing
the full persistence implementation, duplicate failure branches are reduced,
and every Stage 1215 behavioral test passes unchanged.
## Stage 17: Restore Batch Behavioral Coverage
Status: Completed.
### Goal
Re-establish confidence that morning and evening batches preserve their
pre-migration behavior while sharing one Promptkit executor.
### Work
1. Add assembled batch tests proving:
- one executor factory call and one executor per CLI invocation;
- inspection of the full candidate set before collection;
- one weather collection;
- existing morning/evening planning and ordering;
- sequential execution through the shared executor;
- continuation after an independent report failure;
- no retry after capacity rejection;
- distinct identities and debug directories for multiple Daily dates; and
- exact reached paths on successful and failed batch items.
2. Restore notification coverage for disabled notification, suppressed
per-report notification, all-success batch notification, skipped
notification after report failure, and persisted notification failure/status
artifacts.
3. Restore output-directory, Today/Tomorrow naming, dynamic Daily planning,
prior-snapshot, and managed-Markdown upload-source coverage.
4. Adapt useful tests from the deleted batch portions of the app and CLI suites.
Do not restore retired report cases or Scriptorium fakes.
5. Fix only roadmap-defined batch regressions exposed by the restored tests.
### Tests
Run:
```sh
go test ./internal/app ./internal/cli
go test -race ./internal/app
git diff --check
```
### Exit Gate
Morning and evening batches are covered as assembled sequential workflows and
demonstrably preserve collection, planning, continuation, output, debug,
artifact, and notification contracts with one Promptkit executor.
## Stage 18: Restore CLI And Inspection Coverage
Status: Completed.
### Goal
Restore the user-facing command, summary, and historical inspection contracts
removed with the old root test suite.
### Work
1. Add parser and resolver tests for all four generate commands, both batch
commands, shared flags, report-specific date rules, malformed input,
`--llm-debug-dir`, `--quiet`, output paths, and rejection of retired report
names.
2. Add assembled CLI tests for representative successful single and batch
invocations using injected offline boundaries. Verify exactly one executor
construction per action.
3. Cover pre-run errors with no invented run summary, successful and failed
JSON summaries, quiet-mode behavior, safe human status output, partial paths,
and omission of absent notification/debug fields.
4. Restore inspection tests for report listing and v1/v2 metadata, modules,
data packages, prior snapshots, and sources. Include failed v2 runs and v1
fixtures using historical report IDs.
5. Assert that routine output never contains rendered prompts, schema bodies,
data packages, generated bodies, endpoints, full parameters, credentials, or
secret-like dependency errors.
6. Keep tests at stable CLI/app boundaries; do not restore assertions about
private parser formatting or Scriptorium subprocess mechanics.
### Tests
Run:
```sh
go test ./internal/cli ./internal/app ./internal/state
go run ./cmd/weatherreporter --help
git diff --check
```
### Exit Gate
The supported CLI surface, summaries, quiet mode, partial failures, executor
composition, and v1/v2 inspection behavior have deterministic offline coverage.
## Stage 19: Finalize Documentation And Repository Verification
Status: Completed.
### Goal
Close the audit remediation, make roadmap lifecycle state truthful, and verify
the repository against the complete target contract.
### Work
1. Update `docs/roadmap/promptkit.md` from future tense and “unimplemented”
statuses to a completed roadmap record. Describe its old seven-report and
Scriptorium material explicitly as the pre-migration baseline rather than
current behavior.
2. Mark Stages 1219 and this implementation plan complete only after their
exit gates pass. Retain the concise completed-stage history unless the
documentation policy calls for archival in the same change.
3. Review canonical architecture, app, state, CLI, Promptkit integration,
operations, troubleshooting, configuration, and testing documentation
against the corrected implementation. Update only actual current-state
discrepancies; do not duplicate the roadmap.
4. Search current-state code, tests, examples, help, and non-roadmap
documentation for stale Scriptorium terms, retired reports, old artifact
fields, speculative-path descriptions, or claims of missing Promptkit
implementation.
5. Confirm examples contain no credentials or private infrastructure values
and load through config tests.
### Final Verification
Run:
```sh
gofmt -w <all changed Go files>
go mod tidy
go vet ./...
go test -count=1 ./...
go test -race ./...
go run ./cmd/weatherreporter --help
git diff --check
```
Then verify explicitly:
- `go list -m gitea.maximumdirect.net/eric/promptkit` reports `v0.4.0`;
- no committed `go.work`, `replace`, secret fixture, or live-provider test
exists;
- all four prompts inspect at exact version `1.0.0`;
- no runtime prompt requests repair attempts;
- v1 fixtures remain inspectable and new runs write only v2;
- normal artifacts and output contain no sensitive prompt/debug content;
- failed-run metadata, execution artifacts, app results, batch items, and CLI
summaries contain exactly the paths actually reached;
- help exposes only Daily, Today, Tomorrow, Hourly, morning, and evening; and
- managed Markdown remains the only Distributor upload source.
### Exit Gate
Every migration and audit-remediation criterion is demonstrably satisfied,
the restored tests protect the consequential contracts, canonical
documentation describes the corrected implementation, and both roadmap
documents are marked complete.
- Run `GOWORK=off go run ./cmd/weatherreporter --help` and
`GOWORK=off go run ./cmd/weatherreporter compare --help` without credentials
or network access, and confirm that help agrees with `docs/cli.md`.
- Inspect the final diff for accidental generated artifacts, secrets,
workspaces, vendored dependencies, release notes, or unrelated changes.
## Open Questions
None. The roadmap and this completed plan record the decisions used for the
audit remediation.
None. The roadmap and the contracts in Stages 11 through 15 define the
remaining decisions needed to complete the corrective work.

View File

@@ -0,0 +1,398 @@
# LLM Profile Comparison Roadmap
Status: Implemented; retained as the feature roadmap.
## Purpose
Prompt development currently requires separate Weatherreporter invocations to
compare several LLM profiles. Those invocations may collect different weather
snapshots or rebuild inputs at different times, making model output harder to
compare and slowing prompt iteration.
Weatherreporter should provide a first-class `compare` command that resolves
one report, prepares one exact data package, executes the same prompt and data
package concurrently through several explicitly selected Promptkit profiles,
and publishes a self-contained local comparison bundle.
An illustrative invocation is:
```sh
weatherreporter compare daily \
--date 2026-08-24 \
--profile weather-light \
--profile weather-balanced \
--profile weather-deep
```
This is a prompt-development workflow, not an automated model evaluator. Its
output gives a maintainer consistent evidence for human comparison without
assigning scores or selecting a winner.
## Prerequisite
Configurable output directories are implemented. Profile comparison must reuse
the current [configuration reference](../config.md) and [operations
guide](../operations.md) rather than introduce a second destination policy.
## User Intent
The command is intended for deliberate evaluation of multiple profiles,
including sets of eight to twelve candidate models. Concurrency is part of the
feature, not a future optimization. Promptkit should retain ownership of
backend-specific capacity, while Weatherreporter owns comparison-wide
coordination, cancellation, deterministic results, and artifact publication.
Every profile must receive byte-for-byte identical prompt input. Weather data,
derived facts, modules, prompt metadata, and serialized YAML must not be
recollected or rebuilt separately for individual profiles.
Comparison bundles are explicitly requested, operator-owned development
outputs. They are not Weatherreporter state, are never read implicitly by a
later run, and do not weaken the ordinary stateless execution model.
## Command Contract
The command form is:
```text
weatherreporter compare REPORT [options]
```
`REPORT` accepts the implemented generated-text reports: `daily`, `today`,
`tomorrow`, and `hourly`. Report-date behavior matches `generate`: `daily`
requires `--date`, `today` may accept an explicit date or use the current local
date, and the remaining report types retain their existing period policies.
The command accepts the applicable common generation options, including
`--config`, `--units`, `--tz`, `--date`, `--llm-debug-dir`, and `--quiet`, plus:
- repeatable `--profile PROFILE_ID` selections;
- `--out-dir PATH` for the exact comparison-bundle directory; and
- `--replace` to authorize guarded replacement of a recognized existing
comparison bundle.
At least two distinct, nonblank profile IDs are required. Their command-line
order is significant and is preserved in filenames, summaries, and
`comparison.json`. Duplicate profile IDs are rejected rather than silently
deduplicated or executed twice.
Profiles are always explicit for this command. `promptkit.profile` does not add
or replace a comparison selection, but all other effective Promptkit settings,
profile-source precedence, local backend configuration, credential lookup, and
profile overrides remain in force.
The initial feature has no Weatherreporter-specific concurrency flag or
artificial profile-count ceiling. The explicit profile list bounds the
comparison, and Promptkit owns capacity enforcement for each selected backend.
## Preparation And Execution Invariants
A comparison has this logical lifecycle:
1. Parse and validate the report, date, profile list, configuration, output
destination, and replacement authorization.
2. Resolve the report definition, valid period, prompt identity, and default
output name once.
3. Inspect the exact prompt once and preflight every selected profile,
including its effective backend, model, and required credential
availability, before weather collection.
4. Collect weather data exactly once.
5. Build collected and derived facts, the module snapshot, briefing metadata,
and the prompt data package exactly once.
6. Marshal the data package to one immutable YAML byte sequence exactly once.
7. Execute the exact prompt version concurrently for every selected profile,
passing the same immutable YAML bytes to every execution.
8. Validate and render each profile result independently from the shared
deterministic inputs.
9. Assemble results in requested-profile order and publish one coherent
comparison bundle.
This lifecycle describes the required end-state behavior rather than an
implementation-stage sequence.
No profile execution may cause recollection, report re-resolution, module
rebuilding, or data-package remarshalling. Prompt execution may perform
Promptkit-owned validation or repair behavior, but Weatherreporter does not
retry a failed comparison execution independently.
## Concurrency And Cancellation
Weatherreporter starts one execution for each preflighted profile and permits
them to run concurrently through one shared, concurrency-safe Promptkit
executor. Promptkit's engine-local backend pools remain authoritative for
backend concurrency and waiting capacity. Profiles routed to a limited local
backend therefore respect its configured limit, while profiles routed to
other backends may proceed independently.
Weatherreporter must not add a second semaphore that obscures or overrides
Promptkit's backend policy. It must safely coordinate goroutine lifecycles,
result collection, debug callbacks, and output assembly without data races.
One profile failure does not cancel its peers. Provider, capacity, validation,
and rendering failures are recorded for that profile while other executions
continue. Cancellation or deadline expiration of the comparison command is
propagated to every outstanding execution, prevents new publication, and is
joined without leaking goroutines.
Completion order must not affect filenames, manifest order, CLI summaries, or
error aggregation. Those outputs always follow the original `--profile`
order.
## Output Destination
Without `--out-dir`, Weatherreporter derives a comparison directory from the
resolved report's existing default Markdown filename by removing `.md` and
prefixing `comparison-`:
| Report output | Comparison directory |
| --- | --- |
| `today.md` | `comparison-today/` |
| `tomorrow.md` | `comparison-tomorrow/` |
| `hourly.md` | `comparison-hourly/` |
| `daily-2026-08-24.md` | `comparison-daily-2026-08-24/` |
The derived directory is created beneath `output.directory` when configured,
or beneath the present working directory otherwise. An explicit `--out-dir`
is the exact bundle directory, resolves relative to the present working
directory when necessary, and overrides `output.directory` completely.
All destination selection and validation completes before weather collection.
The resolved comparison directory is returned in the command's structured
result.
## Comparison Bundle
A successful three-profile comparison has a flat layout:
```text
comparison-daily-2026-08-24/
├── comparison.json
├── data-package.yml
├── 01-weather-light.md
├── 02-weather-balanced.md
└── 03-weather-deep.md
```
`data-package.yml` contains the exact YAML bytes passed to every Promptkit
execution. It is written once and its SHA-256 digest is recorded in the
manifest.
Each report filename begins with its one-based, zero-padded selection position
and a filesystem-safe representation of the requested logical profile ID. The
safe representation must not permit absolute paths, traversal, separators, or
control characters. The manifest retains the exact case-sensitive profile ID,
so filename normalization never becomes the authority for profile identity.
`comparison.json` is the authoritative index for the bundle. It uses an
explicit schema version and records safe comparison information including:
- comparison identity and start and finish timestamps;
- report ID, resolved valid period, and effective timezone;
- prompt ID, version, and inspected prompt hash;
- the relative data-package filename and SHA-256 digest;
- total, succeeded, and failed profile counts; and
- one ordered result per requested profile containing the exact profile ID,
resolved backend and model, relative report filename when present,
execution and validation status, and safe error information when failed.
The manifest and normal command summary must not contain credentials, provider
request bodies, raw model output, rendered prompts, schemas, provider
endpoints, or other content-rich diagnostics. The explicit data package and
generated reports contain the development material the user requested and
must be handled as operator-owned potentially sensitive output.
## Failure And Publication Policy
Failure before concurrent execution, including invalid profiles, missing
credentials, collection failure, preparation failure, or unsafe destination,
publishes no comparison bundle and performs no model calls where the failure
is discoverable during preflight.
After execution begins, Weatherreporter waits for every non-cancelled profile.
If one or more profiles fail, it still publishes a coherent partial bundle
containing `data-package.yml`, every successfully rendered report, and a
manifest describing all successes and failures. It then returns a non-zero
exit status. A failed profile has no report file unless a future contract
explicitly introduces a separately named diagnostic artifact.
Bundle contents are staged outside the destination and published only after
the manifest is complete. Ordinary publication accepts only an absent or empty
target directory. A nonempty existing directory fails without modification
unless `--replace` is present.
`--replace` may replace only the exact resolved target and must reject broad or
unsafe targets such as a filesystem root, the present working directory, a
symlink, or an unrecognized nonempty directory. A recognized prior bundle must
contain a valid Weatherreporter comparison manifest. Replacement publishes the
new complete or coherent partial bundle as a unit, prevents stale reports from
the prior comparison from surviving, and preserves or restores the prior
bundle if the final replacement operation fails.
An interrupted or cancelled comparison does not replace an existing bundle.
Temporary staging artifacts are cleaned up on ordinary failure and
cancellation without scanning or modifying unrelated directories.
## Prompt Debugging
The existing `--llm-debug-dir` mechanism remains available. Concurrent
comparison executions require distinct, deterministic debug identities that
include the comparison and exact profile selection so callbacks cannot collide
or overwrite another profile's artifacts.
Debug writing must be concurrency-safe and retain the existing permission,
redaction, explicit-opt-in, and path-containment guarantees. Debug artifacts
remain separate from the comparison bundle; the bundle does not implicitly
enable full Promptkit diagnostics.
## Notification Policy
Profile comparisons never invoke Distributor notification, even when
notification is enabled in the effective configuration. Comparison reports
are local development artifacts rather than ordinary report publications.
Adding comparison publication or upload behavior would require a separate
accepted feature scope and explicit operator authorization.
## Architectural End State
Application orchestration exposes a reusable prepared-report boundary that
contains the resolved report, shared collected and derived facts, module
snapshot, briefing metadata, generated-text handler, render inputs, and exact
serialized data package. That boundary is immutable during concurrent profile
execution.
Ordinary `generate` behavior continues to prepare once and execute once.
`compare` prepares once and executes many without duplicating the generation
workflow or calling `GenerateDetailed` in a loop. Shared preparation,
profile-specific Promptkit execution, structured-output validation, rendering,
and artifact publication remain distinct responsibilities.
The Promptkit adapter remains the only owner of dependency-specific types and
engine calls. The CLI owns parsing and user-facing summaries. The configuration
package owns configuration. Application orchestration owns comparison order,
concurrency lifecycle, failure aggregation, and bundle publication. Domain,
prompt-input, generated-text, and template packages retain their existing
deterministic contracts.
## Scope
The completed feature includes:
- the `compare` CLI command for every implemented generated-text report;
- repeatable explicit profile selection and validation;
- configured and CLI output-directory integration through the implemented
destination policy;
- one-time report resolution, collection, deterministic preparation, and YAML
serialization;
- concurrent execution through one Promptkit executor with backend capacity
respected;
- independent validation and rendering with deterministic ordered results;
- the flat, versioned comparison-bundle contract;
- safe filename derivation and data-package hashing;
- coherent partial-result publication and non-zero failure behavior;
- guarded whole-bundle replacement through `--replace`;
- comparison-aware, concurrency-safe optional prompt debugging;
- explicit suppression of Distributor notification;
- structured normal and quiet-mode CLI behavior consistent with existing
commands;
- focused race-safe tests across configuration, CLI, application,
Promptkit-adapter, rendering, and filesystem boundaries; and
- updates to every affected canonical user, operator, architecture,
integration, and internal document.
## Compatibility
The feature is additive. Existing `generate` and `run` commands, report
definitions, profile defaults, configuration, output filenames, notification
behavior, and exit contracts remain unchanged.
The comparison manifest and bundle layout begin as versioned contracts. They
do not become inputs accepted by Weatherreporter, and no backward-compatible
replay or long-term archive guarantee is implied beyond identifying the schema
used to interpret a produced bundle.
## Testing Expectations
Tests should provide durable coverage for:
- report and date parsing consistent with `generate`;
- rejection of fewer than two profiles, blanks, and duplicates;
- inspection of the exact prompt and every profile before collection;
- no collection or model execution after a preflight failure;
- exactly one weather collection and one preparation for several profiles;
- byte-for-byte identical data-package input in every execution;
- observable concurrent execution through a concurrency-safe fake executor;
- respect for Promptkit-owned backend capacity in an assembled adapter test
where that integration adds distinct confidence;
- deterministic filenames, manifest order, summaries, and errors under varied
completion order;
- continuation and coherent partial publication after one profile fails;
- cancellation propagation, goroutine completion, and preservation of an
existing destination;
- destination precedence and each derived default directory;
- safe filename handling for unusual valid profile IDs;
- absent, empty, occupied, symlinked, unsafe, recognized, and unrecognized
replacement targets;
- removal of stale prior report files during authorized whole-bundle
replacement;
- exact package digest and manifest/result consistency;
- concurrency-safe, non-colliding opt-in debug artifacts; and
- absence of Distributor calls for complete and partial comparisons.
Concurrency and replacement behavior require race-enabled and consequential
failure-path coverage. Tests must remain deterministic, offline, credential
free, and independent of real Promptkit providers or machine-specific paths.
## Documentation End State
Once implemented, the [CLI reference](../cli.md) owns command syntax, flags,
summary, and exit behavior. The [operations guide](../operations.md) owns the
bundle lifecycle, replacement procedure, sensitivity guidance, and practical
prompt-comparison workflow. The [architecture policy](../policy/architecture.md)
owns the statelessness, concurrency, notification, and publication invariants.
The [Promptkit integration guide](../integrations/promptkit.md) should describe
the consumer-visible multi-profile execution boundary without duplicating
Promptkit's backend-capacity reference. The versioned manifest and flat bundle
format belong in a focused contract under `docs/integrations/`. App
orchestration, prompt input, generated text, prompt debugging, and bundle
publication mechanics belong in focused documents under `docs/internal/`.
Current-state documentation must not describe profile comparison as available
until the implementation lands.
## Non-Goals
This roadmap does not introduce:
- automatic model scoring, ranking, recommendation, or winner selection;
- semantic or textual diff generation between reports;
- repeated sampling of one profile or statistical evaluation;
- prompt or profile editing through Weatherreporter;
- replaying a saved data package as command input;
- comparing several report types in one command;
- Weatherreporter-owned backend concurrency or queue configuration;
- automatic retries beyond Promptkit's existing execution contract;
- Distributor upload or other external publication;
- comparison history, indexing, retention, cleanup schedules, or implicit
discovery of prior bundles; or
- changes to ordinary report content or normal generation behavior.
Any later automated evaluation, replay, sampling, or publication feature
requires a separate accepted roadmap.
## Completion Criteria
The feature is complete when a maintainer can select several Promptkit
profiles, have them execute concurrently against one exact prepared report
package, and receive a safe, flat, deterministic comparison bundle whose
manifest accurately describes every success and failure. Configured and
explicit destinations must follow the accepted output policy, replacement must
never mix or silently destroy unrelated contents, cancellation and partial
failure must be race-safe, ordinary notification must remain disabled, and all
affected canonical documentation must describe the implemented behavior.
## Open Questions
None. The scope, prerequisites, user intent, and target behavior are defined
above.

View File

@@ -1,516 +0,0 @@
# Promptkit Migration Roadmap
Status: Completed roadmap record.
## Purpose
This roadmap records the scope, decisions, and completed outcome of replacing
the external Scriptorium CLI integration with Promptkit. Canonical
documentation outside `docs/roadmap/` owns the implemented behavior.
## Pre-Migration Baseline
Status: Historical migration input.
Before the migration, Weatherreporter exposed seven report definitions, but
only four had complete prompt-backed report implementations:
- Daily Report: `weather.daily_generated_text`
- Today Report: `weather.today_generated_text`
- Tomorrow Report: `weather.tomorrow_generated_text`
- Hourly Report: `weather.hourly_generated_text`
The three-day, weekend, and storm commands and registry definitions had no
corresponding Scriptorium prompt or schema and never formed complete
operational report products. The `weather.daily_report` Scriptorium prompt was
legacy source material and was not selected by the registry.
The Scriptorium source corpus was retained temporarily under
`docs/roadmap/scriptorium/` as migration input. It contained the four
operational generated-text prompt definitions, their referenced content,
private response schemas, shared instructions, and the unused legacy Daily
Markdown prompt. The temporary corpus was removed after the runtime assets
were reconciled and embedded.
## Implemented End State
Status: Completed.
Weatherreporter pins
`gitea.maximumdirect.net/eric/promptkit` at `v0.4.0` and uses it as the
in-process engine for prompt inspection, prepared execution, provider calls,
and first-pass output validation.
The `scriptorium` executable, subprocess adapter, configuration, runtime
dependency, direct-Markdown execution path, and integration documentation have
been removed. The four operational reports continue to use structured
generated text followed by weatherreporter-owned validation and Markdown
templates.
The unfinished three-day, weekend, and storm reports are not implemented as
part of this migration. Their incomplete CLI, registry, documentation, and
generation declarations are removed from the implemented surface before the
migration is considered complete. Any future implementation of those products
requires separate roadmap scope, prompt and schema design, tests, and
documentation.
Weather selection, forecast derivation, valid periods, module construction,
Recent Changes, generated-text interpretation, Markdown templates, durable
state, inspection, output copies, and Distributor notification remain owned by
weatherreporter.
The four report prompts and private response schemas are versioned embedded
application assets. Operators configure Promptkit profiles without replacing
the report-owned corpus. One Promptkit engine is constructed per CLI
invocation and shared by every report in that invocation, including all
reports in a morning or evening batch.
Promptkit is isolated behind a weatherreporter-owned execution contract.
Promptkit request, result, validation, error, profile, backend, and provider
types do not leak into application orchestration, report definitions, domain
packages, CLI summaries, durable state contracts, or Distributor behavior.
## Goals
Status: Completed migration outcomes.
- Removed the Scriptorium runtime dependency and subprocess boundary.
- Migrated the four operational report prompts to Promptkit `v0.4.0`.
- Used prepared execution to persist preparation provenance before provider work
while executing the exact frozen snapshot.
- Validated report prompt and profile selections before weather collection when
the required information is available.
- Preserved deterministic module snapshots and structured Recent Changes.
- Preserved generated-text domain validation and repository-owned Markdown
rendering.
- Preserved context cancellation, actionable errors, secret redaction, and
inspectable failures.
- Improved durable prompt provenance with prompt, input, profile, model,
validation, usage, and timing metadata.
- Kept content-rich prompt and response diagnostics separate from routine
metadata and CLI output.
- Kept tests offline and deterministic through injected Promptkit model
clients and fixtures.
- Removed incomplete report declarations from the implemented product surface
rather than creating new report products during an integration migration.
## Non-Goals
Status: Completed migration constraints.
The completed migration did not:
- create prompts, schemas, templates, or completed products for three-day,
weekend, or storm reports;
- preserve the unused `weather.daily_report` legacy Markdown prompt as an
active runtime asset;
- preserve a direct-Markdown LLM generation mode;
- move meteorological selection, derivation, thresholds, or comparison logic
into prompts or Promptkit;
- send raw unbounded Weather API responses to the model;
- replace weatherreporter's generated-text domain validation or Markdown
template rendering;
- add a general workflow engine, provider plugin system, or arbitrary backend
registry;
- add automatic provider, validation, repair, or capacity retries;
- add concurrent report generation to the sequential batch workflow;
- expose Promptkit types as weatherreporter contracts;
- keep a production-selectable Scriptorium/Promptkit dual-run mode;
- require Promptkit eager source validation, structured generation errors, or
semantic execution-target fingerprints; or
- use an unpublished Promptkit commit, committed Go workspace, or committed
local module replacement.
## Locked Decisions
Status: Implemented migration decisions.
### Dependency And Upgrade Boundary
- The migration pins the tagged Promptkit `v0.4.0` release.
- Coordinated local development may temporarily use the sibling Promptkit
checkout, but committed module metadata must reference the tagged release.
- The adapter relies on the public root Promptkit package only.
- A future Promptkit upgrade requires explicit review of prepared-execution
lifecycle, prompt and profile inspection, prompt/profile/schema formats,
error identities, validation behavior, capacity behavior, and the outbound
provider contract.
- Promptkit's deferred eager source validation, structured generation errors,
and semantic execution-target fingerprints do not block this migration.
### Operational Report Scope
- The migration preserves these prompt IDs:
`weather.daily_generated_text`, `weather.today_generated_text`,
`weather.tomorrow_generated_text`, and `weather.hourly_generated_text`.
- Each operational report definition selects the exact embedded prompt version
`1.0.0`; execution does not rely on ambiguous single-version lookup.
- Morning and evening batch membership remains based on Today, Tomorrow, and
eligible future Daily reports.
- Three-day, weekend, and storm are removed from current CLI help, parsing,
report registry membership, tests that claim implemented generation, and
non-roadmap documentation.
- The future product concepts may remain under `docs/roadmap/`, but migration
verification does not invent outputs or compare nonexistent prompts.
### Application Boundary
- Promptkit remains an adapter boundary even though it runs in process.
- A weatherreporter-owned contract represents prompt identity, preparation,
execution, output, validation, usage, provenance, and neutral error
categories.
- The Promptkit adapter maps public Promptkit values into that contract.
- App orchestration and test fakes depend on the project-owned contract, not
Promptkit.
- Scriptorium-specific request, result, error, and generation-mode types are
removed rather than renamed and retained.
### Prompt And Schema Ownership
- Weatherreporter embeds the four operational prompt definitions, referenced
prompt content, shared prompt content, and private response schemas.
- Assets remain separate files rather than inline Go strings.
- The temporary corpus under `docs/roadmap/scriptorium/` is migration source
material, not the final runtime location.
- Weatherreporter's existing generated-text domain types, schemas, and
templates remain the canonical application contract. Imported Scriptorium
assets are reconciled with that contract rather than copied blindly or kept
as duplicate runtime schemas.
- The imported Daily schema's incorrect Today `$id` and title are corrected.
- `confidence` is handled consistently across each prompt, provider-facing
schema, generated-text domain type, and template. The existing optional
weatherreporter field remains supported unless a separate domain decision
removes it.
- Prompt input metadata identifies the serialized data package as YAML rather
than JSON.
- Imported `pipeline-weather/...` schema paths are replaced with paths valid
inside the embedded Promptkit schema source.
- Imported `repair_attempts: 2` values are removed or set to zero. The
migration does not rely on Promptkit's internal-only repair capability.
- The unused `weather.daily_report` prompt is not promoted into runtime assets.
- One centralized embedded prompt/schema source is sufficient; Weatherreporter
does not need Notarius's multi-module asset-flattening registry.
### Profiles, Backends, And Credentials
- Execution profiles remain operator-configurable rather than embedded report
policy.
- Each embedded operational prompt declares Promptkit's built-in
`gemini-flash-latest` profile as its default.
- `gemini-flash-latest` is intentionally a moving model alias. The execution
record captures the effective model identity, but operators who require a
pinned model must select an explicit external profile.
- Configuration supports at most one external profile source:
`promptkit.profile_file` or `promptkit.profile_dir`. The two fields are
mutually exclusive.
- A nonblank `promptkit.profile` is the explicit request profile for every
report in the invocation and takes precedence over each prompt's
`default_profile`. A blank value uses the prompt default.
- Promptkit's normal profile-source precedence remains intact: an external
matching profile takes precedence over an embedded built-in profile, and an
invalid matching external profile is an error rather than a reason to fall
back.
- Weatherreporter exposes Promptkit's conventional `local` backend through the
narrow `promptkit.local.endpoint` and
`promptkit.local.concurrency_limit` configuration fields. It does not expose
arbitrary backend registration.
- A configured local endpoint registers the engine-scoped `local` backend. An
operator-supplied external profile selects it with `backend: local` and owns
the model-specific settings; Weatherreporter does not invent a local model
profile.
- Local concurrency defaults to one. A value of zero means unlimited, matching
Promptkit, and a negative value is invalid. Queue capacity and general
backend parameters are not exposed.
- Credential values remain in environment variables or file-backed
environment secrets. Configuration contains only credential source names.
- Provider credentials never appear in logs, errors, CLI output, durable
metadata, preparation records, execution records, or debug summaries.
- Promptkit `InspectProfile` reports structural target and credential
requirements; Weatherreporter owns policy for checking configured
environment availability.
- Promptkit revalidates environment credentials at `RunPrepared`; a successful
preparation does not promise that execution-time credentials remain
available.
### Configuration Contract
The replacement configuration surface is:
```yaml
promptkit:
profile: ""
profile_file: ""
profile_dir: ""
timeout: 2m
local:
endpoint: ""
concurrency_limit: 1
```
- `timeout` remains the transport-wide provider-call safety cap.
- A blank local endpoint leaves the conventional local backend unregistered.
- Scriptorium's `binary`, `config_path`, and `extra_args` settings have no
Promptkit equivalents and are removed.
- Configuration validation rejects simultaneous `profile_file` and
`profile_dir` values, invalid local endpoints, negative concurrency, and
selected profiles that cannot resolve their backend.
### Engine Construction And Inspection
- One Promptkit engine is constructed per CLI invocation at the application
composition boundary.
- Single-report generation and every report in a batch use that same engine.
- Per-report orchestration does not construct a default engine.
- Promptkit backend capacity state and HTTP transport are shared consistently
for the invocation.
- Before collection, `InspectPrompt` checks every selected report's exact ID
and version, declared `data_package` input, default-profile metadata, prompt
hash availability, and declared output contract.
- `InspectPrompt` is a point-in-time structural check. It does not load a JSON
Schema, resolve a profile, or freeze later execution.
- Explicit profile overrides and relevant prompt defaults are checked with
`InspectProfile` before collection when application policy requires them.
- `InspectProfile` is also point-in-time and does not check credential values.
- Successful `PrepareExecution`, not inspection, is the per-run authority for
loaded schema, rendered content, frozen inputs, effective settings, and
durable execution provenance.
### Prompt Input
- Promptkit receives only the curated `data_package` produced by
`internal/promptinput`.
- Weatherreporter serializes the package once, atomically persists those exact
bytes, and supplies the same bytes with a Promptkit inline artifact.
- The managed data-package path may be supplied as non-secret provenance
through the inline artifact URI.
- Weatherreporter does not delegate unrestricted path loading to Promptkit's
default file artifact reader.
- Prompt inspection and adapter tests verify that `data_package` is required
and declared with the chosen YAML media type.
### Prepared Execution
- `Engine.PrepareExecution` replaces Scriptorium render preflight.
- Weatherreporter obtains `PreparedExecution.Details`, maps a safe subset into
its own preparation record, and persists that record before calling
`Engine.RunPrepared`.
- `RunPrepared` executes the frozen prompt, profile, schema, inputs, rendered
messages, target, and validation resources retained by the handle.
- Every acquired handle is followed immediately by `defer handle.Discard()`.
Discard is safe after execution and releases unused private execution state.
- Handles remain adapter-local, engine-bound, one-shot, in-process values.
They are never serialized, persisted, copied into app contracts, or treated
as restartable jobs.
- Preparation and execution use independent contexts. Execution receives the
active report workflow context.
- Capacity is not reserved during preparation. Capacity rejection can
therefore occur after a preparation record has been persisted.
- `RunPrepared` consumes the handle on success and every operational failure.
- Preparation details remain available from the adapter after execution or
discard, but rendered messages are not copied into routine durable state.
- Promptkit execution timing excludes preparation and consumer-held delay.
Weatherreporter records preparation timing and execution timing separately.
### Execution And Validation
- All four operational reports use Promptkit JSON Schema output validation.
- A completed Promptkit validation rejection returns a `RunResult`; the
adapter retains raw output and bounded validation details before failing the
report.
- An operational generation or validation error returns no partial
`RunResult`.
- Weatherreporter's `internal/generatedtext` validation remains the final
report-specific decode and domain boundary.
- Weatherreporter's `internal/reporttemplate` remains responsible for managed
Markdown rendering.
- Weatherreporter atomically persists Promptkit raw output and later artifacts
rather than asking Promptkit to choose managed filesystem paths.
- No Promptkit output-repair behavior is assumed or requested.
## Durable Artifacts And Observability
Status: Implemented design constraints.
Routine durable state retains useful non-secret provenance without persisting
full rendered prompts.
The preparation record contains:
- prompt ID and exact version;
- prompt definition hash;
- rendered prompt hash;
- input hashes;
- selected profile and backend identity;
- effective model identity;
- output contract summary;
- preparation start, end, and duration; and
- the path of the exact persisted data package.
The execution record and run metadata contain, when available:
- Promptkit run ID;
- prompt ID, version, and hashes;
- input hashes;
- selected profile, backend, and model identity;
- generated-content hash;
- token usage;
- execution start, end, and duration;
- validation status and bounded diagnostics; and
- paths of separately persisted raw output, normalized generated text, render
context, managed Markdown, and other artifacts reached by the workflow.
Provider endpoints, full effective model parameter maps, rendered messages,
schema bodies, data-package contents, and generated content do not belong in
routine metadata or CLI summaries.
Rendered messages and other content-rich preparation or response diagnostics
are available only when the operator supplies
`--llm-debug-dir <path>` to a single-report or batch command.
- There is no persistent YAML setting for debug capture.
- The debug root is validated or created before weather collection or provider
work. A requested destination that cannot be secured or written is an error.
- Artifacts are grouped beneath
`<path>/<report-id>/<valid-date>/<run-id>/`.
- Directories and files use owner-only permissions and atomic writes.
- Debug artifacts may contain rendered messages and content-rich preparation
or response diagnostics, but never credentials.
- The debug path appears in command output only when debug capture is enabled;
it is not added to routine durable metadata.
- Debug artifacts are not cache or comparison inputs. Their retention is owned
by the operator who selected the directory.
### Artifact Identities And Versions
Weatherreporter replaces Scriptorium-specific artifact identities rather than
reusing names whose meanings have changed:
- `PromptPreparationArtifact` uses schema version
`weatherreporter.prompt_preparation.v1`, is written as
`prompt_preparation.<runID>.json`, and is referenced by
`preparationPath`.
- `PromptExecutionArtifact` uses schema version
`weatherreporter.prompt_execution.v1`, is written as
`prompt_execution.<runID>.json`, and is referenced by `executionPath`.
- Run metadata advances to `weatherreporter.metadata.v2` and uses those new
path fields.
Preparation files remain beneath the existing configurable `preflight/`
directory, and execution files remain beneath the existing `snapshots/` tree.
The stable physical grouping limits deployment disruption without preserving
misleading Scriptorium-era filenames or field names. Raw generated output,
normalized generated text, render context, managed Markdown, and other
artifacts whose meanings have not changed retain their existing names and
locations.
Run inspection remains able to read `weatherreporter.metadata.v1` and its
legacy `preflightPath` and `generatedTextResultPath` references. New runs write
only the v2 metadata and new artifact names; Weatherreporter does not
dual-write deprecated aliases. CLI summary fields adopt `preparationPath` and
`executionPath` as an explicit, documented contract change.
## Failure Contract
Status: Implemented design constraints.
- A preparation failure produces a redacted weatherreporter-owned failure
receipt with report, RunID, prompt, stage, timing, and classified error
context. It does not fabricate Promptkit preparation details.
- An operational execution failure retains the successful preparation record
and adds a redacted execution failure receipt. No partial Promptkit result or
model output is invented.
- A Promptkit validation rejection retains the returned result, raw generated
output, validation details, and safe provenance before the report fails.
- A later generated-text decode, domain-validation, or template failure
retains every raw and validated artifact reached before that stage.
- Caller cancellation takes precedence when the active workflow context is
canceled.
- `promptkit.CapacityError` is recognized with `errors.As`; its backend ID is
copied into a weatherreporter-owned capacity error while
`ErrCapacityExceeded` remains the classification.
- Capacity rejection is an operational report failure, not invalid model
output, and does not trigger an automatic retry.
- Other Promptkit public error identities are translated into the narrow
weatherreporter error categories needed by CLI, metadata, and batch
behavior. Diagnostic prose is not parsed as a contract.
- Single-report commands return the classified failure with available
inspectable paths.
- Batch runs continue independent later reports under the existing batch
failure policy.
- Any future retry policy belongs to app orchestration, not the adapter.
## Compatibility Requirements
Status: Implemented design constraints.
- Daily, Today, Tomorrow, and Hourly report IDs, prompt IDs, valid periods,
artifact grouping, output names, and Distributor bundle behavior remain
stable.
- Morning and evening batch collection, planning, ordering, and continuation
behavior remains stable.
- Module snapshot and Recent Changes behavior remains deterministic.
- Promptkit receives only the existing curated prompt-input boundary.
- Managed Markdown remains the Distributor upload source.
- RunID lookup and inspection remain available for successful and failed runs.
- Existing managed paths remain stable where their meaning is unchanged.
Scriptorium-specific artifact names or schemas change when retaining them
would misrepresent the Promptkit contract.
- Existing v1 run metadata and referenced artifacts remain inspectable after
the migration. New runs use the v2 metadata and Promptkit-era artifact
identities without dual-writing deprecated aliases.
- Artifact or metadata schema changes are explicit, documented, and covered by
state and inspection tests.
- Prompt or generated content is not added to routine logs or CLI summaries.
- Tests do not require live providers or credentials.
- Removing incomplete three-day, weekend, and storm surfaces is documented as
correction of an unfinished product boundary, not as successful Promptkit
migration of those reports.
## Verification And Completion Criteria
Status: Completed and verified.
Completion was verified by the following outcomes:
- the four operational reports inspect, prepare, and execute through Promptkit
`v0.4.0` using embedded report-owned assets;
- every report uses exact prompt version `1.0.0`, requires the YAML
`data_package`, and declares the expected JSON Schema output contract;
- prepared execution persists a safe preparation record before provider work
and executes the same frozen snapshot;
- deterministic offline adapter and app tests cover success, preparation
failure, credential revalidation, capacity rejection, cancellation, timeout,
generation failure, Promptkit validation rejection, generated-text domain
failure, template failure, and handle discard;
- morning and evening batches construct one engine and preserve current
collection, planning, ordering, continuation, output, and notification
behavior;
- the temporary corpus has been reconciled into one runtime prompt/schema
source without duplicate provider-facing schemas;
- configuration examples load and contain no Scriptorium fields;
- CLI summaries and inspection commands expose the new project-owned artifact
contract without Promptkit types;
- Scriptorium code, configuration, tests, and runtime documentation have been
removed;
- incomplete three-day, weekend, and storm commands, registry entries, tests,
and current-behavior documentation have been removed or moved to roadmap
scope;
- non-roadmap documentation describes only the implemented Promptkit
integration;
- `go test ./...`, required focused or race-enabled checks, CLI help
validation, and `git diff --check` pass; and
- no committed `go.work`, local `replace`, live-provider test, or
secret-bearing fixture remains.
Fixture-based comparison with prior Scriptorium behavior is sufficient.
Production dual-run is not required because model calls are nondeterministic,
costly, and difficult to compare meaningfully.
## Decision Status
Status: Completed.
The roadmap has no remaining open product or architecture questions. Later
changes to this completed scope require new roadmap or decision-record scope
rather than implicit changes to this historical record.

View File

@@ -128,8 +128,7 @@ It is not a source for deterministic weather facts.
| --- | --- | --- | --- |
| `.GeneratedText.Summary` | `string` | `string` | Required. |
| `.GeneratedText.ForecastDiscussion` | `string` | `[]string` | Required; range over the day-style paragraph slice. |
| `.GeneratedText.PrecipitationTiming` | `string` | `string` | Optional prose used by the precipitation partial when deterministic windows exist. |
| `.GeneratedText.Confidence` | `string` | `string` | Optional validated prose; the current templates do not render it. |
| `.GeneratedText.PrecipitationTiming` | `string` | `string` | Required field; an empty string represents no supported prose. The precipitation partial uses nonempty prose only when deterministic windows exist. |
The JSON schema rejects unknown properties and defines the required fields, but
the schema body and validation behavior are documented in [Generated Text

View File

@@ -1,48 +0,0 @@
# Troubleshooting
Keep failed workspace artifacts in place. When a RunID is available, start
with `weatherreporter inspect metadata RUN_ID` and use the paths in its result.
## Prompt inspection or credentials fail before collection
A prompt/version, contract, selected profile, unsupported direct-key profile,
or required environment credential can fail before weather collection. Correct
the configured `promptkit` profile or profile source, confirm the exact
Promptkit asset is available, and supply any reported environment credential.
Do not add provider keys to YAML. See [configuration](config.md).
## Preparation, capacity, or execution fails
A preparation failure occurs before provider work; an execution failure occurs
after preparation. Both leave safe provenance and metadata when reached. A
capacity error for one batch report does not retry that report or prevent later
independent reports. Inspect the preparation or execution path, correct the
profile/backend condition, and create a new run. See [operations](operations.md).
## Generated text fails validation
Raw generated output may be saved but Markdown is not rendered when the JSON
does not match the report schema. Correct the Promptkit prompt/profile behavior
or the matching schema and validator in source control; do not edit raw output
to treat it as validated. See [templates](templates.md).
## Debug capture fails
`--llm-debug-dir` must be an absolute secure directory outside workspace state.
A debug-write failure stops the affected report to avoid continuing without the
requested diagnostic. Repair the named path's ownership or permissions, then
rerun. Treat capture files as sensitive. See [operations](operations.md).
## Weather, state, output, or notification fails
Collection errors precede planning. Later filesystem, output-copy, template,
or Distributor errors retain the reached safe paths in the summary. Repair only
the reported endpoint or path, leave successful managed reports intact, and
rerun the affected report or batch. A batch notification is intentionally
skipped when any report item fails.
## Secrets cannot be loaded
Secret files must be regular non-symlink files directly beneath
`secrets.directory` with valid environment-variable basenames. Correct the
reported file or directory without placing secret values in YAML.

View File

@@ -14,6 +14,9 @@ location:
secrets:
directory: ""
output:
directory: /var/lib/weatherreporter/reports
notify:
distributor:
enabled: false
@@ -40,14 +43,6 @@ promptkit:
local:
concurrency_limit: 1
workspace:
root: workspace
snapshots_dir: snapshots
reports_dir: reports
data_packages_dir: data-packages
preflight_dir: preflight
notifications_dir: notifications
dayparts:
- name: overnight
start: "00:00"
@@ -65,12 +60,6 @@ dayparts:
start: "17:00"
end: "24:00"
recent_change:
temperature_degrees: 5
precip_probability_points: 20
wind_gust_miles_per_hour: 10
precip_timing_shift_minutes: 120
reports:
daily:
distributor:

View File

@@ -0,0 +1,4 @@
id: weather-light
endpoint: http://127.0.0.1:11434/v1
model: weather-local
timeout_seconds: 180

2
go.mod
View File

@@ -6,7 +6,7 @@ require gopkg.in/yaml.v3 v3.0.1
require (
gitea.maximumdirect.net/eric/distributor v0.5.0
gitea.maximumdirect.net/eric/promptkit v0.4.0
gitea.maximumdirect.net/eric/promptkit v0.5.0
)
require (

4
go.sum
View File

@@ -1,7 +1,7 @@
gitea.maximumdirect.net/eric/distributor v0.5.0 h1:+al7Bw+kMv6V35a3Sm5rUtCTQhwOn5b9x3RsclPMKJk=
gitea.maximumdirect.net/eric/distributor v0.5.0/go.mod h1:G03FCFZPHpsUKC6SeMgTdbfNRpPQBdyTtDUj04e1Tu8=
gitea.maximumdirect.net/eric/promptkit v0.4.0 h1:WHRQEt3BVBAR7hQePBaGtNXpzrs59mlr/42nQzwgOz4=
gitea.maximumdirect.net/eric/promptkit v0.4.0/go.mod h1:R95NM6fbMDGDC0/UomgnSBP6ui2ns+8SZb8bESNvrDQ=
gitea.maximumdirect.net/eric/promptkit v0.5.0 h1:jnpazLyyNhWrB2xzwwtUkNUfktkTdkENTwuSPnKiYrc=
gitea.maximumdirect.net/eric/promptkit v0.5.0/go.mod h1:R95NM6fbMDGDC0/UomgnSBP6ui2ns+8SZb8bESNvrDQ=
github.com/aws/aws-sdk-go-v2 v1.41.9 h1:/rYeyO2+HrMztAmxAq9++XJtFMqSIpSsNA0yDGALYq4=
github.com/aws/aws-sdk-go-v2 v1.41.9/go.mod h1:+HsoOEX80qAVUitj1A2DhCNTjmb3edVyuDypb6LNEeo=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.11 h1:h5+3VT69KUBK24grGuuA5saDJTj2IIjLb9au668Fo5I=

View File

@@ -45,6 +45,7 @@ func newAdapter(config Config, additionalOptions ...promptkit.Option) (*Adapter,
options := []promptkit.Option{
promptkit.WithPromptFS(promptassets.PromptFS(), "."),
promptkit.WithSchemaFS(promptassets.SchemaFS(), "."),
promptkit.WithFallbackProfileFS(promptassets.ProfileFS(), "."),
}
if config.ProfileFile != "" {
options = append(options, promptkit.WithProfileFile(config.ProfileFile))
@@ -123,9 +124,7 @@ func (adapter *Adapter) Execute(ctx context.Context, request promptexec.ExecuteR
PromptID: request.PromptID,
PromptVersion: request.PromptVersion,
ProfileID: request.ProfileID,
Inputs: map[string]promptkit.ArtifactRef{
"data_package": promptkit.InlineWithURI(request.DataPackagePath, string(append([]byte(nil), request.DataPackage...))),
},
Inputs: map[string]promptkit.ArtifactRef{"data_package": promptkit.Inline(string(append([]byte(nil), request.DataPackage...)))},
})
if err != nil {
return nil, classifyError(err)
@@ -133,7 +132,7 @@ func (adapter *Adapter) Execute(ctx context.Context, request promptexec.ExecuteR
defer prepared.Discard()
details := prepared.Details()
preparation, debug := preparationValues(details, request.DataPackagePath, request.CaptureDebug)
preparation, debug := preparationValues(details, request.CaptureDebug)
if preparedCallback != nil {
if err := preparedCallback(preparation, debug); err != nil {
return nil, err
@@ -144,7 +143,7 @@ func (adapter *Adapter) Execute(ctx context.Context, request promptexec.ExecuteR
if err != nil {
return nil, classifyError(err)
}
return executionValue(result, request.DataPackagePath, request.CaptureDebug), nil
return executionValue(result, request.CaptureDebug), nil
}
func outputContract(value promptkit.OutputContract) promptexec.OutputContract {
@@ -155,7 +154,7 @@ func outputContract(value promptkit.OutputContract) promptexec.OutputContract {
}
}
func preparationValues(value promptkit.PreparedRun, dataPackagePath string, captureDebug bool) (promptexec.Preparation, *promptexec.PreparationDebug) {
func preparationValues(value promptkit.PreparedRun, captureDebug bool) (promptexec.Preparation, *promptexec.PreparationDebug) {
preparation := promptexec.Preparation{
PromptID: value.PromptID,
PromptVersion: value.PromptVersion,
@@ -169,7 +168,6 @@ func preparationValues(value promptkit.PreparedRun, dataPackagePath string, capt
StartedAt: value.StartTime,
EndedAt: value.EndTime,
Duration: time.Duration(value.DurationMS) * time.Millisecond,
DataPackagePath: dataPackagePath,
}
if !captureDebug {
return preparation, nil
@@ -185,7 +183,7 @@ func preparationValues(value promptkit.PreparedRun, dataPackagePath string, capt
return preparation, debug
}
func executionValue(value *promptkit.RunResult, dataPackagePath string, captureDebug bool) *promptexec.Execution {
func executionValue(value *promptkit.RunResult, captureDebug bool) *promptexec.Execution {
if value == nil {
return nil
}
@@ -213,12 +211,11 @@ func executionValue(value *promptkit.RunResult, dataPackagePath string, captureD
CachedTokens: value.Usage.CachedTokens,
CacheWriteTokens: value.Usage.CacheWriteTokens,
},
StartedAt: value.StartTime,
EndedAt: value.EndTime,
Duration: value.Duration,
Validation: validation,
DataPackagePath: dataPackagePath,
RawOutput: []byte(value.RawOutput),
StartedAt: value.StartTime,
EndedAt: value.EndTime,
Duration: value.Duration,
Validation: validation,
RawOutput: []byte(value.RawOutput),
}
if captureDebug {
execution.Debug = &promptexec.ExecutionDebug{

View File

@@ -68,11 +68,11 @@ func (client *fakeClient) request() promptkit.GenerateRequest {
func TestInspectPromptAndProfile(t *testing.T) {
adapter := newTestAdapter(t, &fakeClient{})
inspection, err := adapter.InspectPrompt(context.Background(), "weather.daily_generated_text", "1.0.0")
inspection, err := adapter.InspectPrompt(context.Background(), "weather.daily_generated_text", "2.0.0")
if err != nil {
t.Fatalf("InspectPrompt() error = %v", err)
}
if inspection.PromptID != "weather.daily_generated_text" || inspection.PromptVersion != "1.0.0" || inspection.DefaultProfileID != "gemini-flash-latest" {
if inspection.PromptID != "weather.daily_generated_text" || inspection.PromptVersion != "2.0.0" || inspection.DefaultProfileID != "weather-balanced" {
t.Fatalf("inspection = %#v", inspection)
}
if len(inspection.Inputs) != 1 || inspection.Inputs[0].Name != "data_package" || !inspection.Inputs[0].Required || inspection.Inputs[0].ContentType != "application/yaml" {
@@ -102,6 +102,107 @@ func TestInspectPromptAndProfile(t *testing.T) {
}
}
func TestEmbeddedProfilesAreAvailableToProductionAndTestAdapters(t *testing.T) {
adapter, err := New(Config{})
if err != nil {
t.Fatalf("New() error = %v", err)
}
for _, want := range []struct {
id string
backend string
model string
}{
{"weather-light", "openrouter", "deepseek/deepseek-v4-flash"},
{"weather-balanced", "openrouter", "~google/gemini-flash-latest"},
{"weather-deep", "openrouter", "~anthropic/claude-sonnet-latest"},
} {
t.Run(want.id, func(t *testing.T) {
assertProfile(t, adapter, want.id, want.backend, want.model)
})
}
testAdapter, err := newAdapterForTest(Config{}, &fakeClient{})
if err != nil {
t.Fatalf("newAdapterForTest() error = %v", err)
}
assertProfile(t, testAdapter, "weather-light", "openrouter", "deepseek/deepseek-v4-flash")
}
func TestConfiguredProfilesOverrideEmbeddedFallbacks(t *testing.T) {
file := writeProfileFile(t, `id: weather-light
endpoint: https://local-file.example/v1
model: file-light
`)
fileAdapter, err := New(Config{ProfileFile: file})
if err != nil {
t.Fatalf("New(profile file) error = %v", err)
}
assertProfile(t, fileAdapter, "weather-light", "", "file-light")
directory := testProfileDirectory(t, `id: weather-light
backend: local
model: directory-light
`)
directoryAdapter, err := New(Config{ProfileDirectory: directory, LocalEndpoint: "https://local-directory.example/v1"})
if err != nil {
t.Fatalf("New(profile directory) error = %v", err)
}
assertProfile(t, directoryAdapter, "weather-light", promptkit.BackendLocal, "directory-light")
}
func TestMaintainedWeatherLightLocalProfileExampleInspectsOffline(t *testing.T) {
adapter, err := New(Config{ProfileFile: filepath.Join("..", "..", "..", "examples", "weather-light-local-profile.yml")})
if err != nil {
t.Fatalf("New() error = %v", err)
}
assertProfile(t, adapter, "weather-light", "", "weather-local")
}
func TestProfileResolutionFallsThroughOnlyWhenTheConfiguredIDIsAbsent(t *testing.T) {
absentAdapter, err := New(Config{ProfileDirectory: testProfileDirectory(t, `id: other-profile
backend: openrouter
model: other-model
`)})
if err != nil {
t.Fatalf("New(absent profile) error = %v", err)
}
assertProfile(t, absentAdapter, "weather-light", "openrouter", "deepseek/deepseek-v4-flash")
malformedAdapter, err := New(Config{ProfileDirectory: testProfileDirectory(t, `id: weather-light
backend: openrouter
`)})
if err != nil {
t.Fatalf("New(malformed profile) error = %v", err)
}
if _, err := malformedAdapter.InspectProfile(context.Background(), "weather-light"); err == nil {
t.Fatal("InspectProfile() error = nil, want malformed configured profile error")
}
}
func TestProfileResolutionPreservesBuiltInAndExplicitPrecedence(t *testing.T) {
adapter, err := New(Config{})
if err != nil {
t.Fatalf("New() error = %v", err)
}
builtin, err := adapter.InspectProfile(context.Background(), "gemini-flash-latest")
if err != nil {
t.Fatalf("InspectProfile(builtin) error = %v", err)
}
if builtin.ProfileID != "gemini-flash-latest" || builtin.BackendID != "openrouter" || builtin.ModelName == "" {
t.Fatalf("builtin profile = %#v", builtin)
}
explicit, err := newAdapter(Config{}, promptkit.WithProfiles(promptkit.Profile{
ID: "weather-light",
Endpoint: "https://explicit.example/v1",
Model: "explicit-light",
}))
if err != nil {
t.Fatalf("newAdapter(explicit profile) error = %v", err)
}
assertProfile(t, explicit, "weather-light", "", "explicit-light")
}
func TestExecuteUsesPreparedInlineDataPackage(t *testing.T) {
client := &fakeClient{response: validResponse()}
adapter := newTestAdapter(t, client)
@@ -109,7 +210,7 @@ func TestExecuteUsesPreparedInlineDataPackage(t *testing.T) {
callbackCalls := 0
result, err := adapter.Execute(context.Background(), request, func(preparation promptexec.Preparation, debug *promptexec.PreparationDebug) error {
callbackCalls++
if preparation.PromptID != request.PromptID || preparation.PromptVersion != request.PromptVersion || preparation.DataPackagePath != request.DataPackagePath || preparation.ModelName != "test-model" {
if preparation.PromptID != request.PromptID || preparation.PromptVersion != request.PromptVersion || preparation.ModelName != "test-model" {
t.Fatalf("preparation = %#v", preparation)
}
if debug != nil {
@@ -126,7 +227,7 @@ func TestExecuteUsesPreparedInlineDataPackage(t *testing.T) {
if callbackCalls != 1 || client.callCount() != 1 {
t.Fatalf("callback/provider calls = %d/%d, want 1/1", callbackCalls, client.callCount())
}
if result == nil || result.Validation.Status != promptexec.ValidationPassed || string(result.RawOutput) != client.response.Content || result.DataPackagePath != request.DataPackagePath {
if result == nil || result.Validation.Status != promptexec.ValidationPassed || string(result.RawOutput) != client.response.Content {
t.Fatalf("result = %#v", result)
}
if result.Debug != nil {
@@ -141,6 +242,43 @@ func TestExecuteUsesPreparedInlineDataPackage(t *testing.T) {
}
}
func TestExecuteEmbeddedHourlyProfileThroughPreparedPath(t *testing.T) {
t.Setenv("OPENROUTER_API_KEY", "test-openrouter-key")
client := &fakeClient{response: hourlyValidResponse()}
adapter, err := newAdapter(Config{}, promptkit.WithLLMClient(client))
if err != nil {
t.Fatalf("newAdapter() error = %v", err)
}
request := promptexec.ExecuteRequest{
PromptID: "weather.hourly_generated_text",
PromptVersion: "2.0.0",
ProfileID: "weather-light",
DataPackage: []byte("report:\n id: hourly\nbriefing: {}\n"),
}
var preparation promptexec.Preparation
prepared := false
result, err := adapter.Execute(context.Background(), request, func(value promptexec.Preparation, _ *promptexec.PreparationDebug) error {
if client.callCount() != 0 {
t.Fatal("provider was called before preparation completed")
}
preparation = value
prepared = true
return nil
})
if err != nil {
t.Fatalf("Execute() error = %v", err)
}
if !prepared || preparation.ProfileID != "weather-light" || preparation.BackendID != "openrouter" || preparation.ModelName != "deepseek/deepseek-v4-flash" {
t.Fatalf("preparation = %#v", preparation)
}
if result == nil || result.ProfileID != "weather-light" || result.BackendID != "openrouter" || result.ModelName != "deepseek/deepseek-v4-flash" || result.Validation.Status != promptexec.ValidationPassed {
t.Fatalf("execution = %#v", result)
}
if client.callCount() != 1 || client.request().Target.Model != "deepseek/deepseek-v4-flash" {
t.Fatalf("provider calls/request = %d/%#v", client.callCount(), client.request())
}
}
func TestExecuteUsesExactInlineDataPackageProvenance(t *testing.T) {
client := &fakeClient{response: validResponse()}
reader := &recordingReader{}
@@ -149,7 +287,7 @@ func TestExecuteUsesExactInlineDataPackageProvenance(t *testing.T) {
if _, err := adapter.Execute(context.Background(), request, nil); err != nil {
t.Fatalf("Execute() error = %v", err)
}
if reader.ref.Type != promptkit.ArtifactRefInline || reader.ref.URI != request.DataPackagePath || reader.ref.Body != string(request.DataPackage) {
if reader.ref.Type != promptkit.ArtifactRefInline || reader.ref.URI != "" || reader.ref.Body != string(request.DataPackage) {
t.Fatalf("artifact ref = %#v, want exact inline data package provenance", reader.ref)
}
}
@@ -339,6 +477,17 @@ func newTestAdapter(t *testing.T, client promptkit.LLMClient) *Adapter {
return newTestAdapterWithOptions(t, client)
}
func assertProfile(t *testing.T, adapter *Adapter, id string, backend string, model string) {
t.Helper()
profile, err := adapter.InspectProfile(context.Background(), id)
if err != nil {
t.Fatalf("InspectProfile(%q) error = %v", id, err)
}
if profile.ProfileID != id || profile.BackendID != backend || profile.ModelName != model {
t.Fatalf("profile = %#v, want %q with backend/model %q/%q", profile, id, backend, model)
}
}
func newTestAdapterWithOptions(t *testing.T, client promptkit.LLMClient, options ...promptkit.Option) *Adapter {
t.Helper()
profiles := testProfileDirectory(t, `id: test-profile
@@ -366,19 +515,34 @@ func testProfileDirectory(t *testing.T, profile string) string {
return profiles
}
func writeProfileFile(t *testing.T, profile string) string {
t.Helper()
path := filepath.Join(t.TempDir(), "profile.yml")
if err := os.WriteFile(path, []byte(profile), 0o600); err != nil {
t.Fatalf("write profile: %v", err)
}
return path
}
func testExecuteRequest() promptexec.ExecuteRequest {
return promptexec.ExecuteRequest{
PromptID: "weather.daily_generated_text",
PromptVersion: "1.0.0",
ProfileID: "test-profile",
DataPackage: []byte("report:\n id: daily\nbriefing: {}\n"),
DataPackagePath: "data-packages/daily/data_package.yaml",
PromptID: "weather.daily_generated_text",
PromptVersion: "2.0.0",
ProfileID: "test-profile",
DataPackage: []byte("report:\n id: daily\nbriefing: {}\n"),
}
}
func validResponse() *promptkit.GenerateResponse {
return &promptkit.GenerateResponse{
Content: `{"summary":"A quiet day is expected.","forecast_discussion":["High pressure keeps conditions settled."],"confidence":"High."}`,
Content: `{"summary":"A quiet day is expected.","forecast_discussion":["High pressure keeps conditions settled."],"precipitation_timing":""}`,
Usage: promptkit.TokenUsage{PromptTokens: 12, CompletionTokens: 8, TotalTokens: 20},
}
}
func hourlyValidResponse() *promptkit.GenerateResponse {
return &promptkit.GenerateResponse{
Content: `{"summary":"A quiet hour is expected.","forecast_discussion":"Conditions remain settled.","precipitation_timing":""}`,
Usage: promptkit.TokenUsage{PromptTokens: 12, CompletionTokens: 8, TotalTokens: 20},
}
}

View File

@@ -9,17 +9,15 @@ import (
distributoradapter "gitea.maximumdirect.net/eric/weatherreporter/internal/adapters/distributor"
"gitea.maximumdirect.net/eric/weatherreporter/internal/briefing"
"gitea.maximumdirect.net/eric/weatherreporter/internal/changes"
"gitea.maximumdirect.net/eric/weatherreporter/internal/collect"
"gitea.maximumdirect.net/eric/weatherreporter/internal/config"
"gitea.maximumdirect.net/eric/weatherreporter/internal/facts"
"gitea.maximumdirect.net/eric/weatherreporter/internal/fileutil"
"gitea.maximumdirect.net/eric/weatherreporter/internal/forecast"
"gitea.maximumdirect.net/eric/weatherreporter/internal/module"
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptdebug"
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec"
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptinput"
"gitea.maximumdirect.net/eric/weatherreporter/internal/report"
"gitea.maximumdirect.net/eric/weatherreporter/internal/state"
"gitea.maximumdirect.net/eric/weatherreporter/internal/timeutil"
"gitea.maximumdirect.net/eric/weatherreporter/internal/weatherdata"
)
@@ -43,6 +41,7 @@ const (
type GenerateRequest struct {
Config config.Config
Report ReportKind
WorkingDir string
OutputPath string
LLMDebugDir string
Now time.Time
@@ -50,26 +49,20 @@ type GenerateRequest struct {
Collector Collector
Notifier Notifier
Executor promptexec.Executor
Store state.Store
}
type BatchRequest struct {
Config config.Config
Batch BatchKind
Now time.Time
WorkingDir string
OutputDir string
LLMDebugDir string
Collector Collector
Executor promptexec.Executor
Store state.Store
Notifier Notifier
}
type FetchBundleRequest struct {
Config config.Config
OutputPath string
}
type ModuleSnapshotRequest struct {
Config config.Config
Resolved report.Resolved
@@ -81,24 +74,22 @@ type ReportFacts struct {
}
type ReportResult struct {
ModuleSnapshot module.Snapshot
ModuleSnapshotPath string
DataPackage promptinput.Package
DataPackagePath string
PreparationPath string
ExecutionPath string
LLMDebugPath string
ReportPath string
OutputPath string
NotificationPath string
Metadata state.Metadata
MetadataPath string
PriorSnapshot *state.PriorSnapshot
RecentChanges []changes.Change
GeneratedTextRawPath string
GeneratedTextPath string
RenderContextPath string
Notification *NotificationResult
ReportID report.ID
ReportName string
PromptID string
PromptVersion string
RunID string
GeneratedAt time.Time
Timezone string
ValidPeriod timeutil.Period
ProfileID string
BackendID string
ModelName string
SourceWarnings []weatherdata.SourceWarning
ValidationStatus promptexec.ValidationStatus
LLMDebugPath string
OutputPath string
Notification *NotificationResult
}
type BatchResult struct {
@@ -119,7 +110,6 @@ type BatchNotificationResult struct {
PipelineID string `json:"pipelineId,omitempty"`
BundleID string `json:"bundleId,omitempty"`
IdempotencyKey string `json:"idempotencyKey,omitempty"`
Path string `json:"path,omitempty"`
IncludedReports []BatchNotificationReport `json:"includedReports,omitempty"`
Error string `json:"error,omitempty"`
}
@@ -132,26 +122,22 @@ type BatchNotificationReport struct {
}
type BatchReportResult struct {
ReportID report.ID `json:"reportId"`
ReportName string `json:"reportName"`
PromptID string `json:"promptId"`
RunID string `json:"runId"`
Status string `json:"status"`
Error string `json:"error,omitempty"`
NotificationStatus string `json:"notificationStatus,omitempty"`
NotificationRunID string `json:"notificationRunId,omitempty"`
NotificationPipelineID string `json:"notificationPipelineId,omitempty"`
NotificationError string `json:"notificationError,omitempty"`
NotificationPath string `json:"notificationPath,omitempty"`
GeneratedAt time.Time `json:"generatedAt"`
ValidPeriod timeutil.Period `json:"validPeriod"`
DataPackagePath string `json:"dataPackagePath,omitempty"`
PreparationPath string `json:"preparationPath,omitempty"`
ExecutionPath string `json:"executionPath,omitempty"`
LLMDebugPath string `json:"llmDebugPath,omitempty"`
ReportPath string `json:"reportPath,omitempty"`
OutputPath string `json:"outputPath,omitempty"`
MetadataPath string `json:"metadataPath,omitempty"`
ReportID report.ID `json:"reportId"`
ReportName string `json:"reportName"`
PromptID string `json:"promptId"`
RunID string `json:"runId"`
Status string `json:"status"`
Error string `json:"error,omitempty"`
GeneratedAt time.Time `json:"generatedAt"`
ValidPeriod timeutil.Period `json:"validPeriod"`
Timezone string `json:"timezone"`
ProfileID string `json:"profileId,omitempty"`
BackendID string `json:"backendId,omitempty"`
ModelName string `json:"modelName,omitempty"`
SourceWarnings []weatherdata.SourceWarning `json:"sourceWarnings,omitempty"`
ValidationStatus promptexec.ValidationStatus `json:"validationStatus,omitempty"`
LLMDebugPath string `json:"llmDebugPath,omitempty"`
OutputPath string `json:"outputPath,omitempty"`
}
type BatchError struct {
@@ -162,13 +148,14 @@ func (e BatchError) Error() string {
if e.Result == nil {
return "batch failed"
}
if batchNotificationFailed(e.Result) && batchReportFailures(e.Result) == 0 {
failedReports := batchReportFailures(e.Result)
if batchNotificationFailed(e.Result) && failedReports == 0 {
if e.Result.Notification.Error != "" {
return fmt.Sprintf("batch %s notification failed: %s", e.Result.Batch, e.Result.Notification.Error)
}
return fmt.Sprintf("batch %s notification failed", e.Result.Batch)
}
return fmt.Sprintf("batch %s failed: %d of %d reports failed", e.Result.Batch, e.Result.Failed, e.Result.Total)
return fmt.Sprintf("batch %s failed: %d of %d reports failed", e.Result.Batch, failedReports, len(e.Result.Reports))
}
func batchNotificationFailed(result *BatchResult) bool {
@@ -261,9 +248,15 @@ func GenerateDetailed(ctx context.Context, req GenerateRequest) (*ReportResult,
if err != nil {
return nil, err
}
debugWriter, err := state.NewPromptDebugWriter(req.LLMDebugDir)
result := initialReportResult(req, resolved, PromptInspectionResult{})
outputPath, err := resolveReportOutputPath(req.WorkingDir, req.OutputPath, req.Config.Output.Directory, resolved)
if err != nil {
return nil, promptexec.NewError(promptexec.InvalidConfiguration, "initialize prompt debug", err)
return result, err
}
req.OutputPath = outputPath
debugWriter, err := promptdebug.NewPromptDebugWriter(req.LLMDebugDir)
if err != nil {
return result, promptexec.NewError(promptexec.InvalidConfiguration, "initialize prompt debug", err)
}
inspection, err := InspectPromptExecution(ctx, PromptInspectionRequest{
Resolved: resolved,
@@ -271,11 +264,12 @@ func GenerateDetailed(ctx context.Context, req GenerateRequest) (*ReportResult,
Promptkit: req.Config.Promptkit,
})
if err != nil {
return nil, err
return result, err
}
result.ProfileID, result.BackendID, result.ModelName = inspection.ProfileID, inspection.BackendID, inspection.ModelName
collection, err := collectWeather(ctx, req.Config, req.Collector)
if err != nil {
return nil, err
return result, err
}
return generatePromptReport(ctx, promptReportRequest{
GenerateRequest: req,
@@ -283,6 +277,7 @@ func GenerateDetailed(ctx context.Context, req GenerateRequest) (*ReportResult,
Collection: *collection,
Inspection: inspection,
DebugWriter: debugWriter,
Result: result,
})
}
@@ -291,7 +286,7 @@ func RunBatch(ctx context.Context, req BatchRequest) error {
if err != nil {
return err
}
if result.Failed > 0 {
if result.Failed > 0 || batchNotificationFailed(result) {
return BatchError{Result: result}
}
return nil
@@ -305,7 +300,12 @@ func RunBatchDetailed(ctx context.Context, req BatchRequest) (*BatchResult, erro
if _, err := report.BatchForCommandName(string(req.Batch)); err != nil {
return nil, err
}
debugWriter, err := state.NewPromptDebugWriter(req.LLMDebugDir)
outputDir, err := resolveOutputDirWithConfigured(req.WorkingDir, req.OutputDir, req.Config.Output.Directory)
if err != nil {
return nil, err
}
req.OutputDir = outputDir
debugWriter, err := promptdebug.NewPromptDebugWriter(req.LLMDebugDir)
if err != nil {
return nil, promptexec.NewError(promptexec.InvalidConfiguration, "initialize prompt debug", err)
}
@@ -329,28 +329,21 @@ func RunBatchDetailed(ctx context.Context, req BatchRequest) (*BatchResult, erro
if err != nil {
return nil, err
}
if err := prepareBatchOutputs(req.OutputDir, plannedReports); err != nil {
return nil, err
}
if req.Batch == BatchEvening || req.Batch == BatchMorning {
store := req.Store
if store == nil {
defaultStore, err := defaultStore(req.Config)
if err != nil {
return nil, err
}
store = defaultStore
}
startedAt := now
result := &BatchResult{Batch: req.Batch, StartedAt: startedAt}
for _, planned := range plannedReports {
resolved := planned.Resolved
item := batchReportResult(planned)
outputPath := plannedBatchOutputPath(req.OutputDir, planned)
reportResult, err := generatePromptReport(ctx, promptReportRequest{
GenerateRequest: GenerateRequest{
Config: req.Config,
OutputPath: outputPath,
OutputPath: planned.OutputPath,
Notifier: req.Notifier,
Executor: req.Executor,
Store: store,
},
Resolved: resolved,
Collection: *collection,
@@ -359,7 +352,7 @@ func RunBatchDetailed(ctx context.Context, req BatchRequest) (*BatchResult, erro
noNotify: true,
})
if reportResult != nil {
copyBatchReportPaths(&item, reportResult)
copyBatchReportDetails(&item, reportResult)
}
if err != nil {
item.Status = "failed"
@@ -372,33 +365,25 @@ func RunBatchDetailed(ctx context.Context, req BatchRequest) (*BatchResult, erro
result.Reports = append(result.Reports, item)
}
result.Total = len(result.Reports)
batchNotification, err := notifyBatch(ctx, req.Config, req.Batch, batchRunID(startedAt, req.Batch), startedAt, result, plannedReports, store, req.Notifier)
batchNotification := notifyBatch(ctx, req.Config, req.Batch, batchRunID(startedAt, req.Batch), startedAt, result, plannedReports, req.Notifier)
if batchNotification != nil {
result.Notification = batchNotification
}
if err != nil {
result.Failed++
}
result.FinishedAt = time.Now()
return result, nil
}
return nil, fmt.Errorf("run is not implemented")
}
func copyBatchReportPaths(item *BatchReportResult, result *ReportResult) {
item.DataPackagePath = result.DataPackagePath
item.PreparationPath = result.PreparationPath
item.ExecutionPath = result.ExecutionPath
func copyBatchReportDetails(item *BatchReportResult, result *ReportResult) {
item.LLMDebugPath = result.LLMDebugPath
item.ReportPath = result.ReportPath
item.OutputPath = result.OutputPath
item.MetadataPath = result.MetadataPath
item.NotificationPath = result.NotificationPath
if result.Notification != nil {
item.NotificationStatus = result.Notification.Status
item.NotificationRunID = result.Notification.RunID
item.NotificationPipelineID = result.Notification.PipelineID
}
item.ProfileID = result.ProfileID
item.BackendID = result.BackendID
item.ModelName = result.ModelName
item.Timezone = result.Timezone
item.SourceWarnings = append([]weatherdata.SourceWarning(nil), result.SourceWarnings...)
item.ValidationStatus = result.ValidationStatus
}
func batchInspectionCandidates(req BatchRequest, now time.Time) ([]report.Resolved, error) {
@@ -440,23 +425,10 @@ func batchReportResult(planned plannedBatchReport) BatchReportResult {
RunID: metadata.RunID,
GeneratedAt: metadata.GeneratedAt,
ValidPeriod: metadata.ValidPeriod,
Timezone: "",
}
}
func plannedBatchOutputPath(outputDir string, planned plannedBatchReport) string {
if outputDir == "" {
return ""
}
outputCopyName := planned.OutputCopyName
if outputCopyName == "" {
outputCopyName = planned.Resolved.Definition.BatchOutputName
}
if outputCopyName == "" {
return ""
}
return filepath.Join(outputDir, outputCopyName)
}
func ResolveGenerate(req GenerateRequest, now time.Time) (report.Resolved, error) {
location, err := timeutil.LoadLocation(req.Config.WeatherAPI.Timezone)
if err != nil {
@@ -489,14 +461,6 @@ func reportRegistry(cfg config.Config) (report.Registry, error) {
return registry, nil
}
func FetchBundle(ctx context.Context, req FetchBundleRequest) (*weatherdata.Bundle, error) {
result, err := collectWeather(ctx, req.Config, nil)
if err != nil {
return nil, err
}
return result.Bundle, nil
}
func collectWeather(ctx context.Context, cfg config.Config, collector Collector) (*collect.Result, error) {
if collector == nil {
collector = defaultCollector{}
@@ -514,137 +478,25 @@ func collectWeather(ctx context.Context, cfg config.Config, collector Collector)
return result, nil
}
func FetchAndSaveBundle(ctx context.Context, req FetchBundleRequest) (*weatherdata.Bundle, error) {
if req.OutputPath == "" {
return nil, fmt.Errorf("output path is required")
func notifyReport(ctx context.Context, cfg config.Config, resolved report.Resolved, outputPath, runID string, generatedAt time.Time, notifier Notifier) (*NotificationResult, error) {
notifier, enabled := reportNotifier(cfg, notifier)
if !enabled {
return nil, nil
}
bundle, err := FetchBundle(ctx, req)
notificationRequest, err := buildNotificationRequest(cfg, resolved, outputPath, runID, generatedAt)
if err != nil {
return nil, err
}
if err := fileutil.WriteJSONAtomic(req.OutputPath, bundle); err != nil {
return nil, fmt.Errorf("save bundle: %w", err)
}
return bundle, nil
}
type finalizeRenderedReportRequest struct {
Config config.Config
Store state.Store
Resolved report.Resolved
Metadata state.Metadata
MetadataPath string
ExecutionArtifact *state.PromptExecutionArtifact
ManagedReportPath string
OutputPath string
Notifier Notifier
GenerationErr error
noNotify bool
}
type finalizeRenderedReportResult struct {
OutputPath string
NotificationPath string
Metadata state.Metadata
MetadataPath string
Notification *NotificationResult
}
func finalizeRenderedReport(ctx context.Context, req finalizeRenderedReportRequest) (finalizeRenderedReportResult, error) {
if req.Store == nil {
return finalizeRenderedReportResult{}, fmt.Errorf("state store is required")
}
if req.ManagedReportPath == "" {
return finalizeRenderedReportResult{}, fmt.Errorf("managed report path is required for report %q", req.Resolved.Definition.ID)
}
if req.ExecutionArtifact == nil {
return finalizeRenderedReportResult{}, fmt.Errorf("prompt execution artifact is required for report %q", req.Resolved.Definition.ID)
}
result := finalizeRenderedReportResult{Metadata: req.Metadata, MetadataPath: req.MetadataPath}
if req.OutputPath != "" && req.GenerationErr == nil {
if req.OutputPath != req.ManagedReportPath {
if err := fileutil.CopyFileAtomic(req.ManagedReportPath, req.OutputPath); err != nil {
return result, err
}
result.OutputPath = req.OutputPath
if err := persistReachedPromptPath(ctx, req.Store, req.Resolved, req.ExecutionArtifact, func(paths *state.PromptExecutionPaths) {
paths.OutputPath = req.OutputPath
}); err != nil {
return result, err
}
} else {
result.OutputPath = req.OutputPath
}
}
metadata := req.Metadata
metadata.RenderedReportPath = req.ManagedReportPath
metadataPath, err := req.Store.SaveMetadata(ctx, metadata)
result, err := notifier.Notify(ctx, notificationRequest)
if err != nil {
return result, err
}
result.Metadata = metadata
result.MetadataPath = metadataPath
if req.GenerationErr != nil {
return result, req.GenerationErr
}
if req.noNotify {
return result, nil
}
notification, notificationPath, err := notifyReport(ctx, req.Config, req.Resolved, req.ManagedReportPath, metadata, req.Notifier, req.Store)
if notificationPath != "" {
result.NotificationPath = notificationPath
result.Notification = notification
metadata.NotificationPath = notificationPath
result.Metadata = metadata
if saveErr := persistReachedPromptPath(ctx, req.Store, req.Resolved, req.ExecutionArtifact, func(paths *state.PromptExecutionPaths) {
paths.NotificationPath = notificationPath
}); saveErr != nil {
return result, saveErr
return result, &NotificationError{
Request: notificationRequest,
Err: fmt.Errorf("notify report %q run %q from output %q: %w", resolved.Definition.ID, runID, outputPath, err),
}
metadataPath, saveErr := req.Store.SaveMetadata(ctx, metadata)
if saveErr != nil {
return result, saveErr
}
result.Metadata = metadata
result.MetadataPath = metadataPath
}
result.Notification = notification
if err != nil {
return result, err
}
return result, nil
}
func notifyReport(ctx context.Context, cfg config.Config, resolved report.Resolved, reportPath string, metadata state.Metadata, notifier Notifier, store state.Store) (*NotificationResult, string, error) {
notifier, enabled := reportNotifier(cfg, notifier)
if !enabled {
return nil, "", nil
}
notificationRequest, err := buildNotificationRequest(cfg, resolved, reportPath, metadata)
if err != nil {
notificationPath, saveErr := saveNotificationArtifact(ctx, store, resolved, cfg, metadata, NotificationRequest{}, nil, err)
if saveErr != nil {
return nil, "", saveErr
}
return nil, notificationPath, err
}
result, err := notifier.Notify(ctx, notificationRequest)
notificationPath, saveErr := saveNotificationArtifact(ctx, store, resolved, cfg, metadata, notificationRequest, result, err)
if saveErr != nil {
return nil, "", saveErr
}
if err != nil {
return result, notificationPath, &NotificationError{
Request: notificationRequest,
Err: fmt.Errorf("notify report %q run %q from managed report %q: %w", resolved.Definition.ID, metadata.RunID, reportPath, err),
}
}
return result, notificationPath, nil
}
func reportNotifier(cfg config.Config, notifier Notifier) (Notifier, bool) {
if !cfg.Notify.Distributor.Enabled {
return noopNotifier{}, false
@@ -657,8 +509,8 @@ func reportNotifier(cfg config.Config, notifier Notifier) (Notifier, bool) {
}, true
}
func buildNotificationRequest(cfg config.Config, resolved report.Resolved, reportPath string, metadata state.Metadata) (NotificationRequest, error) {
values, err := distributorTemplateValuesForReport(cfg, resolved, metadata.RunID, resolved.Definition.BatchOutputName)
func buildNotificationRequest(cfg config.Config, resolved report.Resolved, outputPath, runID string, generatedAt time.Time) (NotificationRequest, error) {
values, err := distributorTemplateValuesForReport(cfg, resolved, runID, filepath.Base(outputPath))
if err != nil {
return NotificationRequest{}, err
}
@@ -675,32 +527,36 @@ func buildNotificationRequest(cfg config.Config, resolved report.Resolved, repor
if err != nil {
return NotificationRequest{}, err
}
bundlePaths, err := renderDistributorReportBundlePaths(cfg, resolved, metadata.RunID, reportPath, values)
bundlePaths, err := renderDistributorReportBundlePaths(cfg, resolved, runID, outputPath, values)
if err != nil {
return NotificationRequest{}, err
}
return NotificationRequest{
ReportID: resolved.Definition.ID,
RunID: metadata.RunID,
RunID: runID,
PipelineID: pipelineID,
BundleID: bundleID,
IdempotencyKey: idempotencyKey,
ReportPath: reportPath,
ReportPath: outputPath,
BundlePaths: bundlePaths,
CreatedAt: metadata.GeneratedAt,
CreatedAt: generatedAt,
}, nil
}
func distributorTemplateValuesForReport(cfg config.Config, resolved report.Resolved, runID string, batchOutputName string) (config.DistributorTemplateValues, error) {
func distributorTemplateValuesForReport(cfg config.Config, resolved report.Resolved, runID string, outputName string) (config.DistributorTemplateValues, error) {
values := config.DistributorTemplateValues{
LocationID: cfg.Location.ID,
ReportID: string(resolved.Definition.ID),
RunID: runID,
ArtifactGroup: resolved.Definition.ArtifactGroup,
BatchOutputName: batchOutputName,
BatchOutputName: outputName,
}
if values.BatchOutputName == "" {
values.BatchOutputName = resolved.Definition.BatchOutputName
var err error
values.BatchOutputName, err = resolved.OutputName()
if err != nil {
return config.DistributorTemplateValues{}, err
}
}
if err := addDistributorValidPeriodValues(&values, resolved.ValidPeriod, cfg.WeatherAPI.Timezone); err != nil {
return config.DistributorTemplateValues{}, err
@@ -757,54 +613,6 @@ func addDistributorValidPeriodValues(values *config.DistributorTemplateValues, p
return nil
}
func saveNotificationArtifact(ctx context.Context, store state.Store, resolved report.Resolved, cfg config.Config, metadata state.Metadata, req NotificationRequest, result *NotificationResult, notifyErr error) (string, error) {
if store == nil {
return "", fmt.Errorf("state store is required")
}
artifact := state.DistributorNotificationArtifact{
SchemaVersion: state.DistributorNotificationSchemaVersion,
RunID: metadata.RunID,
ReportID: resolved.Definition.ID,
AttemptedAt: time.Now(),
Endpoint: cfg.Notify.Distributor.Endpoint,
PipelineID: req.PipelineID,
BundleID: req.BundleID,
IdempotencyKey: req.IdempotencyKey,
SourcePath: req.ReportPath,
BundlePaths: append([]string(nil), req.BundlePaths...),
BundleCreated: req.CreatedAt,
Status: "attempted",
}
if result != nil {
artifact.Status = result.Status
artifact.Upload = &state.DistributorUploadResult{
RunID: result.RunID,
Status: result.UploadStatus,
}
if result.PipelineID != "" || !result.AcceptedAt.IsZero() || result.StartedAt != nil || result.FinishedAt != nil || len(result.Report) > 0 || result.Error != "" {
artifact.RunStatus = &state.DistributorRunStatus{
RunID: result.RunID,
PipelineID: result.PipelineID,
Status: result.Status,
AcceptedAt: result.AcceptedAt,
StartedAt: result.StartedAt,
FinishedAt: result.FinishedAt,
Report: append([]byte(nil), result.Report...),
Error: result.Error,
}
}
artifact.StatusError = result.StatusError
}
if notifyErr != nil {
artifact.Status = "failed"
artifact.Error = notifyErr.Error()
}
if artifact.Status == "" {
artifact.Status = "unknown"
}
return store.SaveDistributorNotification(ctx, resolved, artifact)
}
type noopNotifier struct{}
func (noopNotifier) Notify(context.Context, NotificationRequest) (*NotificationResult, error) {
@@ -951,7 +759,7 @@ func briefingBuildContext(cfg config.Config, resolved report.Resolved, collected
}
}
func promptMetadata(metadata state.Metadata) promptinput.Metadata {
func promptMetadata(metadata briefing.Metadata) promptinput.Metadata {
return promptinput.Metadata{
RunID: metadata.RunID,
ReportID: metadata.ReportID,
@@ -994,32 +802,6 @@ func briefingLocation(cfg config.Config) *briefing.LocationContext {
return &location
}
func defaultStore(cfg config.Config) (*state.FilesystemStore, error) {
return state.NewFilesystemStore(cfg.Workspace)
}
func recentChanges(ctx context.Context, store state.Store, priorSnapshot *state.PriorSnapshot, reportID report.ID, current module.Snapshot, cfg config.RecentChangeConfig) ([]changes.Change, error) {
if priorSnapshot == nil {
return nil, nil
}
previous, err := store.LoadModuleSnapshot(ctx, priorSnapshot.ModuleSnapshotPath)
if err != nil {
return nil, err
}
thresholds := changes.Thresholds{
TemperatureDegrees: cfg.TemperatureDegrees,
PrecipProbabilityPoints: cfg.PrecipProbabilityPoints,
WindGustMilesPerHour: cfg.WindGustMilesPerHour,
PrecipTimingShiftMinutes: cfg.PrecipTimingShiftMinutes,
}
switch reportID {
case report.Daily, report.Today, report.Tomorrow:
return changes.CompareDaily(previous, current, thresholds)
default:
return nil, nil
}
}
func generatedReportError(resolved report.Resolved, runID string, operation string, err error) error {
if err == nil {
return nil

View File

@@ -1,85 +0,0 @@
package app
import (
"context"
"encoding/json"
"errors"
"strings"
"testing"
"gitea.maximumdirect.net/eric/weatherreporter/internal/collect"
"gitea.maximumdirect.net/eric/weatherreporter/internal/config"
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec"
)
func TestRunBatchDetailedInspectsEveryCandidateBeforeCollection(t *testing.T) {
tests := []struct {
name string
batch BatchKind
now string
wantPrompts int
}{
{name: "morning", batch: BatchMorning, now: "2026-05-29T08:00:00-05:00", wantPrompts: 3},
{name: "evening", batch: BatchEvening, now: "2026-05-29T18:00:00-05:00", wantPrompts: 2},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
cfg := config.Defaults()
cfg.Workspace.Root = t.TempDir()
now := mustParse(test.now)
req := BatchRequest{Config: cfg, Batch: test.batch, Now: now}
candidates, err := batchInspectionCandidates(req, now)
if err != nil {
t.Fatalf("batchInspectionCandidates() error = %v", err)
}
executor := &inspectionExecutor{profiles: map[string]promptexec.ProfileInspection{
"default-profile": {ProfileID: "default-profile", BackendID: "local", ModelName: "model"},
}, prompts: map[string]promptexec.PromptInspection{}}
for _, candidate := range candidates {
executor.prompts[candidate.Definition.PromptID] = validPromptInspection(candidate.Definition)
}
collector := collectorFunc(func(context.Context, collect.Request) (*collect.Result, error) {
return nil, errors.New("collection reached")
})
req.Executor = executor
req.Collector = collector
_, err = RunBatchDetailed(context.Background(), req)
if err == nil || err.Error() != "collection reached" {
t.Fatalf("RunBatchDetailed() error = %v, want collection error", err)
}
if len(executor.promptRequests) != test.wantPrompts || len(executor.profileRequests) != 1 {
t.Fatalf("inspection calls = prompts %#v profiles %#v", executor.promptRequests, executor.profileRequests)
}
})
}
}
func TestCopyBatchReportPathsLeavesUnreachedPathsEmpty(t *testing.T) {
item := BatchReportResult{}
copyBatchReportPaths(&item, &ReportResult{
DataPackagePath: "/runs/daily/data_package.yaml",
PreparationPath: "/runs/daily/preparation.json",
})
data, err := json.Marshal(item)
if err != nil {
t.Fatalf("Marshal() error = %v", err)
}
text := string(data)
for _, omitted := range []string{"executionPath", "reportPath", "outputPath", "metadataPath", "notificationPath"} {
if strings.Contains(text, omitted) {
t.Fatalf("batch item includes unreached field %q:\n%s", omitted, text)
}
}
if !strings.Contains(text, "dataPackagePath") || !strings.Contains(text, "preparationPath") {
t.Fatalf("batch item omits reached paths:\n%s", text)
}
}
type collectorFunc func(context.Context, collect.Request) (*collect.Result, error)
func (f collectorFunc) Run(ctx context.Context, req collect.Request) (*collect.Result, error) {
return f(ctx, req)
}
var _ Collector = collectorFunc(nil)

View File

@@ -0,0 +1,236 @@
package app
import (
"context"
"errors"
"os"
"path/filepath"
"strings"
"testing"
"gitea.maximumdirect.net/eric/weatherreporter/internal/config"
)
func TestRunBatchDetailedKeepsSuccessfulOutputAndSkipsNotificationAfterPartialFailure(t *testing.T) {
bundle := generationBundle(t)
bundle.Hourly.Periods = bundle.Hourly.Periods[:1]
notifier := &generationNotifier{}
executor := &generationExecutor{failedPrompt: generationDefinitionForPrompt("weather.tomorrow_generated_text").PromptID}
result, err := RunBatchDetailed(context.Background(), BatchRequest{
Config: generationDistributorConfig(), Batch: BatchMorning,
Now: generationTime("2026-05-29T08:30:00-05:00"), WorkingDir: t.TempDir(), OutputDir: t.TempDir(),
Collector: &generationCollector{bundle: &bundle}, Executor: executor, Notifier: notifier,
})
if err != nil || result == nil || result.Total != 2 || result.Succeeded != 1 || result.Failed != 1 || result.Notification == nil || result.Notification.Status != "skipped" || notifier.batchCalls != 0 {
t.Fatalf("RunBatchDetailed() result/error/notifier = %#v/%v/%#v", result, err, notifier)
}
if result.Reports[0].Status != "succeeded" || result.Reports[0].OutputPath == "" || result.Reports[1].Status != "failed" || result.Reports[1].OutputPath != "" {
t.Fatalf("report results = %#v", result.Reports)
}
if data, readErr := os.ReadFile(result.Reports[0].OutputPath); readErr != nil || len(data) == 0 {
t.Fatalf("successful output = %q, error = %v", data, readErr)
}
}
func TestRunBatchDetailedNotifiesOnlyAfterAllOutputsExist(t *testing.T) {
bundle := generationBundle(t)
bundle.Hourly.Periods = bundle.Hourly.Periods[:1]
outputDir := t.TempDir()
notifier := &generationNotifier{}
result, err := RunBatchDetailed(context.Background(), BatchRequest{
Config: generationDistributorConfig(), Batch: BatchMorning,
Now: generationTime("2026-05-29T08:30:00-05:00"), WorkingDir: t.TempDir(), OutputDir: outputDir,
Collector: &generationCollector{bundle: &bundle}, Executor: &generationExecutor{}, Notifier: notifier,
})
if err != nil || result == nil || result.Total != 2 || result.Succeeded != 2 || result.Failed != 0 || notifier.batchCalls != 1 || result.Notification == nil || result.Notification.Status != "succeeded" {
t.Fatalf("RunBatchDetailed() result/error/notifier = %#v/%v/%#v", result, err, notifier)
}
if len(notifier.batchRequest.Files) < 2 || len(notifier.batchRequest.IncludedReports) != 2 {
t.Fatalf("batch notification = %#v", notifier.batchRequest)
}
if result.Reports[0].OutputPath == result.Reports[1].OutputPath {
t.Fatalf("batch reports share output path %q", result.Reports[0].OutputPath)
}
for _, file := range notifier.batchRequest.Files {
if filepath.Dir(file.SourcePath) != outputDir || file.BundlePath == "" {
t.Fatalf("notification file = %#v", file)
}
if _, statErr := os.Stat(file.SourcePath); statErr != nil {
t.Fatalf("notification source %q: %v", file.SourcePath, statErr)
}
}
}
func TestRunBatchDetailedUsesDefaultAndConfiguredOutputDirectories(t *testing.T) {
tests := []struct {
name string
directory func(t *testing.T, workingDir string) string
wantDir func(t *testing.T, workingDir string, configuredDir string) string
}{
{
name: "working directory default",
directory: func(_ *testing.T, _ string) string {
return ""
},
wantDir: func(_ *testing.T, workingDir string, _ string) string {
return workingDir
},
},
{
name: "absolute directory",
directory: func(t *testing.T, _ string) string {
return filepath.Join(t.TempDir(), "reports")
},
wantDir: func(_ *testing.T, _ string, configuredDir string) string {
return configuredDir
},
},
{
name: "relative directory",
directory: func(_ *testing.T, _ string) string {
return "configured/../reports"
},
wantDir: func(_ *testing.T, workingDir string, _ string) string {
return filepath.Join(workingDir, "reports")
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
workingDir := t.TempDir()
configuredDir := tt.directory(t, workingDir)
cfg := generationDistributorConfig()
cfg.Output.Directory = configuredDir
bundle := generationBundle(t)
bundle.Hourly.Periods = bundle.Hourly.Periods[:1]
notifier := &generationNotifier{}
result, err := RunBatchDetailed(context.Background(), BatchRequest{
Config: cfg, Batch: BatchMorning,
Now: generationTime("2026-05-29T08:30:00-05:00"), WorkingDir: workingDir,
Collector: &generationCollector{bundle: &bundle}, Executor: &generationExecutor{}, Notifier: notifier,
})
wantDir := tt.wantDir(t, workingDir, configuredDir)
if err != nil || result == nil || result.Succeeded != len(result.Reports) || notifier.batchCalls != 1 {
t.Fatalf("RunBatchDetailed() result/error/notifier = %#v/%v/%#v", result, err, notifier)
}
for _, item := range result.Reports {
if filepath.Dir(item.OutputPath) != wantDir {
t.Fatalf("report output %q, want directory %q", item.OutputPath, wantDir)
}
}
for _, file := range notifier.batchRequest.Files {
if filepath.Dir(file.SourcePath) != wantDir {
t.Fatalf("notification source %q, want directory %q", file.SourcePath, wantDir)
}
}
})
}
}
func TestRunBatchDetailedExplicitOutputDirectoryIgnoresConfiguredDirectory(t *testing.T) {
configuredPath := filepath.Join(t.TempDir(), "not-a-directory")
if err := os.WriteFile(configuredPath, []byte("not a directory"), 0o600); err != nil {
t.Fatal(err)
}
explicitDir := t.TempDir()
cfg := generationDistributorConfig()
cfg.Output.Directory = configuredPath
bundle := generationBundle(t)
bundle.Hourly.Periods = bundle.Hourly.Periods[:1]
result, err := RunBatchDetailed(context.Background(), BatchRequest{
Config: cfg, Batch: BatchMorning,
Now: generationTime("2026-05-29T08:30:00-05:00"), WorkingDir: t.TempDir(), OutputDir: explicitDir,
Collector: &generationCollector{bundle: &bundle}, Executor: &generationExecutor{}, Notifier: &generationNotifier{},
})
if err != nil || result == nil || result.Succeeded != len(result.Reports) {
t.Fatalf("RunBatchDetailed() result/error = %#v/%v", result, err)
}
for _, item := range result.Reports {
if filepath.Dir(item.OutputPath) != explicitDir {
t.Fatalf("report output %q, want directory %q", item.OutputPath, explicitDir)
}
}
}
func TestRunBatchDetailedPreflightsAllOutputPaths(t *testing.T) {
bundle := generationBundle(t)
bundle.Hourly.Periods = bundle.Hourly.Periods[:1]
outputDir := t.TempDir()
if err := os.Mkdir(filepath.Join(outputDir, "tomorrow.md"), 0o700); err != nil {
t.Fatal(err)
}
todayPath := filepath.Join(outputDir, "today.md")
const previousReport = "previous report"
if err := os.WriteFile(todayPath, []byte(previousReport), 0o600); err != nil {
t.Fatal(err)
}
executor := &generationExecutor{}
promptInspectedBeforeCollection := false
collector := &generationCollector{
bundle: &bundle,
beforeRun: func() {
promptInspectedBeforeCollection = executor.promptInspections > 0
},
}
result, err := RunBatchDetailed(context.Background(), BatchRequest{
Config: generationDistributorConfig(), Batch: BatchMorning,
Now: generationTime("2026-05-29T08:30:00-05:00"), WorkingDir: t.TempDir(), OutputDir: outputDir,
Collector: collector, Executor: executor, Notifier: &generationNotifier{},
})
if err == nil || result != nil || !collector.called || !promptInspectedBeforeCollection || executor.called {
t.Fatalf("RunBatchDetailed() result/error/collection/inspection/execution = %#v/%v/%t/%t/%t", result, err, collector.called, promptInspectedBeforeCollection, executor.called)
}
if data, readErr := os.ReadFile(todayPath); readErr != nil || string(data) != previousReport {
t.Fatalf("earlier output = %q, error = %v", data, readErr)
}
if info, statErr := os.Stat(filepath.Join(outputDir, "tomorrow.md")); statErr != nil || !info.IsDir() {
t.Fatalf("blocked output info/error = %#v/%v", info, statErr)
}
}
func TestRunBatchDetailedRetainsReportCountsWhenNotificationFails(t *testing.T) {
bundle := generationBundle(t)
bundle.Hourly.Periods = bundle.Hourly.Periods[:1]
outputDir := t.TempDir()
notifier := &generationNotifier{batchErr: errors.New("distributor unavailable")}
result, err := RunBatchDetailed(context.Background(), BatchRequest{
Config: generationDistributorConfig(), Batch: BatchMorning,
Now: generationTime("2026-05-29T08:30:00-05:00"), WorkingDir: t.TempDir(), OutputDir: outputDir,
Collector: &generationCollector{bundle: &bundle}, Executor: &generationExecutor{}, Notifier: notifier,
})
if err != nil || result == nil || result.Total != len(result.Reports) || result.Succeeded != len(result.Reports) || result.Failed != 0 || result.Notification == nil || result.Notification.Status != "failed" {
t.Fatalf("RunBatchDetailed() result/error = %#v/%v", result, err)
}
for _, item := range result.Reports {
if item.Status != "succeeded" || item.OutputPath == "" {
t.Fatalf("report result = %#v", item)
}
if _, statErr := os.Stat(item.OutputPath); statErr != nil {
t.Fatalf("published output %q: %v", item.OutputPath, statErr)
}
}
}
func TestRunBatchReturnsNotificationFailureWithoutReportFailureWording(t *testing.T) {
bundle := generationBundle(t)
bundle.Hourly.Periods = bundle.Hourly.Periods[:1]
err := RunBatch(context.Background(), BatchRequest{
Config: generationDistributorConfig(), Batch: BatchMorning,
Now: generationTime("2026-05-29T08:30:00-05:00"), WorkingDir: t.TempDir(), OutputDir: t.TempDir(),
Collector: &generationCollector{bundle: &bundle}, Executor: &generationExecutor{}, Notifier: &generationNotifier{batchErr: errors.New("distributor unavailable")},
})
var batchErr BatchError
if !errors.As(err, &batchErr) || batchErr.Result == nil || batchErr.Result.Failed != 0 || batchErr.Result.Notification == nil || batchErr.Result.Notification.Status != "failed" || !strings.Contains(err.Error(), "notification failed") || strings.Contains(err.Error(), "reports failed") {
t.Fatalf("RunBatch() error/result = %v/%#v", err, batchErr.Result)
}
}
func generationDistributorConfig() config.Config {
cfg := generationConfig()
cfg.Notify.Distributor.Enabled = true
cfg.Notify.Distributor.PipelineIDTemplate = "weather"
return cfg
}

View File

@@ -3,12 +3,12 @@ package app
import (
"context"
"fmt"
"path/filepath"
"time"
distributoradapter "gitea.maximumdirect.net/eric/weatherreporter/internal/adapters/distributor"
"gitea.maximumdirect.net/eric/weatherreporter/internal/config"
"gitea.maximumdirect.net/eric/weatherreporter/internal/report"
"gitea.maximumdirect.net/eric/weatherreporter/internal/state"
"gitea.maximumdirect.net/eric/weatherreporter/internal/timeutil"
)
@@ -46,39 +46,31 @@ func batchRunID(startedAt time.Time, batch BatchKind) string {
return startedAt.UTC().Format(runIDTimestampLayout) + "_" + string(batch)
}
func notifyBatch(ctx context.Context, cfg config.Config, batch BatchKind, runID string, startedAt time.Time, result *BatchResult, planned []plannedBatchReport, store state.Store, notifier Notifier) (*BatchNotificationResult, error) {
func notifyBatch(ctx context.Context, cfg config.Config, batch BatchKind, runID string, startedAt time.Time, result *BatchResult, planned []plannedBatchReport, notifier Notifier) *BatchNotificationResult {
if !cfg.Notify.Distributor.Enabled {
return nil, nil
return nil
}
if !cfg.Notify.Distributor.Batch.Enabled {
return nil, nil
return nil
}
if result == nil {
return nil, fmt.Errorf("batch result is required")
return failedBatchNotificationResult(batchNotificationRequest{}, fmt.Errorf("batch result is required"))
}
if result.Failed > 0 {
return &BatchNotificationResult{
Status: "skipped",
Reason: "one or more reports failed",
}, nil
}
}
req, err := buildBatchNotificationRequest(cfg, batch, runID, startedAt, result.Reports, planned)
if err != nil {
path, saveErr := saveBatchNotificationArtifact(ctx, store, cfg, batch, runID, startedAt, batchNotificationRequest{}, nil, err)
if saveErr != nil {
return nil, saveErr
}
return failedBatchNotificationResult(batchNotificationRequest{}, path, err), err
return failedBatchNotificationResult(batchNotificationRequest{}, err)
}
batchNotifier, err := resolveBatchNotifier(cfg, notifier)
if err != nil {
path, saveErr := saveBatchNotificationArtifact(ctx, store, cfg, batch, runID, startedAt, req, nil, err)
if saveErr != nil {
return nil, saveErr
}
return failedBatchNotificationResult(req, path, err), err
return failedBatchNotificationResult(req, err)
}
notification, notifyErr := batchNotifier.NotifyBatch(ctx, req)
@@ -86,18 +78,13 @@ func notifyBatch(ctx context.Context, cfg config.Config, batch BatchKind, runID
if notifyErr != nil {
wrappedErr = fmt.Errorf("notify batch %q run %q bundle %q: %w", batch, runID, req.BundleID, notifyErr)
}
path, saveErr := saveBatchNotificationArtifact(ctx, store, cfg, batch, runID, startedAt, req, notification, wrappedErr)
if saveErr != nil {
return nil, saveErr
}
batchResult := batchNotificationResult(req, notification, path)
batchResult := batchNotificationResult(req, notification)
if wrappedErr != nil {
batchResult.Status = "failed"
batchResult.Error = wrappedErr.Error()
return batchResult, wrappedErr
return batchResult
}
return batchResult, nil
return batchResult
}
func resolveBatchNotifier(cfg config.Config, notifier Notifier) (batchNotifier, error) {
@@ -153,15 +140,15 @@ func buildBatchNotificationRequest(cfg config.Config, batch BatchKind, runID str
if item.ReportID != plannedReport.Resolved.Definition.ID {
return batchNotificationRequest{}, fmt.Errorf("batch notification report %q run %q does not match planned report %q", item.ReportID, item.RunID, plannedReport.Resolved.Definition.ID)
}
if item.ReportPath == "" {
return batchNotificationRequest{}, fmt.Errorf("batch notification report %q run %q is missing managed report path", item.ReportID, item.RunID)
if item.OutputPath == "" {
return batchNotificationRequest{}, fmt.Errorf("batch notification report %q run %q is missing output path", item.ReportID, item.RunID)
}
values, err := distributorTemplateValuesForReport(cfg, plannedReport.Resolved, item.RunID, plannedReport.OutputCopyName)
values, err := distributorTemplateValuesForReport(cfg, plannedReport.Resolved, item.RunID, filepath.Base(item.OutputPath))
if err != nil {
return batchNotificationRequest{}, fmt.Errorf("batch notification report %q run %q source path %q: %w", item.ReportID, item.RunID, item.ReportPath, err)
return batchNotificationRequest{}, fmt.Errorf("batch notification report %q run %q source path %q: %w", item.ReportID, item.RunID, item.OutputPath, err)
}
bundlePaths, err := renderDistributorReportBundlePaths(cfg, plannedReport.Resolved, item.RunID, item.ReportPath, values)
bundlePaths, err := renderDistributorReportBundlePaths(cfg, plannedReport.Resolved, item.RunID, item.OutputPath, values)
if err != nil {
return batchNotificationRequest{}, err
}
@@ -169,18 +156,18 @@ func buildBatchNotificationRequest(cfg config.Config, batch BatchKind, runID str
included := BatchNotificationReport{
ReportID: item.ReportID,
RunID: item.RunID,
SourcePath: item.ReportPath,
SourcePath: item.OutputPath,
BundlePaths: append([]string(nil), bundlePaths...),
}
for _, bundlePath := range bundlePaths {
file := batchNotificationFile{
ReportID: item.ReportID,
RunID: item.RunID,
SourcePath: item.ReportPath,
SourcePath: item.OutputPath,
BundlePath: bundlePath,
}
if previous, ok := seenBundlePaths[bundlePath]; ok {
return batchNotificationRequest{}, fmt.Errorf("batch notification duplicate bundle path %q for report %q run %q source path %q; already used by report %q run %q source path %q", bundlePath, item.ReportID, item.RunID, item.ReportPath, previous.ReportID, previous.RunID, previous.SourcePath)
return batchNotificationRequest{}, fmt.Errorf("batch notification duplicate bundle path %q for report %q run %q source path %q; already used by report %q run %q source path %q", bundlePath, item.ReportID, item.RunID, item.OutputPath, previous.ReportID, previous.RunID, previous.SourcePath)
}
seenBundlePaths[bundlePath] = file
req.Files = append(req.Files, file)
@@ -225,13 +212,12 @@ func batchDistributorUploadRequest(req batchNotificationRequest) distributoradap
}
}
func batchNotificationResult(req batchNotificationRequest, result *NotificationResult, path string) *BatchNotificationResult {
func batchNotificationResult(req batchNotificationRequest, result *NotificationResult) *BatchNotificationResult {
notification := &BatchNotificationResult{
Status: "unknown",
PipelineID: req.PipelineID,
BundleID: req.BundleID,
IdempotencyKey: req.IdempotencyKey,
Path: path,
IncludedReports: append([]BatchNotificationReport(nil), req.IncludedReports...),
}
if result != nil {
@@ -256,8 +242,8 @@ func batchNotificationResult(req batchNotificationRequest, result *NotificationR
return notification
}
func failedBatchNotificationResult(req batchNotificationRequest, path string, err error) *BatchNotificationResult {
notification := batchNotificationResult(req, nil, path)
func failedBatchNotificationResult(req batchNotificationRequest, err error) *BatchNotificationResult {
notification := batchNotificationResult(req, nil)
notification.Status = "failed"
if err != nil {
notification.Error = err.Error()
@@ -265,78 +251,6 @@ func failedBatchNotificationResult(req batchNotificationRequest, path string, er
return notification
}
func saveBatchNotificationArtifact(ctx context.Context, store state.Store, cfg config.Config, batch BatchKind, runID string, startedAt time.Time, req batchNotificationRequest, result *NotificationResult, notifyErr error) (string, error) {
if store == nil {
return "", fmt.Errorf("state store is required")
}
location, err := timeutil.LoadLocation(cfg.WeatherAPI.Timezone)
if err != nil {
return "", fmt.Errorf("load batch notification timezone: %w", err)
}
artifact := state.BatchDistributorNotificationArtifact{
SchemaVersion: state.BatchDistributorNotificationSchemaVersion,
Batch: string(batch),
BatchRunID: runID,
AttemptedAt: time.Now(),
Endpoint: cfg.Notify.Distributor.Endpoint,
PipelineID: req.PipelineID,
BundleID: req.BundleID,
IdempotencyKey: req.IdempotencyKey,
BundleCreated: req.CreatedAt,
Reports: batchNotificationReportArtifacts(req.IncludedReports),
Status: "attempted",
}
if result != nil {
artifact.Status = result.Status
artifact.Upload = &state.DistributorUploadResult{
RunID: result.RunID,
Status: result.UploadStatus,
}
if result.PipelineID != "" || !result.AcceptedAt.IsZero() || result.StartedAt != nil || result.FinishedAt != nil || len(result.Report) > 0 || result.Error != "" {
artifact.RunStatus = &state.DistributorRunStatus{
RunID: result.RunID,
PipelineID: result.PipelineID,
Status: result.Status,
AcceptedAt: result.AcceptedAt,
StartedAt: result.StartedAt,
FinishedAt: result.FinishedAt,
Report: append([]byte(nil), result.Report...),
Error: result.Error,
}
}
artifact.StatusError = result.StatusError
}
if notifyErr != nil {
artifact.Status = "failed"
artifact.Error = notifyErr.Error()
}
if artifact.Status == "" {
artifact.Status = "unknown"
}
return store.SaveBatchDistributorNotification(ctx, state.BatchDistributorNotificationRef{
Batch: string(batch),
BatchRunID: runID,
StartedAt: startedAt,
Location: location,
}, artifact)
}
func batchNotificationReportArtifacts(reports []BatchNotificationReport) []state.BatchDistributorNotificationReportArtifact {
if len(reports) == 0 {
return nil
}
artifacts := make([]state.BatchDistributorNotificationReportArtifact, 0, len(reports))
for _, item := range reports {
artifacts = append(artifacts, state.BatchDistributorNotificationReportArtifact{
ReportID: item.ReportID,
RunID: item.RunID,
SourcePath: item.SourcePath,
BundlePaths: append([]string(nil), item.BundlePaths...),
})
}
return artifacts
}
func renderBatchNotificationIdentity(cfg config.Config, batch BatchKind, runID string, startedAt time.Time) (batchNotificationIdentity, error) {
values, err := batchNotificationTemplateValues(cfg, batch, runID, startedAt)
if err != nil {

View File

@@ -11,8 +11,8 @@ import (
)
type plannedBatchReport struct {
Resolved report.Resolved
OutputCopyName string
Resolved report.Resolved
OutputPath string
}
func planBatchRun(req BatchRequest, now time.Time, collection collect.Result) ([]plannedBatchReport, error) {
@@ -36,16 +36,16 @@ func planBatchRun(req BatchRequest, now time.Time, collection collect.Result) ([
var planned []plannedBatchReport
switch batch {
case report.Morning:
planned, err = appendPlannedReport(planned, registry, report.Today, resolveReq, "")
planned, err = appendPlannedReport(planned, registry, report.Today, resolveReq)
if err != nil {
return nil, err
}
planned, err = appendPlannedReport(planned, registry, report.Tomorrow, resolveReq, "")
planned, err = appendPlannedReport(planned, registry, report.Tomorrow, resolveReq)
if err != nil {
return nil, err
}
case report.Evening:
planned, err = appendPlannedReport(planned, registry, report.Tomorrow, resolveReq, "")
planned, err = appendPlannedReport(planned, registry, report.Tomorrow, resolveReq)
if err != nil {
return nil, err
}
@@ -60,8 +60,7 @@ func planBatchRun(req BatchRequest, now time.Time, collection collect.Result) ([
for _, date := range eligibleDailyDates(hourly, now, location) {
dailyReq := resolveReq
dailyReq.Date = date
outputCopyName := "daily-" + date.In(location).Format(timeutil.DateLayout) + ".md"
planned, err = appendPlannedReport(planned, registry, report.Daily, dailyReq, outputCopyName)
planned, err = appendPlannedReport(planned, registry, report.Daily, dailyReq)
if err != nil {
return nil, err
}
@@ -69,15 +68,12 @@ func planBatchRun(req BatchRequest, now time.Time, collection collect.Result) ([
return planned, nil
}
func appendPlannedReport(planned []plannedBatchReport, registry report.Registry, id report.ID, req report.ResolveRequest, outputCopyName string) ([]plannedBatchReport, error) {
func appendPlannedReport(planned []plannedBatchReport, registry report.Registry, id report.ID, req report.ResolveRequest) ([]plannedBatchReport, error) {
resolved, err := registry.Resolve(id, req)
if err != nil {
return nil, err
}
return append(planned, plannedBatchReport{
Resolved: resolved,
OutputCopyName: outputCopyName,
}), nil
return append(planned, plannedBatchReport{Resolved: resolved}), nil
}
func eligibleDailyDates(hourly *weatherdata.ForecastRun, now time.Time, location *time.Location) []time.Time {

View File

@@ -55,7 +55,7 @@ func TestPlanBatchRunDynamicDailyDatesStartAfterTomorrow(t *testing.T) {
assertPlanningPeriod(t, daily[1].Resolved.ValidPeriod, "2026-06-01T00:00:00-05:00", "2026-06-02T00:00:00-05:00")
}
func TestPlanBatchRunDynamicDailyOutputCopyNames(t *testing.T) {
func TestPlanBatchRunUsesResolvedOutputNames(t *testing.T) {
location := mustLoadTestLocation(t, "America/Chicago")
hourly := hourlyRun(fullDayPeriods(t, "2026-05-31", location)...)
@@ -68,11 +68,19 @@ func TestPlanBatchRunDynamicDailyOutputCopyNames(t *testing.T) {
if len(daily) != 1 {
t.Fatalf("daily reports = %#v, want one Daily report", daily)
}
if daily[0].OutputCopyName != "daily-2026-05-31.md" {
t.Fatalf("OutputCopyName = %q, want date-qualified Daily name", daily[0].OutputCopyName)
outputName, err := daily[0].Resolved.OutputName()
if err != nil {
t.Fatalf("OutputName() error = %v", err)
}
if planned[0].OutputCopyName != "" {
t.Fatalf("Tomorrow OutputCopyName = %q, want definition batch output name to apply later", planned[0].OutputCopyName)
if outputName != "daily-2026-05-31.md" {
t.Fatalf("Daily output name = %q, want date-qualified name", outputName)
}
outputName, err = planned[0].Resolved.OutputName()
if err != nil {
t.Fatalf("OutputName() error = %v", err)
}
if outputName != "tomorrow.md" {
t.Fatalf("Tomorrow output name = %q, want tomorrow.md", outputName)
}
}

View File

@@ -1,500 +0,0 @@
package app
import (
"bytes"
"context"
"encoding/json"
"errors"
"os"
"path/filepath"
"strings"
"testing"
"time"
"gitea.maximumdirect.net/eric/weatherreporter/internal/collect"
"gitea.maximumdirect.net/eric/weatherreporter/internal/config"
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec"
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptinput"
"gitea.maximumdirect.net/eric/weatherreporter/internal/report"
"gitea.maximumdirect.net/eric/weatherreporter/internal/state"
"gitea.maximumdirect.net/eric/weatherreporter/internal/weatherdata"
)
type assembledBatchExecutor struct {
definitions map[string]report.Definition
promptRequests []string
profileRequests []string
executeRequests []promptexec.ExecuteRequest
failures map[int]error
active int
maxActive int
}
func newAssembledBatchExecutor() *assembledBatchExecutor {
definitions := make(map[string]report.Definition)
for _, definition := range report.DefaultRegistry().All() {
definitions[definition.PromptID] = definition
}
return &assembledBatchExecutor{definitions: definitions, failures: make(map[int]error)}
}
func (e *assembledBatchExecutor) InspectPrompt(_ context.Context, id, version string) (promptexec.PromptInspection, error) {
e.promptRequests = append(e.promptRequests, id+"@"+version)
definition, ok := e.definitions[id]
if !ok || definition.PromptVersion != version {
return promptexec.PromptInspection{}, errors.New("unexpected prompt inspection")
}
return validPromptInspection(definition), nil
}
func (e *assembledBatchExecutor) InspectProfile(_ context.Context, id string) (promptexec.ProfileInspection, error) {
e.profileRequests = append(e.profileRequests, id)
return promptexec.ProfileInspection{ProfileID: id, BackendID: "fixture", ModelName: "fixture-model"}, nil
}
func (e *assembledBatchExecutor) Execute(_ context.Context, req promptexec.ExecuteRequest, callback promptexec.PreparationCallback) (*promptexec.Execution, error) {
call := len(e.executeRequests)
e.executeRequests = append(e.executeRequests, req)
e.active++
if e.active > e.maxActive {
e.maxActive = e.active
}
defer func() { e.active-- }()
stamp := time.Date(2026, 5, 29, 15, 0, 0, 0, time.UTC)
preparation := promptexec.Preparation{
PromptID: req.PromptID, PromptVersion: req.PromptVersion, PromptHash: "prompt-hash",
RenderedPromptHash: "rendered-hash", ProfileID: req.ProfileID, BackendID: "fixture",
ModelName: "fixture-model", DataPackagePath: req.DataPackagePath, StartedAt: stamp, EndedAt: stamp,
}
if err := callback(preparation, nil); err != nil {
return nil, err
}
if err := e.failures[call]; err != nil {
return nil, err
}
definition := e.definitions[req.PromptID]
return &promptexec.Execution{
RunID: "provider-run", PromptID: req.PromptID, PromptVersion: req.PromptVersion,
PromptHash: "prompt-hash", RenderedPromptHash: "rendered-hash", ProfileID: req.ProfileID,
BackendID: "fixture", ModelName: "fixture-model", GeneratedHash: "generated-hash",
StartedAt: stamp, EndedAt: stamp, DataPackagePath: req.DataPackagePath,
RawOutput: []byte(generatedTextForPrompt(req.PromptID)),
Validation: promptexec.NewValidation(promptexec.ValidationPassed, "json_schema", definition.GeneratedTextSchemaID+".generated_text.schema.json", nil),
}, nil
}
type assembledBatchNotifier struct {
reportRequests []NotificationRequest
batchRequests []batchNotificationRequest
batchResult *NotificationResult
batchErr error
}
func (n *assembledBatchNotifier) Notify(_ context.Context, req NotificationRequest) (*NotificationResult, error) {
n.reportRequests = append(n.reportRequests, req)
return nil, errors.New("per-report notification must be suppressed")
}
func (n *assembledBatchNotifier) NotifyBatch(_ context.Context, req batchNotificationRequest) (*NotificationResult, error) {
n.batchRequests = append(n.batchRequests, req)
if n.batchErr != nil {
return nil, n.batchErr
}
if n.batchResult != nil {
result := *n.batchResult
if result.PipelineID == "" {
result.PipelineID = req.PipelineID
}
if result.BundleID == "" {
result.BundleID = req.BundleID
}
if result.IdempotencyKey == "" {
result.IdempotencyKey = req.IdempotencyKey
}
return &result, nil
}
return &NotificationResult{
RunID: "batch-notification-run", PipelineID: req.PipelineID, BundleID: req.BundleID,
IdempotencyKey: req.IdempotencyKey, Status: "succeeded", UploadStatus: "accepted",
}, nil
}
func TestRunBatchDetailedExecutesRetainedReportsSequentially(t *testing.T) {
tests := []struct {
name string
batch BatchKind
now time.Time
wantIDs []report.ID
wantCopies []string
}{
{name: "morning", batch: BatchMorning, now: workflowTime("2026-05-29T08:00:00-05:00"), wantIDs: []report.ID{report.Today, report.Tomorrow, report.Daily}, wantCopies: []string{"today.md", "tomorrow.md", "daily-2026-05-31.md"}},
{name: "evening", batch: BatchEvening, now: workflowTime("2026-05-29T18:00:00-05:00"), wantIDs: []report.ID{report.Tomorrow, report.Daily}, wantCopies: []string{"tomorrow.md", "daily-2026-05-31.md"}},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
cfg := assembledBatchConfig(t, false)
bundle := assembledBatchBundle(t, "2026-05-31")
collector := &workflowCollector{result: &collect.Result{Bundle: &bundle}}
executor := newAssembledBatchExecutor()
outputDir := filepath.Join(t.TempDir(), "output")
debugRoot := filepath.Join(t.TempDir(), "debug")
result, err := RunBatchDetailed(context.Background(), BatchRequest{
Config: cfg, Batch: test.batch, Now: test.now, OutputDir: outputDir, LLMDebugDir: debugRoot,
Collector: collector, Executor: executor,
})
if err != nil {
t.Fatalf("RunBatchDetailed() error = %v", err)
}
if collector.calls != 1 || result.Total != len(test.wantIDs) || result.Succeeded != len(test.wantIDs) || result.Failed != 0 {
t.Fatalf("collection/summary = %d/%d/%d/%d", collector.calls, result.Total, result.Succeeded, result.Failed)
}
if len(executor.executeRequests) != len(test.wantIDs) || executor.maxActive != 1 {
t.Fatalf("executor calls/max active = %d/%d, want %d/1", len(executor.executeRequests), executor.maxActive, len(test.wantIDs))
}
if len(executor.profileRequests) != 1 {
t.Fatalf("profile inspections = %#v, want one shared profile inspection", executor.profileRequests)
}
for index, item := range result.Reports {
if item.ReportID != test.wantIDs[index] || item.Status != "succeeded" {
t.Fatalf("report %d = %s/%s, want %s/succeeded", index, item.ReportID, item.Status, test.wantIDs[index])
}
if executor.executeRequests[index].PromptID != item.PromptID {
t.Fatalf("execution %d prompt = %q, want item prompt %q", index, executor.executeRequests[index].PromptID, item.PromptID)
}
if item.DataPackagePath == "" || item.PreparationPath == "" || item.ExecutionPath == "" || item.LLMDebugPath == "" || item.ReportPath == "" || item.OutputPath == "" || item.MetadataPath == "" {
t.Fatalf("successful report paths = %#v", item)
}
assertBatchItemMatchesMetadata(t, item)
if filepath.Base(item.OutputPath) != test.wantCopies[index] {
t.Fatalf("output copy = %q, want %q", item.OutputPath, test.wantCopies[index])
}
assertBatchPathsExist(t, item.DataPackagePath, item.PreparationPath, item.ExecutionPath, item.LLMDebugPath, item.ReportPath, item.OutputPath, item.MetadataPath)
managed, readErr := os.ReadFile(item.ReportPath)
if readErr != nil {
t.Fatalf("read managed report: %v", readErr)
}
copied, readErr := os.ReadFile(item.OutputPath)
if readErr != nil || !bytes.Equal(managed, copied) {
t.Fatalf("output copy mismatch/error = %v", readErr)
}
}
})
}
}
func TestRunBatchDetailedContinuesAfterCapacityRejection(t *testing.T) {
cfg := assembledBatchConfig(t, true)
bundle := assembledBatchBundle(t, "2026-05-31")
collector := &workflowCollector{result: &collect.Result{Bundle: &bundle}}
executor := newAssembledBatchExecutor()
executor.failures[1] = promptexec.NewError(promptexec.Capacity, "capacity rejected", nil)
notifier := &assembledBatchNotifier{}
result, err := RunBatchDetailed(context.Background(), BatchRequest{
Config: cfg, Batch: BatchMorning, Now: workflowTime("2026-05-29T08:00:00-05:00"),
OutputDir: filepath.Join(t.TempDir(), "output"), LLMDebugDir: filepath.Join(t.TempDir(), "debug"),
Collector: collector, Executor: executor, Notifier: notifier,
})
if err != nil {
t.Fatalf("RunBatchDetailed() error = %v", err)
}
if collector.calls != 1 || len(executor.executeRequests) != 3 || result.Total != 3 || result.Succeeded != 2 || result.Failed != 1 {
t.Fatalf("collection/execution/summary = %d/%d/%d/%d/%d", collector.calls, len(executor.executeRequests), result.Total, result.Succeeded, result.Failed)
}
if len(notifier.reportRequests) != 0 || len(notifier.batchRequests) != 0 || result.Notification == nil || result.Notification.Status != "skipped" {
t.Fatalf("notification state = reports %d batches %d result %#v", len(notifier.reportRequests), len(notifier.batchRequests), result.Notification)
}
for index, item := range result.Reports {
if index == 1 {
if item.ReportID != report.Tomorrow || item.Status != "failed" || !strings.Contains(item.Error, string(promptexec.Capacity)) {
t.Fatalf("failed item = %#v", item)
}
if item.DataPackagePath == "" || item.PreparationPath == "" || item.ExecutionPath == "" || item.LLMDebugPath == "" || item.MetadataPath == "" || item.ReportPath != "" || item.OutputPath != "" {
t.Fatalf("failed item reached paths = %#v", item)
}
assertBatchItemMatchesMetadata(t, item)
continue
}
if item.Status != "succeeded" || item.ReportPath == "" || item.OutputPath == "" {
t.Fatalf("continued item %d = %#v", index, item)
}
}
}
func TestRunBatchDetailedNotificationLifecycle(t *testing.T) {
t.Run("disabled", func(t *testing.T) {
cfg := assembledBatchConfig(t, false)
bundle := assembledBatchBundle(t, "2026-05-31")
notifier := &assembledBatchNotifier{}
result, err := RunBatchDetailed(context.Background(), BatchRequest{
Config: cfg, Batch: BatchEvening, Now: workflowTime("2026-05-29T18:00:00-05:00"),
Collector: &workflowCollector{result: &collect.Result{Bundle: &bundle}}, Executor: newAssembledBatchExecutor(), Notifier: notifier,
})
if err != nil || result.Notification != nil || result.Failed != 0 || len(notifier.reportRequests) != 0 || len(notifier.batchRequests) != 0 {
t.Fatalf("result/error/requests = %#v/%v/%d/%d", result, err, len(notifier.reportRequests), len(notifier.batchRequests))
}
})
t.Run("batch disabled", func(t *testing.T) {
cfg := assembledBatchConfig(t, true)
cfg.Notify.Distributor.Batch.Enabled = false
bundle := assembledBatchBundle(t, "2026-05-31")
notifier := &assembledBatchNotifier{}
result, err := RunBatchDetailed(context.Background(), BatchRequest{
Config: cfg, Batch: BatchEvening, Now: workflowTime("2026-05-29T18:00:00-05:00"),
Collector: &workflowCollector{result: &collect.Result{Bundle: &bundle}}, Executor: newAssembledBatchExecutor(), Notifier: notifier,
})
if err != nil || result.Notification != nil || len(notifier.reportRequests) != 0 || len(notifier.batchRequests) != 0 {
t.Fatalf("result/error/requests = %#v/%v/%d/%d", result, err, len(notifier.reportRequests), len(notifier.batchRequests))
}
})
t.Run("all success", func(t *testing.T) {
cfg := assembledBatchConfig(t, true)
bundle := assembledBatchBundle(t, "2026-05-31")
notifier := &assembledBatchNotifier{batchResult: &NotificationResult{
RunID: "batch-notification-run", Status: "succeeded", UploadStatus: "accepted",
Report: []byte(`{"actions":[{"action":"replace_older"}]}`),
}}
outputDir := filepath.Join(t.TempDir(), "output")
result, err := RunBatchDetailed(context.Background(), BatchRequest{
Config: cfg, Batch: BatchMorning, Now: workflowTime("2026-05-29T08:00:00-05:00"), OutputDir: outputDir,
Collector: &workflowCollector{result: &collect.Result{Bundle: &bundle}}, Executor: newAssembledBatchExecutor(), Notifier: notifier,
})
if err != nil {
t.Fatalf("RunBatchDetailed() error = %v", err)
}
if result.Failed != 0 || result.Notification == nil || result.Notification.Status != "succeeded" || result.Notification.Path == "" || len(notifier.reportRequests) != 0 || len(notifier.batchRequests) != 1 {
t.Fatalf("notification result/requests = %#v/%d/%d", result.Notification, len(notifier.reportRequests), len(notifier.batchRequests))
}
managedPaths := make(map[string]struct{}, len(result.Reports))
for _, item := range result.Reports {
managedPaths[item.ReportPath] = struct{}{}
if item.NotificationPath != "" {
t.Fatalf("report item contains per-report notification path: %#v", item)
}
}
request := notifier.batchRequests[0]
if len(request.IncludedReports) != len(result.Reports) {
t.Fatalf("included reports = %d, want %d", len(request.IncludedReports), len(result.Reports))
}
for _, file := range request.Files {
if _, ok := managedPaths[file.SourcePath]; !ok || strings.HasPrefix(file.SourcePath, outputDir+string(filepath.Separator)) || file.BundlePath == "" {
t.Fatalf("notification file = %#v, want managed Markdown source", file)
}
}
artifact := readBatchNotificationArtifact(t, result.Notification.Path)
if artifact.Status != "succeeded" || artifact.Upload == nil || artifact.Upload.RunID != "batch-notification-run" || artifact.RunStatus == nil || len(artifact.Reports) != len(result.Reports) {
t.Fatalf("notification artifact = %#v", artifact)
}
})
t.Run("upload failure", func(t *testing.T) {
cfg := assembledBatchConfig(t, true)
bundle := assembledBatchBundle(t, "2026-05-31")
notifier := &assembledBatchNotifier{batchErr: errors.New("batch upload rejected")}
result, err := RunBatchDetailed(context.Background(), BatchRequest{
Config: cfg, Batch: BatchEvening, Now: workflowTime("2026-05-29T18:00:00-05:00"),
Collector: &workflowCollector{result: &collect.Result{Bundle: &bundle}}, Executor: newAssembledBatchExecutor(), Notifier: notifier,
})
if err != nil {
t.Fatalf("RunBatchDetailed() error = %v", err)
}
if result.Succeeded != 2 || result.Failed != 1 || result.Notification == nil || result.Notification.Status != "failed" || result.Notification.Path == "" {
t.Fatalf("result = %#v, want successful reports and failed notification", result)
}
for _, item := range result.Reports {
if item.Status != "succeeded" {
t.Fatalf("report item = %#v, want success despite notification failure", item)
}
}
artifact := readBatchNotificationArtifact(t, result.Notification.Path)
if artifact.Status != "failed" || !strings.Contains(artifact.Error, "batch upload rejected") {
t.Fatalf("notification artifact = %#v", artifact)
}
})
t.Run("status report", func(t *testing.T) {
cfg := assembledBatchConfig(t, true)
bundle := assembledBatchBundle(t, "2026-05-31")
notifier := &assembledBatchNotifier{batchResult: &NotificationResult{
RunID: "batch-notification-run", Status: "accepted", UploadStatus: "accepted",
StatusError: "status lookup unavailable", Report: []byte(`{"actions":[{"action":"replace_older"}]}`),
}}
result, err := RunBatchDetailed(context.Background(), BatchRequest{
Config: cfg, Batch: BatchEvening, Now: workflowTime("2026-05-29T18:00:00-05:00"),
Collector: &workflowCollector{result: &collect.Result{Bundle: &bundle}}, Executor: newAssembledBatchExecutor(), Notifier: notifier,
})
if err != nil || result.Failed != 0 || result.Notification == nil || result.Notification.Path == "" {
t.Fatalf("result/error = %#v/%v", result, err)
}
artifact := readBatchNotificationArtifact(t, result.Notification.Path)
if artifact.StatusError != "status lookup unavailable" || artifact.RunStatus == nil || !bytes.Contains(artifact.RunStatus.Report, []byte("replace_older")) {
t.Fatalf("notification artifact = %#v", artifact)
}
})
}
func TestRunBatchDetailedKeepsDynamicDailyArtifactsDistinct(t *testing.T) {
cfg := assembledBatchConfig(t, false)
bundle := assembledBatchBundle(t, "2026-05-31", "2026-06-01")
outputDir := filepath.Join(t.TempDir(), "output")
debugRoot := filepath.Join(t.TempDir(), "debug")
result, err := RunBatchDetailed(context.Background(), BatchRequest{
Config: cfg, Batch: BatchEvening, Now: workflowTime("2026-05-29T18:00:00-05:00"), OutputDir: outputDir, LLMDebugDir: debugRoot,
Collector: &workflowCollector{result: &collect.Result{Bundle: &bundle}}, Executor: newAssembledBatchExecutor(),
})
if err != nil || result.Failed != 0 || len(result.Reports) != 3 {
t.Fatalf("result/error = %#v/%v", result, err)
}
seenRuns := make(map[string]struct{})
seenDebug := make(map[string]struct{})
dailyDates := make(map[string]BatchReportResult)
for _, item := range result.Reports {
if _, exists := seenRuns[item.RunID]; exists {
t.Fatalf("duplicate run ID %q", item.RunID)
}
seenRuns[item.RunID] = struct{}{}
if _, exists := seenDebug[item.LLMDebugPath]; exists {
t.Fatalf("duplicate debug path %q", item.LLMDebugPath)
}
seenDebug[item.LLMDebugPath] = struct{}{}
if item.ReportID == report.Daily {
date := item.ValidPeriod.Start.In(mustLoadTestLocation(t, "America/Chicago")).Format("2006-01-02")
dailyDates[date] = item
}
}
for _, date := range []string{"2026-05-31", "2026-06-01"} {
item, ok := dailyDates[date]
if !ok {
t.Fatalf("daily items = %#v, want %s", dailyDates, date)
}
if !strings.HasSuffix(item.RunID, "_daily_"+date) || item.OutputPath != filepath.Join(outputDir, "daily-"+date+".md") {
t.Fatalf("daily identity/output = %q/%q", item.RunID, item.OutputPath)
}
wantDebugPrefix := filepath.Join(debugRoot, "daily", date, item.RunID)
if item.LLMDebugPath != wantDebugPrefix {
t.Fatalf("daily debug path = %q, want %q", item.LLMDebugPath, wantDebugPrefix)
}
assertBatchPathsExist(t, filepath.Join(item.LLMDebugPath, "preparation.json"), filepath.Join(item.LLMDebugPath, "execution.json"))
}
}
func TestRunBatchDetailedUsesPriorSnapshotsForPromptPackages(t *testing.T) {
cfg := assembledBatchConfig(t, false)
firstBundle := assembledBatchBundle(t, "2026-05-31")
setWorkflowTemperatures(&firstBundle, 45)
first, err := RunBatchDetailed(context.Background(), BatchRequest{
Config: cfg, Batch: BatchMorning, Now: workflowTime("2026-05-29T08:00:00-05:00"),
Collector: &workflowCollector{result: &collect.Result{Bundle: &firstBundle}}, Executor: newAssembledBatchExecutor(),
})
if err != nil || first.Failed != 0 {
t.Fatalf("first result/error = %#v/%v", first, err)
}
secondBundle := assembledBatchBundle(t, "2026-05-31")
setWorkflowTemperatures(&secondBundle, 85)
second, err := RunBatchDetailed(context.Background(), BatchRequest{
Config: cfg, Batch: BatchMorning, Now: workflowTime("2026-05-29T08:30:00-05:00"),
Collector: &workflowCollector{result: &collect.Result{Bundle: &secondBundle}}, Executor: newAssembledBatchExecutor(),
})
if err != nil || second.Failed != 0 || len(second.Reports) != len(first.Reports) {
t.Fatalf("second result/error = %#v/%v", second, err)
}
for _, item := range second.Reports {
pkg := loadBatchDataPackage(t, item.DataPackagePath)
if len(pkg.RecentChanges.Items) == 0 {
t.Fatalf("report %s data package has no changes from prior snapshot", item.ReportID)
}
}
}
func assembledBatchConfig(t *testing.T, notify bool) config.Config {
t.Helper()
cfg := workflowConfig(t)
cfg.Notify.Distributor.Enabled = notify
cfg.Notify.Distributor.Batch.Enabled = notify
return cfg
}
func assembledBatchBundle(t *testing.T, dates ...string) weatherdata.Bundle {
t.Helper()
bundle := workflowBundle(t)
location := mustLoadTestLocation(t, "America/Chicago")
for _, date := range dates {
periods := fullDayPeriods(t, date, location)
for index := range periods {
temperature := float64(60 + index)
periods[index].TemperatureF = &temperature
periods[index].TextDescription = "Partly cloudy"
}
bundle.Hourly.Periods = append(bundle.Hourly.Periods, periods...)
}
return bundle
}
func generatedTextForPrompt(promptID string) string {
switch promptID {
case "weather.today_generated_text":
return validTodayWorkflowJSON()
case "weather.tomorrow_generated_text":
return validTomorrowWorkflowJSON()
case "weather.daily_generated_text":
return validDailyWorkflowJSON()
case "weather.hourly_generated_text":
return validHourlyWorkflowJSON()
default:
return ""
}
}
func assertBatchPathsExist(t *testing.T, paths ...string) {
t.Helper()
for _, path := range paths {
if _, err := os.Stat(path); err != nil {
t.Fatalf("expected path %q: %v", path, err)
}
}
}
func assertBatchItemMatchesMetadata(t *testing.T, item BatchReportResult) {
t.Helper()
data, err := os.ReadFile(item.MetadataPath)
if err != nil {
t.Fatalf("read metadata %q: %v", item.MetadataPath, err)
}
var metadata state.Metadata
if err := json.Unmarshal(data, &metadata); err != nil {
t.Fatalf("decode metadata %q: %v", item.MetadataPath, err)
}
if item.ReportID != metadata.ReportID || item.RunID != metadata.RunID ||
item.DataPackagePath != metadata.DataPackagePath || item.PreparationPath != metadata.PreparationPath ||
item.ExecutionPath != metadata.ExecutionPath || item.ReportPath != metadata.RenderedReportPath ||
item.NotificationPath != metadata.NotificationPath {
t.Fatalf("batch item paths do not exactly match metadata: item=%#v metadata=%#v", item, metadata)
}
}
func readBatchNotificationArtifact(t *testing.T, path string) state.BatchDistributorNotificationArtifact {
t.Helper()
data, err := os.ReadFile(path)
if err != nil {
t.Fatalf("read batch notification artifact: %v", err)
}
var artifact state.BatchDistributorNotificationArtifact
if err := json.Unmarshal(data, &artifact); err != nil {
t.Fatalf("decode batch notification artifact: %v", err)
}
return artifact
}
func loadBatchDataPackage(t *testing.T, path string) promptinput.Package {
t.Helper()
data, err := os.ReadFile(path)
if err != nil {
t.Fatalf("read data package: %v", err)
}
pkg, err := promptinput.LoadYAML(data)
if err != nil {
t.Fatalf("LoadYAML() error = %v", err)
}
return pkg
}

240
internal/app/comparison.go Normal file
View File

@@ -0,0 +1,240 @@
package app
import (
"context"
"fmt"
"os"
"path/filepath"
"time"
"gitea.maximumdirect.net/eric/weatherreporter/internal/comparison"
"gitea.maximumdirect.net/eric/weatherreporter/internal/config"
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptdebug"
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec"
"gitea.maximumdirect.net/eric/weatherreporter/internal/report"
"gitea.maximumdirect.net/eric/weatherreporter/internal/timeutil"
)
// ComparisonRequest describes one explicit, multi-profile report comparison.
// It deliberately does not accept a notifier: comparison publication is local.
type ComparisonRequest struct {
Config config.Config
Report ReportKind
ProfileIDs []string
WorkingDir string
OutputDir string
Replace bool
LLMDebugDir string
Date time.Time
Clock timeutil.Clock
Collector Collector
Executor promptexec.Executor
}
// ComparisonResult records the resolved comparison and profile outcomes.
type ComparisonResult struct {
ComparisonID string
ReportID report.ID
ReportName string
PromptID string
PromptVersion string
PromptHash string
StartedAt time.Time
FinishedAt time.Time
Timezone string
ValidPeriod timeutil.Period
OutputDirectory string
ManifestPath string
DataPackagePath string
Total int
Succeeded int
Failed int
Results []ComparisonProfileResult
}
// ComparisonProfileResult records one explicitly selected profile.
type ComparisonProfileResult struct {
Position int
ProfileID string
BackendID string
ModelName string
Status string
ValidationStatus promptexec.ValidationStatus
ReportPath string
LLMDebugPath string
Error *comparison.SafeError
}
type comparisonPublisher func(context.Context, comparison.DestinationPlan, comparison.LogicalBundle) (comparison.PublicationResult, error)
// CompareDetailed assembles, executes, and atomically publishes a comparison
// bundle. Profile failures publish a complete partial bundle. Failures before
// commit leave the destination untouched; a post-commit cleanup failure leaves
// the new bundle installed and returns its artifact paths with an error.
func CompareDetailed(ctx context.Context, req ComparisonRequest) (*ComparisonResult, error) {
return compareDetailed(ctx, req, comparison.Publish)
}
func compareDetailed(ctx context.Context, req ComparisonRequest, publish comparisonPublisher) (*ComparisonResult, error) {
if err := comparison.ValidateProfileIDs(req.ProfileIDs); err != nil {
return nil, err
}
clock := req.Clock
if clock == nil {
clock = timeutil.SystemClock{}
}
now := clock.Now()
resolved, err := ResolveGenerate(GenerateRequest{Config: req.Config, Report: req.Report, Date: req.Date}, now)
if err != nil {
return nil, err
}
metadata := resolved.Metadata()
comparisonID, err := comparison.BuildComparisonID(metadata.RunID)
if err != nil {
return nil, fmt.Errorf("build comparison identity: %w", err)
}
result := initialComparisonResult(req, resolved, comparisonID, now.UTC())
defer func() {
if result.FinishedAt.IsZero() {
finalizeComparisonResult(result, clock)
}
}()
outputName, err := resolved.OutputName()
if err != nil {
return result, fmt.Errorf("resolve comparison output name: %w", err)
}
outputDirectory, err := resolveComparisonOutputDirectory(req.WorkingDir, req.OutputDir, req.Config.Output.Directory, outputName)
if err != nil {
return result, err
}
result.OutputDirectory = outputDirectory
_, err = comparison.PlanDestination(req.WorkingDir, outputDirectory, req.Replace)
if err != nil {
return result, fmt.Errorf("preflight comparison destination: %w", err)
}
debugWriter, err := promptdebug.NewPromptDebugWriter(req.LLMDebugDir)
if err != nil {
return result, promptexec.NewError(promptexec.InvalidConfiguration, "initialize prompt debug", err)
}
inspection, err := InspectComparisonExecution(ctx, ComparisonInspectionRequest{
Resolved: resolved, ProfileIDs: req.ProfileIDs, Executor: req.Executor, LookupEnv: os.LookupEnv,
})
result.PromptID, result.PromptVersion, result.PromptHash = inspection.PromptID, inspection.PromptVersion, inspection.PromptHash
if err != nil {
return result, err
}
collection, err := collectWeather(ctx, req.Config, req.Collector)
if err != nil {
return result, err
}
prepared, err := prepareReport(prepareReportRequest{Config: req.Config, Resolved: resolved, Collection: *collection})
if err != nil {
return result, fmt.Errorf("prepare comparison report: %w", err)
}
executed := executeComparisonProfiles(ctx, comparisonExecutionRequest{
Prepared: prepared, Inspection: inspection, ComparisonID: comparisonID, DebugWriter: debugWriter, Executor: req.Executor,
})
finalizeComparisonResult(result, clock)
copyComparisonOutcomes(result, executed.Outcomes, false)
if executed.Canceled {
return result, fmt.Errorf("comparison execution: %w", ctx.Err())
}
bundle := comparisonBundle(result, prepared.dataPackageCopy(), executed.Outcomes)
if err := bundle.Validate(); err != nil {
return result, fmt.Errorf("build comparison bundle: %w", err)
}
publicationPlan, err := comparison.PlanDestination(req.WorkingDir, outputDirectory, req.Replace)
if err != nil {
return result, fmt.Errorf("re-preflight comparison destination: %w", err)
}
publication, err := publish(ctx, publicationPlan, bundle)
if publication.Committed {
result.OutputDirectory = publicationPlan.Target
result.ManifestPath = filepath.Join(publicationPlan.Target, comparison.ManifestFilename)
result.DataPackagePath = filepath.Join(publicationPlan.Target, comparison.DataPackageFilename)
copyComparisonOutcomes(result, executed.Outcomes, true)
}
if err != nil {
return result, fmt.Errorf("publish comparison bundle: %w", err)
}
if result.Failed > 0 {
return result, fmt.Errorf("comparison completed with %d failed profiles", result.Failed)
}
return result, nil
}
func finalizeComparisonResult(result *ComparisonResult, clock timeutil.Clock) {
finishedAt := clock.Now().UTC()
if finishedAt.IsZero() {
finishedAt = time.Unix(0, 1).UTC()
}
if finishedAt.Before(result.StartedAt) {
finishedAt = result.StartedAt
}
result.FinishedAt = finishedAt
}
func initialComparisonResult(req ComparisonRequest, resolved report.Resolved, comparisonID string, startedAt time.Time) *ComparisonResult {
metadata := resolved.Metadata()
return &ComparisonResult{
ComparisonID: comparisonID,
ReportID: resolved.Definition.ID,
ReportName: resolved.Definition.Name,
StartedAt: startedAt,
Timezone: req.Config.WeatherAPI.Timezone,
ValidPeriod: metadata.ValidPeriod,
}
}
func copyComparisonOutcomes(result *ComparisonResult, outcomes []comparisonProfileOutcome, published bool) {
result.Results = make([]ComparisonProfileResult, len(outcomes))
result.Total, result.Succeeded, result.Failed = len(outcomes), 0, 0
for index, outcome := range outcomes {
profile := ComparisonProfileResult{
Position: outcome.Position, ProfileID: outcome.ProfileID, BackendID: outcome.BackendID, ModelName: outcome.ModelName,
Status: outcome.Status, ValidationStatus: outcome.ValidationStatus, LLMDebugPath: outcome.LLMDebugPath, Error: outcome.Error,
}
if published && outcome.Status == comparison.StatusSucceeded {
profile.ReportPath = filepath.Join(result.OutputDirectory, outcome.ReportPath)
}
result.Results[index] = profile
if outcome.Status == comparison.StatusSucceeded {
result.Succeeded++
} else {
result.Failed++
}
}
}
func comparisonBundle(result *ComparisonResult, dataPackage []byte, outcomes []comparisonProfileOutcome) comparison.LogicalBundle {
manifest := comparison.Manifest{
SchemaVersion: comparison.SchemaVersion, ComparisonID: result.ComparisonID,
StartedAt: result.StartedAt.UTC(), FinishedAt: result.FinishedAt.UTC(),
ReportID: string(result.ReportID), Timezone: result.Timezone,
ValidPeriod: comparison.ValidPeriod{Start: result.ValidPeriod.Start, End: result.ValidPeriod.End},
PromptID: result.PromptID, PromptVersion: result.PromptVersion, PromptHash: result.PromptHash,
DataPackage: comparison.DataPackageReference{Path: comparison.DataPackageFilename, SHA256: comparison.SHA256(dataPackage)},
Total: result.Total, Succeeded: result.Succeeded, Failed: result.Failed,
Results: make([]comparison.Result, len(outcomes)),
}
bundle := comparison.LogicalBundle{Manifest: manifest, DataPackage: dataPackage}
for index, outcome := range outcomes {
manifestResult := comparison.Result{
Position: outcome.Position, ProfileID: outcome.ProfileID, BackendID: outcome.BackendID, ModelName: outcome.ModelName,
Status: outcome.Status, ValidationStatus: string(outcome.ValidationStatus), Error: outcome.Error,
}
if outcome.Status == comparison.StatusSucceeded {
manifestResult.ReportPath = outcome.ReportPath
bundle.Reports = append(bundle.Reports, comparison.BundleReport{Position: outcome.Position, Path: outcome.ReportPath, Markdown: outcome.Markdown})
}
bundle.Manifest.Results[index] = manifestResult
}
return bundle
}

View File

@@ -0,0 +1,165 @@
package app
import (
"context"
"errors"
"fmt"
"sync"
"gitea.maximumdirect.net/eric/weatherreporter/internal/comparison"
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptdebug"
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec"
)
type comparisonExecutionRequest struct {
Prepared preparedReport
Inspection ComparisonInspectionResult
ComparisonID string
DebugWriter *promptdebug.PromptDebugWriter
Executor promptexec.Executor
}
type comparisonExecutionResult struct {
Outcomes []comparisonProfileOutcome
Canceled bool
}
type comparisonProfileOutcome struct {
Position int
ProfileID string
BackendID string
ModelName string
Status string
ValidationStatus promptexec.ValidationStatus
ReportPath string
Markdown []byte
LLMDebugPath string
Error *comparison.SafeError
}
func executeComparisonProfiles(ctx context.Context, req comparisonExecutionRequest) comparisonExecutionResult {
profiles := req.Inspection.Profiles
result := comparisonExecutionResult{Outcomes: make([]comparisonProfileOutcome, len(profiles))}
for index, profile := range profiles {
result.Outcomes[index] = comparisonProfileOutcome{
Position: index + 1,
ProfileID: profile.ProfileID,
BackendID: profile.BackendID,
ModelName: profile.ModelName,
Status: comparison.StatusFailed,
}
}
var waitGroup sync.WaitGroup
for index, profile := range profiles {
if err := ctx.Err(); err != nil {
result.Canceled = true
markUnstartedComparisonOutcomes(result.Outcomes[index:], err)
break
}
index, profile := index, profile
waitGroup.Add(1)
go func() {
defer waitGroup.Done()
result.Outcomes[index] = executeComparisonProfile(ctx, req, index, profile)
}()
}
waitGroup.Wait()
if err := ctx.Err(); err != nil {
result.Canceled = true
for index := range result.Outcomes {
if result.Outcomes[index].Status != comparison.StatusSucceeded {
markCanceledComparisonOutcome(&result.Outcomes[index], err)
}
}
}
return result
}
func executeComparisonProfile(ctx context.Context, req comparisonExecutionRequest, index int, profile ComparisonProfileInspection) comparisonProfileOutcome {
position := index + 1
outcome := comparisonProfileOutcome{
Position: position, ProfileID: profile.ProfileID, BackendID: profile.BackendID, ModelName: profile.ModelName,
Status: comparison.StatusFailed,
}
debugRef := promptdebug.PromptDebugRef{
ReportID: req.Prepared.resolved.Definition.ID,
ValidDate: req.Prepared.resolved.ValidPeriod.Start.Format("2006-01-02"),
RunID: comparisonDebugRunID(req.ComparisonID, position, len(req.Inspection.Profiles), profile.ProfileID),
}
execution, markdown, err := executePreparedProfile(ctx, profileExecutionRequest{
Prepared: req.Prepared,
Prompt: PromptInspectionResult{
PromptID: req.Inspection.PromptID, PromptVersion: req.Inspection.PromptVersion, PromptHash: req.Inspection.PromptHash,
},
Profile: promptexec.ProfileInspection{ProfileID: profile.ProfileID, BackendID: profile.BackendID, ModelName: profile.ModelName},
Executor: req.Executor, DebugWriter: req.DebugWriter, DebugRef: &debugRef,
})
outcome.ProfileID, outcome.BackendID, outcome.ModelName = execution.ProfileID, execution.BackendID, execution.ModelName
outcome.ValidationStatus = execution.ValidationStatus
outcome.LLMDebugPath = execution.LLMDebugPath
if err != nil {
safe := comparisonSafeExecutionError(err)
outcome.Error = &safe
return outcome
}
reportPath, err := comparison.ReportFilename(position, len(req.Inspection.Profiles), profile.ProfileID)
if err != nil {
safe := comparison.NewSafeError("application", "derive comparison report filename failed")
outcome.Error = &safe
return outcome
}
outcome.Status = comparison.StatusSucceeded
outcome.ReportPath = reportPath
outcome.Markdown = append([]byte(nil), markdown...)
return outcome
}
func comparisonDebugRunID(comparisonID string, position, profileCount int, profileID string) string {
return fmt.Sprintf("%s_%0*d-%s", comparisonID, comparison.OrdinalWidth(profileCount), position, comparison.ProfileSlug(profileID))
}
func markUnstartedComparisonOutcomes(outcomes []comparisonProfileOutcome, err error) {
for index := range outcomes {
markCanceledComparisonOutcome(&outcomes[index], err)
}
}
func markCanceledComparisonOutcome(outcome *comparisonProfileOutcome, err error) {
outcome.Status = comparison.StatusFailed
outcome.ValidationStatus = promptexec.ValidationSkipped
outcome.ReportPath = ""
outcome.Markdown = nil
safe := comparisonSafeExecutionError(err)
outcome.Error = &safe
}
func comparisonSafeExecutionError(err error) comparison.SafeError {
category := promptexec.CategoryOf(err)
if category == "" {
switch {
case errors.Is(err, context.Canceled):
category = promptexec.Canceled
case errors.Is(err, context.DeadlineExceeded):
category = promptexec.DeadlineExceeded
}
}
if category == "" {
return comparison.NewSafeError("application", comparisonExecutionMessage(err))
}
return comparison.NewSafeError(string(category), comparisonExecutionMessage(err))
}
func comparisonExecutionMessage(err error) string {
if errors.Is(err, context.Canceled) {
return "profile execution canceled"
}
if errors.Is(err, context.DeadlineExceeded) {
return "profile execution deadline exceeded"
}
var execution *profileExecutionError
if errors.As(err, &execution) {
return comparison.TruncateErrorMessage(execution.operation + " failed")
}
return "profile execution failed"
}

View File

@@ -0,0 +1,300 @@
package app
import (
"context"
"errors"
"fmt"
"os"
"path/filepath"
"reflect"
"sync"
"testing"
"time"
"gitea.maximumdirect.net/eric/weatherreporter/internal/comparison"
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptdebug"
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec"
)
func TestExecuteComparisonProfilesRunsOrderedProfilesConcurrently(t *testing.T) {
prepared, prompt := preparedDailyProfile(t)
profiles := comparisonProfiles(10)
executor := newBarrierExecutor(profiles)
results := make(chan comparisonExecutionResult, 1)
go func() {
results <- executeComparisonProfiles(context.Background(), comparisonExecutionRequest{
Prepared: prepared, Inspection: comparisonInspection(prompt, profiles), ComparisonID: "comparison_daily", Executor: executor,
})
}()
waitForProfileStarts(t, executor, profiles, results)
if executor.maximumInFlight() < 2 {
t.Fatalf("maximum in-flight executions = %d, want overlap", executor.maximumInFlight())
}
for index := len(profiles) - 1; index >= 0; index-- {
executor.release(profiles[index].ProfileID)
}
result := <-results
if result.Canceled || len(result.Outcomes) != len(profiles) {
t.Fatalf("result = %#v", result)
}
for index, profile := range profiles {
outcome := result.Outcomes[index]
wantPath, err := comparison.ReportFilename(index+1, len(profiles), profile.ProfileID)
if err != nil {
t.Fatal(err)
}
if outcome.Position != index+1 || outcome.ProfileID != profile.ProfileID || outcome.Status != comparison.StatusSucceeded || outcome.ValidationStatus != promptexec.ValidationPassed || outcome.ReportPath != wantPath || len(outcome.Markdown) == 0 || outcome.Error != nil {
t.Fatalf("outcome[%d] = %#v", index, outcome)
}
request, ok := executor.request(profile.ProfileID)
if !ok || request.PromptVersion != prompt.PromptVersion || !bytesEqual(request.DataPackage, prepared.dataPackage) {
t.Fatalf("request for %q = %#v, want prompt version %q and shared data package", profile.ProfileID, request, prompt.PromptVersion)
}
}
}
func TestExecuteComparisonProfilesContinuesAfterProfileFailure(t *testing.T) {
prepared, prompt := preparedDailyProfile(t)
profiles := comparisonProfiles(3)
executor := newBarrierExecutor(profiles)
executor.setError(profiles[1].ProfileID, errors.New("provider response body must not escape"))
results := make(chan comparisonExecutionResult, 1)
go func() {
results <- executeComparisonProfiles(context.Background(), comparisonExecutionRequest{
Prepared: prepared, Inspection: comparisonInspection(prompt, profiles), ComparisonID: "comparison_daily", Executor: executor,
})
}()
waitForProfileStarts(t, executor, profiles, results)
for _, profile := range profiles {
executor.release(profile.ProfileID)
}
result := <-results
if result.Canceled || result.Outcomes[0].Status != comparison.StatusSucceeded || result.Outcomes[1].Status != comparison.StatusFailed || result.Outcomes[2].Status != comparison.StatusSucceeded {
t.Fatalf("outcomes = %#v", result.Outcomes)
}
failure := result.Outcomes[1]
if failure.Error == nil || failure.Error.Category != string(promptexec.Generation) || failure.Error.Message != "execute prompt failed" || failure.ReportPath != "" || len(failure.Markdown) != 0 {
t.Fatalf("failure outcome = %#v", failure)
}
}
func TestExecuteComparisonProfilesPropagatesCancellationAndJoins(t *testing.T) {
prepared, prompt := preparedDailyProfile(t)
profiles := comparisonProfiles(4)
executor := newBarrierExecutor(profiles)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
results := make(chan comparisonExecutionResult, 1)
go func() {
results <- executeComparisonProfiles(ctx, comparisonExecutionRequest{
Prepared: prepared, Inspection: comparisonInspection(prompt, profiles), ComparisonID: "comparison_daily", Executor: executor,
})
}()
waitForProfileStarts(t, executor, profiles, results)
cancel()
result := <-results
if !result.Canceled || executor.inFlightCount() != 0 {
t.Fatalf("result/in-flight = %#v/%d", result, executor.inFlightCount())
}
for _, outcome := range result.Outcomes {
if outcome.Status != comparison.StatusFailed || outcome.Error == nil || outcome.Error.Category != string(promptexec.Canceled) || outcome.ValidationStatus != promptexec.ValidationSkipped || outcome.ReportPath != "" || len(outcome.Markdown) != 0 {
t.Fatalf("canceled outcome = %#v", outcome)
}
}
}
func TestExecuteComparisonProfilesUsesDistinctDeterministicDebugReferences(t *testing.T) {
prepared, prompt := preparedDailyProfile(t)
profiles := []ComparisonProfileInspection{
{ProfileID: "light.one", BackendID: "local", ModelName: "light"},
{ProfileID: "deep/two", BackendID: "cloud", ModelName: "deep"},
}
debugWriter, err := promptdebug.NewPromptDebugWriter(t.TempDir())
if err != nil {
t.Fatalf("NewPromptDebugWriter() error = %v", err)
}
executor := newBarrierExecutor(profiles)
results := make(chan comparisonExecutionResult, 1)
go func() {
results <- executeComparisonProfiles(context.Background(), comparisonExecutionRequest{
Prepared: prepared, Inspection: comparisonInspection(prompt, profiles), ComparisonID: "comparison_daily", DebugWriter: debugWriter, Executor: executor,
})
}()
waitForProfileStarts(t, executor, profiles, results)
for _, profile := range profiles {
executor.release(profile.ProfileID)
}
result := <-results
paths := map[string]struct{}{}
for index, outcome := range result.Outcomes {
wantName := fmt.Sprintf("comparison_daily_%0*d-%s", comparison.OrdinalWidth(len(profiles)), index+1, comparison.ProfileSlug(outcome.ProfileID))
if filepath.Base(outcome.LLMDebugPath) != wantName {
t.Fatalf("debug path = %q, want base %q", outcome.LLMDebugPath, wantName)
}
if _, err := os.Stat(filepath.Join(outcome.LLMDebugPath, "preparation.json")); err != nil {
t.Fatalf("preparation artifact %q: %v", outcome.LLMDebugPath, err)
}
paths[outcome.LLMDebugPath] = struct{}{}
}
if len(paths) != len(profiles) {
t.Fatalf("debug paths = %#v", paths)
}
}
type barrierExecutor struct {
mu sync.Mutex
started chan string
callbackFailures chan error
releases map[string]chan struct{}
requests map[string]promptexec.ExecuteRequest
errors map[string]error
inFlight int
maximum int
}
func newBarrierExecutor(profiles []ComparisonProfileInspection) *barrierExecutor {
releases := make(map[string]chan struct{}, len(profiles))
for _, profile := range profiles {
releases[profile.ProfileID] = make(chan struct{})
}
return &barrierExecutor{
started: make(chan string, len(profiles)), callbackFailures: make(chan error, len(profiles)), releases: releases,
requests: make(map[string]promptexec.ExecuteRequest, len(profiles)), errors: map[string]error{},
}
}
func (e *barrierExecutor) InspectPrompt(context.Context, string, string) (promptexec.PromptInspection, error) {
return promptexec.PromptInspection{}, errors.New("unexpected prompt inspection")
}
func (e *barrierExecutor) InspectProfile(context.Context, string) (promptexec.ProfileInspection, error) {
return promptexec.ProfileInspection{}, errors.New("unexpected profile inspection")
}
func (e *barrierExecutor) Execute(ctx context.Context, req promptexec.ExecuteRequest, callback promptexec.PreparationCallback) (*promptexec.Execution, error) {
stamp := time.Date(2026, 5, 29, 15, 0, 0, 0, time.UTC)
if err := callback(promptexec.Preparation{PromptID: req.PromptID, PromptVersion: req.PromptVersion, PromptHash: "prompt-hash", ProfileID: req.ProfileID, BackendID: "backend-" + req.ProfileID, ModelName: "model-" + req.ProfileID, StartedAt: stamp, EndedAt: stamp}, nil); err != nil {
e.callbackFailures <- err
return nil, err
}
e.mu.Lock()
e.requests[req.ProfileID] = promptexec.ExecuteRequest{PromptID: req.PromptID, PromptVersion: req.PromptVersion, ProfileID: req.ProfileID, DataPackage: append([]byte(nil), req.DataPackage...), CaptureDebug: req.CaptureDebug}
e.inFlight++
if e.inFlight > e.maximum {
e.maximum = e.inFlight
}
release := e.releases[req.ProfileID]
e.mu.Unlock()
e.started <- req.ProfileID
select {
case <-release:
case <-ctx.Done():
e.mu.Lock()
e.inFlight--
e.mu.Unlock()
return nil, ctx.Err()
}
e.mu.Lock()
e.inFlight--
err := e.errors[req.ProfileID]
e.mu.Unlock()
if err != nil {
return nil, err
}
return &promptexec.Execution{
PromptID: req.PromptID, PromptVersion: req.PromptVersion, PromptHash: "prompt-hash",
ProfileID: req.ProfileID, BackendID: "backend-" + req.ProfileID, ModelName: "model-" + req.ProfileID,
StartedAt: stamp, EndedAt: stamp, RawOutput: comparisonRawOutput(),
Validation: promptexec.NewValidation(promptexec.ValidationPassed, "json_schema", generationDefinitionForPrompt(req.PromptID).GeneratedTextSchemaID+".generated_text.schema.json", nil),
}, nil
}
func (e *barrierExecutor) request(profileID string) (promptexec.ExecuteRequest, bool) {
e.mu.Lock()
defer e.mu.Unlock()
request, ok := e.requests[profileID]
return request, ok
}
func (e *barrierExecutor) setError(profileID string, err error) {
e.mu.Lock()
defer e.mu.Unlock()
e.errors[profileID] = err
}
func (e *barrierExecutor) release(profileID string) {
close(e.releases[profileID])
}
func (e *barrierExecutor) releaseAll() {
for _, release := range e.releases {
select {
case <-release:
default:
close(release)
}
}
}
func (e *barrierExecutor) maximumInFlight() int {
e.mu.Lock()
defer e.mu.Unlock()
return e.maximum
}
func (e *barrierExecutor) inFlightCount() int {
e.mu.Lock()
defer e.mu.Unlock()
return e.inFlight
}
func waitForProfileStarts(t *testing.T, executor *barrierExecutor, profiles []ComparisonProfileInspection, results <-chan comparisonExecutionResult) {
t.Helper()
timeout := time.NewTimer(5 * time.Second)
defer timeout.Stop()
seen := map[string]struct{}{}
for range profiles {
var profileID string
select {
case profileID = <-executor.started:
case err := <-executor.callbackFailures:
executor.releaseAll()
select {
case result := <-results:
t.Fatalf("comparison profile preparation failed before executor entry: %v; result: %#v", err, result)
case <-timeout.C:
t.Fatalf("comparison profile preparation failed before executor entry: %v; comparison did not finish", err)
}
case result := <-results:
t.Fatalf("comparison completed before all profiles started: %#v", result)
case <-timeout.C:
t.Fatal("timed out waiting for comparison profile starts")
}
if _, duplicate := seen[profileID]; duplicate {
t.Fatalf("duplicate execution start for %q", profileID)
}
seen[profileID] = struct{}{}
}
}
func comparisonProfiles(count int) []ComparisonProfileInspection {
profiles := make([]ComparisonProfileInspection, 0, count)
for index := 1; index <= count; index++ {
profiles = append(profiles, ComparisonProfileInspection{ProfileID: fmt.Sprintf("profile.%02d", index), BackendID: "backend", ModelName: "model"})
}
return profiles
}
func comparisonInspection(prompt PromptInspectionResult, profiles []ComparisonProfileInspection) ComparisonInspectionResult {
return ComparisonInspectionResult{PromptID: prompt.PromptID, PromptVersion: prompt.PromptVersion, PromptHash: prompt.PromptHash, Profiles: profiles}
}
func comparisonRawOutput() []byte {
return []byte(`{"summary":"Showers are possible during the selected day.","forecast_discussion":["A front will keep rain chances in the forecast."],"precipitation_timing":"Rain is most likely during the afternoon."}`)
}
func bytesEqual(left, right []byte) bool {
return reflect.DeepEqual(left, right)
}
var _ promptexec.Executor = (*barrierExecutor)(nil)

View File

@@ -0,0 +1,332 @@
package app
import (
"context"
"encoding/json"
"errors"
"os"
"path/filepath"
"strings"
"testing"
"time"
"gitea.maximumdirect.net/eric/weatherreporter/internal/comparison"
"gitea.maximumdirect.net/eric/weatherreporter/internal/config"
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec"
"gitea.maximumdirect.net/eric/weatherreporter/internal/timeutil"
"gitea.maximumdirect.net/eric/weatherreporter/internal/weatherdata"
)
func TestCompareDetailedPublishesOneCoherentBundle(t *testing.T) {
cfg := comparisonConfig()
bundle := generationBundle(t)
workingDir := t.TempDir()
executor := &generationExecutor{}
inspectedBeforeCollection := false
result, err := CompareDetailed(context.Background(), ComparisonRequest{
Config: cfg, Report: ReportDaily, ProfileIDs: []string{"weather-light", "weather-deep"},
WorkingDir: workingDir, Date: generationTime("2026-05-29T12:00:00-05:00"),
Clock: timeutil.FixedClock{Time: generationTime("2026-05-29T08:30:00-05:00")},
Collector: &generationCollector{bundle: &bundle, beforeRun: func() {
inspectedBeforeCollection = executor.promptInspections == 1 && executor.profileInspections == 2
}}, Executor: executor,
})
if err != nil {
t.Fatalf("CompareDetailed() error = %v", err)
}
if result == nil || result.Total != 2 || result.Succeeded != 2 || result.Failed != 0 || executor.promptInspections != 1 || executor.profileInspections != 2 || executor.executeCalls != 2 || !inspectedBeforeCollection || result.ManifestPath == "" || result.DataPackagePath == "" {
t.Fatalf("result/executor = %#v/%#v", result, executor)
}
if result.OutputDirectory != filepath.Dir(result.ManifestPath) || !filepath.IsAbs(result.ManifestPath) || !filepath.IsAbs(result.DataPackagePath) {
t.Fatalf("published paths = %#v", result)
}
for index, profile := range result.Results {
if profile.Position != index+1 || profile.Status != comparison.StatusSucceeded || !filepath.IsAbs(profile.ReportPath) || profile.Error != nil {
t.Fatalf("profile result = %#v", profile)
}
}
data, readErr := os.ReadFile(result.ManifestPath)
if readErr != nil {
t.Fatal(readErr)
}
var manifest comparison.Manifest
if err := json.Unmarshal(data, &manifest); err != nil {
t.Fatal(err)
}
if manifest.ComparisonID != result.ComparisonID || manifest.Total != result.Total || manifest.Succeeded != result.Succeeded || manifest.DataPackage.SHA256 == "" || len(manifest.Results) != 2 {
t.Fatalf("manifest = %#v", manifest)
}
if manifest.Results[0].ReportPath != filepath.Base(result.Results[0].ReportPath) || manifest.Results[1].ReportPath != filepath.Base(result.Results[1].ReportPath) {
t.Fatalf("manifest report paths = %#v", manifest.Results)
}
}
func TestCompareDetailedPublishesPartialBundleAndReturnsAggregateError(t *testing.T) {
cfg := comparisonConfig()
bundle := generationBundle(t)
executor := &generationExecutor{executeErrors: map[string]error{"weather-deep": errors.New("provider detail must not escape")}}
result, err := CompareDetailed(context.Background(), ComparisonRequest{
Config: cfg, Report: ReportDaily, ProfileIDs: []string{"weather-light", "weather-deep", "weather-fallback"},
WorkingDir: t.TempDir(), Date: generationTime("2026-05-29T12:00:00-05:00"),
Clock: timeutil.FixedClock{Time: generationTime("2026-05-29T08:30:00-05:00")},
Collector: &generationCollector{bundle: &bundle}, Executor: executor,
})
if err == nil || err.Error() != "comparison completed with 1 failed profiles" || result == nil || result.Total != 3 || result.Succeeded != 2 || result.Failed != 1 {
t.Fatalf("CompareDetailed() result/error = %#v/%v", result, err)
}
failure := result.Results[1]
if failure.Status != comparison.StatusFailed || failure.ReportPath != "" || failure.Error == nil || strings.Contains(failure.Error.Message, "provider detail") {
t.Fatalf("failure = %#v", failure)
}
if _, statErr := os.Stat(result.ManifestPath); statErr != nil {
t.Fatalf("partial manifest: %v", statErr)
}
if _, statErr := os.Stat(filepath.Join(result.OutputDirectory, filepath.Base(result.Results[0].ReportPath))); statErr != nil {
t.Fatalf("successful partial report: %v", statErr)
}
}
func TestCompareDetailedRetainsCommittedPathsWhenBackupCleanupFails(t *testing.T) {
bundle := generationBundle(t)
backupPath := filepath.Join(t.TempDir(), ".comparison-daily.backup-retained")
cleanupCause := errors.New("backup cleanup failed")
publish := func(context.Context, comparison.DestinationPlan, comparison.LogicalBundle) (comparison.PublicationResult, error) {
return comparison.PublicationResult{Committed: true, RetainedBackupPath: backupPath}, &comparison.PublicationCleanupError{RetainedBackupPath: backupPath, Err: cleanupCause}
}
result, err := compareDetailed(context.Background(), ComparisonRequest{
Config: comparisonConfig(), Report: ReportDaily, ProfileIDs: []string{"weather-light", "weather-deep"},
WorkingDir: t.TempDir(), Date: generationTime("2026-05-29T12:00:00-05:00"),
Clock: timeutil.FixedClock{Time: generationTime("2026-05-29T08:30:00-05:00")},
Collector: &generationCollector{bundle: &bundle}, Executor: &generationExecutor{},
}, publish)
var cleanupErr *comparison.PublicationCleanupError
if result == nil || !errors.As(err, &cleanupErr) || !errors.Is(err, cleanupCause) || cleanupErr.RetainedBackupPath != backupPath || !filepath.IsAbs(result.ManifestPath) || !filepath.IsAbs(result.DataPackagePath) {
t.Fatalf("CompareDetailed() result/error = %#v/%v", result, err)
}
for _, profile := range result.Results {
if profile.Status == comparison.StatusSucceeded && !filepath.IsAbs(profile.ReportPath) {
t.Fatalf("published profile result = %#v", profile)
}
}
}
func TestCompareDetailedPreflightsBeforePromptOrCollection(t *testing.T) {
invalidDestination := filepath.Join(t.TempDir(), "not-a-directory")
if err := os.WriteFile(invalidDestination, []byte("x"), 0o600); err != nil {
t.Fatal(err)
}
bundle := generationBundle(t)
collector := &generationCollector{bundle: &bundle}
executor := &generationExecutor{}
result, err := CompareDetailed(context.Background(), ComparisonRequest{
Config: comparisonConfig(), Report: ReportDaily, ProfileIDs: []string{"weather-light", "weather-deep"},
WorkingDir: t.TempDir(), OutputDir: invalidDestination, Date: generationTime("2026-05-29T12:00:00-05:00"),
Clock: timeutil.FixedClock{Time: generationTime("2026-05-29T08:30:00-05:00")}, Collector: collector, Executor: executor,
})
if err == nil || result == nil || collector.called || executor.promptInspections != 0 || executor.executeCalls != 0 || result.ManifestPath != "" {
t.Fatalf("result/error/collector/executor = %#v/%v/%#v/%#v", result, err, collector, executor)
}
}
func TestCompareDetailedFinalizesUnpublishedFailures(t *testing.T) {
for _, test := range []struct {
name string
prepare func(t *testing.T, outputDirectory string)
debugDir string
executor *generationExecutor
collector *generationCollector
wantPrompt bool
wantCollection bool
}{
{
name: "destination preflight",
prepare: func(t *testing.T, outputDirectory string) {
t.Helper()
if err := os.WriteFile(outputDirectory, []byte("not a directory"), 0o600); err != nil {
t.Fatal(err)
}
},
executor: &generationExecutor{},
},
{
name: "debug initialization",
debugDir: "relative-debug-directory",
executor: &generationExecutor{},
collector: &generationCollector{},
},
{
name: "prompt preflight",
executor: &generationExecutor{inspectErr: promptexec.NewError(promptexec.PromptLoad, "unsafe prompt detail", errors.New("unsafe cause"))},
collector: &generationCollector{},
wantPrompt: false,
},
{
name: "profile preflight",
executor: &generationExecutor{profileInspectErrors: map[string]error{
"weather-deep": promptexec.NewError(promptexec.MissingCredential, "profile credential is unavailable", errors.New("unsafe cause")),
}},
collector: &generationCollector{},
wantPrompt: true,
},
{
name: "collection",
executor: &generationExecutor{},
collector: &generationCollector{err: errors.New("collection failed")},
wantPrompt: true,
wantCollection: true,
},
{
name: "preparation",
executor: &generationExecutor{},
collector: &generationCollector{bundle: &weatherdata.Bundle{}},
wantPrompt: true,
wantCollection: true,
},
} {
t.Run(test.name, func(t *testing.T) {
workingDirectory := t.TempDir()
outputDirectory := filepath.Join(workingDirectory, "comparison-output")
if test.prepare != nil {
test.prepare(t, outputDirectory)
}
collector := test.collector
if collector == nil {
bundle := generationBundle(t)
collector = &generationCollector{bundle: &bundle}
}
result, err := CompareDetailed(context.Background(), ComparisonRequest{
Config: comparisonConfig(), Report: ReportDaily, ProfileIDs: []string{"weather-light", "weather-deep"},
WorkingDir: workingDirectory, OutputDir: outputDirectory, LLMDebugDir: test.debugDir,
Date: generationTime("2026-05-29T12:00:00-05:00"), Clock: timeutil.FixedClock{Time: generationTime("2026-05-29T08:30:00-05:00")},
Collector: collector, Executor: test.executor,
})
if err == nil {
t.Fatal("CompareDetailed() error = nil")
}
assertUnpublishedComparisonResult(t, result, outputDirectory)
if (result.PromptID != "") != test.wantPrompt || (result.PromptHash != "") != test.wantPrompt {
t.Fatalf("prompt identity = %q/%q, want resolved=%t", result.PromptID, result.PromptHash, test.wantPrompt)
}
if collector.called != test.wantCollection || test.executor.executeCalls != 0 {
t.Fatalf("collection/execution = %t/%d, want collection=%t and no execution", collector.called, test.executor.executeCalls, test.wantCollection)
}
})
}
}
func TestCompareDetailedLeavesDestinationWhenCollectionOrPreparationFails(t *testing.T) {
collectionErr := errors.New("weather collection failed")
for _, test := range []struct {
name string
collector *generationCollector
}{
{name: "collection", collector: &generationCollector{err: collectionErr}},
{name: "preparation", collector: &generationCollector{bundle: &weatherdata.Bundle{}}},
} {
t.Run(test.name, func(t *testing.T) {
workingDir := t.TempDir()
executor := &generationExecutor{}
result, err := CompareDetailed(context.Background(), ComparisonRequest{
Config: comparisonConfig(), Report: ReportDaily, ProfileIDs: []string{"weather-light", "weather-deep"},
WorkingDir: workingDir, Date: generationTime("2026-05-29T12:00:00-05:00"),
Clock: timeutil.FixedClock{Time: generationTime("2026-05-29T08:30:00-05:00")}, Collector: test.collector, Executor: executor,
})
if err == nil || result == nil || executor.executeCalls != 0 || result.ManifestPath != "" || result.DataPackagePath != "" {
t.Fatalf("CompareDetailed() result/error/executor = %#v/%v/%#v", result, err, executor)
}
if _, statErr := os.Stat(filepath.Join(workingDir, "comparison-daily-2026-05-29")); !os.IsNotExist(statErr) {
t.Fatalf("comparison destination stat error = %v", statErr)
}
})
}
}
func TestCompareDetailedPublishesManifestWhenEveryProfileFails(t *testing.T) {
bundle := generationBundle(t)
executor := &generationExecutor{executeErrors: map[string]error{
"weather-light": errors.New("first provider failure"), "weather-deep": errors.New("second provider failure"),
}}
result, err := CompareDetailed(context.Background(), ComparisonRequest{
Config: comparisonConfig(), Report: ReportDaily, ProfileIDs: []string{"weather-light", "weather-deep"},
WorkingDir: t.TempDir(), Date: generationTime("2026-05-29T12:00:00-05:00"),
Clock: timeutil.FixedClock{Time: generationTime("2026-05-29T08:30:00-05:00")},
Collector: &generationCollector{bundle: &bundle}, Executor: executor,
})
if err == nil || err.Error() != "comparison completed with 2 failed profiles" || result == nil || result.Succeeded != 0 || result.Failed != 2 || result.ManifestPath == "" {
t.Fatalf("CompareDetailed() result/error = %#v/%v", result, err)
}
for _, profile := range result.Results {
if profile.ReportPath != "" || profile.Error == nil {
t.Fatalf("failed profile = %#v", profile)
}
}
}
func TestCompareDetailedCancellationPreservesPublishedBundle(t *testing.T) {
workingDir := t.TempDir()
bundle := generationBundle(t)
request := ComparisonRequest{
Config: comparisonConfig(), Report: ReportDaily, ProfileIDs: []string{"weather-light", "weather-deep"},
WorkingDir: workingDir, Date: generationTime("2026-05-29T12:00:00-05:00"),
Clock: timeutil.FixedClock{Time: generationTime("2026-05-29T08:30:00-05:00")}, Collector: &generationCollector{bundle: &bundle}, Executor: &generationExecutor{},
}
previous, err := CompareDetailed(context.Background(), request)
if err != nil {
t.Fatalf("initial CompareDetailed() error = %v", err)
}
before, err := os.ReadFile(previous.ManifestPath)
if err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithCancel(context.Background())
request.Replace = true
request.Executor = &generationExecutor{cancelBeforeReturn: cancel}
result, err := CompareDetailed(ctx, request)
if !errors.Is(err, context.Canceled) || result == nil || result.ManifestPath != "" || result.DataPackagePath != "" {
t.Fatalf("canceled CompareDetailed() result/error = %#v/%v", result, err)
}
after, readErr := os.ReadFile(previous.ManifestPath)
if readErr != nil || string(after) != string(before) {
t.Fatalf("published manifest changed = %q, error = %v", after, readErr)
}
}
func TestCompareDetailedLeavesExistingBundleWhenPublicationPreflightChanges(t *testing.T) {
workingDir := t.TempDir()
target := filepath.Join(workingDir, "comparison-output")
bundle := generationBundle(t)
executor := &generationExecutor{beforeExecute: func(promptexec.ExecuteRequest) {
_ = os.WriteFile(target, []byte("changed"), 0o600)
}}
result, err := CompareDetailed(context.Background(), ComparisonRequest{
Config: comparisonConfig(), Report: ReportDaily, ProfileIDs: []string{"weather-light", "weather-deep"},
WorkingDir: workingDir, OutputDir: target, Date: generationTime("2026-05-29T12:00:00-05:00"),
Clock: timeutil.FixedClock{Time: generationTime("2026-05-29T08:30:00-05:00")}, Collector: &generationCollector{bundle: &bundle}, Executor: executor,
})
if err == nil || result == nil || result.ManifestPath != "" || result.DataPackagePath != "" || result.Results[0].ReportPath != "" {
t.Fatalf("CompareDetailed() result/error = %#v/%v", result, err)
}
data, readErr := os.ReadFile(target)
if readErr != nil || string(data) != "changed" {
t.Fatalf("destination = %q, error = %v", data, readErr)
}
}
func comparisonConfig() config.Config {
cfg := config.Defaults()
cfg.WeatherAPI.Timezone, cfg.Location.ID = "America/Chicago", "home"
return cfg
}
func assertUnpublishedComparisonResult(t *testing.T, result *ComparisonResult, outputDirectory string) {
t.Helper()
if result == nil || result.OutputDirectory != outputDirectory || !filepath.IsAbs(result.OutputDirectory) || result.FinishedAt.IsZero() || result.FinishedAt.Location() != time.UTC || result.FinishedAt.Before(result.StartedAt) || result.ManifestPath != "" || result.DataPackagePath != "" {
t.Fatalf("unpublished comparison result = %#v", result)
}
for _, profile := range result.Results {
if profile.ReportPath != "" {
t.Fatalf("unpublished profile result = %#v", profile)
}
}
}

View File

@@ -0,0 +1,448 @@
package app
import (
"context"
"encoding/json"
"errors"
"os"
"path/filepath"
"sync"
"testing"
"time"
"gitea.maximumdirect.net/eric/weatherreporter/internal/collect"
"gitea.maximumdirect.net/eric/weatherreporter/internal/config"
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec"
"gitea.maximumdirect.net/eric/weatherreporter/internal/report"
"gitea.maximumdirect.net/eric/weatherreporter/internal/weatherdata"
)
type generationCollector struct {
bundle *weatherdata.Bundle
err error
called bool
calls int
beforeRun func()
}
func (c *generationCollector) Run(context.Context, collect.Request) (*collect.Result, error) {
if c.beforeRun != nil {
c.beforeRun()
}
c.called = true
c.calls++
return &collect.Result{Bundle: c.bundle}, c.err
}
type generationExecutor struct {
called bool
executeCalls int
promptInspections int
profileInspections int
inspectErr error
profileInspectErrors map[string]error
executeErr error
executeErrors map[string]error
beforeExecute func(promptexec.ExecuteRequest)
cancelBeforeReturn context.CancelFunc
validation promptexec.ValidationStatus
rawOutput []byte
failedPrompt string
}
var generationExecutorMu sync.Mutex
func (e *generationExecutor) InspectPrompt(_ context.Context, id, version string) (promptexec.PromptInspection, error) {
generationExecutorMu.Lock()
defer generationExecutorMu.Unlock()
e.promptInspections++
if e.inspectErr != nil {
return promptexec.PromptInspection{}, e.inspectErr
}
definition := generationDefinitionForPrompt(id)
return promptexec.PromptInspection{PromptID: id, PromptVersion: version, PromptHash: generationPromptHash, DefaultProfileID: "fixture", Inputs: []promptexec.InputDefinition{{Name: "data_package", Required: true, ContentType: "application/yaml"}}, Output: promptexec.OutputContract{Format: "json", ValidationMode: "json_schema", SchemaPath: definition.GeneratedTextSchemaID + ".generated_text.schema.json"}}, nil
}
func (e *generationExecutor) InspectProfile(_ context.Context, id string) (promptexec.ProfileInspection, error) {
generationExecutorMu.Lock()
defer generationExecutorMu.Unlock()
e.profileInspections++
if err := e.profileInspectErrors[id]; err != nil {
return promptexec.ProfileInspection{}, err
}
return promptexec.ProfileInspection{ProfileID: id, BackendID: "fixture", ModelName: "fixture-model"}, nil
}
func (e *generationExecutor) Execute(_ context.Context, req promptexec.ExecuteRequest, callback promptexec.PreparationCallback) (*promptexec.Execution, error) {
stamp := time.Date(2026, 5, 29, 15, 0, 0, 0, time.UTC)
if err := callback(promptexec.Preparation{PromptID: req.PromptID, PromptVersion: req.PromptVersion, PromptHash: generationPromptHash, RenderedPromptHash: "rendered-hash", ProfileID: req.ProfileID, BackendID: "fixture", ModelName: "fixture-model", StartedAt: stamp, EndedAt: stamp}, nil); err != nil {
return nil, err
}
generationExecutorMu.Lock()
e.called = true
e.executeCalls++
beforeExecute := e.beforeExecute
profileErr := e.executeErrors[req.ProfileID]
executeErr := e.executeErr
status := e.validation
rawOutput := append([]byte(nil), e.rawOutput...)
failedPrompt := e.failedPrompt
cancelBeforeReturn := e.cancelBeforeReturn
generationExecutorMu.Unlock()
if beforeExecute != nil {
beforeExecute(req)
}
if profileErr != nil {
return nil, profileErr
}
if executeErr != nil {
return nil, executeErr
}
if status == "" {
status = promptexec.ValidationPassed
}
if failedPrompt == req.PromptID {
status = promptexec.ValidationFailed
}
if rawOutput == nil {
rawOutput = []byte(`{"summary":"Showers are possible during the selected day.","forecast_discussion":["A front will keep rain chances in the forecast."],"precipitation_timing":"Rain is most likely during the afternoon."}`)
}
if cancelBeforeReturn != nil {
cancelBeforeReturn()
}
return &promptexec.Execution{RunID: "provider-run", PromptID: req.PromptID, PromptVersion: req.PromptVersion, PromptHash: generationPromptHash, RenderedPromptHash: "rendered-hash", ProfileID: req.ProfileID, BackendID: "fixture", ModelName: "fixture-model", StartedAt: stamp, EndedAt: stamp, RawOutput: rawOutput, Validation: promptexec.NewValidation(status, "json_schema", generationDefinitionForPrompt(req.PromptID).GeneratedTextSchemaID+".generated_text.schema.json", nil)}, nil
}
const generationPromptHash = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"
func generationDefinitionForPrompt(promptID string) report.Definition {
for _, definition := range report.DefaultRegistry().All() {
if definition.PromptID == promptID {
return definition
}
}
panic("unknown fixture prompt " + promptID)
}
func TestGenerateDetailedPublishesOnlySelectedOutput(t *testing.T) {
cfg := config.Defaults()
cfg.WeatherAPI.Timezone, cfg.Location.ID = "America/Chicago", "home"
bundle := generationBundle(t)
executor := &generationExecutor{}
collector := &generationCollector{bundle: &bundle}
workingDir := t.TempDir()
result, err := GenerateDetailed(context.Background(), GenerateRequest{Config: cfg, Report: ReportDaily, Date: generationTime("2026-05-29T12:00:00-05:00"), Now: generationTime("2026-05-29T08:30:00-05:00"), WorkingDir: workingDir, Collector: collector, Executor: executor})
if err != nil {
t.Fatalf("GenerateDetailed() error = %v", err)
}
if !executor.called || executor.executeCalls != 1 || collector.calls != 1 || result.OutputPath != filepath.Join(workingDir, "daily-2026-05-29.md") || result.ValidationStatus != promptexec.ValidationPassed || result.ProfileID == "" || result.BackendID == "" || result.ModelName == "" {
t.Fatalf("result = %#v", result)
}
if result.LLMDebugPath != "" {
t.Fatalf("unexpected debug output = %q", result.LLMDebugPath)
}
if _, err := os.Stat(filepath.Join(workingDir, "workspace")); !os.IsNotExist(err) {
t.Fatalf("unexpected default state directory: %v", err)
}
data, err := os.ReadFile(result.OutputPath)
if err != nil || len(data) == 0 {
t.Fatalf("output = %q, error = %v", data, err)
}
}
func TestGenerateDetailedUsesConfiguredOutputDirectory(t *testing.T) {
tests := []struct {
name string
directory func(t *testing.T, workingDir string) string
wantDir func(t *testing.T, workingDir string, configuredDir string) string
}{
{
name: "absolute directory",
directory: func(t *testing.T, _ string) string {
return filepath.Join(t.TempDir(), "reports")
},
wantDir: func(_ *testing.T, _ string, configuredDir string) string {
return configuredDir
},
},
{
name: "relative directory",
directory: func(_ *testing.T, _ string) string {
return "configured/../reports"
},
wantDir: func(_ *testing.T, workingDir string, _ string) string {
return filepath.Join(workingDir, "reports")
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
workingDir := t.TempDir()
configuredDir := tt.directory(t, workingDir)
cfg := generationDistributorConfig()
cfg.Output.Directory = configuredDir
bundle := generationBundle(t)
notifier := &generationNotifier{}
result, err := GenerateDetailed(context.Background(), GenerateRequest{
Config: cfg, Report: ReportDaily,
Date: generationTime("2026-05-29T12:00:00-05:00"), Now: generationTime("2026-05-29T08:30:00-05:00"),
WorkingDir: workingDir, Collector: &generationCollector{bundle: &bundle}, Executor: &generationExecutor{}, Notifier: notifier,
})
wantPath := filepath.Join(tt.wantDir(t, workingDir, configuredDir), "daily-2026-05-29.md")
if err != nil || result == nil || result.OutputPath != wantPath || notifier.request.ReportPath != wantPath {
t.Fatalf("GenerateDetailed() result/error/notification = %#v/%v/%#v", result, err, notifier.request)
}
if info, statErr := os.Stat(filepath.Dir(wantPath)); statErr != nil || !info.IsDir() {
t.Fatalf("configured output directory info/error = %#v/%v", info, statErr)
}
if _, statErr := os.Stat(wantPath); statErr != nil {
t.Fatalf("output %q: %v", wantPath, statErr)
}
})
}
}
func TestGenerateDetailedExplicitOutputPathIgnoresConfiguredDirectory(t *testing.T) {
configuredPath := filepath.Join(t.TempDir(), "not-a-directory")
if err := os.WriteFile(configuredPath, []byte("not a directory"), 0o600); err != nil {
t.Fatal(err)
}
explicitPath := filepath.Join(t.TempDir(), "explicit.md")
cfg := generationConfig()
cfg.Output.Directory = configuredPath
bundle := generationBundle(t)
result, err := GenerateDetailed(context.Background(), GenerateRequest{
Config: cfg, Report: ReportDaily,
Date: generationTime("2026-05-29T12:00:00-05:00"), Now: generationTime("2026-05-29T08:30:00-05:00"),
WorkingDir: t.TempDir(), OutputPath: explicitPath, Collector: &generationCollector{bundle: &bundle}, Executor: &generationExecutor{},
})
if err != nil || result == nil || result.OutputPath != explicitPath {
t.Fatalf("GenerateDetailed() result/error = %#v/%v", result, err)
}
if _, statErr := os.Stat(explicitPath); statErr != nil {
t.Fatalf("explicit output %q: %v", explicitPath, statErr)
}
}
func TestGenerateDetailedRejectsConfiguredNonDirectoryBeforeWork(t *testing.T) {
configuredPath := filepath.Join(t.TempDir(), "not-a-directory")
if err := os.WriteFile(configuredPath, []byte("not a directory"), 0o600); err != nil {
t.Fatal(err)
}
cfg := generationDistributorConfig()
cfg.Output.Directory = configuredPath
bundle := generationBundle(t)
collector := &generationCollector{bundle: &bundle}
executor := &generationExecutor{}
notifier := &generationNotifier{}
result, err := GenerateDetailed(context.Background(), GenerateRequest{
Config: cfg, Report: ReportDaily,
Date: generationTime("2026-05-29T12:00:00-05:00"), Now: generationTime("2026-05-29T08:30:00-05:00"),
WorkingDir: t.TempDir(), Collector: collector, Executor: executor, Notifier: notifier,
})
if err == nil || result == nil || collector.called || executor.promptInspections != 0 || executor.called || notifier.calls != 0 {
t.Fatalf("GenerateDetailed() result/error/collector/executor/notifier = %#v/%v/%t/%#v/%#v", result, err, collector.called, executor, notifier)
}
if data, readErr := os.ReadFile(configuredPath); readErr != nil || string(data) != "not a directory" {
t.Fatalf("configured path = %q, error = %v", data, readErr)
}
}
func TestGenerateDetailedReturnsResolvedResultWhenCollectionFails(t *testing.T) {
cfg := config.Defaults()
cfg.WeatherAPI.Timezone, cfg.Location.ID = "America/Chicago", "home"
collectionErr := errors.New("weather source unavailable")
result, err := GenerateDetailed(context.Background(), GenerateRequest{
Config: cfg, Report: ReportDaily, Date: generationTime("2026-05-29T12:00:00-05:00"), Now: generationTime("2026-05-29T08:30:00-05:00"),
WorkingDir: t.TempDir(), Collector: &generationCollector{err: collectionErr}, Executor: &generationExecutor{},
})
if !errors.Is(err, collectionErr) {
t.Fatalf("GenerateDetailed() error = %v, want %v", err, collectionErr)
}
if result == nil || result.ReportID != report.Daily || result.RunID == "" || result.ProfileID != "fixture" || result.BackendID != "fixture" || result.ModelName != "fixture-model" || result.OutputPath != "" {
t.Fatalf("result = %#v", result)
}
}
func TestGenerateDetailedInspectsPromptBeforeCollectingWeather(t *testing.T) {
cfg := generationConfig()
inspectionErr := errors.New("profile is invalid")
collector := &generationCollector{bundle: generationBundlePointer(t)}
result, err := GenerateDetailed(context.Background(), GenerateRequest{Config: cfg, Report: ReportDaily, Date: generationTime("2026-05-29T12:00:00-05:00"), Now: generationTime("2026-05-29T08:30:00-05:00"), WorkingDir: t.TempDir(), Collector: collector, Executor: &generationExecutor{inspectErr: inspectionErr}})
if !errors.Is(err, inspectionErr) || collector.called || result == nil {
t.Fatalf("GenerateDetailed() result/error/collector-called = %#v/%v/%t", result, err, collector.called)
}
}
func TestGenerateDetailedPreservesDestinationBeforePublish(t *testing.T) {
for _, scenario := range []struct {
name string
executor generationExecutor
}{
{name: "generation", executor: generationExecutor{executeErr: errors.New("provider unavailable")}},
{name: "render", executor: generationExecutor{rawOutput: []byte(`{"summary":""}`)}},
} {
t.Run(scenario.name, func(t *testing.T) {
outputPath := filepath.Join(t.TempDir(), "daily.md")
if err := os.WriteFile(outputPath, []byte("previous report"), 0o600); err != nil {
t.Fatal(err)
}
bundle := generationBundle(t)
result, err := GenerateDetailed(context.Background(), GenerateRequest{Config: generationConfig(), Report: ReportDaily, Date: generationTime("2026-05-29T12:00:00-05:00"), Now: generationTime("2026-05-29T08:30:00-05:00"), WorkingDir: t.TempDir(), OutputPath: outputPath, Collector: &generationCollector{bundle: &bundle}, Executor: &scenario.executor})
data, readErr := os.ReadFile(outputPath)
if err == nil || result == nil || readErr != nil || string(data) != "previous report" {
t.Fatalf("GenerateDetailed() result/error/output = %#v/%v/%q (%v)", result, err, data, readErr)
}
})
}
}
func TestGenerateDetailedPreservesDestinationWhenContextCancelsBeforePublication(t *testing.T) {
outputPath := filepath.Join(t.TempDir(), "daily.md")
const previousReport = "previous report"
if err := os.WriteFile(outputPath, []byte(previousReport), 0o600); err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithCancel(context.Background())
bundle := generationBundle(t)
result, err := GenerateDetailed(ctx, GenerateRequest{
Config: generationConfig(), Report: ReportDaily,
Date: generationTime("2026-05-29T12:00:00-05:00"), Now: generationTime("2026-05-29T08:30:00-05:00"),
WorkingDir: t.TempDir(), OutputPath: outputPath, Collector: &generationCollector{bundle: &bundle}, Executor: &generationExecutor{cancelBeforeReturn: cancel},
})
data, readErr := os.ReadFile(outputPath)
if !errors.Is(err, context.Canceled) || promptexec.CategoryOf(err) != promptexec.Canceled || result == nil || result.OutputPath != "" || readErr != nil || string(data) != previousReport {
t.Fatalf("GenerateDetailed() result/error/output = %#v/%v/%q (%v)", result, err, data, readErr)
}
}
func TestGenerateDetailedPreservesDestinationWhenContextDeadlineExpiresBeforePublication(t *testing.T) {
outputPath := filepath.Join(t.TempDir(), "daily.md")
const previousReport = "previous report"
if err := os.WriteFile(outputPath, []byte(previousReport), 0o600); err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithDeadline(context.Background(), time.Unix(0, 0))
defer cancel()
bundle := generationBundle(t)
result, err := GenerateDetailed(ctx, GenerateRequest{
Config: generationConfig(), Report: ReportDaily,
Date: generationTime("2026-05-29T12:00:00-05:00"), Now: generationTime("2026-05-29T08:30:00-05:00"),
WorkingDir: t.TempDir(), OutputPath: outputPath, Collector: &generationCollector{bundle: &bundle}, Executor: &generationExecutor{},
})
data, readErr := os.ReadFile(outputPath)
if !errors.Is(err, context.DeadlineExceeded) || promptexec.CategoryOf(err) != promptexec.DeadlineExceeded || result == nil || result.OutputPath != "" || readErr != nil || string(data) != previousReport {
t.Fatalf("GenerateDetailed() result/error/output = %#v/%v/%q (%v)", result, err, data, readErr)
}
}
func TestGenerateDetailedRetainsPublishedOutputWhenNotificationFails(t *testing.T) {
cfg := generationConfig()
cfg.Notify.Distributor.Enabled = true
cfg.Notify.Distributor.PipelineIDTemplate = "weather"
bundle := generationBundle(t)
outputPath := filepath.Join(t.TempDir(), "daily.md")
notifier := &generationNotifier{err: errors.New("distributor unavailable")}
result, err := GenerateDetailed(context.Background(), GenerateRequest{Config: cfg, Report: ReportDaily, Date: generationTime("2026-05-29T12:00:00-05:00"), Now: generationTime("2026-05-29T08:30:00-05:00"), WorkingDir: t.TempDir(), OutputPath: outputPath, Collector: &generationCollector{bundle: &bundle}, Executor: &generationExecutor{}, Notifier: notifier})
if err == nil || result == nil || result.OutputPath != outputPath || notifier.request.ReportPath != outputPath || len(notifier.request.BundlePaths) == 0 {
t.Fatalf("GenerateDetailed() result/error/request = %#v/%v/%#v", result, err, notifier.request)
}
if data, readErr := os.ReadFile(outputPath); readErr != nil || len(data) == 0 {
t.Fatalf("published output = %q, error = %v", data, readErr)
}
}
func TestGenerateDetailedDoesNotReplaceDirectoryOutput(t *testing.T) {
bundle := generationBundle(t)
outputPath := filepath.Join(t.TempDir(), "daily.md")
if err := os.Mkdir(outputPath, 0o700); err != nil {
t.Fatal(err)
}
result, err := GenerateDetailed(context.Background(), GenerateRequest{Config: generationConfig(), Report: ReportDaily, Date: generationTime("2026-05-29T12:00:00-05:00"), Now: generationTime("2026-05-29T08:30:00-05:00"), WorkingDir: t.TempDir(), OutputPath: outputPath, Collector: &generationCollector{bundle: &bundle}, Executor: &generationExecutor{}})
info, statErr := os.Stat(outputPath)
if err == nil || result == nil || statErr != nil || !info.IsDir() {
t.Fatalf("GenerateDetailed() result/error/output-info = %#v/%v/%#v (%v)", result, err, info, statErr)
}
}
func TestGenerateDetailedWritesRequestedPromptDebugArtifacts(t *testing.T) {
bundle := generationBundle(t)
debugRoot := t.TempDir()
result, err := GenerateDetailed(context.Background(), GenerateRequest{
Config: generationConfig(), Report: ReportDaily,
Date: generationTime("2026-05-29T12:00:00-05:00"), Now: generationTime("2026-05-29T08:30:00-05:00"),
WorkingDir: t.TempDir(), LLMDebugDir: debugRoot, Collector: &generationCollector{bundle: &bundle}, Executor: &generationExecutor{},
})
if err != nil || result == nil || result.LLMDebugPath == "" {
t.Fatalf("GenerateDetailed() result/error = %#v/%v", result, err)
}
for _, name := range []string{"preparation.json", "execution.json"} {
if _, statErr := os.Stat(filepath.Join(result.LLMDebugPath, name)); statErr != nil {
t.Fatalf("debug artifact %q: %v", name, statErr)
}
}
}
func generationConfig() config.Config {
cfg := config.Defaults()
cfg.WeatherAPI.Timezone, cfg.Location.ID = "America/Chicago", "home"
return cfg
}
func generationBundlePointer(t *testing.T) *weatherdata.Bundle {
bundle := generationBundle(t)
return &bundle
}
type generationNotifier struct {
err error
batchErr error
calls int
request NotificationRequest
batchRequest batchNotificationRequest
batchCalls int
}
func (n *generationNotifier) Notify(_ context.Context, request NotificationRequest) (*NotificationResult, error) {
n.calls++
n.request = request
if n.err != nil {
return nil, n.err
}
return &NotificationResult{Status: "succeeded"}, nil
}
func (n *generationNotifier) NotifyBatch(_ context.Context, request batchNotificationRequest) (*NotificationResult, error) {
n.batchCalls++
n.batchRequest = request
for _, file := range request.Files {
if _, err := os.Stat(file.SourcePath); err != nil {
return nil, err
}
}
return &NotificationResult{Status: "succeeded", PipelineID: request.PipelineID, BundleID: request.BundleID}, n.batchErr
}
func generationBundle(t *testing.T) weatherdata.Bundle {
t.Helper()
data, err := os.ReadFile(filepath.Join("..", "forecast", "testdata", "daily_bundle.json"))
if err != nil {
t.Fatalf("read bundle fixture: %v", err)
}
var bundle weatherdata.Bundle
if err := json.Unmarshal(data, &bundle); err != nil {
t.Fatalf("decode bundle fixture: %v", err)
}
return bundle
}
func generationTime(value string) time.Time {
parsed, _ := time.Parse(time.RFC3339, value)
return parsed
}
var _ promptexec.Executor = (*generationExecutor)(nil)
var _ Collector = (*generationCollector)(nil)
var _ Notifier = (*generationNotifier)(nil)
var _ = report.Daily

View File

@@ -1,126 +0,0 @@
package app
import (
"context"
"fmt"
"gitea.maximumdirect.net/eric/weatherreporter/internal/briefing"
"gitea.maximumdirect.net/eric/weatherreporter/internal/config"
"gitea.maximumdirect.net/eric/weatherreporter/internal/module"
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptinput"
"gitea.maximumdirect.net/eric/weatherreporter/internal/report"
"gitea.maximumdirect.net/eric/weatherreporter/internal/state"
"gitea.maximumdirect.net/eric/weatherreporter/internal/timeutil"
"gitea.maximumdirect.net/eric/weatherreporter/internal/weatherdata"
)
type InspectReportsRequest struct {
Config config.Config
Limit int
}
type InspectRunRequest struct {
Config config.Config
RunID string
}
type SourceInspection struct {
RunID string `json:"runId"`
ReportID report.ID `json:"reportId"`
SourceLocation string `json:"sourceLocation,omitempty"`
Sources []briefing.SourceMetadata `json:"sources,omitempty"`
Warnings []weatherdata.SourceWarning `json:"warnings,omitempty"`
}
func InspectReports(ctx context.Context, req InspectReportsRequest) ([]state.ReportRecord, error) {
store, err := defaultStore(req.Config)
if err != nil {
return nil, err
}
return store.ListReports(ctx, req.Limit)
}
func InspectMetadata(ctx context.Context, req InspectRunRequest) (state.Metadata, error) {
inspection, err := inspectRun(ctx, req)
return inspection.metadata, err
}
func InspectModules(ctx context.Context, req InspectRunRequest) (module.Snapshot, error) {
inspection, err := inspectRun(ctx, req)
if err != nil {
return module.Snapshot{}, err
}
return inspection.store.LoadModuleSnapshot(ctx, inspection.metadata.ModuleSnapshotPath)
}
func InspectDataPackage(ctx context.Context, req InspectRunRequest) (promptinput.Package, error) {
inspection, err := inspectRun(ctx, req)
if err != nil {
return promptinput.Package{}, err
}
return inspection.store.LoadDataPackage(ctx, inspection.metadata.DataPackagePath)
}
func InspectPriorSnapshot(ctx context.Context, req InspectRunRequest) (*state.PriorSnapshot, error) {
inspection, err := inspectRun(ctx, req)
if err != nil {
return nil, err
}
resolved, err := resolvedFromMetadata(inspection.metadata)
if err != nil {
return nil, err
}
return inspection.store.FindPriorSnapshot(ctx, resolved)
}
func InspectSources(ctx context.Context, req InspectRunRequest) (SourceInspection, error) {
inspection, err := inspectRun(ctx, req)
if err != nil {
return SourceInspection{}, err
}
metadata := inspection.metadata
return SourceInspection{
RunID: metadata.RunID,
ReportID: metadata.ReportID,
SourceLocation: metadata.SourceLocation,
Sources: metadata.Sources,
Warnings: metadata.SourceWarnings,
}, nil
}
type runInspection struct {
store *state.FilesystemStore
metadata state.Metadata
}
func inspectRun(ctx context.Context, req InspectRunRequest) (runInspection, error) {
store, err := defaultStore(req.Config)
if err != nil {
return runInspection{}, err
}
metadata, _, err := store.LoadMetadataByRunID(ctx, req.RunID)
if err != nil {
return runInspection{}, err
}
return runInspection{store: store, metadata: metadata}, nil
}
func resolvedFromMetadata(metadata state.Metadata) (report.Resolved, error) {
definition, err := report.DefaultRegistry().Lookup(metadata.ReportID)
if err != nil {
return report.Resolved{}, err
}
location, err := timeutil.LoadLocation(metadata.Timezone)
if err != nil {
return report.Resolved{}, err
}
if !metadata.ValidPeriod.IsValid() {
return report.Resolved{}, fmt.Errorf("metadata valid period for run id %q is invalid", metadata.RunID)
}
return report.Resolved{
Definition: definition,
GeneratedAt: metadata.GeneratedAt,
Timezone: location.String(),
ValidPeriod: metadata.ValidPeriod,
}, nil
}

193
internal/app/output.go Normal file
View File

@@ -0,0 +1,193 @@
package app
import (
"fmt"
"os"
"path/filepath"
"strings"
"gitea.maximumdirect.net/eric/weatherreporter/internal/comparison"
"gitea.maximumdirect.net/eric/weatherreporter/internal/report"
)
func plannedBatchOutputPath(outputDir string, planned plannedBatchReport) (string, error) {
outputName, err := planned.Resolved.OutputName()
if err != nil {
return "", err
}
return validateOutputPath(filepath.Join(outputDir, outputName))
}
func prepareBatchOutputs(outputDir string, plannedReports []plannedBatchReport) error {
for index := range plannedReports {
outputPath, err := plannedBatchOutputPath(outputDir, plannedReports[index])
if err != nil {
return err
}
plannedReports[index].OutputPath = outputPath
}
return nil
}
func resolveReportOutputPath(workingDir, override, configuredDir string, resolved report.Resolved) (string, error) {
outputName, err := resolved.OutputName()
if err != nil {
return "", err
}
if override != "" {
return resolveOutputPath(workingDir, override, outputName)
}
outputDir, err := resolveOutputDir(workingDir, configuredDir)
if err != nil {
return "", err
}
return validateOutputPath(filepath.Join(outputDir, outputName))
}
func resolveOutputDirWithConfigured(workingDir, override, configuredDir string) (string, error) {
directory := configuredDir
if override != "" {
directory = override
}
return resolveOutputDir(workingDir, directory)
}
func resolveComparisonOutputDirectory(workingDir, override, configuredDir, reportOutputName string) (string, error) {
workingDir, err := validateWorkingDir(workingDir)
if err != nil {
return "", err
}
if override != "" {
return resolveComparisonDirectoryPath(workingDir, override)
}
outputDir, err := resolveOutputDir(workingDir, configuredDir)
if err != nil {
return "", err
}
name, err := comparison.DefaultDirectoryName(reportOutputName)
if err != nil {
return "", err
}
return filepath.Join(outputDir, name), nil
}
func resolveComparisonDirectoryPath(workingDir, directory string) (string, error) {
if strings.TrimSpace(directory) == "" {
return "", fmt.Errorf("comparison output directory is required")
}
if !filepath.IsAbs(directory) {
directory = filepath.Join(workingDir, directory)
}
return filepath.Clean(directory), nil
}
func resolveOutputDir(workingDir, override string) (string, error) {
workingDir, err := validateWorkingDir(workingDir)
if err != nil {
return "", err
}
if override == "" {
return workingDir, nil
}
if strings.TrimSpace(override) == "" {
return "", fmt.Errorf("output directory is required")
}
directory := override
if !filepath.IsAbs(directory) {
directory = filepath.Join(workingDir, directory)
}
directory = filepath.Clean(directory)
if err := preflightOutputDirectory(directory); err != nil {
return "", err
}
return directory, nil
}
func preflightOutputDirectory(directory string) error {
info, err := os.Stat(directory)
if err == nil {
if !info.IsDir() {
return fmt.Errorf("output directory %q is not a directory", directory)
}
return nil
}
if !os.IsNotExist(err) {
return fmt.Errorf("inspect output directory %q: %w", directory, err)
}
// A missing directory is valid, but os.Stat also reports ErrNotExist for a
// dangling symlink. Walk to the first existing component so invalid links
// fail preflight instead of being discovered only during publication.
for component := directory; ; component = filepath.Dir(component) {
componentInfo, componentErr := os.Lstat(component)
if componentErr == nil {
if componentInfo.Mode()&os.ModeSymlink != 0 {
targetInfo, targetErr := os.Stat(component)
if targetErr != nil {
return fmt.Errorf("inspect output directory %q at %q: %w", directory, component, targetErr)
}
if !targetInfo.IsDir() {
return fmt.Errorf("output directory %q has non-directory path component %q", directory, component)
}
return nil
}
if !componentInfo.IsDir() {
return fmt.Errorf("output directory %q has non-directory path component %q", directory, component)
}
return nil
}
if !os.IsNotExist(componentErr) {
return fmt.Errorf("inspect output directory %q at %q: %w", directory, component, componentErr)
}
if filepath.Dir(component) == component {
return fmt.Errorf("inspect output directory %q: no existing directory ancestor", directory)
}
}
}
func resolveOutputPath(workingDir, override, defaultName string) (string, error) {
workingDir, err := validateWorkingDir(workingDir)
if err != nil {
return "", err
}
path := override
if path == "" {
path = defaultName
}
if strings.TrimSpace(path) == "" {
return "", fmt.Errorf("final output path is required")
}
if !filepath.IsAbs(path) {
path = filepath.Join(workingDir, path)
}
return validateOutputPath(path)
}
func validateWorkingDir(workingDir string) (string, error) {
if strings.TrimSpace(workingDir) == "" {
return "", fmt.Errorf("working directory is required")
}
if !filepath.IsAbs(workingDir) {
return "", fmt.Errorf("working directory %q must be absolute", workingDir)
}
return filepath.Clean(workingDir), nil
}
func validateOutputPath(path string) (string, error) {
if strings.TrimSpace(path) == "" {
return "", fmt.Errorf("final output path is required")
}
path = filepath.Clean(path)
if !filepath.IsAbs(path) {
return "", fmt.Errorf("final output path %q must be absolute", path)
}
if filepath.Dir(path) == path {
return "", fmt.Errorf("final output path %q must not be a filesystem root", path)
}
if info, err := os.Stat(path); err == nil && info.IsDir() {
return "", fmt.Errorf("final output path %q is a directory", path)
} else if err != nil && !os.IsNotExist(err) {
return "", fmt.Errorf("inspect final output path %q: %w", path, err)
}
return path, nil
}

View File

@@ -0,0 +1,75 @@
package app
import (
"os"
"path/filepath"
"testing"
)
func TestResolveComparisonOutputDirectory(t *testing.T) {
workingDir := t.TempDir()
configured := filepath.Join(workingDir, "configured")
blocked := filepath.Join(workingDir, "not-a-directory")
if err := os.WriteFile(blocked, []byte("blocked"), 0o600); err != nil {
t.Fatal(err)
}
tests := []struct {
name string
override string
configuredDir string
reportOutputName string
want string
wantErr bool
}{
{name: "working directory default", reportOutputName: "today.md", want: filepath.Join(workingDir, "comparison-today")},
{name: "configured relative directory", configuredDir: "configured", reportOutputName: "tomorrow.md", want: filepath.Join(configured, "comparison-tomorrow")},
{name: "configured absolute directory", configuredDir: configured, reportOutputName: "hourly.md", want: filepath.Join(configured, "comparison-hourly")},
{name: "relative explicit directory", override: "exact", configuredDir: blocked, reportOutputName: "daily-2026-08-24.md", want: filepath.Join(workingDir, "exact")},
{name: "absolute explicit directory", override: filepath.Join(workingDir, "absolute"), reportOutputName: "today.md", want: filepath.Join(workingDir, "absolute")},
{name: "invalid report suffix", reportOutputName: "today.txt", wantErr: true},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
got, err := resolveComparisonOutputDirectory(workingDir, test.override, test.configuredDir, test.reportOutputName)
if (err != nil) != test.wantErr {
t.Fatalf("resolveComparisonOutputDirectory() error = %v, want error %t", err, test.wantErr)
}
if !test.wantErr && got != test.want {
t.Fatalf("resolveComparisonOutputDirectory() = %q, want %q", got, test.want)
}
})
}
}
func TestResolveOutputDirRejectsDanglingSymlinkComponents(t *testing.T) {
workingDir := t.TempDir()
dangling := filepath.Join(workingDir, "dangling")
if err := os.Symlink(filepath.Join(workingDir, "missing"), dangling); err != nil {
t.Fatal(err)
}
for _, directory := range []string{dangling, filepath.Join(dangling, "reports")} {
t.Run(filepath.Base(directory), func(t *testing.T) {
if _, err := resolveOutputDir(workingDir, directory); err == nil {
t.Fatalf("resolveOutputDir(%q) error = nil, want dangling symlink error", directory)
}
})
}
}
func TestResolveOutputDirAllowsMissingDirectoryBelowValidSymlink(t *testing.T) {
workingDir := t.TempDir()
target := t.TempDir()
link := filepath.Join(workingDir, "linked")
if err := os.Symlink(target, link); err != nil {
t.Fatal(err)
}
directory := filepath.Join(link, "reports")
got, err := resolveOutputDir(workingDir, directory)
if err != nil || got != directory {
t.Fatalf("resolveOutputDir() = %q, %v, want %q, nil", got, err, directory)
}
}

View File

@@ -0,0 +1,150 @@
package app
import (
"encoding/json"
"fmt"
"gitea.maximumdirect.net/eric/weatherreporter/internal/briefing"
"gitea.maximumdirect.net/eric/weatherreporter/internal/collect"
"gitea.maximumdirect.net/eric/weatherreporter/internal/config"
"gitea.maximumdirect.net/eric/weatherreporter/internal/generatedtext"
"gitea.maximumdirect.net/eric/weatherreporter/internal/module"
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptinput"
"gitea.maximumdirect.net/eric/weatherreporter/internal/report"
"gitea.maximumdirect.net/eric/weatherreporter/internal/weatherdata"
)
// preparedReport contains the immutable deterministic inputs shared by prompt
// executions for one resolved report.
type preparedReport struct {
resolved report.Resolved
reportFacts ReportFacts
moduleSnapshot module.Snapshot
briefingMetadata briefing.Metadata
sourceWarnings []weatherdata.SourceWarning
dataPackage []byte
handler generatedtext.Handler
}
type prepareReportRequest struct {
Config config.Config
Resolved report.Resolved
Collection collect.Result
}
type preparationError struct {
operation string
err error
}
func (e *preparationError) Error() string {
return e.operation + ": " + e.err.Error()
}
func (e *preparationError) Unwrap() error {
return e.err
}
func prepareReport(req prepareReportRequest) (preparedReport, error) {
if req.Collection.Bundle == nil {
return preparedReport{}, &preparationError{operation: "prepare report", err: fmt.Errorf("collected weather bundle is required")}
}
reportFacts, err := BuildReportFacts(ModuleSnapshotRequest{Config: req.Config, Resolved: req.Resolved}, req.Collection.Bundle)
if err != nil {
return preparedReport{}, &preparationError{operation: "build report facts", err: err}
}
moduleSnapshot, err := BuildModuleSnapshotFromFacts(ModuleSnapshotRequest{Config: req.Config, Resolved: req.Resolved}, reportFacts)
if err != nil {
return preparedReport{}, &preparationError{operation: "build module snapshot", err: err}
}
metadata := briefing.BuildMetadata(briefingBuildContext(req.Config, req.Resolved, reportFacts.Collected))
dataPackage, err := promptinput.Build(promptinput.BuildRequest{Metadata: promptMetadata(metadata), Modules: moduleSnapshot})
if err != nil {
return preparedReport{}, &preparationError{operation: "build data package", err: err}
}
serializedDataPackage, err := promptinput.MarshalYAML(dataPackage)
if err != nil {
return preparedReport{}, &preparationError{operation: "marshal data package", err: err}
}
handler, err := generatedtext.LookupDefinition(req.Resolved.Definition)
if err != nil {
return preparedReport{}, &preparationError{operation: "lookup generated text catalog", err: err}
}
clonedFacts, err := clonePreparedValue(reportFacts)
if err != nil {
return preparedReport{}, &preparationError{operation: "copy prepared report facts", err: err}
}
clonedSnapshot, err := clonePreparedValue(moduleSnapshot)
if err != nil {
return preparedReport{}, &preparationError{operation: "copy prepared module snapshot", err: err}
}
clonedMetadata, err := clonePreparedValue(metadata)
if err != nil {
return preparedReport{}, &preparationError{operation: "copy prepared briefing metadata", err: err}
}
prepared := preparedReport{
resolved: cloneResolved(req.Resolved),
reportFacts: clonedFacts,
moduleSnapshot: clonedSnapshot,
briefingMetadata: clonedMetadata,
sourceWarnings: append([]weatherdata.SourceWarning(nil), clonedMetadata.SourceWarnings...),
dataPackage: append([]byte(nil), serializedDataPackage...),
handler: handler,
}
return prepared, nil
}
func cloneResolved(value report.Resolved) report.Resolved {
cloned := value
cloned.Definition.DistributorPathTemplates = append([]string(nil), value.Definition.DistributorPathTemplates...)
cloned.Definition.Modules = make([]module.ConfigItem, len(value.Definition.Modules))
for i, item := range value.Definition.Modules {
cloned.Definition.Modules[i] = item
switch options := item.Options.(type) {
case module.AreaForecastDiscussionOptions:
options.Sections = append([]string(nil), options.Sections...)
cloned.Definition.Modules[i].Options = options
}
}
return cloned
}
func (p preparedReport) dataPackageCopy() []byte {
return append([]byte(nil), p.dataPackage...)
}
func (p preparedReport) sourceWarningsCopy() []weatherdata.SourceWarning {
return append([]weatherdata.SourceWarning(nil), p.sourceWarnings...)
}
func (p preparedReport) renderInputs() (briefing.Metadata, module.Snapshot, ReportFacts, error) {
metadata, err := clonePreparedValue(p.briefingMetadata)
if err != nil {
return briefing.Metadata{}, module.Snapshot{}, ReportFacts{}, err
}
snapshot, err := clonePreparedValue(p.moduleSnapshot)
if err != nil {
return briefing.Metadata{}, module.Snapshot{}, ReportFacts{}, err
}
reportFacts, err := clonePreparedValue(p.reportFacts)
if err != nil {
return briefing.Metadata{}, module.Snapshot{}, ReportFacts{}, err
}
return metadata, snapshot, reportFacts, nil
}
func clonePreparedValue[T any](value T) (T, error) {
encoded, err := json.Marshal(value)
if err != nil {
var zero T
return zero, fmt.Errorf("marshal immutable prepared value: %w", err)
}
var cloned T
if err := json.Unmarshal(encoded, &cloned); err != nil {
var zero T
return zero, fmt.Errorf("unmarshal immutable prepared value: %w", err)
}
return cloned, nil
}

View File

@@ -0,0 +1,64 @@
package app
import (
"bytes"
"reflect"
"testing"
"gitea.maximumdirect.net/eric/weatherreporter/internal/collect"
"gitea.maximumdirect.net/eric/weatherreporter/internal/weatherdata"
)
func TestPrepareReportBuildsImmutableDeterministicInputs(t *testing.T) {
cfg := generationConfig()
bundle := generationBundle(t)
resolved, err := ResolveGenerate(GenerateRequest{
Config: cfg, Report: ReportDaily,
Date: generationTime("2026-05-29T12:00:00-05:00"), Now: generationTime("2026-05-29T08:30:00-05:00"),
}, generationTime("2026-05-29T08:30:00-05:00"))
if err != nil {
t.Fatalf("ResolveGenerate() error = %v", err)
}
request := prepareReportRequest{Config: cfg, Resolved: resolved, Collection: collect.Result{Bundle: &bundle}}
prepared, err := prepareReport(request)
if err != nil {
t.Fatalf("prepareReport() error = %v", err)
}
repeated, err := prepareReport(request)
if err != nil {
t.Fatalf("second prepareReport() error = %v", err)
}
if len(prepared.dataPackage) == 0 || !bytes.Equal(prepared.dataPackage, repeated.dataPackage) || !reflect.DeepEqual(prepared.briefingMetadata, repeated.briefingMetadata) {
t.Fatalf("prepared package/metadata are not deterministic: %q/%#v", prepared.dataPackage, prepared.briefingMetadata)
}
originalDataPackage := append([]byte(nil), prepared.dataPackage...)
originalMetadata := prepared.briefingMetadata
originalWarnings := append([]weatherdata.SourceWarning(nil), prepared.sourceWarnings...)
metadata, snapshot, reportFacts, err := prepared.renderInputs()
if err != nil {
t.Fatalf("renderInputs() error = %v", err)
}
metadata.SourceWarnings = append(metadata.SourceWarnings, weatherdata.SourceWarning{Source: "test", Message: "consumer mutation"})
snapshot.Outputs = nil
reportFacts.Collected.Hourly.Periods[0].TextDescription = "consumer mutation"
bundle.Hourly.Periods[0].TextDescription = "mutated after preparation"
bundle.Warnings = append(bundle.Warnings, weatherdata.SourceWarning{Source: "test", Message: "mutated warning"})
if len(bundle.Sources) > 0 {
if bundle.Sources[0].Query == nil {
bundle.Sources[0].Query = map[string]string{}
}
bundle.Sources[0].Query["mutated"] = "true"
}
if !bytes.Equal(prepared.dataPackage, originalDataPackage) || !reflect.DeepEqual(prepared.briefingMetadata, originalMetadata) || !reflect.DeepEqual(prepared.sourceWarnings, originalWarnings) {
t.Fatalf("prepared values changed after caller mutation: %#v", prepared)
}
if prepared.reportFacts.Collected.Hourly.Periods[0].TextDescription == "mutated after preparation" {
t.Fatal("prepared report facts retain caller-owned weather data")
}
if prepared.reportFacts.Collected.Hourly.Periods[0].TextDescription == "consumer mutation" || len(prepared.moduleSnapshot.Outputs) == 0 {
t.Fatal("prepared report values retain consumer mutation")
}
}

View File

@@ -0,0 +1,127 @@
package app
import (
"context"
"fmt"
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptdebug"
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec"
)
type profileExecutionRequest struct {
Prepared preparedReport
Prompt PromptInspectionResult
Profile promptexec.ProfileInspection
Executor promptexec.Executor
DebugWriter *promptdebug.PromptDebugWriter
DebugRef *promptdebug.PromptDebugRef
}
type profileExecutionOutcome struct {
ProfileID string
BackendID string
ModelName string
ValidationStatus promptexec.ValidationStatus
LLMDebugPath string
}
type profileExecutionError struct {
operation string
err error
callbackFailure bool
}
func (e *profileExecutionError) Error() string {
return e.operation + ": " + e.err.Error()
}
func (e *profileExecutionError) Unwrap() error {
return e.err
}
func executePreparedProfile(ctx context.Context, req profileExecutionRequest) (profileExecutionOutcome, []byte, error) {
outcome := profileExecutionOutcome{
ProfileID: req.Profile.ProfileID,
BackendID: req.Profile.BackendID,
ModelName: req.Profile.ModelName,
}
if req.Executor == nil {
return outcome, nil, &profileExecutionError{operation: "execute prompt", err: promptexec.NewError(promptexec.InvalidConfiguration, "prompt executor is required", nil)}
}
callbackFailed := false
preparationCallback := func(preparation promptexec.Preparation, debug *promptexec.PreparationDebug) error {
outcome.ProfileID, outcome.BackendID, outcome.ModelName = preparation.ProfileID, preparation.BackendID, preparation.ModelName
if req.DebugWriter == nil || !req.DebugWriter.Enabled() {
return nil
}
if req.DebugRef == nil {
callbackFailed = true
return promptDebugWriteError(fmt.Errorf("prompt debug reference is required"))
}
path, err := req.DebugWriter.WritePreparation(*req.DebugRef, preparation, debug)
if err != nil {
callbackFailed = true
return promptDebugWriteError(err)
}
outcome.LLMDebugPath = path
return nil
}
captureDebug := req.DebugWriter != nil && req.DebugWriter.Enabled()
execution, err := req.Executor.Execute(ctx, promptexec.ExecuteRequest{
PromptID: req.Prompt.PromptID,
PromptVersion: req.Prompt.PromptVersion,
ProfileID: req.Profile.ProfileID,
DataPackage: req.Prepared.dataPackageCopy(),
CaptureDebug: captureDebug,
}, preparationCallback)
if err != nil {
if callbackFailed {
return outcome, nil, &profileExecutionError{operation: "execute prompt", err: err, callbackFailure: true}
}
return outcome, nil, &profileExecutionError{operation: "execute prompt", err: classifiedPromptError("prompt execution failed", err)}
}
if execution == nil {
return outcome, nil, &profileExecutionError{operation: "execute prompt", err: promptexec.NewError(promptexec.Generation, "prompt executor returned no execution", nil)}
}
outcome.ValidationStatus = execution.Validation.Status
if req.DebugWriter != nil && req.DebugWriter.Enabled() {
if req.DebugRef == nil {
return outcome, nil, &profileExecutionError{operation: "write prompt debug", err: promptDebugWriteError(fmt.Errorf("prompt debug reference is required"))}
}
path, err := req.DebugWriter.WriteExecution(*req.DebugRef, *execution)
if err != nil {
return outcome, nil, &profileExecutionError{operation: "write prompt debug", err: promptDebugWriteError(err)}
}
if path != "" {
outcome.LLMDebugPath = path
}
}
if execution.Validation.Status != promptexec.ValidationPassed && execution.Validation.Status != promptexec.ValidationFailed {
return outcome, nil, &profileExecutionError{operation: "validate prompt execution", err: promptexec.NewError(promptexec.OperationalValidation, "prompt execution did not complete validation", nil)}
}
if execution.Validation.Status == promptexec.ValidationFailed {
return outcome, nil, &profileExecutionError{operation: "validate prompt execution", err: promptexec.NewError(promptexec.ValidationRejected, "prompt output did not satisfy its schema", nil)}
}
generatedText, _, err := req.Prepared.handler.Validate(execution.RawOutput)
if err != nil {
return outcome, nil, &profileExecutionError{operation: "validate generated text", err: err}
}
metadata, snapshot, reportFacts, err := req.Prepared.renderInputs()
if err != nil {
return outcome, nil, &profileExecutionError{operation: "copy prepared render inputs", err: err}
}
renderContext, err := req.Prepared.handler.BuildRenderContext(metadata, snapshot, reportFacts.Collected, reportFacts.Derived, generatedText)
if err != nil {
return outcome, nil, &profileExecutionError{operation: "build render context", err: err}
}
rendered, err := req.Prepared.handler.Render(renderContext)
if err != nil {
return outcome, nil, &profileExecutionError{operation: "render template", err: err}
}
return outcome, rendered, nil
}

View File

@@ -0,0 +1,70 @@
package app
import (
"context"
"errors"
"os"
"path/filepath"
"testing"
"gitea.maximumdirect.net/eric/weatherreporter/internal/collect"
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptdebug"
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec"
)
func TestExecutePreparedProfileRendersWithoutPublishing(t *testing.T) {
prepared, inspection := preparedDailyProfile(t)
executor := &generationExecutor{}
outputPath := filepath.Join(t.TempDir(), "report.md")
outcome, rendered, err := executePreparedProfile(context.Background(), profileExecutionRequest{
Prepared: prepared, Prompt: inspection,
Profile: promptexec.ProfileInspection{ProfileID: inspection.ProfileID, BackendID: inspection.BackendID, ModelName: inspection.ModelName},
Executor: executor,
})
if err != nil {
t.Fatalf("executePreparedProfile() error = %v", err)
}
if len(rendered) == 0 || outcome.ValidationStatus != promptexec.ValidationPassed || outcome.ProfileID != inspection.ProfileID || executor.executeCalls != 1 {
t.Fatalf("outcome/rendered/execution calls = %#v/%q/%d", outcome, rendered, executor.executeCalls)
}
if _, statErr := os.Stat(outputPath); !os.IsNotExist(statErr) {
t.Fatalf("execution unexpectedly published %q: %v", outputPath, statErr)
}
}
func TestExecutePreparedProfileKeepsDebugCallbackFailureLocal(t *testing.T) {
prepared, inspection := preparedDailyProfile(t)
debugWriter, err := promptdebug.NewPromptDebugWriter(t.TempDir())
if err != nil {
t.Fatalf("NewPromptDebugWriter() error = %v", err)
}
executor := &generationExecutor{}
outcome, rendered, err := executePreparedProfile(context.Background(), profileExecutionRequest{
Prepared: prepared, Prompt: inspection,
Profile: promptexec.ProfileInspection{ProfileID: inspection.ProfileID, BackendID: inspection.BackendID, ModelName: inspection.ModelName},
Executor: executor, DebugWriter: debugWriter,
DebugRef: &promptdebug.PromptDebugRef{ReportID: inspectionResolved(t).Definition.ID, ValidDate: "2026-05-29", RunID: "invalid/path"},
})
var executionErr *profileExecutionError
if err == nil || !errors.As(err, &executionErr) || !executionErr.callbackFailure || promptexec.CategoryOf(err) != promptexec.InvalidConfiguration || len(rendered) != 0 || executor.executeCalls != 0 || outcome.LLMDebugPath != "" {
t.Fatalf("outcome/rendered/error/execution calls = %#v/%q/%v/%d", outcome, rendered, err, executor.executeCalls)
}
}
func preparedDailyProfile(t *testing.T) (preparedReport, PromptInspectionResult) {
t.Helper()
cfg := generationConfig()
resolved, err := ResolveGenerate(GenerateRequest{
Config: cfg, Report: ReportDaily,
Date: generationTime("2026-05-29T12:00:00-05:00"), Now: generationTime("2026-05-29T08:30:00-05:00"),
}, generationTime("2026-05-29T08:30:00-05:00"))
if err != nil {
t.Fatalf("ResolveGenerate() error = %v", err)
}
bundle := generationBundle(t)
prepared, err := prepareReport(prepareReportRequest{Config: cfg, Resolved: resolved, Collection: collect.Result{Bundle: &bundle}})
if err != nil {
t.Fatalf("prepareReport() error = %v", err)
}
return prepared, PromptInspectionResult{PromptID: resolved.Definition.PromptID, PromptVersion: resolved.Definition.PromptVersion, PromptHash: "prompt-hash", ProfileID: "fixture", BackendID: "fixture", ModelName: "fixture-model"}
}

View File

@@ -1,552 +0,0 @@
package app
import (
"context"
"encoding/json"
"errors"
"os"
"path/filepath"
"strings"
"testing"
"time"
"gitea.maximumdirect.net/eric/weatherreporter/internal/collect"
"gitea.maximumdirect.net/eric/weatherreporter/internal/config"
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec"
"gitea.maximumdirect.net/eric/weatherreporter/internal/report"
"gitea.maximumdirect.net/eric/weatherreporter/internal/state"
"gitea.maximumdirect.net/eric/weatherreporter/internal/weatherdata"
)
const (
failPromptExecution = "prompt execution"
failMetadata = "metadata"
failGeneratedText = "generated text"
failRenderContext = "render context"
failRenderedReportPath = "rendered report path"
failDistributorNotification = "distributor notification"
)
type failingPersistenceStore struct {
state.Store
failOperation string
failExecutionCall int
failMetadataCall int
executionCalls int
metadataCalls int
renderedReportPath string
}
func (s *failingPersistenceStore) SavePromptExecution(ctx context.Context, resolved report.Resolved, artifact state.PromptExecutionArtifact) (string, error) {
s.executionCalls++
if s.failOperation == failPromptExecution && (s.failExecutionCall == 0 || s.executionCalls == s.failExecutionCall) {
return "", errors.New("injected prompt execution persistence failure")
}
return s.Store.SavePromptExecution(ctx, resolved, artifact)
}
func (s *failingPersistenceStore) SaveGeneratedText(ctx context.Context, resolved report.Resolved, data []byte) (string, error) {
if s.failOperation == failGeneratedText {
return "", errors.New("injected generated text persistence failure")
}
return s.Store.SaveGeneratedText(ctx, resolved, data)
}
func (s *failingPersistenceStore) SaveRenderContext(ctx context.Context, resolved report.Resolved, value any) (string, error) {
if s.failOperation == failRenderContext {
return "", errors.New("injected render context persistence failure")
}
return s.Store.SaveRenderContext(ctx, resolved, value)
}
func (s *failingPersistenceStore) PrepareRenderedReport(ctx context.Context, resolved report.Resolved) (string, error) {
if s.failOperation == failRenderedReportPath {
return s.renderedReportPath, nil
}
return s.Store.PrepareRenderedReport(ctx, resolved)
}
func (s *failingPersistenceStore) SaveDistributorNotification(ctx context.Context, resolved report.Resolved, artifact state.DistributorNotificationArtifact) (string, error) {
if s.failOperation == failDistributorNotification {
return "", errors.New("injected notification persistence failure")
}
return s.Store.SaveDistributorNotification(ctx, resolved, artifact)
}
func (s *failingPersistenceStore) SaveMetadata(ctx context.Context, metadata state.Metadata) (string, error) {
s.metadataCalls++
if s.failOperation == failMetadata && s.metadataCalls == s.failMetadataCall {
return "", errors.New("injected metadata persistence failure")
}
return s.Store.SaveMetadata(ctx, metadata)
}
type artifactPathExecutor struct {
beforePreparationErr error
afterPreparationErr error
validation promptexec.ValidationStatus
}
func (e artifactPathExecutor) InspectPrompt(context.Context, string, string) (promptexec.PromptInspection, error) {
return promptexec.PromptInspection{}, errors.New("unexpected inspection")
}
func (e artifactPathExecutor) InspectProfile(context.Context, string) (promptexec.ProfileInspection, error) {
return promptexec.ProfileInspection{}, errors.New("unexpected inspection")
}
func (e artifactPathExecutor) Execute(_ context.Context, req promptexec.ExecuteRequest, callback promptexec.PreparationCallback) (*promptexec.Execution, error) {
if e.beforePreparationErr != nil {
return nil, e.beforePreparationErr
}
now := time.Date(2026, 5, 29, 15, 0, 0, 0, time.UTC)
if err := callback(promptexec.Preparation{
PromptID: req.PromptID, PromptVersion: req.PromptVersion, PromptHash: "prompt-hash",
RenderedPromptHash: "rendered-hash", ProfileID: req.ProfileID, BackendID: "test",
ModelName: "test-model", DataPackagePath: req.DataPackagePath, StartedAt: now, EndedAt: now,
}, nil); err != nil {
return nil, err
}
if e.afterPreparationErr != nil {
return nil, e.afterPreparationErr
}
validation := e.validation
if validation == "" {
validation = promptexec.ValidationPassed
}
return &promptexec.Execution{
RunID: "provider-run", PromptID: req.PromptID, PromptVersion: req.PromptVersion,
PromptHash: "prompt-hash", RenderedPromptHash: "rendered-hash", ProfileID: req.ProfileID,
BackendID: "test", ModelName: "test-model", GeneratedHash: "generated-hash",
StartedAt: now, EndedAt: now, DataPackagePath: req.DataPackagePath,
RawOutput: []byte(`{"summary":"Showers are possible during the selected day.","forecast_discussion":["A front will keep rain chances in the forecast."],"precipitation_timing":"Rain is most likely during the afternoon.","confidence":"Medium"}`),
Validation: promptexec.NewValidation(validation, "json_schema", "daily.generated_text.schema.json", nil),
}, nil
}
type successfulNotifier struct{}
func (successfulNotifier) Notify(context.Context, NotificationRequest) (*NotificationResult, error) {
return &NotificationResult{RunID: "notification-run", Status: "succeeded", UploadStatus: "accepted"}, nil
}
type failingNotifier struct{}
func (failingNotifier) Notify(context.Context, NotificationRequest) (*NotificationResult, error) {
return nil, errors.New("injected notification failure")
}
func TestGeneratePromptReportReturnsOnlyReachedArtifactPaths(t *testing.T) {
tests := []struct {
name string
failOperation string
failMetadataCall int
outputCopy bool
notify bool
want reachedPromptArtifacts
}{
{name: "preparation then metadata", failOperation: failMetadata, failMetadataCall: 1, want: reachedPromptArtifacts{preparation: true}},
{name: "raw output then execution", failOperation: failPromptExecution, want: reachedPromptArtifacts{preparation: true, metadata: true, raw: true}},
{name: "execution then metadata", failOperation: failMetadata, failMetadataCall: 2, want: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true}},
{name: "normalized output then metadata", failOperation: failMetadata, failMetadataCall: 3, want: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true}},
{name: "render context then metadata", failOperation: failMetadata, failMetadataCall: 4, want: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true, renderContext: true}},
{name: "managed report then metadata", failOperation: failMetadata, failMetadataCall: 5, want: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true, renderContext: true, report: true}},
{name: "output copy then metadata", failOperation: failMetadata, failMetadataCall: 5, outputCopy: true, want: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true, renderContext: true, report: true, output: true}},
{name: "notification then metadata", failOperation: failMetadata, failMetadataCall: 6, notify: true, want: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true, renderContext: true, report: true, notification: true}},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
req, paths := promptArtifactRequest(t, artifactPathExecutor{})
store := &failingPersistenceStore{Store: req.Store, failOperation: test.failOperation, failMetadataCall: test.failMetadataCall}
req.Store = store
if test.outputCopy {
req.OutputPath = filepath.Join(t.TempDir(), "daily.md")
paths.output = req.OutputPath
}
if test.notify {
req.Config.Notify.Distributor.Enabled = true
req.Config.Notify.Distributor.PipelineIDTemplate = "weatherreporter"
req.Notifier = successfulNotifier{}
req.noNotify = false
}
result, err := generatePromptReport(context.Background(), req)
if err == nil || result == nil {
t.Fatalf("generatePromptReport() result/error = %#v/%v, want partial result and failure", result, err)
}
assertReachedPromptArtifacts(t, result, paths, test.want)
})
}
}
func TestGeneratePromptReportFailureReceiptsExposeReachedPaths(t *testing.T) {
tests := []struct {
name string
executor artifactPathExecutor
want reachedPromptArtifacts
wantExecutionStatus state.PromptExecutionStatus
wantExecutionPaths state.PromptExecutionPaths
wantRawExecution bool
}{
{
name: "preparation failure",
executor: artifactPathExecutor{beforePreparationErr: promptexec.NewError(promptexec.Generation, "prepare failed", nil)},
want: reachedPromptArtifacts{preparation: true, metadata: true},
},
{
name: "operational execution failure",
executor: artifactPathExecutor{afterPreparationErr: promptexec.NewError(promptexec.Generation, "provider failed", nil)},
want: reachedPromptArtifacts{preparation: true, execution: true, metadata: true},
wantExecutionStatus: state.PromptExecutionFailed,
},
{
name: "completed validation rejection",
executor: artifactPathExecutor{validation: promptexec.ValidationFailed},
want: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true},
wantExecutionStatus: state.PromptExecutionValidationRejected,
wantRawExecution: true,
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
req, paths := promptArtifactRequest(t, test.executor)
if test.wantRawExecution {
test.wantExecutionPaths.RawOutputPath = paths.GeneratedTextRaw
}
result, err := generatePromptReport(context.Background(), req)
if err == nil || result == nil {
t.Fatalf("generatePromptReport() result/error = %#v/%v, want partial result and failure", result, err)
}
assertReachedPromptArtifacts(t, result, paths, test.want)
if test.wantExecutionStatus != "" {
artifact, loadErr := req.Store.LoadPromptExecution(context.Background(), result.ExecutionPath)
if loadErr != nil {
t.Fatalf("LoadPromptExecution() error = %v", loadErr)
}
if artifact.Status != test.wantExecutionStatus || artifact.Paths != test.wantExecutionPaths {
t.Fatalf("execution outcome/paths = %q/%#v, want %q/%#v", artifact.Status, artifact.Paths, test.wantExecutionStatus, test.wantExecutionPaths)
}
}
})
}
}
func TestCompletedExecutionArtifactTracksDownstreamLifecycle(t *testing.T) {
req, paths := promptArtifactRequest(t, artifactPathExecutor{})
req.OutputPath = filepath.Join(t.TempDir(), "daily.md")
paths.output = req.OutputPath
req.Config.Notify.Distributor.Enabled = true
req.Config.Notify.Distributor.PipelineIDTemplate = "weatherreporter"
req.Notifier = successfulNotifier{}
req.noNotify = false
result, err := generatePromptReport(context.Background(), req)
if err != nil {
t.Fatalf("generatePromptReport() error = %v", err)
}
want := state.PromptExecutionPaths{
RawOutputPath: paths.GeneratedTextRaw, GeneratedTextPath: paths.GeneratedText,
RenderContextPath: paths.RenderContext, RenderedReportPath: paths.RenderedReport,
OutputPath: paths.output, NotificationPath: paths.Notification,
}
assertPersistedExecutionPaths(t, req.Store, result.ExecutionPath, want)
data, err := os.ReadFile(result.ExecutionPath)
if err != nil {
t.Fatalf("read execution artifact: %v", err)
}
text := string(data)
for _, forbidden := range []string{
"Showers are possible during the selected day", `"rawOutput":`, `"debug":`,
`"renderedMessages":`, `"structuredSchema":`, `"endpoint":`, `"parametersJSON":`,
"credential", "secret-value",
} {
if strings.Contains(text, forbidden) {
t.Fatalf("execution artifact contains unsafe generated or provider detail %q:\n%s", forbidden, text)
}
}
}
func TestCompletedExecutionArtifactRetainsLastPersistedCheckpoint(t *testing.T) {
tests := []struct {
name string
failOperation string
failExecutionCall int
failMetadataCall int
requestOutput bool
failOutputCopy bool
notify bool
notificationFailure bool
wantExecution reachedExecutionArtifacts
wantResult reachedPromptArtifacts
}{
{
name: "normalized text write", failOperation: failGeneratedText,
wantExecution: reachedExecutionArtifacts{raw: true},
wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true},
},
{
name: "normalized text checkpoint", failOperation: failPromptExecution, failExecutionCall: 2,
wantExecution: reachedExecutionArtifacts{raw: true},
wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true},
},
{
name: "normalized text metadata", failOperation: failMetadata, failMetadataCall: 3,
wantExecution: reachedExecutionArtifacts{raw: true, normalized: true},
wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true},
},
{
name: "render context write", failOperation: failRenderContext,
wantExecution: reachedExecutionArtifacts{raw: true, normalized: true},
wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true},
},
{
name: "render context checkpoint", failOperation: failPromptExecution, failExecutionCall: 3,
wantExecution: reachedExecutionArtifacts{raw: true, normalized: true},
wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true, renderContext: true},
},
{
name: "render context metadata", failOperation: failMetadata, failMetadataCall: 4,
wantExecution: reachedExecutionArtifacts{raw: true, normalized: true, renderContext: true},
wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true, renderContext: true},
},
{
name: "managed report write", failOperation: failRenderedReportPath,
wantExecution: reachedExecutionArtifacts{raw: true, normalized: true, renderContext: true},
wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true, renderContext: true},
},
{
name: "managed report checkpoint", failOperation: failPromptExecution, failExecutionCall: 4,
wantExecution: reachedExecutionArtifacts{raw: true, normalized: true, renderContext: true},
wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true, renderContext: true, report: true},
},
{
name: "output copy write", requestOutput: true, failOutputCopy: true,
wantExecution: reachedExecutionArtifacts{raw: true, normalized: true, renderContext: true, report: true},
wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true, renderContext: true, report: true},
},
{
name: "output copy checkpoint", failOperation: failPromptExecution, failExecutionCall: 5, requestOutput: true,
wantExecution: reachedExecutionArtifacts{raw: true, normalized: true, renderContext: true, report: true},
wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true, renderContext: true, report: true, output: true},
},
{
name: "output copy metadata", failOperation: failMetadata, failMetadataCall: 5, requestOutput: true,
wantExecution: reachedExecutionArtifacts{raw: true, normalized: true, renderContext: true, report: true, output: true},
wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true, renderContext: true, report: true, output: true},
},
{
name: "notification artifact write", failOperation: failDistributorNotification, requestOutput: true, notify: true,
wantExecution: reachedExecutionArtifacts{raw: true, normalized: true, renderContext: true, report: true, output: true},
wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true, renderContext: true, report: true, output: true},
},
{
name: "notification checkpoint", failOperation: failPromptExecution, failExecutionCall: 6, requestOutput: true, notify: true,
wantExecution: reachedExecutionArtifacts{raw: true, normalized: true, renderContext: true, report: true, output: true},
wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true, renderContext: true, report: true, output: true, notification: true},
},
{
name: "notification metadata", failOperation: failMetadata, failMetadataCall: 6, requestOutput: true, notify: true,
wantExecution: reachedExecutionArtifacts{raw: true, normalized: true, renderContext: true, report: true, output: true, notification: true},
wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true, renderContext: true, report: true, output: true, notification: true},
},
{
name: "notification operation", requestOutput: true, notify: true, notificationFailure: true,
wantExecution: reachedExecutionArtifacts{raw: true, normalized: true, renderContext: true, report: true, output: true, notification: true},
wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true, renderContext: true, report: true, output: true, notification: true},
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
req, paths := promptArtifactRequest(t, artifactPathExecutor{})
store := &failingPersistenceStore{
Store: req.Store, failOperation: test.failOperation,
failExecutionCall: test.failExecutionCall, failMetadataCall: test.failMetadataCall,
}
if test.failOperation == failRenderedReportPath {
store.renderedReportPath = t.TempDir()
}
req.Store = store
if test.requestOutput {
req.OutputPath = filepath.Join(t.TempDir(), "daily.md")
paths.output = req.OutputPath
}
if test.failOutputCopy {
blocker := filepath.Join(t.TempDir(), "not-a-directory")
if err := os.WriteFile(blocker, []byte("block"), 0o600); err != nil {
t.Fatalf("write output blocker: %v", err)
}
req.OutputPath = filepath.Join(blocker, "daily.md")
paths.output = req.OutputPath
}
if test.notify {
req.Config.Notify.Distributor.Enabled = true
req.Config.Notify.Distributor.PipelineIDTemplate = "weatherreporter"
req.Notifier = successfulNotifier{}
req.noNotify = false
}
if test.notificationFailure {
req.Notifier = failingNotifier{}
}
result, err := generatePromptReport(context.Background(), req)
if err == nil || result == nil {
t.Fatalf("generatePromptReport() result/error = %#v/%v, want partial result and failure", result, err)
}
assertReachedPromptArtifacts(t, result, paths, test.wantResult)
assertPersistedExecutionPaths(t, store, result.ExecutionPath, executionPathsFor(paths, test.wantExecution))
})
}
}
type reachedExecutionArtifacts struct {
raw bool
normalized bool
renderContext bool
report bool
output bool
notification bool
}
func executionPathsFor(paths promptArtifactPaths, reached reachedExecutionArtifacts) state.PromptExecutionPaths {
result := state.PromptExecutionPaths{}
if reached.raw {
result.RawOutputPath = paths.GeneratedTextRaw
}
if reached.normalized {
result.GeneratedTextPath = paths.GeneratedText
}
if reached.renderContext {
result.RenderContextPath = paths.RenderContext
}
if reached.report {
result.RenderedReportPath = paths.RenderedReport
}
if reached.output {
result.OutputPath = paths.output
}
if reached.notification {
result.NotificationPath = paths.Notification
}
return result
}
func assertPersistedExecutionPaths(t *testing.T, store state.Store, path string, want state.PromptExecutionPaths) {
t.Helper()
artifact, err := store.LoadPromptExecution(context.Background(), path)
if err != nil {
t.Fatalf("LoadPromptExecution() error = %v", err)
}
if artifact.Status != state.PromptExecutionSucceeded || artifact.Validation == nil || artifact.Validation.Status != promptexec.ValidationPassed {
t.Fatalf("execution outcome changed after downstream write: %#v", artifact)
}
if artifact.Provenance == nil || artifact.Provenance.RunID != "provider-run" || artifact.Provenance.PromptHash != "prompt-hash" {
t.Fatalf("execution provenance changed after downstream write: %#v", artifact.Provenance)
}
if artifact.Paths != want {
t.Fatalf("execution paths = %#v, want %#v", artifact.Paths, want)
}
}
type promptArtifactPaths struct {
state.ArtifactPaths
output string
}
type reachedPromptArtifacts struct {
preparation bool
execution bool
metadata bool
raw bool
normalized bool
renderContext bool
report bool
output bool
notification bool
}
func promptArtifactRequest(t *testing.T, executor promptexec.Executor) (promptReportRequest, promptArtifactPaths) {
t.Helper()
cfg := config.Defaults()
cfg.Workspace.Root = t.TempDir()
resolved, err := ResolveGenerate(GenerateRequest{
Config: cfg, Report: ReportDaily, Date: mustParse("2026-05-29T12:00:00-05:00"),
}, mustParse("2026-05-29T05:00:00-05:00"))
if err != nil {
t.Fatalf("ResolveGenerate() error = %v", err)
}
bundleData, err := os.ReadFile(filepath.Join("..", "forecast", "testdata", "daily_bundle.json"))
if err != nil {
t.Fatalf("read daily fixture: %v", err)
}
var bundle weatherdata.Bundle
if err := json.Unmarshal(bundleData, &bundle); err != nil {
t.Fatalf("decode daily fixture: %v", err)
}
filesystemStore, err := state.NewFilesystemStore(cfg.Workspace)
if err != nil {
t.Fatalf("NewFilesystemStore() error = %v", err)
}
paths, err := filesystemStore.Paths(resolved)
if err != nil {
t.Fatalf("Paths() error = %v", err)
}
debugWriter, err := state.NewPromptDebugWriter("")
if err != nil {
t.Fatalf("NewPromptDebugWriter() error = %v", err)
}
return promptReportRequest{
GenerateRequest: GenerateRequest{Config: cfg, Report: ReportDaily, Executor: executor, Store: filesystemStore},
Resolved: resolved, Collection: collect.Result{Bundle: &bundle},
Inspection: PromptInspectionResult{
PromptID: resolved.Definition.PromptID, PromptVersion: resolved.Definition.PromptVersion,
PromptHash: "prompt-hash", ProfileID: "test-profile", BackendID: "test", ModelName: "test-model",
},
DebugWriter: debugWriter, noNotify: true,
}, promptArtifactPaths{ArtifactPaths: paths}
}
func assertReachedPromptArtifacts(t *testing.T, result *ReportResult, paths promptArtifactPaths, want reachedPromptArtifacts) {
t.Helper()
if result.ModuleSnapshotPath != paths.ModuleSnapshot || result.DataPackagePath != paths.DataPackage {
t.Fatalf("base paths = module %q data %q, want %q and %q", result.ModuleSnapshotPath, result.DataPackagePath, paths.ModuleSnapshot, paths.DataPackage)
}
if result.Metadata.ModuleSnapshotPath != paths.ModuleSnapshot || result.Metadata.DataPackagePath != paths.DataPackage || result.Metadata.MetadataPath != paths.Metadata {
t.Fatalf("metadata base paths = %#v, want reached module/data paths and metadata destination", result.Metadata)
}
checks := []struct {
name string
got string
metadataGot string
inMetadata bool
path string
want bool
}{
{"preparation", result.PreparationPath, result.Metadata.PreparationPath, true, paths.Preparation, want.preparation},
{"execution", result.ExecutionPath, result.Metadata.ExecutionPath, true, paths.Execution, want.execution},
{"metadata", result.MetadataPath, "", false, paths.Metadata, want.metadata},
{"raw", result.GeneratedTextRawPath, result.Metadata.GeneratedTextRawPath, true, paths.GeneratedTextRaw, want.raw},
{"normalized", result.GeneratedTextPath, result.Metadata.GeneratedTextPath, true, paths.GeneratedText, want.normalized},
{"render context", result.RenderContextPath, result.Metadata.RenderContextPath, true, paths.RenderContext, want.renderContext},
{"report", result.ReportPath, result.Metadata.RenderedReportPath, true, paths.RenderedReport, want.report},
{"output", result.OutputPath, "", false, paths.output, want.output},
{"notification", result.NotificationPath, result.Metadata.NotificationPath, true, paths.Notification, want.notification},
}
for _, check := range checks {
if check.want && check.got != check.path {
t.Errorf("%s path = %q, want reached path %q", check.name, check.got, check.path)
}
if check.want && check.inMetadata && check.metadataGot != check.path {
t.Errorf("metadata %s path = %q, want reached path %q", check.name, check.metadataGot, check.path)
}
if !check.want && check.got != "" {
t.Errorf("%s path = %q, want empty because artifact was not reached", check.name, check.got)
}
if !check.want && check.inMetadata && check.metadataGot != "" {
t.Errorf("metadata %s path = %q, want empty because artifact was not reached", check.name, check.metadataGot)
}
}
}

View File

@@ -2,18 +2,14 @@ package app
import (
"context"
"errors"
"fmt"
"time"
"gitea.maximumdirect.net/eric/weatherreporter/internal/briefing"
"gitea.maximumdirect.net/eric/weatherreporter/internal/collect"
"gitea.maximumdirect.net/eric/weatherreporter/internal/fileutil"
"gitea.maximumdirect.net/eric/weatherreporter/internal/generatedtext"
"gitea.maximumdirect.net/eric/weatherreporter/internal/module"
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptdebug"
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec"
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptinput"
"gitea.maximumdirect.net/eric/weatherreporter/internal/report"
"gitea.maximumdirect.net/eric/weatherreporter/internal/state"
)
type promptReportRequest struct {
@@ -21,392 +17,107 @@ type promptReportRequest struct {
Resolved report.Resolved
Collection collect.Result
Inspection PromptInspectionResult
DebugWriter *state.PromptDebugWriter
DebugWriter *promptdebug.PromptDebugWriter
Result *ReportResult
noNotify bool
}
func generatePromptReport(ctx context.Context, req promptReportRequest) (*ReportResult, error) {
workflow, err := newPromptReportWorkflow(ctx, req)
if err != nil {
return nil, err
}
if err := workflow.buildInputs(); err != nil {
return workflow.result, err
}
execution, executeErr := workflow.executePrompt()
if executeErr != nil {
return workflow.result, workflow.handleExecutionFailure(executeErr)
}
if execution == nil {
err := promptexec.NewError(promptexec.Generation, "prompt executor returned no execution", nil)
if saveErr := workflow.persistOperationalExecutionFailure(err); saveErr != nil {
return workflow.result, saveErr
}
return workflow.result, workflow.reportError("execute prompt", err)
}
if err := workflow.persistExecutionDebug(*execution); err != nil {
return workflow.result, err
}
if execution.Validation.Status != promptexec.ValidationPassed && execution.Validation.Status != promptexec.ValidationFailed {
err := promptexec.NewError(promptexec.OperationalValidation, "prompt execution did not complete validation", nil)
if saveErr := workflow.persistOperationalExecutionFailure(err); saveErr != nil {
return workflow.result, saveErr
}
return workflow.result, workflow.reportError("validate prompt execution", err)
}
if err := workflow.persistCompletedExecution(*execution); err != nil {
return workflow.result, err
}
if execution.Validation.Status == promptexec.ValidationFailed {
err := promptexec.NewError(promptexec.ValidationRejected, "prompt output did not satisfy its schema", nil)
return workflow.result, workflow.reportError("validate prompt execution", err)
}
rendered, err := workflow.persistGeneratedContent(execution.RawOutput)
if err != nil {
return workflow.result, err
}
return workflow.finalizeReport(rendered)
}
type promptReportWorkflow struct {
ctx context.Context
req promptReportRequest
store state.Store
result *ReportResult
metadata state.Metadata
briefingMetadata briefing.Metadata
reportFacts ReportFacts
moduleSnapshot module.Snapshot
dataPackageBytes []byte
handler generatedtext.Handler
executionArtifact state.PromptExecutionArtifact
debugRef state.PromptDebugRef
prepared bool
callbackFailed bool
}
func newPromptReportWorkflow(ctx context.Context, req promptReportRequest) (*promptReportWorkflow, error) {
if req.Collection.Bundle == nil {
return nil, fmt.Errorf("collected weather bundle is required")
}
store := req.Store
var err error
if store == nil {
store, err = defaultStore(req.Config)
if err != nil {
return nil, err
}
result := req.Result
if result == nil {
result = initialReportResult(req.GenerateRequest, req.Resolved, req.Inspection)
}
return &promptReportWorkflow{ctx: ctx, req: req, store: store}, nil
}
func (w *promptReportWorkflow) buildInputs() error {
paths, err := w.store.Paths(w.req.Resolved)
prepared, err := prepareReport(prepareReportRequest{Config: req.Config, Resolved: req.Resolved, Collection: req.Collection})
if err != nil {
return err
return result, generatedPreparationError(req.Resolved, result.RunID, err)
}
w.result = &ReportResult{}
priorSnapshot, err := w.store.FindPriorSnapshot(w.ctx, w.req.Resolved)
if err != nil {
return err
}
w.reportFacts, err = BuildReportFacts(ModuleSnapshotRequest{Config: w.req.Config, Resolved: w.req.Resolved}, w.req.Collection.Bundle)
if err != nil {
return generatedReportError(w.req.Resolved, w.req.Resolved.Metadata().RunID, "build report facts", err)
}
w.moduleSnapshot, err = BuildModuleSnapshotFromFacts(ModuleSnapshotRequest{Config: w.req.Config, Resolved: w.req.Resolved}, w.reportFacts)
if err != nil {
return generatedReportError(w.req.Resolved, w.req.Resolved.Metadata().RunID, "build module snapshot", err)
}
moduleSnapshotPath, err := w.store.SaveModuleSnapshot(w.ctx, w.req.Resolved, w.moduleSnapshot)
if err != nil {
return err
}
w.result.ModuleSnapshot = w.moduleSnapshot
w.result.ModuleSnapshotPath = moduleSnapshotPath
w.result.PriorSnapshot = priorSnapshot
recent, err := recentChanges(w.ctx, w.store, priorSnapshot, w.req.Resolved.Definition.ID, w.moduleSnapshot, w.req.Config.RecentChange)
if err != nil {
return err
}
w.result.RecentChanges = recent
w.briefingMetadata = briefing.BuildMetadata(briefingBuildContext(w.req.Config, w.req.Resolved, w.reportFacts.Collected))
w.metadata = state.BuildPromptMetadataFromBriefingMetadata(w.req.Resolved, w.briefingMetadata, state.ArtifactPaths{
ModuleSnapshot: moduleSnapshotPath,
Metadata: paths.Metadata,
result.SourceWarnings = prepared.sourceWarningsCopy()
debugRef := promptdebug.PromptDebugRef{ReportID: result.ReportID, ValidDate: prepared.resolved.ValidPeriod.Start.Format("2006-01-02"), RunID: result.RunID}
outcome, rendered, err := executePreparedProfile(ctx, profileExecutionRequest{
Prepared: prepared,
Prompt: req.Inspection,
Profile: promptexec.ProfileInspection{
ProfileID: req.Inspection.ProfileID,
BackendID: req.Inspection.BackendID,
ModelName: req.Inspection.ModelName,
},
Executor: req.Executor, DebugWriter: req.DebugWriter, DebugRef: &debugRef,
})
w.result.Metadata = w.metadata
dataPackage, err := promptinput.Build(promptinput.BuildRequest{
Metadata: promptMetadata(w.metadata), Modules: w.moduleSnapshot, RecentChanges: recent,
result.ProfileID, result.BackendID, result.ModelName = outcome.ProfileID, outcome.BackendID, outcome.ModelName
result.ValidationStatus = outcome.ValidationStatus
result.LLMDebugPath = outcome.LLMDebugPath
if err != nil {
return result, generatedProfileExecutionError(req.Resolved, result.RunID, err)
}
return publishPromptReport(ctx, promptPublicationRequest{
GenerateRequest: req.GenerateRequest,
Resolved: req.Resolved,
OutputPath: req.OutputPath,
Result: result,
Markdown: rendered,
suppressNotification: req.noNotify,
})
if err != nil {
return w.reportError("build data package", err)
}
w.dataPackageBytes, err = promptinput.MarshalYAML(dataPackage)
if err != nil {
return err
}
dataPackagePath, err := w.store.SaveDataPackageBytes(w.ctx, w.req.Resolved, w.dataPackageBytes)
if err != nil {
return err
}
w.metadata.DataPackagePath = dataPackagePath
w.result.DataPackage = dataPackage
w.result.DataPackagePath = dataPackagePath
w.result.Metadata = w.metadata
w.handler, err = generatedtext.LookupDefinition(w.req.Resolved.Definition)
if err != nil {
return w.reportError("lookup generated text catalog", err)
}
w.debugRef = state.PromptDebugRef{
ReportID: w.req.Resolved.Definition.ID, ValidDate: w.req.Resolved.ValidPeriod.Start.Format("2006-01-02"), RunID: w.metadata.RunID,
}
return nil
}
func (w *promptReportWorkflow) executePrompt() (*promptexec.Execution, error) {
return w.req.Executor.Execute(w.ctx, promptexec.ExecuteRequest{
PromptID: w.req.Inspection.PromptID, PromptVersion: w.req.Inspection.PromptVersion,
ProfileID: w.req.Inspection.ProfileID, DataPackage: w.dataPackageBytes,
DataPackagePath: w.result.DataPackagePath, CaptureDebug: w.req.DebugWriter.Enabled(),
}, w.persistPreparation)
func initialReportResult(req GenerateRequest, resolved report.Resolved, inspection PromptInspectionResult) *ReportResult {
metadata := resolved.Metadata()
return &ReportResult{
ReportID: resolved.Definition.ID, ReportName: resolved.Definition.Name,
PromptID: resolved.Definition.PromptID, PromptVersion: resolved.Definition.PromptVersion,
RunID: metadata.RunID, GeneratedAt: metadata.GeneratedAt, Timezone: req.Config.WeatherAPI.Timezone,
ValidPeriod: metadata.ValidPeriod,
ProfileID: inspection.ProfileID, BackendID: inspection.BackendID, ModelName: inspection.ModelName,
}
}
func (w *promptReportWorkflow) persistPreparation(preparation promptexec.Preparation, debug *promptexec.PreparationDebug) error {
artifact := state.PromptPreparationArtifact{
SchemaVersion: state.PromptPreparationSchemaVersion, Status: state.PromptPreparationSucceeded,
ReportID: w.req.Resolved.Definition.ID, RunID: w.metadata.RunID,
PromptID: w.req.Inspection.PromptID, PromptVersion: w.req.Inspection.PromptVersion,
DataPackagePath: w.result.DataPackagePath, Preparation: &preparation,
StartedAt: preparation.StartedAt, EndedAt: preparation.EndedAt, Duration: preparation.Duration,
}
path, err := w.store.SavePromptPreparation(w.ctx, w.req.Resolved, artifact)
if err != nil {
w.callbackFailed = true
return err
}
w.prepared = true
w.result.PreparationPath = path
w.metadata.PreparationPath = path
w.result.Metadata = w.metadata
debugPath, err := w.req.DebugWriter.WritePreparation(w.debugRef, preparation, debug)
if err != nil {
w.callbackFailed = true
return promptDebugWriteError(err)
}
if debugPath != "" {
w.result.LLMDebugPath = debugPath
}
if err := w.saveMetadata(); err != nil {
w.callbackFailed = true
return err
}
return nil
type promptPublicationRequest struct {
GenerateRequest
Resolved report.Resolved
OutputPath string
Result *ReportResult
Markdown []byte
suppressNotification bool
}
func (w *promptReportWorkflow) handleExecutionFailure(executeErr error) error {
if w.callbackFailed {
return executeErr
func publishPromptReport(ctx context.Context, req promptPublicationRequest) (*ReportResult, error) {
if err := publicationContextError(ctx); err != nil {
return req.Result, generatedReportError(req.Resolved, req.Result.RunID, "publish report", err)
}
executeErr = classifiedPromptError("prompt execution failed", executeErr)
if !w.prepared {
if err := w.persistPreparationFailure(executeErr); err != nil {
return err
if err := fileutil.WriteFileAtomic(req.OutputPath, req.Markdown); err != nil {
return req.Result, err
}
req.Result.OutputPath = req.OutputPath
if req.suppressNotification {
return req.Result, nil
}
notification, err := notifyReport(ctx, req.Config, req.Resolved, req.Result.OutputPath, req.Result.RunID, req.Result.GeneratedAt, req.Notifier)
req.Result.Notification = notification
if err != nil {
return req.Result, err
}
return req.Result, nil
}
func generatedPreparationError(resolved report.Resolved, runID string, err error) error {
var preparation *preparationError
if errors.As(err, &preparation) {
return generatedReportError(resolved, runID, preparation.operation, preparation.err)
}
return generatedReportError(resolved, runID, "prepare report", err)
}
func generatedProfileExecutionError(resolved report.Resolved, runID string, err error) error {
var execution *profileExecutionError
if errors.As(err, &execution) {
if execution.callbackFailure {
return execution.err
}
return w.reportError("prepare prompt", executeErr)
}
if promptexec.CategoryOf(executeErr) != "" {
if err := w.persistOperationalExecutionFailure(executeErr); err != nil {
return err
}
}
return w.reportError("execute prompt", executeErr)
}
func (w *promptReportWorkflow) persistPreparationFailure(executeErr error) error {
startedAt, endedAt := time.Now(), time.Now()
artifact := state.PromptPreparationArtifact{
SchemaVersion: state.PromptPreparationSchemaVersion, Status: state.PromptPreparationFailed,
ReportID: w.req.Resolved.Definition.ID, RunID: w.metadata.RunID,
PromptID: w.req.Inspection.PromptID, PromptVersion: w.req.Inspection.PromptVersion,
DataPackagePath: w.result.DataPackagePath, StartedAt: startedAt, EndedAt: endedAt,
Error: state.NewPromptArtifactError(executeErr),
}
path, err := w.store.SavePromptPreparation(w.ctx, w.req.Resolved, artifact)
if err != nil {
return err
}
w.result.PreparationPath = path
w.metadata.PreparationPath = path
w.result.Metadata = w.metadata
return w.saveMetadata()
}
func (w *promptReportWorkflow) persistOperationalExecutionFailure(executeErr error) error {
artifact := failedPromptExecutionArtifact(w.req.Resolved, w.metadata, w.req.Inspection, executeErr)
path, err := w.store.SavePromptExecution(w.ctx, w.req.Resolved, artifact)
if err != nil {
return err
}
w.result.ExecutionPath = path
w.metadata.ExecutionPath = path
w.result.Metadata = w.metadata
return w.saveMetadata()
}
func (w *promptReportWorkflow) persistExecutionDebug(execution promptexec.Execution) error {
debugPath, err := w.req.DebugWriter.WriteExecution(w.debugRef, execution)
if err != nil {
return w.reportError("write prompt debug", promptDebugWriteError(err))
}
if debugPath != "" {
w.result.LLMDebugPath = debugPath
}
return nil
}
func (w *promptReportWorkflow) persistCompletedExecution(execution promptexec.Execution) error {
rawPath, err := w.store.SaveGeneratedTextRaw(w.ctx, w.req.Resolved, execution.RawOutput)
if err != nil {
return err
}
w.result.GeneratedTextRawPath = rawPath
w.metadata.GeneratedTextRawPath = rawPath
w.result.Metadata = w.metadata
w.executionArtifact = state.PromptExecutionArtifact{
SchemaVersion: state.PromptExecutionSchemaVersion,
ReportID: w.req.Resolved.Definition.ID, RunID: w.metadata.RunID,
PromptID: w.req.Inspection.PromptID, PromptVersion: w.req.Inspection.PromptVersion,
Provenance: ptr(state.PromptExecutionProvenanceFrom(execution)), Validation: &execution.Validation,
Paths: state.PromptExecutionPaths{RawOutputPath: rawPath},
StartedAt: execution.StartedAt, EndedAt: execution.EndedAt, Duration: execution.Duration,
}
if execution.Validation.Status == promptexec.ValidationPassed {
w.executionArtifact.Status = state.PromptExecutionSucceeded
} else {
w.executionArtifact.Status = state.PromptExecutionValidationRejected
}
executionPath, err := w.store.SavePromptExecution(w.ctx, w.req.Resolved, w.executionArtifact)
if err != nil {
return err
}
w.result.ExecutionPath = executionPath
w.metadata.ExecutionPath = executionPath
w.result.Metadata = w.metadata
return w.saveMetadata()
}
func (w *promptReportWorkflow) persistGeneratedContent(raw []byte) ([]byte, error) {
generatedText, normalized, err := w.handler.Validate(raw)
if err != nil {
return nil, w.reportError("validate generated text", err)
}
generatedTextPath, err := w.store.SaveGeneratedText(w.ctx, w.req.Resolved, normalized)
if err != nil {
return nil, err
}
w.result.GeneratedTextPath = generatedTextPath
w.metadata.GeneratedTextPath = generatedTextPath
w.result.Metadata = w.metadata
if err := w.persistReachedPathAndMetadata(func(paths *state.PromptExecutionPaths) { paths.GeneratedTextPath = generatedTextPath }); err != nil {
return nil, err
}
renderContext, err := w.handler.BuildRenderContext(w.briefingMetadata, w.moduleSnapshot, w.reportFacts.Collected, w.reportFacts.Derived, generatedText)
if err != nil {
return nil, w.reportError("build render context", err)
}
renderContextPath, err := w.store.SaveRenderContext(w.ctx, w.req.Resolved, renderContext)
if err != nil {
return nil, err
}
w.result.RenderContextPath = renderContextPath
w.metadata.RenderContextPath = renderContextPath
w.result.Metadata = w.metadata
if err := w.persistReachedPathAndMetadata(func(paths *state.PromptExecutionPaths) { paths.RenderContextPath = renderContextPath }); err != nil {
return nil, err
}
rendered, err := w.handler.Render(renderContext)
if err != nil {
return nil, w.reportError("render template", err)
}
return rendered, nil
}
func (w *promptReportWorkflow) finalizeReport(rendered []byte) (*ReportResult, error) {
reportPath, err := w.store.PrepareRenderedReport(w.ctx, w.req.Resolved)
if err != nil {
return w.result, err
}
if err := fileutil.WriteFileAtomic(reportPath, rendered); err != nil {
return w.result, err
}
w.result.ReportPath = reportPath
w.metadata.RenderedReportPath = reportPath
w.result.Metadata = w.metadata
if err := w.persistReachedPath(func(paths *state.PromptExecutionPaths) { paths.RenderedReportPath = reportPath }); err != nil {
return w.result, err
}
finalized, err := finalizeRenderedReport(w.ctx, finalizeRenderedReportRequest{
Config: w.req.Config, Store: w.store, Resolved: w.req.Resolved, Metadata: w.metadata, MetadataPath: w.result.MetadataPath,
ExecutionArtifact: &w.executionArtifact, ManagedReportPath: reportPath, OutputPath: w.req.OutputPath,
Notifier: w.req.Notifier, noNotify: w.req.noNotify,
})
w.result.OutputPath, w.result.NotificationPath = finalized.OutputPath, finalized.NotificationPath
w.result.Metadata, w.result.MetadataPath, w.result.Notification = finalized.Metadata, finalized.MetadataPath, finalized.Notification
return w.result, err
}
func (w *promptReportWorkflow) persistReachedPath(update func(*state.PromptExecutionPaths)) error {
return persistReachedPromptPath(w.ctx, w.store, w.req.Resolved, &w.executionArtifact, update)
}
func (w *promptReportWorkflow) persistReachedPathAndMetadata(update func(*state.PromptExecutionPaths)) error {
if err := w.persistReachedPath(update); err != nil {
return err
}
return w.saveMetadata()
}
func (w *promptReportWorkflow) saveMetadata() error {
path, err := w.store.SaveMetadata(w.ctx, w.metadata)
if err != nil {
return err
}
w.result.Metadata = w.metadata
w.result.MetadataPath = path
return nil
}
func (w *promptReportWorkflow) reportError(operation string, err error) error {
return generatedReportError(w.req.Resolved, w.metadata.RunID, operation, err)
}
func persistReachedPromptPath(
ctx context.Context,
store state.Store,
resolved report.Resolved,
artifact *state.PromptExecutionArtifact,
update func(*state.PromptExecutionPaths),
) error {
update(&artifact.Paths)
_, err := store.SavePromptExecution(ctx, resolved, *artifact)
return err
}
func failedPromptExecutionArtifact(resolved report.Resolved, metadata state.Metadata, inspection PromptInspectionResult, err error) state.PromptExecutionArtifact {
now := time.Now()
return state.PromptExecutionArtifact{
SchemaVersion: state.PromptExecutionSchemaVersion, Status: state.PromptExecutionFailed,
ReportID: resolved.Definition.ID, RunID: metadata.RunID, PromptID: inspection.PromptID,
PromptVersion: inspection.PromptVersion, StartedAt: now, EndedAt: now,
Error: state.NewPromptArtifactError(err),
return generatedReportError(resolved, runID, execution.operation, execution.err)
}
return generatedReportError(resolved, runID, "execute prompt", err)
}
func classifiedPromptError(operation string, err error) error {
@@ -416,8 +127,16 @@ func classifiedPromptError(operation string, err error) error {
return promptexec.NewError(promptexec.Generation, operation, err)
}
func publicationContextError(ctx context.Context) error {
if err := ctx.Err(); err != nil {
if errors.Is(err, context.DeadlineExceeded) {
return promptexec.NewError(promptexec.DeadlineExceeded, "context expired before output publication", err)
}
return promptexec.NewError(promptexec.Canceled, "context canceled before output publication", err)
}
return nil
}
func promptDebugWriteError(err error) error {
return promptexec.NewError(promptexec.InvalidConfiguration, "write requested prompt debug artifact", err)
}
func ptr[T any](value T) *T { return &value }

View File

@@ -5,6 +5,7 @@ import (
"os"
"strings"
"gitea.maximumdirect.net/eric/weatherreporter/internal/comparison"
"gitea.maximumdirect.net/eric/weatherreporter/internal/config"
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec"
"gitea.maximumdirect.net/eric/weatherreporter/internal/report"
@@ -39,6 +40,32 @@ type PromptExecutionsInspectionRequest struct {
LookupEnv func(string) (string, bool)
}
// ComparisonInspectionRequest contains the explicit profile selection for one
// resolved prompt comparison. It intentionally has no configured profile field.
type ComparisonInspectionRequest struct {
Resolved report.Resolved
ProfileIDs []string
Executor promptexec.Executor
LookupEnv func(string) (string, bool)
}
// ComparisonInspectionResult contains the safe, shared prompt identity and
// ordered effective profile identities for a comparison.
type ComparisonInspectionResult struct {
PromptID string
PromptVersion string
PromptHash string
Profiles []ComparisonProfileInspection
}
// ComparisonProfileInspection contains one requested profile's safe effective
// execution identity.
type ComparisonProfileInspection struct {
ProfileID string
BackendID string
ModelName string
}
// InspectPromptExecution validates the exact prompt and profile needed for a
// report before collection, execution, or durable writes begin.
func InspectPromptExecution(ctx context.Context, req PromptInspectionRequest) (PromptInspectionResult, error) {
@@ -64,21 +91,9 @@ func InspectPromptExecutions(ctx context.Context, req PromptExecutionsInspection
profiles := map[string]promptexec.ProfileInspection{}
for _, resolved := range req.Resolved {
definition := resolved.Definition
if strings.TrimSpace(definition.PromptID) == "" || strings.TrimSpace(definition.PromptVersion) == "" {
return nil, promptexec.NewError(promptexec.InvalidConfiguration, "report prompt identity is incomplete", nil)
}
inspection, err := req.Executor.InspectPrompt(ctx, definition.PromptID, definition.PromptVersion)
inspection, err := inspectPromptContract(ctx, req.Executor, definition)
if err != nil {
return nil, promptInspectionError("prompt inspection failed", err)
}
if inspection.PromptID != definition.PromptID || inspection.PromptVersion != definition.PromptVersion {
return nil, promptexec.NewError(promptexec.InvalidConfiguration, "prompt inspection did not return the requested prompt version", nil)
}
if !validPromptInput(inspection.Inputs) {
return nil, promptexec.NewError(promptexec.InvalidConfiguration, "prompt must declare exactly one required application/yaml data_package input", nil)
}
if !validPromptOutput(definition, inspection.Output) {
return nil, promptexec.NewError(promptexec.InvalidConfiguration, "prompt must declare the report JSON Schema output contract", nil)
return nil, err
}
profileID := req.Promptkit.Profile
if profileID == "" {
@@ -103,6 +118,62 @@ func InspectPromptExecutions(ctx context.Context, req PromptExecutionsInspection
return results, nil
}
// InspectComparisonExecution validates one exact prompt and every explicitly
// requested profile before collection or model execution. Profiles are
// inspected sequentially in request order. If a profile fails, the returned
// partial result retains the prompt identity and successfully inspected prefix.
func InspectComparisonExecution(ctx context.Context, req ComparisonInspectionRequest) (ComparisonInspectionResult, error) {
if err := comparison.ValidateProfileIDs(req.ProfileIDs); err != nil {
return ComparisonInspectionResult{}, promptexec.NewError(promptexec.InvalidRequest, "comparison profile selection is invalid", err)
}
if req.Executor == nil {
return ComparisonInspectionResult{}, promptexec.NewError(promptexec.InvalidConfiguration, "prompt executor is required", nil)
}
inspection, err := inspectPromptContract(ctx, req.Executor, req.Resolved.Definition)
if err != nil {
return ComparisonInspectionResult{}, comparisonInspectionError("comparison prompt inspection failed", err)
}
result := ComparisonInspectionResult{
PromptID: inspection.PromptID,
PromptVersion: inspection.PromptVersion,
PromptHash: inspection.PromptHash,
Profiles: make([]ComparisonProfileInspection, 0, len(req.ProfileIDs)),
}
for _, profileID := range req.ProfileIDs {
profile, err := inspectPromptProfile(ctx, req.Executor, profileID, req.LookupEnv)
if err != nil {
return result, comparisonInspectionError("comparison profile inspection failed", err)
}
result.Profiles = append(result.Profiles, ComparisonProfileInspection{
ProfileID: profile.ProfileID,
BackendID: profile.BackendID,
ModelName: profile.ModelName,
})
}
return result, nil
}
func inspectPromptContract(ctx context.Context, executor promptexec.Executor, definition report.Definition) (promptexec.PromptInspection, error) {
if strings.TrimSpace(definition.PromptID) == "" || strings.TrimSpace(definition.PromptVersion) == "" {
return promptexec.PromptInspection{}, promptexec.NewError(promptexec.InvalidConfiguration, "report prompt identity is incomplete", nil)
}
inspection, err := executor.InspectPrompt(ctx, definition.PromptID, definition.PromptVersion)
if err != nil {
return promptexec.PromptInspection{}, promptInspectionError("prompt inspection failed", err)
}
if inspection.PromptID != definition.PromptID || inspection.PromptVersion != definition.PromptVersion {
return promptexec.PromptInspection{}, promptexec.NewError(promptexec.InvalidConfiguration, "prompt inspection did not return the requested prompt version", nil)
}
if !validPromptInput(inspection.Inputs) {
return promptexec.PromptInspection{}, promptexec.NewError(promptexec.InvalidConfiguration, "prompt must declare exactly one required application/yaml data_package input", nil)
}
if !validPromptOutput(definition, inspection.Output) {
return promptexec.PromptInspection{}, promptexec.NewError(promptexec.InvalidConfiguration, "prompt must declare the report JSON Schema output contract", nil)
}
return inspection, nil
}
func inspectPromptProfile(ctx context.Context, executor promptexec.Executor, profileID string, lookupEnv func(string) (string, bool)) (promptexec.ProfileInspection, error) {
profile, err := executor.InspectProfile(ctx, profileID)
if err != nil {
@@ -140,3 +211,11 @@ func promptInspectionError(operation string, err error) error {
}
return promptexec.NewError(promptexec.InvalidConfiguration, operation, err)
}
func comparisonInspectionError(operation string, err error) error {
category := promptexec.CategoryOf(err)
if category == "" {
category = promptexec.InvalidConfiguration
}
return promptexec.NewError(category, operation, err)
}

View File

@@ -3,6 +3,7 @@ package app
import (
"context"
"errors"
"reflect"
"strings"
"testing"
"time"
@@ -131,6 +132,98 @@ func TestInspectPromptExecutionsReusesEffectiveProfile(t *testing.T) {
}
}
func TestInspectComparisonExecutionPreservesOrderedExplicitProfiles(t *testing.T) {
resolved := inspectionResolved(t)
executor := &inspectionExecutor{
prompt: validPromptInspection(resolved.Definition),
profiles: map[string]promptexec.ProfileInspection{
"weather-light": {ProfileID: "weather-light", BackendID: "local", ModelName: "light-model"},
"weather-deep": {ProfileID: "weather-deep", BackendID: "cloud", ModelName: "deep-model"},
},
}
profileIDs := []string{"weather-light", "weather-deep"}
result, err := InspectComparisonExecution(context.Background(), ComparisonInspectionRequest{
Resolved: resolved, ProfileIDs: profileIDs, Executor: executor,
})
if err != nil {
t.Fatalf("InspectComparisonExecution() error = %v", err)
}
if result.PromptID != resolved.Definition.PromptID || result.PromptVersion != resolved.Definition.PromptVersion || result.PromptHash != "prompt-hash" {
t.Fatalf("prompt result = %#v", result)
}
if !reflect.DeepEqual(executor.profileRequests, profileIDs) || len(executor.promptRequests) != 1 || executor.executeRequests != 0 {
t.Fatalf("prompt/profile/execute requests = %#v/%#v/%d", executor.promptRequests, executor.profileRequests, executor.executeRequests)
}
wantProfiles := []ComparisonProfileInspection{
{ProfileID: "weather-light", BackendID: "local", ModelName: "light-model"},
{ProfileID: "weather-deep", BackendID: "cloud", ModelName: "deep-model"},
}
if !reflect.DeepEqual(result.Profiles, wantProfiles) {
t.Fatalf("profiles = %#v, want %#v", result.Profiles, wantProfiles)
}
}
func TestInspectComparisonExecutionRejectsInvalidProfilesBeforeInspection(t *testing.T) {
resolved := inspectionResolved(t)
for _, profileIDs := range [][]string{
{"weather-light"},
{"weather-light", " \t"},
{"weather-light", "weather-light"},
} {
t.Run(strings.Join(profileIDs, ","), func(t *testing.T) {
executor := &inspectionExecutor{prompt: validPromptInspection(resolved.Definition)}
_, err := InspectComparisonExecution(context.Background(), ComparisonInspectionRequest{
Resolved: resolved, ProfileIDs: profileIDs, Executor: executor,
})
if err == nil || promptexec.CategoryOf(err) != promptexec.InvalidRequest {
t.Fatalf("error/category = %v/%q, want invalid request", err, promptexec.CategoryOf(err))
}
if len(executor.promptRequests) != 0 || len(executor.profileRequests) != 0 || executor.executeRequests != 0 {
t.Fatalf("invalid profile selection performed prompt/profile/execution work: %#v/%#v/%d", executor.promptRequests, executor.profileRequests, executor.executeRequests)
}
})
}
}
func TestInspectComparisonExecutionStopsAtFirstProfileFailure(t *testing.T) {
resolved := inspectionResolved(t)
executor := &inspectionExecutor{
prompt: validPromptInspection(resolved.Definition),
profiles: map[string]promptexec.ProfileInspection{
"weather-light": {ProfileID: "weather-light", BackendID: "local", ModelName: "light-model"},
"missing-key": {ProfileID: "missing-key", APIKeyEnv: "PROMPT_API_KEY"},
"weather-deep": {ProfileID: "weather-deep", BackendID: "cloud", ModelName: "deep-model"},
},
}
result, err := InspectComparisonExecution(context.Background(), ComparisonInspectionRequest{
Resolved: resolved, ProfileIDs: []string{"weather-light", "missing-key", "weather-deep"}, Executor: executor,
LookupEnv: func(string) (string, bool) { return "", false },
})
if err == nil || promptexec.CategoryOf(err) != promptexec.MissingCredential {
t.Fatalf("error/category = %v/%q, want missing credential", err, promptexec.CategoryOf(err))
}
if !reflect.DeepEqual(executor.profileRequests, []string{"weather-light", "missing-key"}) || len(executor.promptRequests) != 1 || executor.executeRequests != 0 {
t.Fatalf("prompt/profile/execute requests = %#v/%#v/%d", executor.promptRequests, executor.profileRequests, executor.executeRequests)
}
if result.PromptID != resolved.Definition.PromptID || result.PromptVersion != resolved.Definition.PromptVersion || result.PromptHash == "" || len(result.Profiles) != 1 || result.Profiles[0].ProfileID != "weather-light" {
t.Fatalf("partial inspection result = %#v", result)
}
}
func TestInspectComparisonExecutionStopsBeforeProfileInspectionWhenPromptFails(t *testing.T) {
resolved := inspectionResolved(t)
executor := &inspectionExecutor{promptErr: promptexec.NewError(promptexec.PromptNotFound, "prompt is unavailable", nil)}
_, err := InspectComparisonExecution(context.Background(), ComparisonInspectionRequest{
Resolved: resolved, ProfileIDs: []string{"weather-light", "weather-deep"}, Executor: executor,
})
if err == nil || promptexec.CategoryOf(err) != promptexec.PromptNotFound || !strings.Contains(err.Error(), "comparison prompt") {
t.Fatalf("error/category = %v/%q, want prompt-context prompt not found", err, promptexec.CategoryOf(err))
}
if len(executor.promptRequests) != 1 || len(executor.profileRequests) != 0 || executor.executeRequests != 0 {
t.Fatalf("prompt/profile/execute requests = %#v/%#v/%d", executor.promptRequests, executor.profileRequests, executor.executeRequests)
}
}
type inspectionPromptRequest struct {
id string
version string
@@ -143,6 +236,7 @@ type inspectionExecutor struct {
promptErr error
promptRequests []inspectionPromptRequest
profileRequests []string
executeRequests int
}
func (e *inspectionExecutor) InspectPrompt(_ context.Context, id string, version string) (promptexec.PromptInspection, error) {
@@ -166,6 +260,7 @@ func (e *inspectionExecutor) InspectProfile(_ context.Context, id string) (promp
}
func (e *inspectionExecutor) Execute(context.Context, promptexec.ExecuteRequest, promptexec.PreparationCallback) (*promptexec.Execution, error) {
e.executeRequests++
return nil, errors.New("unexpected execution")
}
@@ -189,3 +284,13 @@ func validPromptInspection(definition report.Definition) promptexec.PromptInspec
Output: promptexec.OutputContract{Format: "json", ValidationMode: "json_schema", SchemaPath: definition.GeneratedTextSchemaID + ".generated_text.schema.json"},
}
}
func logicalPromptInspection(definition report.Definition) promptexec.PromptInspection {
inspection := validPromptInspection(definition)
if definition.ID == report.Hourly {
inspection.DefaultProfileID = "weather-light"
} else {
inspection.DefaultProfileID = "weather-balanced"
}
return inspection
}

View File

@@ -0,0 +1,73 @@
package app_test
import (
"context"
"os"
"path/filepath"
"testing"
"time"
promptkitadapter "gitea.maximumdirect.net/eric/weatherreporter/internal/adapters/promptkit"
"gitea.maximumdirect.net/eric/weatherreporter/internal/app"
"gitea.maximumdirect.net/eric/weatherreporter/internal/config"
"gitea.maximumdirect.net/eric/weatherreporter/internal/report"
)
func TestPromptInspectionResolvesEmbeddedAndOverriddenProfilesOffline(t *testing.T) {
lookupEnv := func(string) (string, bool) { return "test-key", true }
inspect := func(t *testing.T, adapter *promptkitadapter.Adapter, id report.ID, profile string, wantID string, wantBackend string, wantModel string) {
t.Helper()
result, err := app.InspectPromptExecution(context.Background(), app.PromptInspectionRequest{
Resolved: resolvedPromptProfile(t, id),
Executor: adapter,
Promptkit: config.PromptkitConfig{Profile: profile},
LookupEnv: lookupEnv,
})
if err != nil {
t.Fatalf("InspectPromptExecution() error = %v", err)
}
if result.ProfileID != wantID || result.BackendID != wantBackend || result.ModelName != wantModel {
t.Fatalf("inspection = %#v, want profile/backend/model %q/%q/%q", result, wantID, wantBackend, wantModel)
}
}
embedded, err := promptkitadapter.New(promptkitadapter.Config{})
if err != nil {
t.Fatalf("New(embedded) error = %v", err)
}
inspect(t, embedded, report.Hourly, "", "weather-light", "openrouter", "deepseek/deepseek-v4-flash")
inspect(t, embedded, report.Daily, "", "weather-balanced", "openrouter", "~google/gemini-flash-latest")
inspect(t, embedded, report.Daily, "weather-deep", "weather-deep", "openrouter", "~anthropic/claude-sonnet-latest")
override, err := promptkitadapter.New(promptkitadapter.Config{ProfileFile: writeProfileFile(t, `id: weather-light
endpoint: https://local.example/v1
model: local-weather
`)})
if err != nil {
t.Fatalf("New(override) error = %v", err)
}
inspect(t, override, report.Hourly, "", "weather-light", "", "local-weather")
}
func resolvedPromptProfile(t *testing.T, id report.ID) report.Resolved {
t.Helper()
now := time.Date(2026, 5, 29, 12, 0, 0, 0, time.UTC)
request := report.ResolveRequest{Now: now, Location: time.UTC}
if id == report.Daily {
request.Date = now
}
resolved, err := report.DefaultRegistry().Resolve(id, request)
if err != nil {
t.Fatalf("Resolve(%q) error = %v", id, err)
}
return resolved
}
func writeProfileFile(t *testing.T, profile string) string {
t.Helper()
path := filepath.Join(t.TempDir(), "profile.yml")
if err := os.WriteFile(path, []byte(profile), 0o600); err != nil {
t.Fatalf("write profile: %v", err)
}
return path
}

View File

@@ -1,659 +0,0 @@
package app
import (
"bytes"
"context"
"encoding/json"
"errors"
"os"
"path/filepath"
"strings"
"testing"
"time"
"gitea.maximumdirect.net/eric/weatherreporter/internal/collect"
"gitea.maximumdirect.net/eric/weatherreporter/internal/config"
"gitea.maximumdirect.net/eric/weatherreporter/internal/module"
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec"
"gitea.maximumdirect.net/eric/weatherreporter/internal/report"
"gitea.maximumdirect.net/eric/weatherreporter/internal/state"
"gitea.maximumdirect.net/eric/weatherreporter/internal/weatherdata"
)
type workflowCollector struct {
result *collect.Result
err error
calls int
}
func (c *workflowCollector) Run(context.Context, collect.Request) (*collect.Result, error) {
c.calls++
if c.err != nil {
return nil, c.err
}
return c.result, nil
}
type workflowExecutor struct {
definition report.Definition
raw []byte
inspectionErr error
profile promptexec.ProfileInspection
beforePreparationErr error
afterCallbackErr error
afterPreparationErr error
validation promptexec.ValidationStatus
executeCalls int
providerCalls int
request promptexec.ExecuteRequest
beforeProvider func()
preparationDebug *promptexec.PreparationDebug
executionDebug *promptexec.ExecutionDebug
}
func (e *workflowExecutor) InspectPrompt(_ context.Context, id, version string) (promptexec.PromptInspection, error) {
if e.inspectionErr != nil {
return promptexec.PromptInspection{}, e.inspectionErr
}
if id != e.definition.PromptID || version != e.definition.PromptVersion {
return promptexec.PromptInspection{}, errors.New("unexpected prompt identity")
}
return validPromptInspection(e.definition), nil
}
func (e *workflowExecutor) InspectProfile(_ context.Context, id string) (promptexec.ProfileInspection, error) {
profile := e.profile
if profile.ProfileID == "" {
profile = promptexec.ProfileInspection{ProfileID: id, BackendID: "fixture", ModelName: "fixture-model"}
}
return profile, nil
}
func (e *workflowExecutor) Execute(_ context.Context, req promptexec.ExecuteRequest, callback promptexec.PreparationCallback) (*promptexec.Execution, error) {
e.executeCalls++
e.request = req
if e.beforePreparationErr != nil {
return nil, e.beforePreparationErr
}
stamp := time.Date(2026, 5, 29, 15, 0, 0, 0, time.UTC)
preparation := promptexec.Preparation{
PromptID: req.PromptID, PromptVersion: req.PromptVersion, PromptHash: "prompt-hash",
RenderedPromptHash: "rendered-hash", ProfileID: req.ProfileID, BackendID: "fixture",
ModelName: "fixture-model", DataPackagePath: req.DataPackagePath, StartedAt: stamp, EndedAt: stamp,
}
if err := callback(preparation, e.preparationDebug); err != nil {
return nil, err
}
if e.afterCallbackErr != nil {
return nil, e.afterCallbackErr
}
if e.beforeProvider != nil {
e.beforeProvider()
}
e.providerCalls++
if e.afterPreparationErr != nil {
return nil, e.afterPreparationErr
}
validation := e.validation
if validation == "" {
validation = promptexec.ValidationPassed
}
return &promptexec.Execution{
RunID: "provider-run", PromptID: req.PromptID, PromptVersion: req.PromptVersion,
PromptHash: "prompt-hash", RenderedPromptHash: "rendered-hash", ProfileID: req.ProfileID,
BackendID: "fixture", ModelName: "fixture-model", GeneratedHash: "generated-hash",
StartedAt: stamp, EndedAt: stamp, DataPackagePath: req.DataPackagePath, RawOutput: e.raw,
Debug: e.executionDebug,
Validation: promptexec.NewValidation(validation, "json_schema", e.definition.GeneratedTextSchemaID+".generated_text.schema.json", nil),
}, nil
}
type workflowNotifier struct {
requests []NotificationRequest
err error
}
func (n *workflowNotifier) Notify(_ context.Context, req NotificationRequest) (*NotificationResult, error) {
n.requests = append(n.requests, req)
if n.err != nil {
return nil, n.err
}
return &NotificationResult{
RunID: "notification-run", PipelineID: req.PipelineID, BundleID: req.BundleID,
IdempotencyKey: req.IdempotencyKey, Status: "succeeded", UploadStatus: "accepted",
}, nil
}
func TestGenerateDetailedCompletesRetainedReportWorkflows(t *testing.T) {
tests := []struct {
name string
kind ReportKind
id report.ID
date time.Time
raw string
wantOutput string
}{
{name: "daily", kind: ReportDaily, id: report.Daily, date: workflowTime("2026-05-29T12:00:00-05:00"), raw: validDailyWorkflowJSON(), wantOutput: "Showers are possible during the selected day."},
{name: "today", kind: ReportToday, id: report.Today, raw: validTodayWorkflowJSON(), wantOutput: "Today starts with showers before improving."},
{name: "tomorrow", kind: ReportTomorrow, id: report.Tomorrow, raw: validTomorrowWorkflowJSON(), wantOutput: "Tomorrow starts with showers before improving."},
{name: "hourly", kind: ReportHourly, id: report.Hourly, raw: validHourlyWorkflowJSON(), wantOutput: "Storm chances increase through late morning."},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
cfg := workflowConfig(t)
definition := report.DefaultRegistry().MustLookup(test.id)
bundle := workflowBundle(t)
collector := &workflowCollector{result: &collect.Result{Bundle: &bundle}}
executor := &workflowExecutor{definition: definition, raw: []byte(test.raw)}
notifier := &workflowNotifier{}
outputPath := filepath.Join(t.TempDir(), test.name+".md")
result, err := GenerateDetailed(context.Background(), GenerateRequest{
Config: cfg, Report: test.kind, Date: test.date, Now: workflowTime("2026-05-29T08:30:00-05:00"),
OutputPath: outputPath, Collector: collector, Executor: executor, Notifier: notifier,
})
if err != nil {
t.Fatalf("GenerateDetailed() error = %v", err)
}
if result.Metadata.ReportID != test.id || result.Metadata.PromptID != definition.PromptID {
t.Fatalf("metadata identity = %q/%q, want %q/%q", result.Metadata.ReportID, result.Metadata.PromptID, test.id, definition.PromptID)
}
if executor.request.PromptVersion != definition.PromptVersion {
t.Fatalf("prompt version = %q, want %q", executor.request.PromptVersion, definition.PromptVersion)
}
filesystem, storeErr := state.NewFilesystemStore(cfg.Workspace)
if storeErr != nil {
t.Fatalf("NewFilesystemStore() error = %v", storeErr)
}
preparation, loadErr := filesystem.LoadPromptPreparation(context.Background(), result.PreparationPath)
if loadErr != nil || preparation.PromptVersion != definition.PromptVersion {
t.Fatalf("persisted preparation prompt version = %q, error %v, want %q", preparation.PromptVersion, loadErr, definition.PromptVersion)
}
if collector.calls != 1 || executor.executeCalls != 1 || executor.providerCalls != 1 {
t.Fatalf("calls = collect %d execute %d provider %d, want one each", collector.calls, executor.executeCalls, executor.providerCalls)
}
persisted, readErr := os.ReadFile(result.DataPackagePath)
if readErr != nil {
t.Fatalf("read data package: %v", readErr)
}
if !bytes.Equal(executor.request.DataPackage, persisted) {
t.Fatal("executor data package differs from exact persisted YAML bytes")
}
managed, readErr := os.ReadFile(result.ReportPath)
if readErr != nil || !strings.Contains(string(managed), test.wantOutput) {
t.Fatalf("managed report = %q, error %v, want generated template output %q", managed, readErr, test.wantOutput)
}
copied, readErr := os.ReadFile(outputPath)
if readErr != nil || !bytes.Equal(copied, managed) || result.OutputPath != outputPath {
t.Fatalf("output copy mismatch/error/path = %v/%q", readErr, result.OutputPath)
}
if len(notifier.requests) != 1 || notifier.requests[0].ReportPath != result.ReportPath || notifier.requests[0].ReportPath == outputPath {
t.Fatalf("notification requests = %#v, want managed report source", notifier.requests)
}
wantPipeline := "reports." + string(test.id) + "." + definition.ArtifactGroup
if notifier.requests[0].PipelineID != wantPipeline {
t.Fatalf("pipeline = %q, want %q", notifier.requests[0].PipelineID, wantPipeline)
}
validDate := result.Metadata.ValidPeriod.Start.Format("2006-01-02")
wantBundlePaths := workflowBundlePaths(test.id, validDate, result.Metadata.RunID)
if strings.Join(notifier.requests[0].BundlePaths, "\n") != strings.Join(wantBundlePaths, "\n") {
t.Fatalf("bundle paths = %#v, want %#v", notifier.requests[0].BundlePaths, wantBundlePaths)
}
managedName := filepath.Base(result.ReportPath)
if !strings.HasPrefix(managedName, "report.") || !strings.Contains(managedName, "_"+test.name) || !strings.HasSuffix(managedName, ".md") || filepath.Base(result.OutputPath) != test.name+".md" {
t.Fatalf("output names = managed %q copy %q", result.ReportPath, result.OutputPath)
}
})
}
}
type preparationFailingStore struct {
state.Store
}
type renderContextFailingStore struct {
state.Store
}
func (s renderContextFailingStore) SaveModuleSnapshot(ctx context.Context, resolved report.Resolved, snapshot module.Snapshot) (string, error) {
path, err := s.Store.SaveModuleSnapshot(ctx, resolved, snapshot)
if err != nil {
return "", err
}
for index := range snapshot.Outputs {
snapshot.Outputs[index].Value = "invalid module value"
}
return path, nil
}
func (s preparationFailingStore) SavePromptPreparation(context.Context, report.Resolved, state.PromptPreparationArtifact) (string, error) {
return "", errors.New("injected preparation persistence failure")
}
func TestGenerateDetailedStopsAtConsequentialPromptFailures(t *testing.T) {
tests := []struct {
name string
configure func(*workflowExecutor)
wantCategory promptexec.ErrorCategory
wantPreparation bool
wantExecution bool
wantRaw bool
wantProviderCall int
}{
{name: "preparation", configure: func(e *workflowExecutor) {
e.beforePreparationErr = promptexec.NewError(promptexec.Generation, "preparation failed", nil)
}, wantCategory: promptexec.Generation, wantPreparation: true},
{name: "credential disappears", configure: func(e *workflowExecutor) {
e.afterCallbackErr = promptexec.NewError(promptexec.MissingCredential, "credential unavailable", nil)
}, wantCategory: promptexec.MissingCredential, wantPreparation: true, wantExecution: true},
{name: "capacity is not retried", configure: func(e *workflowExecutor) {
e.afterPreparationErr = promptexec.NewError(promptexec.Capacity, "capacity rejected", nil)
}, wantCategory: promptexec.Capacity, wantPreparation: true, wantExecution: true, wantProviderCall: 1},
{name: "canceled", configure: func(e *workflowExecutor) {
e.afterPreparationErr = promptexec.NewError(promptexec.Canceled, "request canceled", context.Canceled)
}, wantCategory: promptexec.Canceled, wantPreparation: true, wantExecution: true, wantProviderCall: 1},
{name: "deadline", configure: func(e *workflowExecutor) {
e.afterPreparationErr = promptexec.NewError(promptexec.DeadlineExceeded, "deadline exceeded", context.DeadlineExceeded)
}, wantCategory: promptexec.DeadlineExceeded, wantPreparation: true, wantExecution: true, wantProviderCall: 1},
{name: "generation", configure: func(e *workflowExecutor) {
e.afterPreparationErr = promptexec.NewError(promptexec.Generation, "generation failed", nil)
}, wantCategory: promptexec.Generation, wantPreparation: true, wantExecution: true, wantProviderCall: 1},
{name: "operational validation error", configure: func(e *workflowExecutor) {
e.afterPreparationErr = promptexec.NewError(promptexec.OperationalValidation, "validator failed", nil)
}, wantCategory: promptexec.OperationalValidation, wantPreparation: true, wantExecution: true, wantProviderCall: 1},
{name: "operational validation incomplete", configure: func(e *workflowExecutor) { e.validation = promptexec.ValidationSkipped }, wantCategory: promptexec.OperationalValidation, wantPreparation: true, wantExecution: true, wantProviderCall: 1},
{name: "schema rejection", configure: func(e *workflowExecutor) { e.validation = promptexec.ValidationFailed }, wantCategory: promptexec.ValidationRejected, wantPreparation: true, wantExecution: true, wantRaw: true, wantProviderCall: 1},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
cfg := workflowConfig(t)
cfg.Notify.Distributor.Enabled = false
definition := report.DefaultRegistry().MustLookup(report.Daily)
executor := &workflowExecutor{definition: definition, raw: []byte(validDailyWorkflowJSON())}
test.configure(executor)
bundle := workflowBundle(t)
collector := &workflowCollector{result: &collect.Result{Bundle: &bundle}}
result, err := GenerateDetailed(context.Background(), GenerateRequest{
Config: cfg, Report: ReportDaily, Date: workflowTime("2026-05-29T12:00:00-05:00"), Now: workflowTime("2026-05-29T08:30:00-05:00"),
Collector: collector, Executor: executor,
})
if err == nil || result == nil || promptexec.CategoryOf(err) != test.wantCategory {
t.Fatalf("result/error/category = %#v/%v/%q, want partial result and %q", result, err, promptexec.CategoryOf(err), test.wantCategory)
}
if (result.PreparationPath != "") != test.wantPreparation || (result.ExecutionPath != "") != test.wantExecution || (result.GeneratedTextRawPath != "") != test.wantRaw {
t.Fatalf("paths = preparation %q execution %q raw %q", result.PreparationPath, result.ExecutionPath, result.GeneratedTextRawPath)
}
if executor.executeCalls != 1 || executor.providerCalls != test.wantProviderCall {
t.Fatalf("calls = execute %d provider %d, want 1/%d", executor.executeCalls, executor.providerCalls, test.wantProviderCall)
}
})
}
}
func TestGenerateDetailedRejectsInspectionAndCredentialsBeforeCollection(t *testing.T) {
tests := []struct {
name string
configure func(*workflowExecutor)
wantCategory promptexec.ErrorCategory
}{
{name: "inspection", configure: func(e *workflowExecutor) { e.inspectionErr = errors.New("inspection unavailable") }, wantCategory: promptexec.InvalidConfiguration},
{name: "credential", configure: func(e *workflowExecutor) {
e.profile = promptexec.ProfileInspection{ProfileID: "default-profile", CredentialRequired: true}
}, wantCategory: promptexec.MissingCredential},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
cfg := workflowConfig(t)
definition := report.DefaultRegistry().MustLookup(report.Daily)
executor := &workflowExecutor{definition: definition}
test.configure(executor)
collector := &workflowCollector{err: errors.New("collector must not run")}
result, err := GenerateDetailed(context.Background(), GenerateRequest{
Config: cfg, Report: ReportDaily, Date: workflowTime("2026-05-29T12:00:00-05:00"), Now: workflowTime("2026-05-29T08:30:00-05:00"),
Collector: collector, Executor: executor,
})
if err == nil || result != nil || promptexec.CategoryOf(err) != test.wantCategory || collector.calls != 0 || executor.executeCalls != 0 {
t.Fatalf("result/error/category/collect/execute = %#v/%v/%q/%d/%d", result, err, promptexec.CategoryOf(err), collector.calls, executor.executeCalls)
}
entries, readErr := os.ReadDir(cfg.Workspace.Root)
if readErr != nil || len(entries) != 0 {
t.Fatalf("workspace entries/error = %#v/%v, want no writes before collection", entries, readErr)
}
})
}
}
func TestGenerateDetailedStopsProviderWhenPreparationCannotPersist(t *testing.T) {
cfg := workflowConfig(t)
cfg.Notify.Distributor.Enabled = false
filesystem, err := state.NewFilesystemStore(cfg.Workspace)
if err != nil {
t.Fatalf("NewFilesystemStore() error = %v", err)
}
definition := report.DefaultRegistry().MustLookup(report.Daily)
executor := &workflowExecutor{definition: definition, raw: []byte(validDailyWorkflowJSON())}
bundle := workflowBundle(t)
result, err := GenerateDetailed(context.Background(), GenerateRequest{
Config: cfg, Report: ReportDaily, Date: workflowTime("2026-05-29T12:00:00-05:00"), Now: workflowTime("2026-05-29T08:30:00-05:00"),
Collector: &workflowCollector{result: &collect.Result{Bundle: &bundle}}, Executor: executor, Store: preparationFailingStore{Store: filesystem},
})
if err == nil || result == nil || result.PreparationPath != "" || executor.providerCalls != 0 {
t.Fatalf("result/error/preparation/provider = %#v/%v/%q/%d", result, err, result.PreparationPath, executor.providerCalls)
}
}
func TestGenerateDetailedPersistsPreparationBeforeProviderExecution(t *testing.T) {
cfg := workflowConfig(t)
cfg.Notify.Distributor.Enabled = false
now := workflowTime("2026-05-29T08:30:00-05:00")
request := GenerateRequest{Config: cfg, Report: ReportDaily, Date: workflowTime("2026-05-29T12:00:00-05:00"), Now: now}
resolved, err := ResolveGenerate(request, now)
if err != nil {
t.Fatalf("ResolveGenerate() error = %v", err)
}
filesystem, err := state.NewFilesystemStore(cfg.Workspace)
if err != nil {
t.Fatalf("NewFilesystemStore() error = %v", err)
}
paths, err := filesystem.Paths(resolved)
if err != nil {
t.Fatalf("Paths() error = %v", err)
}
checked := false
executor := &workflowExecutor{definition: resolved.Definition, raw: []byte(validDailyWorkflowJSON())}
executor.beforeProvider = func() {
checked = true
if _, statErr := os.Stat(paths.Preparation); statErr != nil {
t.Fatalf("preparation was not durable before provider execution: %v", statErr)
}
}
bundle := workflowBundle(t)
request.Collector = &workflowCollector{result: &collect.Result{Bundle: &bundle}}
request.Executor = executor
request.Store = filesystem
result, err := GenerateDetailed(context.Background(), request)
if err != nil || result == nil || !checked || result.OutputPath != "" {
t.Fatalf("result/error/checked/output = %#v/%v/%t/%q", result, err, checked, result.OutputPath)
}
}
func TestGenerateDetailedRetainsInspectableArtifactsAfterApplicationFailures(t *testing.T) {
tests := []struct {
name string
raw string
configure func(*GenerateRequest, *workflowNotifier)
wantRaw bool
wantNormalized bool
wantContext bool
wantReport bool
wantOutput bool
wantNotify bool
}{
{name: "generated text decode", raw: `{`, wantRaw: true},
{name: "generated text domain", raw: `{}`, wantRaw: true},
{name: "render context build", raw: validDailyWorkflowJSON(), configure: func(req *GenerateRequest, _ *workflowNotifier) {
req.Store = renderContextFailingStore{Store: req.Store}
}, wantRaw: true, wantNormalized: true},
{name: "render context persistence", raw: validDailyWorkflowJSON(), configure: func(req *GenerateRequest, _ *workflowNotifier) {
req.Store = &failingPersistenceStore{Store: req.Store, failOperation: failRenderContext}
}, wantRaw: true, wantNormalized: true},
{name: "template write", raw: validDailyWorkflowJSON(), configure: func(req *GenerateRequest, _ *workflowNotifier) {
blocker := filepath.Join(t.TempDir(), "report-blocker")
if err := os.Mkdir(blocker, 0o700); err != nil {
t.Fatalf("create report blocker: %v", err)
}
req.Store = &failingPersistenceStore{Store: req.Store, failOperation: failRenderedReportPath, renderedReportPath: blocker}
}, wantRaw: true, wantNormalized: true, wantContext: true},
{name: "output copy", raw: validDailyWorkflowJSON(), configure: func(req *GenerateRequest, _ *workflowNotifier) {
req.OutputPath = t.TempDir()
}, wantRaw: true, wantNormalized: true, wantContext: true, wantReport: true},
{name: "notification", raw: validDailyWorkflowJSON(), configure: func(_ *GenerateRequest, notifier *workflowNotifier) {
notifier.err = errors.New("notification rejected")
}, wantRaw: true, wantNormalized: true, wantContext: true, wantReport: true, wantOutput: true, wantNotify: true},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
cfg := workflowConfig(t)
definition := report.DefaultRegistry().MustLookup(report.Daily)
executor := &workflowExecutor{definition: definition, raw: []byte(test.raw)}
notifier := &workflowNotifier{}
bundle := workflowBundle(t)
filesystem, err := state.NewFilesystemStore(cfg.Workspace)
if err != nil {
t.Fatalf("NewFilesystemStore() error = %v", err)
}
req := GenerateRequest{
Config: cfg, Report: ReportDaily, Date: workflowTime("2026-05-29T12:00:00-05:00"), Now: workflowTime("2026-05-29T08:30:00-05:00"),
OutputPath: filepath.Join(t.TempDir(), "daily.md"), Collector: &workflowCollector{result: &collect.Result{Bundle: &bundle}},
Executor: executor, Notifier: notifier, Store: filesystem,
}
if test.configure != nil {
test.configure(&req, notifier)
}
result, err := GenerateDetailed(context.Background(), req)
if err == nil || result == nil {
t.Fatalf("result/error = %#v/%v, want partial result and error", result, err)
}
if (result.GeneratedTextRawPath != "") != test.wantRaw || (result.GeneratedTextPath != "") != test.wantNormalized ||
(result.RenderContextPath != "") != test.wantContext || (result.ReportPath != "") != test.wantReport ||
(result.OutputPath != "") != test.wantOutput || (result.NotificationPath != "") != test.wantNotify {
t.Fatalf("reached paths = raw %q normalized %q context %q report %q output %q notification %q", result.GeneratedTextRawPath, result.GeneratedTextPath, result.RenderContextPath, result.ReportPath, result.OutputPath, result.NotificationPath)
}
if test.wantRaw {
persisted, readErr := os.ReadFile(result.GeneratedTextRawPath)
if readErr != nil || !bytes.Equal(persisted, []byte(test.raw)) {
t.Fatalf("retained raw output = %q, error %v", persisted, readErr)
}
}
if test.name == "notification" && (len(notifier.requests) != 1 || notifier.requests[0].ReportPath != result.ReportPath) {
t.Fatalf("notification requests = %#v", notifier.requests)
}
})
}
}
func TestGenerateDetailedDebugFailuresRespectProviderBoundary(t *testing.T) {
tests := []struct {
name string
createCollision func(string, report.Resolved) error
wantProviderCalls int
wantPreparationFile bool
}{
{
name: "preparation debug",
createCollision: func(root string, resolved report.Resolved) error {
path := workflowDebugRunPath(root, resolved)
if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
return err
}
return os.WriteFile(path, []byte("not a directory"), 0o600)
},
},
{
name: "execution debug", wantProviderCalls: 1, wantPreparationFile: true,
createCollision: func(root string, resolved report.Resolved) error {
return os.MkdirAll(filepath.Join(workflowDebugRunPath(root, resolved), "execution.json"), 0o700)
},
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
cfg := workflowConfig(t)
cfg.Notify.Distributor.Enabled = false
now := workflowTime("2026-05-29T08:30:00-05:00")
request := GenerateRequest{Config: cfg, Report: ReportDaily, Date: workflowTime("2026-05-29T12:00:00-05:00"), Now: now}
resolved, err := ResolveGenerate(request, now)
if err != nil {
t.Fatalf("ResolveGenerate() error = %v", err)
}
debugRoot := filepath.Join(t.TempDir(), "prompt-debug")
if err := test.createCollision(debugRoot, resolved); err != nil {
t.Fatalf("create debug collision: %v", err)
}
bundle := workflowBundle(t)
executor := &workflowExecutor{definition: resolved.Definition, raw: []byte(validDailyWorkflowJSON())}
request.Collector = &workflowCollector{result: &collect.Result{Bundle: &bundle}}
request.Executor = executor
request.LLMDebugDir = debugRoot
result, err := GenerateDetailed(context.Background(), request)
if err == nil || result == nil || promptexec.CategoryOf(err) != promptexec.InvalidConfiguration {
t.Fatalf("result/error/category = %#v/%v/%q", result, err, promptexec.CategoryOf(err))
}
if executor.providerCalls != test.wantProviderCalls || result.PreparationPath == "" || result.ExecutionPath != "" || result.GeneratedTextRawPath != "" {
t.Fatalf("provider/preparation/metadata/execution/raw = %d/%q/%q/%q/%q", executor.providerCalls, result.PreparationPath, result.MetadataPath, result.ExecutionPath, result.GeneratedTextRawPath)
}
if test.wantPreparationFile && result.MetadataPath == "" {
t.Fatal("execution debug failure lost previously persisted metadata")
}
preparationDebug := filepath.Join(workflowDebugRunPath(debugRoot, resolved), "preparation.json")
_, statErr := os.Stat(preparationDebug)
if (statErr == nil) != test.wantPreparationFile {
t.Fatalf("preparation debug stat error = %v, want file %t", statErr, test.wantPreparationFile)
}
})
}
}
func workflowDebugRunPath(root string, resolved report.Resolved) string {
return filepath.Join(root, string(resolved.Definition.ID), resolved.ValidPeriod.Start.Format("2006-01-02"), resolved.Metadata().RunID)
}
func workflowBundlePaths(id report.ID, validDate, runID string) []string {
switch id {
case report.Daily:
return []string{"daily/" + validDate + "/" + runID + ".md", "daily/" + validDate + "/index.md"}
case report.Today:
return []string{"daily/" + validDate + "/" + runID + ".md", "daily/" + validDate + "/index.md", "today/index.md"}
case report.Tomorrow:
return []string{"daily/" + validDate + "/" + runID + ".md", "daily/" + validDate + "/index.md", "tomorrow/index.md"}
case report.Hourly:
return []string{"hourly/index.md"}
default:
return nil
}
}
func TestGenerateDetailedSelectsPriorSnapshotsForRetainedReports(t *testing.T) {
tests := []struct {
name string
kind ReportKind
id report.ID
date time.Time
raw string
wantPrior bool
wantRecentChanges bool
}{
{name: "daily", kind: ReportDaily, id: report.Daily, date: workflowTime("2026-05-29T12:00:00-05:00"), raw: validDailyWorkflowJSON(), wantPrior: true, wantRecentChanges: true},
{name: "today", kind: ReportToday, id: report.Today, raw: validTodayWorkflowJSON(), wantPrior: true, wantRecentChanges: true},
{name: "tomorrow", kind: ReportTomorrow, id: report.Tomorrow, raw: validTomorrowWorkflowJSON(), wantPrior: true, wantRecentChanges: true},
{name: "hourly", kind: ReportHourly, id: report.Hourly, raw: validHourlyWorkflowJSON()},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
cfg := workflowConfig(t)
cfg.Notify.Distributor.Enabled = false
definition := report.DefaultRegistry().MustLookup(test.id)
firstBundle := workflowBundle(t)
setWorkflowTemperatures(&firstBundle, 45)
first, err := GenerateDetailed(context.Background(), GenerateRequest{
Config: cfg, Report: test.kind, Date: test.date, Now: workflowTime("2026-05-29T08:00:00-05:00"),
Collector: &workflowCollector{result: &collect.Result{Bundle: &firstBundle}},
Executor: &workflowExecutor{definition: definition, raw: []byte(test.raw)},
})
if err != nil {
t.Fatalf("first GenerateDetailed() error = %v", err)
}
secondBundle := workflowBundle(t)
setWorkflowTemperatures(&secondBundle, 85)
second, err := GenerateDetailed(context.Background(), GenerateRequest{
Config: cfg, Report: test.kind, Date: test.date, Now: workflowTime("2026-05-29T08:30:00-05:00"),
Collector: &workflowCollector{result: &collect.Result{Bundle: &secondBundle}},
Executor: &workflowExecutor{definition: definition, raw: []byte(test.raw)},
})
if err != nil {
t.Fatalf("second GenerateDetailed() error = %v", err)
}
if test.wantPrior && (second.PriorSnapshot == nil || second.PriorSnapshot.Metadata.RunID != first.Metadata.RunID || second.PriorSnapshot.Metadata.ReportID != test.id) {
t.Fatalf("prior snapshot = %#v, want first %s run %q", second.PriorSnapshot, test.id, first.Metadata.RunID)
}
if !test.wantPrior && second.PriorSnapshot != nil {
t.Fatalf("prior snapshot = %#v, want none for non-overlapping rolling window", second.PriorSnapshot)
}
if (len(second.RecentChanges) > 0) != test.wantRecentChanges {
t.Fatalf("recent changes = %#v, want present %t", second.RecentChanges, test.wantRecentChanges)
}
if (len(second.DataPackage.RecentChanges.Items) > 0) != test.wantRecentChanges {
t.Fatalf("data package recent changes = %#v, want present %t", second.DataPackage.RecentChanges.Items, test.wantRecentChanges)
}
})
}
}
func setWorkflowTemperatures(bundle *weatherdata.Bundle, temperature float64) {
for index := range bundle.Hourly.Periods {
value := temperature
bundle.Hourly.Periods[index].TemperatureF = &value
}
}
func workflowConfig(t *testing.T) config.Config {
t.Helper()
cfg := config.Defaults()
cfg.Workspace.Root = t.TempDir()
cfg.WeatherAPI.Timezone = "America/Chicago"
cfg.Location.ID = "home"
cfg.Location.Name = "Testville"
cfg.Location.Region = "MO"
cfg.Notify.Distributor.Enabled = true
cfg.Notify.Distributor.PipelineIDTemplate = "reports.{report_id}.{artifact_group}"
return cfg
}
func workflowBundle(t *testing.T) weatherdata.Bundle {
t.Helper()
data, err := os.ReadFile(filepath.Join("..", "forecast", "testdata", "daily_bundle.json"))
if err != nil {
t.Fatalf("read bundle fixture: %v", err)
}
var bundle weatherdata.Bundle
if err := json.Unmarshal(data, &bundle); err != nil {
t.Fatalf("decode bundle fixture: %v", err)
}
future := bundle.Hourly.Periods[0]
future.StartTime = workflowTime("2026-05-30T06:00:00-05:00")
future.EndTime = workflowTime("2026-05-30T07:00:00-05:00")
bundle.Hourly.Periods = append(bundle.Hourly.Periods, future)
futureNarrative := bundle.Narrative.Periods[0]
futureNarrative.StartTime = workflowTime("2026-05-30T06:00:00-05:00")
futureNarrative.EndTime = workflowTime("2026-05-30T18:00:00-05:00")
futureNarrative.Name = "Tomorrow"
bundle.Narrative.Periods = append(bundle.Narrative.Periods, futureNarrative)
return bundle
}
func workflowTime(value string) time.Time {
parsed, err := time.Parse(time.RFC3339, value)
if err != nil {
panic(err)
}
return parsed
}
func validHourlyWorkflowJSON() string {
return `{"summary":"Storm chances increase through late morning.","forecast_discussion":"A front will keep the region unsettled.","precipitation_timing":"A cold front is moving into the region.","confidence":"Medium"}`
}
func validTomorrowWorkflowJSON() string {
return `{"summary":"Tomorrow starts with showers before improving.","forecast_discussion":["Morning showers should taper as drier air arrives.","Afternoon conditions trend quieter."],"precipitation_timing":"The best rain chance is during the morning."}`
}
func validTodayWorkflowJSON() string {
return `{"summary":"Today starts with showers before improving.","forecast_discussion":["Morning showers should taper as drier air arrives.","Afternoon conditions trend quieter."],"precipitation_timing":"The best rain chance is during the morning."}`
}
func validDailyWorkflowJSON() string {
return `{"summary":"Showers are possible during the selected day.","forecast_discussion":["A front will keep rain chances in the forecast.","Temperatures stay seasonable by afternoon."],"precipitation_timing":"Rain is most likely during the afternoon.","confidence":"Medium"}`
}

View File

@@ -1,313 +0,0 @@
// Package changes compares structured module snapshots.
package changes
import (
"fmt"
"math"
"sort"
"strconv"
"strings"
"time"
"gitea.maximumdirect.net/eric/weatherreporter/internal/module"
)
type Thresholds struct {
TemperatureDegrees float64
PrecipProbabilityPoints int
WindGustMilesPerHour int
PrecipTimingShiftMinutes int
}
type Change struct {
Type string `json:"type"`
Message string `json:"message"`
Previous string `json:"previous,omitempty"`
Current string `json:"current,omitempty"`
}
func CompareDaily(previous module.Snapshot, current module.Snapshot, thresholds Thresholds) ([]Change, error) {
previousSummary, err := requiredStanza[dailySummaryStanza](previous, "derived_daily_summary")
if err != nil {
return nil, fmt.Errorf("previous daily summary: %w", err)
}
currentSummary, err := requiredStanza[dailySummaryStanza](current, "derived_daily_summary")
if err != nil {
return nil, fmt.Errorf("current daily summary: %w", err)
}
previousDayparts, err := requiredStanza[map[string]daypartSummaryStanza](previous, "derived_daypart_summaries")
if err != nil {
return nil, fmt.Errorf("previous daypart summaries: %w", err)
}
currentDayparts, err := requiredStanza[map[string]daypartSummaryStanza](current, "derived_daypart_summaries")
if err != nil {
return nil, fmt.Errorf("current daypart summaries: %w", err)
}
previousAlerts, _, err := module.StanzaValue[alertDigestStanza](previous, "alert_digest")
if err != nil {
return nil, err
}
currentAlerts, _, err := module.StanzaValue[alertDigestStanza](current, "alert_digest")
if err != nil {
return nil, err
}
previousTiming, previousHasTiming, err := module.StanzaValue[precipTimingStanza](previous, "precip_timing")
if err != nil {
return nil, err
}
currentTiming, currentHasTiming, err := module.StanzaValue[precipTimingStanza](current, "precip_timing")
if err != nil {
return nil, err
}
var changes []Change
changes = append(changes, compareTemperatureValues("Low", previousSummary.LowTempF, currentSummary.LowTempF, thresholds.TemperatureDegrees)...)
changes = append(changes, compareTemperatureValues("High", previousSummary.HighTempF, currentSummary.HighTempF, thresholds.TemperatureDegrees)...)
changes = append(changes, comparePrecipitationValues(previousSummary.DailyPrecipitationProbability, currentSummary.DailyPrecipitationProbability, thresholds.PrecipProbabilityPoints, "")...)
if previousHasTiming && currentHasTiming {
changes = append(changes, comparePrecipTiming(previousTiming.MaxPopTime, currentTiming.MaxPopTime, thresholds.PrecipTimingShiftMinutes, "")...)
}
changes = append(changes, compareWindValues(previousSummary.MaxWindGustMph, currentSummary.MaxWindGustMph, thresholds.WindGustMilesPerHour, "")...)
changes = append(changes, compareAlerts(previousAlerts.Relevant, currentAlerts.Relevant)...)
changes = append(changes, compareIndicators(aggregateIndicators(previousDayparts), aggregateIndicators(currentDayparts), "")...)
sortChanges(changes)
return changes, nil
}
type dailySummaryStanza struct {
Date string `json:"date,omitempty"`
HighTempF *int `json:"high_temp_f,omitempty"`
LowTempF *int `json:"low_temp_f,omitempty"`
DailyPrecipitationProbability *int `json:"daily_precipitation_probability,omitempty"`
MaxWindGustMph *int `json:"max_wind_gust_mph,omitempty"`
}
type daypartSummaryStanza struct {
Date string `json:"date,omitempty"`
PeriodBegins string `json:"period_begins,omitempty"`
PeriodEnds string `json:"period_ends,omitempty"`
TempRangeF string `json:"temp_range_f,omitempty"`
MaxPopPercent *int `json:"max_pop_percent,omitempty"`
MaxPopTime string `json:"max_pop_time,omitempty"`
MaxWindGustMph *int `json:"max_wind_gust_mph,omitempty"`
Snow bool `json:"snow,omitempty"`
Ice bool `json:"ice,omitempty"`
}
type precipTimingStanza struct {
MaxPopPercent *int `json:"max_pop_percent,omitempty"`
MaxPopTime string `json:"max_pop_time,omitempty"`
}
type alertDigestStanza struct {
Relevant []alertSummaryStanza `json:"relevant,omitempty"`
}
type alertSummaryStanza struct {
Event string `json:"event,omitempty"`
Headline string `json:"headline,omitempty"`
}
type indicators struct {
Snow bool
Ice bool
}
func requiredStanza[T any](snapshot module.Snapshot, name string) (T, error) {
value, ok, err := module.StanzaValue[T](snapshot, name)
if err != nil {
return value, err
}
if !ok {
return value, fmt.Errorf("stanza %q is required", name)
}
return value, nil
}
func compareTemperatureValues(label string, previous *int, current *int, threshold float64) []Change {
if previous == nil || current == nil {
return nil
}
if !differenceAtLeast(float64(*previous), float64(*current), threshold) {
return nil
}
return []Change{{
Type: "temperature_shift",
Message: fmt.Sprintf("%s temperature changed from %d to %d.", label, *previous, *current),
Previous: fmt.Sprintf("%d", *previous),
Current: fmt.Sprintf("%d", *current),
}}
}
func comparePrecipitationValues(previous *int, current *int, threshold int, prefix string) []Change {
if previous == nil || current == nil {
return nil
}
previousCategory := precipitationCategory(float64(*previous))
currentCategory := precipitationCategory(float64(*current))
if previousCategory == currentCategory && !differenceAtLeast(float64(*previous), float64(*current), float64(threshold)) {
return nil
}
changeType := prefix + "precip_probability_change"
return []Change{{
Type: changeType,
Message: fmt.Sprintf("Peak precipitation chance changed from %d%% (%s) to %d%% (%s).", *previous, previousCategory, *current, currentCategory),
Previous: fmt.Sprintf("%d%% %s", *previous, previousCategory),
Current: fmt.Sprintf("%d%% %s", *current, currentCategory),
}}
}
func comparePrecipTiming(previous string, current string, thresholdMinutes int, prefix string) []Change {
if thresholdMinutes <= 0 || previous == "" || current == "" || previous == current {
return nil
}
previousTime, previousOK := parseClock(previous)
currentTime, currentOK := parseClock(current)
if !previousOK || !currentOK {
return nil
}
if int(math.Abs(currentTime.Sub(previousTime).Minutes())) < thresholdMinutes {
return nil
}
return []Change{{
Type: prefix + "precip_timing_shift",
Message: fmt.Sprintf("Peak precipitation timing shifted from %s to %s.", previous, current),
Previous: previous,
Current: current,
}}
}
func compareWindValues(previous *int, current *int, threshold int, prefix string) []Change {
if previous == nil || current == nil || !differenceAtLeast(float64(*previous), float64(*current), float64(threshold)) {
return nil
}
return []Change{{
Type: prefix + "wind_gust_change",
Message: fmt.Sprintf("Peak wind gust changed from %d mph to %d mph.", *previous, *current),
Previous: fmt.Sprintf("%d mph", *previous),
Current: fmt.Sprintf("%d mph", *current),
}}
}
func compareAlerts(previous []alertSummaryStanza, current []alertSummaryStanza) []Change {
previousSet := alertSet(previous)
currentSet := alertSet(current)
var changes []Change
for event := range currentSet {
if _, ok := previousSet[event]; !ok {
changes = append(changes, Change{Type: "alert_added", Message: fmt.Sprintf("Alert added: %s.", event), Current: event})
}
}
for event := range previousSet {
if _, ok := currentSet[event]; !ok {
changes = append(changes, Change{Type: "alert_removed", Message: fmt.Sprintf("Alert removed: %s.", event), Previous: event})
}
}
sortChanges(changes)
return changes
}
func compareIndicators(previous indicators, current indicators, prefix string) []Change {
var changes []Change
for _, item := range []struct {
name string
previous bool
current bool
}{
{name: "snow", previous: previous.Snow, current: current.Snow},
{name: "ice", previous: previous.Ice, current: current.Ice},
} {
if item.previous == item.current {
continue
}
changeType := prefix + item.name + "_risk_change"
if item.current {
changes = append(changes, Change{Type: changeType, Message: fmt.Sprintf("%s risk is now present.", item.name), Current: "present"})
} else {
changes = append(changes, Change{Type: changeType, Message: fmt.Sprintf("%s risk is no longer present.", item.name), Previous: "present"})
}
}
return changes
}
func aggregateIndicators(dayparts map[string]daypartSummaryStanza) indicators {
out := indicators{}
for _, daypart := range dayparts {
out.Snow = out.Snow || daypart.Snow
out.Ice = out.Ice || daypart.Ice
}
return out
}
func alertSet(alerts []alertSummaryStanza) map[string]struct{} {
out := map[string]struct{}{}
for _, alert := range alerts {
event := alert.Event
if event == "" {
event = alert.Headline
}
if event != "" {
out[event] = struct{}{}
}
}
return out
}
func precipitationCategory(value float64) string {
switch {
case value >= 70:
return "high"
case value >= 50:
return "likely"
case value >= 20:
return "possible"
default:
return "low"
}
}
func differenceAtLeast(previous float64, current float64, threshold float64) bool {
if threshold <= 0 {
return previous != current
}
return math.Abs(current-previous) >= threshold
}
func sortChanges(items []Change) {
sort.SliceStable(items, func(i, j int) bool {
if items[i].Type == items[j].Type {
return items[i].Message < items[j].Message
}
return items[i].Type < items[j].Type
})
}
func parseClock(value string) (time.Time, bool) {
value = strings.TrimSpace(value)
for _, layout := range []string{"3 PM", "3:04 PM", "15:04"} {
if parsed, err := time.Parse(layout, value); err == nil {
return parsed, true
}
}
return time.Time{}, false
}
func parseTempRange(value string) (*int, *int) {
value = strings.TrimSpace(value)
if value == "" {
return nil, nil
}
parts := strings.Split(value, "-")
if len(parts) == 1 {
if parsed, err := strconv.Atoi(strings.TrimSpace(parts[0])); err == nil {
return &parsed, &parsed
}
return nil, nil
}
minValue, minErr := strconv.Atoi(strings.TrimSpace(parts[0]))
maxValue, maxErr := strconv.Atoi(strings.TrimSpace(parts[len(parts)-1]))
if minErr != nil || maxErr != nil {
return nil, nil
}
return &minValue, &maxValue
}

View File

@@ -1,138 +0,0 @@
package changes
import (
"strings"
"testing"
"gitea.maximumdirect.net/eric/weatherreporter/internal/module"
)
func TestCompareDailyNoMeaningfulChanges(t *testing.T) {
previous := dailySnapshot(t, 60, 70, 30, "8 AM", nil, false)
current := dailySnapshot(t, 61, 71, 35, "8:30 AM", nil, false)
changes, err := CompareDaily(previous, current, testThresholds())
if err != nil {
t.Fatalf("CompareDaily() error = %v", err)
}
if len(changes) != 0 {
t.Fatalf("changes = %#v, want none", changes)
}
}
func TestCompareDailyTemperatureThreshold(t *testing.T) {
previous := dailySnapshot(t, 50, 70, 10, "8 AM", nil, false)
current := dailySnapshot(t, 58, 79, 10, "8 AM", nil, false)
changes, err := CompareDaily(previous, current, testThresholds())
if err != nil {
t.Fatalf("CompareDaily() error = %v", err)
}
if countType(changes, "temperature_shift") != 2 {
t.Fatalf("changes = %#v, want low and high temperature changes", changes)
}
}
func TestCompareDailyPrecipTimingShift(t *testing.T) {
previous := dailySnapshot(t, 60, 70, 60, "8 AM", nil, false)
current := dailySnapshot(t, 60, 70, 60, "11 AM", nil, false)
changes, err := CompareDaily(previous, current, testThresholds())
if err != nil {
t.Fatalf("CompareDaily() error = %v", err)
}
if countType(changes, "precip_timing_shift") != 1 {
t.Fatalf("changes = %#v, want timing shift", changes)
}
}
func TestCompareDailyAlertAddedAndRemoved(t *testing.T) {
previous := dailySnapshot(t, 60, 70, 10, "8 AM", []string{"Wind Advisory"}, false)
current := dailySnapshot(t, 60, 70, 10, "8 AM", []string{"Flood Watch"}, false)
changes, err := CompareDaily(previous, current, testThresholds())
if err != nil {
t.Fatalf("CompareDaily() error = %v", err)
}
if countType(changes, "alert_added") != 1 || countType(changes, "alert_removed") != 1 {
t.Fatalf("changes = %#v, want one alert added and one removed", changes)
}
}
func TestCompareDailyIndicatorChange(t *testing.T) {
previous := dailySnapshot(t, 60, 70, 10, "8 AM", nil, false)
current := dailySnapshot(t, 60, 70, 10, "8 AM", nil, true)
changes, err := CompareDaily(previous, current, testThresholds())
if err != nil {
t.Fatalf("CompareDaily() error = %v", err)
}
if countType(changes, "snow_risk_change") != 1 {
t.Fatalf("changes = %#v, want snow risk change", changes)
}
}
func TestCompareDailyRequiresComparisonStanzas(t *testing.T) {
_, err := CompareDaily(snapshot(t), dailySnapshot(t, 60, 70, 10, "8 AM", nil, false), testThresholds())
if err == nil {
t.Fatal("CompareDaily() error = nil, want missing stanza error")
}
if !strings.Contains(err.Error(), "derived_daily_summary") {
t.Fatalf("error = %q, want derived_daily_summary context", err.Error())
}
}
func dailySnapshot(t *testing.T, low int, high int, precip int, precipTime string, alerts []string, snow bool) module.Snapshot {
t.Helper()
relevant := make([]alertSummaryStanza, 0, len(alerts))
for _, alert := range alerts {
relevant = append(relevant, alertSummaryStanza{Event: alert})
}
return snapshot(t,
module.Output{ID: module.DerivedDailySummary, StanzaName: "derived_daily_summary", Value: dailySummaryStanza{
Date: "2026-05-29",
HighTempF: &high,
LowTempF: &low,
DailyPrecipitationProbability: &precip,
}},
module.Output{ID: module.DerivedDaypartSummaries, StanzaName: "derived_daypart_summaries", Value: map[string]daypartSummaryStanza{
"morning": {
Date: "2026-05-29",
PeriodBegins: "2026-05-29 at 6:00 AM",
PeriodEnds: "2026-05-29 at 10:00 AM",
TempRangeF: "60-70",
Snow: snow,
},
}},
module.Output{ID: module.PrecipTiming, StanzaName: "precip_timing", Value: precipTimingStanza{MaxPopPercent: &precip, MaxPopTime: precipTime}},
module.Output{ID: module.AlertDigest, StanzaName: "alert_digest", Value: alertDigestStanza{Relevant: relevant}},
)
}
func snapshot(t *testing.T, outputs ...module.Output) module.Snapshot {
t.Helper()
snapshot, err := module.NewSnapshot(outputs)
if err != nil {
t.Fatalf("NewSnapshot() error = %v", err)
}
return snapshot
}
func testThresholds() Thresholds {
return Thresholds{
TemperatureDegrees: 5,
PrecipProbabilityPoints: 20,
WindGustMilesPerHour: 10,
PrecipTimingShiftMinutes: 120,
}
}
func countType(changes []Change, changeType string) int {
var count int
for _, change := range changes {
if change.Type == changeType {
count++
}
}
return count
}

View File

@@ -0,0 +1,368 @@
package cli
import (
"bytes"
"context"
"encoding/json"
"errors"
"os"
"path/filepath"
"reflect"
"strings"
"testing"
"time"
"gitea.maximumdirect.net/eric/weatherreporter/internal/app"
"gitea.maximumdirect.net/eric/weatherreporter/internal/comparison"
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec"
"gitea.maximumdirect.net/eric/weatherreporter/internal/timeutil"
)
func TestParseComparisonFlagsPreservesProfileOrder(t *testing.T) {
opts, err := parseComparisonFlags(app.ReportDaily, []string{
"--profile", "weather-light", "--profile=weather-balanced", "--profile", "weather-deep",
"--date", "2026-05-29", "--out-dir", "reports", "--replace", "--quiet",
})
if err != nil {
t.Fatalf("parseComparisonFlags() error = %v", err)
}
wantProfiles := []string{"weather-light", "weather-balanced", "weather-deep"}
if !reflect.DeepEqual([]string(opts.ProfileIDs), wantProfiles) || opts.Date != "2026-05-29" || opts.OutputDir != "reports" || !opts.Replace || !opts.Quiet {
t.Fatalf("options = %#v", opts)
}
}
func TestResolveComparisonActionBuildsExplicitRequest(t *testing.T) {
workingDir := t.TempDir()
configPath := comparisonConfigPath(t, "weather_api:\n base_url: https://weather.api.example.com/\n units: metric\n timezone: America/Chicago\noutput:\n directory: configured-reports\npromptkit:\n profile: configured-default\n")
clock := timeutil.FixedClock{Time: time.Date(2026, 5, 29, 8, 30, 0, 0, time.UTC)}
var factoryConfig PromptExecutorConfig
factoryCalls := 0
executor := &factoryExecutor{}
runner := Runner{
Clock: clock, WorkingDir: workingDir,
ExecutorFactory: func(value PromptExecutorConfig) (promptexec.Executor, error) {
factoryCalls++
factoryConfig = value
return executor, nil
},
}
req, opts, err := runner.resolveComparisonAction([]string{
"daily", "--date", "2026-05-29", "--profile", "weather-light", "--profile", "weather-deep",
"--out-dir", "bundles/../comparison", "--replace", "--llm-debug-dir", "/tmp/debug", "--units", "imperial", "--tz", "America/New_York", "--config", configPath,
})
if err != nil {
t.Fatalf("resolveComparisonAction() error = %v", err)
}
if factoryCalls != 1 || req.Executor != executor || req.Clock != clock || req.WorkingDir != workingDir || req.OutputDir != filepath.Join(workingDir, "comparison") || !req.Replace || req.LLMDebugDir != "/tmp/debug" {
t.Fatalf("request/factory calls = %#v/%#v/%d", req, factoryConfig, factoryCalls)
}
if got, want := req.ProfileIDs, []string{"weather-light", "weather-deep"}; !reflect.DeepEqual(got, want) {
t.Fatalf("profile IDs = %#v, want %#v", got, want)
}
if req.Config.WeatherAPI.Units != "imperial" || req.Config.WeatherAPI.Timezone != "America/New_York" || req.Config.Output.Directory != "configured-reports" || factoryConfig.Profile != "" || opts.Quiet {
t.Fatalf("configuration/options/factory config = %#v/%#v/%#v", req.Config, opts, factoryConfig)
}
location, loadErr := time.LoadLocation("America/New_York")
if loadErr != nil || req.Date.Location().String() != location.String() || req.Date.Format("2006-01-02") != "2026-05-29" {
t.Fatalf("date/location = %v/%v", req.Date, loadErr)
}
}
func TestResolveComparisonActionMatchesReportDatePolicies(t *testing.T) {
configPath := comparisonConfigPath(t, "weather_api:\n base_url: https://weather.api.example.com/\n timezone: America/Chicago\n")
runner := comparisonRunner(t, t.TempDir())
for _, test := range []struct {
name string
args []string
wantDay string
wantErr bool
}{
{name: "daily requires date", args: []string{"daily", "--profile", "one", "--profile", "two", "--config", configPath}, wantErr: true},
{name: "today uses current local date", args: []string{"today", "--profile", "one", "--profile", "two", "--config", configPath}, wantDay: "2026-05-29"},
{name: "today accepts date", args: []string{"today", "--date", "2026-05-30", "--profile", "one", "--profile", "two", "--config", configPath}, wantDay: "2026-05-30"},
{name: "tomorrow rejects date", args: []string{"tomorrow", "--date", "2026-05-30", "--profile", "one", "--profile", "two", "--config", configPath}, wantErr: true},
{name: "hourly rejects date", args: []string{"hourly", "--date", "2026-05-30", "--profile", "one", "--profile", "two", "--config", configPath}, wantErr: true},
} {
t.Run(test.name, func(t *testing.T) {
req, _, err := runner.resolveComparisonAction(test.args)
if test.wantErr {
if err == nil {
t.Fatal("resolveComparisonAction() error = nil")
}
return
}
if err != nil || req.Date.Format("2006-01-02") != test.wantDay {
t.Fatalf("request/error = %#v/%v", req, err)
}
})
}
}
func TestResolveComparisonActionLeavesConfiguredOutputWithoutOverride(t *testing.T) {
workingDir := t.TempDir()
configPath := comparisonConfigPath(t, "weather_api:\n base_url: https://weather.api.example.com/\noutput:\n directory: configured/../reports\n")
req, _, err := comparisonRunner(t, workingDir).resolveComparisonAction([]string{
"daily", "--date", "2026-05-29", "--profile", "weather-light", "--profile", "weather-deep", "--config", configPath,
})
if err != nil || req.OutputDir != "" || req.Config.Output.Directory != "configured/../reports" {
t.Fatalf("request/error = %#v/%v", req, err)
}
}
func TestComparisonInputFailuresDoNotConstructOrExecute(t *testing.T) {
for _, test := range []struct {
name string
args []string
}{
{name: "missing report", args: nil},
{name: "unknown report", args: []string{"unknown", "--profile", "one", "--profile", "two"}},
{name: "single profile", args: []string{"today", "--profile", "one"}},
{name: "blank profile", args: []string{"today", "--profile", "one", "--profile", " \t"}},
{name: "duplicate profile", args: []string{"today", "--profile", "one", "--profile", "one"}},
{name: "unexpected argument", args: []string{"today", "extra", "--profile", "one", "--profile", "two"}},
{name: "unsupported output flag", args: []string{"today", "--out", "report.md", "--profile", "one", "--profile", "two"}},
{name: "configuration failure", args: []string{"today", "--profile", "one", "--profile", "two", "--config", filepath.Join(t.TempDir(), "missing.yml")}},
} {
t.Run(test.name, func(t *testing.T) {
factoryCalls, applicationCalls := 0, 0
runner := Runner{
Clock: timeutil.FixedClock{Time: time.Date(2026, 5, 29, 8, 30, 0, 0, time.UTC)},
ExecutorFactory: func(PromptExecutorConfig) (promptexec.Executor, error) {
factoryCalls++
return &factoryExecutor{}, nil
},
compareDetailed: func(context.Context, app.ComparisonRequest) (*app.ComparisonResult, error) {
applicationCalls++
return nil, nil
},
}
var stdout, stderr bytes.Buffer
if err := runner.Run(context.Background(), append([]string{"compare"}, test.args...), &stdout, &stderr); err == nil {
t.Fatal("Run() error = nil")
}
if factoryCalls != 0 || applicationCalls != 0 || stdout.Len() != 0 || stderr.Len() != 0 {
t.Fatalf("factory/application calls/output = %d/%d/%q/%q", factoryCalls, applicationCalls, stdout.String(), stderr.String())
}
})
}
}
func TestCompareCommandUsesOneExecutorAndInjectedApplication(t *testing.T) {
workingDir := t.TempDir()
configPath := comparisonConfigPath(t, "weather_api:\n base_url: https://weather.api.example.com/\n")
executor := &factoryExecutor{}
factoryCalls, applicationCalls := 0, 0
var received app.ComparisonRequest
wantResult := &app.ComparisonResult{ComparisonID: "comparison_test"}
comparisonErr := errors.New("comparison completed with 1 failed profiles")
runner := Runner{
Clock: timeutil.FixedClock{Time: time.Date(2026, 5, 29, 8, 30, 0, 0, time.UTC)}, WorkingDir: workingDir,
ExecutorFactory: func(PromptExecutorConfig) (promptexec.Executor, error) {
factoryCalls++
return executor, nil
},
compareDetailed: func(_ context.Context, req app.ComparisonRequest) (*app.ComparisonResult, error) {
applicationCalls++
received = req
return wantResult, comparisonErr
},
}
var stdout, stderr bytes.Buffer
err := runner.Run(context.Background(), []string{
"compare", "daily", "--date", "2026-05-29", "--profile", "weather-light", "--profile", "weather-deep", "--out-dir", "comparison", "--replace", "--config", configPath,
}, &stdout, &stderr)
if !errors.Is(err, comparisonErr) || factoryCalls != 1 || applicationCalls != 1 || received.Executor != executor || received.OutputDir != filepath.Join(workingDir, "comparison") || !received.Replace || stdout.Len() == 0 || stderr.Len() != 0 {
t.Fatalf("error/calls/request/output = %v/%d/%d/%#v/%q/%q", err, factoryCalls, applicationCalls, received, stdout.String(), stderr.String())
}
if !reflect.DeepEqual(received.ProfileIDs, []string{"weather-light", "weather-deep"}) {
t.Fatalf("profile IDs = %#v", received.ProfileIDs)
}
}
func TestCompareCommandWritesStructuredPartialFailure(t *testing.T) {
workingDir := t.TempDir()
configPath := comparisonConfigPath(t, "weather_api:\n base_url: https://weather.api.example.com/\n")
profileFailure := comparison.NewSafeError("generation", "execute prompt failed")
result := comparisonResult("/reports/comparison-daily", []app.ComparisonProfileResult{
{Position: 1, ProfileID: "weather-light", BackendID: "local", ModelName: "light", Status: comparison.StatusSucceeded, ValidationStatus: promptexec.ValidationPassed, ReportPath: "/reports/comparison-daily/01-weather-light.md", LLMDebugPath: "/debug/comparison-light"},
{Position: 2, ProfileID: "weather-deep", BackendID: "cloud", ModelName: "deep", Status: comparison.StatusFailed, Error: &profileFailure},
})
partialErr := errors.New("comparison completed with 1 failed profiles")
factoryCalls, applicationCalls := 0, 0
runner := Runner{
Clock: timeutil.FixedClock{Time: time.Date(2026, 5, 29, 8, 30, 0, 0, time.UTC)}, WorkingDir: workingDir,
ExecutorFactory: func(PromptExecutorConfig) (promptexec.Executor, error) {
factoryCalls++
return &factoryExecutor{}, nil
},
compareDetailed: func(context.Context, app.ComparisonRequest) (*app.ComparisonResult, error) {
applicationCalls++
return result, partialErr
},
}
var stdout, stderr bytes.Buffer
err := runner.Run(context.Background(), []string{"compare", "daily", "--date", "2026-05-29", "--profile", "weather-light", "--profile", "weather-deep", "--config", configPath}, &stdout, &stderr)
if !errors.Is(err, partialErr) || factoryCalls != 1 || applicationCalls != 1 || stderr.Len() != 0 {
t.Fatalf("error/calls/stderr = %v/%d/%d/%q", err, factoryCalls, applicationCalls, stderr.String())
}
var summary comparisonSummary
if err := json.Unmarshal(stdout.Bytes(), &summary); err != nil {
t.Fatalf("decode summary: %v\n%s", err, stdout.String())
}
if summary.Command != commandCompare || summary.Status != summaryStatusFailed || summary.Error == nil || summary.Error.Message != partialErr.Error() || summary.OutputDirectory != result.OutputDirectory || len(summary.Results) != 2 || summary.Results[0].ReportPath != result.Results[0].ReportPath || summary.Results[0].LLMDebugPath != result.Results[0].LLMDebugPath || summary.Results[1].Error == nil {
t.Fatalf("summary = %#v", summary)
}
}
func TestCompareCommandReportsCommittedBundleWhenCleanupFails(t *testing.T) {
workingDir := t.TempDir()
configPath := comparisonConfigPath(t, "weather_api:\n base_url: https://weather.api.example.com/\n")
outputDirectory := filepath.Join(workingDir, "comparison-daily")
result := comparisonResult(outputDirectory, []app.ComparisonProfileResult{
{Position: 1, ProfileID: "weather-light", ModelName: "light", Status: comparison.StatusSucceeded, ValidationStatus: promptexec.ValidationPassed, ReportPath: filepath.Join(outputDirectory, "01-weather-light.md")},
{Position: 2, ProfileID: "weather-deep", ModelName: "deep", Status: comparison.StatusSucceeded, ValidationStatus: promptexec.ValidationPassed, ReportPath: filepath.Join(outputDirectory, "02-weather-deep.md")},
})
backupPath := filepath.Join(workingDir, ".comparison-daily.backup-retained")
cleanupCause := errors.New("filesystem cleanup detail")
cleanupErr := &comparison.PublicationCleanupError{RetainedBackupPath: backupPath, Err: cleanupCause}
runner := comparisonRunner(t, workingDir)
runner.compareDetailed = func(context.Context, app.ComparisonRequest) (*app.ComparisonResult, error) {
return result, cleanupErr
}
var stdout, stderr bytes.Buffer
err := runner.Run(context.Background(), []string{"compare", "daily", "--date", "2026-05-29", "--profile", "weather-light", "--profile", "weather-deep", "--config", configPath}, &stdout, &stderr)
if !errors.Is(err, cleanupCause) || stderr.Len() != 0 {
t.Fatalf("Run() error/stderr = %v/%q", err, stderr.String())
}
var summary comparisonSummary
if err := json.Unmarshal(stdout.Bytes(), &summary); err != nil {
t.Fatalf("decode summary: %v\n%s", err, stdout.String())
}
if summary.Status != summaryStatusFailed || summary.Error == nil || summary.Error.Category != "publication_cleanup" || summary.Error.Message != "comparison published but cleanup did not complete" || summary.ManifestPath != result.ManifestPath || summary.DataPackagePath != result.DataPackagePath || summary.Results[0].ReportPath == "" {
t.Fatalf("summary = %#v", summary)
}
for _, unsafe := range []string{cleanupCause.Error(), backupPath} {
if strings.Contains(stdout.String(), unsafe) {
t.Fatalf("summary contains unsafe recovery detail %q: %s", unsafe, stdout.String())
}
}
}
func TestCompareCommandWritesSuccessForDefaultAndExplicitDestinations(t *testing.T) {
workingDir := t.TempDir()
configPath := comparisonConfigPath(t, "weather_api:\n base_url: https://weather.api.example.com/\noutput:\n directory: configured-reports\n")
for _, test := range []struct {
name string
outputArgument []string
outputDirectory string
wantRequestPath string
}{
{name: "configured default", outputDirectory: filepath.Join(workingDir, "configured-reports", "comparison-daily-2026-05-29")},
{name: "explicit directory", outputArgument: []string{"--out-dir", "published"}, outputDirectory: filepath.Join(workingDir, "published"), wantRequestPath: filepath.Join(workingDir, "published")},
} {
t.Run(test.name, func(t *testing.T) {
var received app.ComparisonRequest
runner := comparisonRunner(t, workingDir)
runner.compareDetailed = func(_ context.Context, req app.ComparisonRequest) (*app.ComparisonResult, error) {
received = req
return comparisonResult(test.outputDirectory, []app.ComparisonProfileResult{
{Position: 1, ProfileID: "weather-light", ModelName: "light", Status: comparison.StatusSucceeded, ValidationStatus: promptexec.ValidationPassed, ReportPath: filepath.Join(test.outputDirectory, "01-weather-light.md")},
{Position: 2, ProfileID: "weather-deep", ModelName: "deep", Status: comparison.StatusSucceeded, ValidationStatus: promptexec.ValidationPassed, ReportPath: filepath.Join(test.outputDirectory, "02-weather-deep.md")},
}), nil
}
args := []string{"compare", "daily", "--date", "2026-05-29", "--profile", "weather-light", "--profile", "weather-deep", "--config", configPath}
args = append(args, test.outputArgument...)
var stdout, stderr bytes.Buffer
if err := runner.Run(context.Background(), args, &stdout, &stderr); err != nil {
t.Fatalf("Run() error = %v", err)
}
var summary comparisonSummary
if err := json.Unmarshal(stdout.Bytes(), &summary); err != nil || summary.Status != summaryStatusSucceeded || summary.OutputDirectory != test.outputDirectory || stderr.Len() != 0 || received.OutputDir != test.wantRequestPath {
t.Fatalf("summary/error/stderr/request = %#v/%v/%q/%#v", summary, err, stderr.String(), received)
}
})
}
}
func TestCompareCommandQuietPreservesFailure(t *testing.T) {
configPath := comparisonConfigPath(t, "weather_api:\n base_url: https://weather.api.example.com/\n")
failure := errors.New("comparison completed with 2 failed profiles")
calls := 0
runner := comparisonRunner(t, t.TempDir())
runner.compareDetailed = func(context.Context, app.ComparisonRequest) (*app.ComparisonResult, error) {
calls++
return comparisonResult("/reports/comparison-daily", nil), failure
}
var stdout, stderr bytes.Buffer
err := runner.Run(context.Background(), []string{"compare", "daily", "--date", "2026-05-29", "--profile", "weather-light", "--profile", "weather-deep", "--quiet", "--config", configPath}, &stdout, &stderr)
if !errors.Is(err, failure) || calls != 1 || stdout.Len() != 0 || stderr.Len() != 0 {
t.Fatalf("error/calls/stdout/stderr = %v/%d/%q/%q", err, calls, stdout.String(), stderr.String())
}
}
func TestCompareCommandLeavesPreExecutionFailuresUnstructured(t *testing.T) {
runner := comparisonRunner(t, t.TempDir())
called := false
runner.compareDetailed = func(context.Context, app.ComparisonRequest) (*app.ComparisonResult, error) {
called = true
return nil, nil
}
var stdout, stderr bytes.Buffer
err := runner.Run(context.Background(), []string{"compare", "today", "--profile", "only-one"}, &stdout, &stderr)
if err == nil || called || stdout.Len() != 0 || stderr.Len() != 0 {
t.Fatalf("error/called/stdout/stderr = %v/%t/%q/%q", err, called, stdout.String(), stderr.String())
}
}
func TestCompareHelpIncludesCommand(t *testing.T) {
var stdout, stderr bytes.Buffer
if err := (Runner{}).Run(context.Background(), []string{"--help"}, &stdout, &stderr); err != nil || !strings.Contains(stdout.String(), "weatherreporter compare REPORT") || !strings.Contains(stdout.String(), "--profile PROFILE") {
t.Fatalf("help/error = %q/%v", stdout.String(), err)
}
}
func TestCompareHelpDoesNotRequireConfiguration(t *testing.T) {
var stdout, stderr bytes.Buffer
if err := (Runner{}).Run(context.Background(), []string{"compare", "--help"}, &stdout, &stderr); err != nil || !strings.Contains(stdout.String(), "weatherreporter compare REPORT") || stderr.Len() != 0 {
t.Fatalf("help/error/stderr = %q/%v/%q", stdout.String(), err, stderr.String())
}
}
func comparisonRunner(t *testing.T, workingDir string) Runner {
t.Helper()
return Runner{
Clock: timeutil.FixedClock{Time: time.Date(2026, 5, 29, 8, 30, 0, 0, time.UTC)}, WorkingDir: workingDir,
ExecutorFactory: func(PromptExecutorConfig) (promptexec.Executor, error) { return &factoryExecutor{}, nil },
}
}
func comparisonConfigPath(t *testing.T, contents string) string {
t.Helper()
path := filepath.Join(t.TempDir(), "config.yml")
if err := os.WriteFile(path, []byte(contents), 0o600); err != nil {
t.Fatal(err)
}
return path
}
func comparisonResult(outputDirectory string, results []app.ComparisonProfileResult) *app.ComparisonResult {
started := time.Date(2026, 5, 29, 13, 30, 0, 0, time.UTC)
result := &app.ComparisonResult{
ComparisonID: "comparison_run-123", ReportID: "daily", ReportName: "Daily Report",
PromptID: "weather.daily_generated_text", PromptVersion: "2.0.0", PromptHash: strings.Repeat("a", 64),
StartedAt: started, FinishedAt: started.Add(time.Minute), Timezone: "America/Chicago",
ValidPeriod: timeutil.Period{Start: started, End: started.Add(24 * time.Hour)}, OutputDirectory: outputDirectory,
ManifestPath: filepath.Join(outputDirectory, "comparison.json"), DataPackagePath: filepath.Join(outputDirectory, "data-package.yml"),
Results: append([]app.ComparisonProfileResult(nil), results...), Total: len(results),
}
for _, profile := range results {
if profile.Status == "succeeded" {
result.Succeeded++
} else {
result.Failed++
}
}
return result
}

View File

@@ -33,21 +33,11 @@ func writeBatchStatus(stderr io.Writer, result *app.BatchResult) {
return
}
for _, item := range result.Reports {
notificationFields := ""
if item.NotificationStatus != "" {
notificationFields += fmt.Sprintf(" notificationStatus=%q", item.NotificationStatus)
}
if item.NotificationRunID != "" {
notificationFields += fmt.Sprintf(" notificationRunId=%q", item.NotificationRunID)
}
if item.NotificationError != "" {
notificationFields += fmt.Sprintf(" notificationError=%q", item.NotificationError)
}
if item.Status == "failed" {
_, _ = fmt.Fprintf(stderr, "report=%s status=failed error=%q%s\n", item.ReportID, item.Error, notificationFields)
_, _ = fmt.Fprintf(stderr, "report=%s status=failed error=%q\n", item.ReportID, item.Error)
continue
}
_, _ = fmt.Fprintf(stderr, "report=%s status=succeeded output=%q%s\n", item.ReportID, item.OutputPath, notificationFields)
_, _ = fmt.Fprintf(stderr, "report=%s status=succeeded output=%q\n", item.ReportID, item.OutputPath)
}
if result.Notification != nil {
_, _ = fmt.Fprintf(stderr, "batchNotification status=%q", result.Notification.Status)
@@ -63,9 +53,6 @@ func writeBatchStatus(stderr io.Writer, result *app.BatchResult) {
if result.Notification.BundleID != "" {
_, _ = fmt.Fprintf(stderr, " bundleId=%q", result.Notification.BundleID)
}
if result.Notification.Path != "" {
_, _ = fmt.Fprintf(stderr, " path=%q", result.Notification.Path)
}
if result.Notification.Error != "" {
_, _ = fmt.Fprintf(stderr, " error=%q", result.Notification.Error)
}

View File

@@ -1,43 +1,50 @@
package cli
import (
"context"
"errors"
"fmt"
"strconv"
"strings"
"time"
"gitea.maximumdirect.net/eric/weatherreporter/internal/app"
"gitea.maximumdirect.net/eric/weatherreporter/internal/comparison"
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec"
"gitea.maximumdirect.net/eric/weatherreporter/internal/report"
"gitea.maximumdirect.net/eric/weatherreporter/internal/timeutil"
"gitea.maximumdirect.net/eric/weatherreporter/internal/weatherdata"
)
const (
commandGenerate = "generate"
commandRun = "run"
commandCompare = "compare"
summaryStatusSucceeded = "succeeded"
summaryStatusFailed = "failed"
)
type generateSummary struct {
Command string `json:"command"`
ReportID report.ID `json:"reportId"`
ReportName string `json:"reportName"`
PromptID string `json:"promptId"`
RunID string `json:"runId"`
Status string `json:"status"`
GeneratedAt time.Time `json:"generatedAt"`
ValidPeriod timeutil.Period `json:"validPeriod"`
ReportPath string `json:"reportPath,omitempty"`
OutputPath string `json:"outputPath,omitempty"`
MetadataPath string `json:"metadataPath,omitempty"`
DataPackagePath string `json:"dataPackagePath,omitempty"`
PreparationPath string `json:"preparationPath,omitempty"`
ExecutionPath string `json:"executionPath,omitempty"`
LLMDebugPath string `json:"llmDebugPath,omitempty"`
GeneratedTextRawPath string `json:"generatedTextRawPath,omitempty"`
GeneratedTextPath string `json:"generatedTextPath,omitempty"`
RenderContextPath string `json:"renderContextPath,omitempty"`
NotificationPath string `json:"notificationPath,omitempty"`
Notification *generateNotificationSummary `json:"notification,omitempty"`
Error string `json:"error,omitempty"`
Command string `json:"command"`
ReportID report.ID `json:"reportId"`
ReportName string `json:"reportName"`
PromptID string `json:"promptId"`
RunID string `json:"runId"`
Status string `json:"status"`
GeneratedAt time.Time `json:"generatedAt"`
ValidPeriod timeutil.Period `json:"validPeriod"`
OutputPath string `json:"outputPath,omitempty"`
LLMDebugPath string `json:"llmDebugPath,omitempty"`
PromptVersion string `json:"promptVersion"`
Timezone string `json:"timezone"`
ProfileID string `json:"profileId,omitempty"`
BackendID string `json:"backendId,omitempty"`
ModelName string `json:"modelName,omitempty"`
SourceWarnings []weatherdata.SourceWarning `json:"sourceWarnings,omitempty"`
ValidationStatus string `json:"validationStatus,omitempty"`
Notification *generateNotificationSummary `json:"notification,omitempty"`
Error string `json:"error,omitempty"`
}
type generateNotificationSummary struct {
@@ -68,31 +75,61 @@ type batchSummary struct {
Error string `json:"error,omitempty"`
}
type comparisonSummary struct {
Command string `json:"command"`
ComparisonID string `json:"comparisonId"`
ReportID report.ID `json:"reportId"`
ReportName string `json:"reportName"`
PromptID string `json:"promptId"`
PromptVersion string `json:"promptVersion"`
PromptHash string `json:"promptHash"`
Status string `json:"status"`
StartedAt time.Time `json:"startedAt"`
FinishedAt time.Time `json:"finishedAt"`
Timezone string `json:"timezone"`
ValidPeriod timeutil.Period `json:"validPeriod"`
OutputDirectory string `json:"outputDirectory"`
ManifestPath string `json:"manifestPath,omitempty"`
DataPackagePath string `json:"dataPackagePath,omitempty"`
Total int `json:"total"`
Succeeded int `json:"succeeded"`
Failed int `json:"failed"`
Results []comparisonProfileSummary `json:"results"`
Error *comparison.SafeError `json:"error,omitempty"`
}
type comparisonProfileSummary struct {
Position int `json:"position"`
ProfileID string `json:"profileId"`
BackendID string `json:"backendId,omitempty"`
ModelName string `json:"modelName"`
Status string `json:"status"`
ValidationStatus string `json:"validationStatus,omitempty"`
ReportPath string `json:"reportPath,omitempty"`
LLMDebugPath string `json:"llmDebugPath,omitempty"`
Error *comparison.SafeError `json:"error,omitempty"`
}
func newGenerateSummary(result *app.ReportResult, err error) generateSummary {
summary := generateSummary{Command: commandGenerate}
if result == nil {
return summary
}
metadata := result.Metadata
summary.ReportID = metadata.ReportID
summary.ReportName = reportName(metadata.ReportID)
summary.PromptID = metadata.PromptID
summary.RunID = metadata.RunID
summary.ReportID = result.ReportID
summary.ReportName = result.ReportName
summary.PromptID = result.PromptID
summary.PromptVersion = result.PromptVersion
summary.RunID = result.RunID
summary.Status = summaryStatusSucceeded
summary.GeneratedAt = metadata.GeneratedAt
summary.ValidPeriod = metadata.ValidPeriod
summary.ReportPath = result.ReportPath
summary.GeneratedAt = result.GeneratedAt
summary.ValidPeriod = result.ValidPeriod
summary.Timezone = result.Timezone
summary.ProfileID, summary.BackendID, summary.ModelName = result.ProfileID, result.BackendID, result.ModelName
summary.SourceWarnings = append([]weatherdata.SourceWarning(nil), result.SourceWarnings...)
summary.ValidationStatus = string(result.ValidationStatus)
summary.OutputPath = result.OutputPath
summary.MetadataPath = result.MetadataPath
summary.DataPackagePath = result.DataPackagePath
summary.PreparationPath = result.PreparationPath
summary.ExecutionPath = result.ExecutionPath
summary.LLMDebugPath = result.LLMDebugPath
summary.GeneratedTextRawPath = result.GeneratedTextRawPath
summary.GeneratedTextPath = result.GeneratedTextPath
summary.RenderContextPath = result.RenderContextPath
summary.NotificationPath = result.NotificationPath
summary.Notification = newGenerateNotificationSummary(result.Notification)
if err != nil {
summary.Status = summaryStatusFailed
@@ -145,6 +182,87 @@ func newBatchSummary(result *app.BatchResult) batchSummary {
return summary
}
func newComparisonSummary(result *app.ComparisonResult, err error) comparisonSummary {
summary := comparisonSummary{Command: commandCompare, Results: []comparisonProfileSummary{}}
if result == nil {
return summary
}
summary.ComparisonID = result.ComparisonID
summary.ReportID, summary.ReportName = result.ReportID, result.ReportName
summary.PromptID, summary.PromptVersion, summary.PromptHash = result.PromptID, result.PromptVersion, result.PromptHash
summary.StartedAt, summary.FinishedAt = result.StartedAt, result.FinishedAt
summary.Timezone, summary.ValidPeriod = result.Timezone, result.ValidPeriod
summary.OutputDirectory = result.OutputDirectory
summary.ManifestPath, summary.DataPackagePath = result.ManifestPath, result.DataPackagePath
summary.Total, summary.Succeeded, summary.Failed = result.Total, result.Succeeded, result.Failed
for _, profile := range result.Results {
summary.Results = append(summary.Results, comparisonProfileSummary{
Position: profile.Position, ProfileID: profile.ProfileID, BackendID: profile.BackendID, ModelName: profile.ModelName,
Status: profile.Status, ValidationStatus: string(profile.ValidationStatus), ReportPath: profile.ReportPath,
LLMDebugPath: profile.LLMDebugPath, Error: profile.Error,
})
}
summary.Status = comparisonSummaryStatus(result, err)
if err != nil {
summary.Error = safeComparisonSummaryError(err)
}
return summary
}
func comparisonSummaryStatus(result *app.ComparisonResult, err error) string {
if result == nil || err != nil || result.Total < 2 || result.Succeeded != result.Total || result.Failed != 0 || result.ManifestPath == "" || result.DataPackagePath == "" {
return summaryStatusFailed
}
return summaryStatusSucceeded
}
func safeComparisonSummaryError(err error) *comparison.SafeError {
if aggregate, ok := comparisonAggregateErrorMessage(err); ok {
safe := comparison.NewSafeError("application", aggregate)
return &safe
}
var cleanupErr *comparison.PublicationCleanupError
if errors.As(err, &cleanupErr) {
safe := comparison.NewSafeError("publication_cleanup", "comparison published but cleanup did not complete")
return &safe
}
var destinationErr *comparison.DestinationError
if errors.As(err, &destinationErr) {
safe := comparison.NewSafeError("destination_"+string(destinationErr.Kind), "comparison destination preflight failed")
return &safe
}
if category := promptexec.CategoryOf(err); category != "" {
safe := comparison.NewSafeError(string(category), "comparison prompt operation failed")
return &safe
}
if errors.Is(err, context.DeadlineExceeded) {
safe := comparison.NewSafeError("deadline_exceeded", "comparison deadline exceeded")
return &safe
}
if errors.Is(err, context.Canceled) {
safe := comparison.NewSafeError("canceled", "comparison canceled")
return &safe
}
safe := comparison.NewSafeError("application", "comparison did not complete")
return &safe
}
func comparisonAggregateErrorMessage(err error) (string, bool) {
const prefix = "comparison completed with "
const suffix = " failed profiles"
for candidate := err; candidate != nil; candidate = errors.Unwrap(candidate) {
value := candidate.Error()
if !strings.HasPrefix(value, prefix) || !strings.HasSuffix(value, suffix) {
continue
}
count, parseErr := strconv.Atoi(strings.TrimSuffix(strings.TrimPrefix(value, prefix), suffix))
if parseErr == nil && count > 0 {
return fmt.Sprintf("comparison completed with %d failed profiles", count), true
}
}
return "", false
}
func batchSummaryStatus(result *app.BatchResult) string {
if result == nil {
return ""

View File

@@ -1,260 +1,211 @@
package cli
import (
"context"
"encoding/json"
"errors"
"fmt"
"strings"
"testing"
"time"
"gitea.maximumdirect.net/eric/weatherreporter/internal/app"
"gitea.maximumdirect.net/eric/weatherreporter/internal/comparison"
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec"
"gitea.maximumdirect.net/eric/weatherreporter/internal/report"
"gitea.maximumdirect.net/eric/weatherreporter/internal/state"
"gitea.maximumdirect.net/eric/weatherreporter/internal/timeutil"
"gitea.maximumdirect.net/eric/weatherreporter/internal/weatherdata"
)
func TestNewGenerateSummaryForGeneratedTextReport(t *testing.T) {
func TestGenerateSummaryUsesActiveResultFields(t *testing.T) {
generatedAt := time.Date(2026, 5, 29, 13, 30, 0, 0, time.UTC)
acceptedAt := generatedAt.Add(time.Minute)
startedAt := acceptedAt.Add(time.Minute)
finishedAt := startedAt.Add(time.Minute)
result := &app.ReportResult{
DataPackagePath: "/runs/hourly/data_package.yaml",
PreparationPath: "/runs/hourly/preparation.json",
ExecutionPath: "/runs/hourly/execution.json",
LLMDebugPath: "/operator-debug/hourly/2026-05-29/run-123",
ReportPath: "/runs/hourly/report.md",
OutputPath: "/copies/hourly.md",
MetadataPath: "/runs/hourly/metadata.json",
GeneratedTextRawPath: "/runs/hourly/generated_text_raw.json",
GeneratedTextPath: "/runs/hourly/generated_text.json",
RenderContextPath: "/runs/hourly/render_context.json",
NotificationPath: "/runs/hourly/notification.json",
Metadata: state.Metadata{
ReportID: report.Hourly,
PromptID: "weather.hourly_generated_text",
RunID: "20260529T133000Z_hourly",
GeneratedAt: generatedAt,
ValidPeriod: testSummaryPeriod(generatedAt),
},
Notification: &app.NotificationResult{
Status: "succeeded",
UploadStatus: "accepted",
RunID: "distributor-run",
PipelineID: "weatherreporter.hourly",
BundleID: "weatherreporter.home.hourly",
IdempotencyKey: "weatherreporter.home.hourly.20260529T133000Z_hourly",
AcceptedAt: acceptedAt,
StartedAt: &startedAt,
FinishedAt: &finishedAt,
Report: []byte(`{"actions":[{"action":"replace_older"}]}`),
},
}
summary := newGenerateSummary(result, nil)
if summary.Command != "generate" || summary.Status != "succeeded" {
t.Fatalf("summary command/status = %q/%q, want generate/succeeded", summary.Command, summary.Status)
}
if summary.ReportID != report.Hourly || summary.ReportName != "Hourly Report" || summary.PromptID != "weather.hourly_generated_text" || summary.RunID != "20260529T133000Z_hourly" {
t.Fatalf("summary identity = %#v, want hourly report identity", summary)
}
if summary.PreparationPath == "" || summary.ExecutionPath == "" || summary.LLMDebugPath == "" || summary.GeneratedTextRawPath == "" || summary.GeneratedTextPath == "" || summary.RenderContextPath == "" {
t.Fatalf("generated-text paths = %#v, want generated-text artifact paths", summary)
}
if summary.Notification == nil || summary.Notification.RunID != "distributor-run" || summary.Notification.AcceptedAt == nil || !summary.Notification.AcceptedAt.Equal(acceptedAt) {
t.Fatalf("notification = %#v, want summarized distributor result", summary.Notification)
summary := newGenerateSummary(&app.ReportResult{ReportID: report.Daily, ReportName: "Daily Report", PromptID: "weather.daily_generated_text", PromptVersion: "2.0.0", RunID: "run-123", GeneratedAt: generatedAt, Timezone: "America/Chicago", ValidPeriod: timeutil.Period{Start: generatedAt, End: generatedAt.Add(24 * time.Hour)}, ProfileID: "weather-balanced", BackendID: "openrouter", ModelName: "model", SourceWarnings: []weatherdata.SourceWarning{{Source: "alerts", Message: "source unavailable"}}, ValidationStatus: promptexec.ValidationPassed, OutputPath: "/reports/daily.md"}, nil)
if summary.OutputPath == "" || summary.ProfileID == "" || summary.ValidationStatus != string(promptexec.ValidationPassed) || len(summary.SourceWarnings) != 1 {
t.Fatalf("summary = %#v", summary)
}
data, err := json.Marshal(summary)
if err != nil {
t.Fatalf("Marshal() error = %v", err)
}
if strings.Contains(string(data), "replace_older") || strings.Contains(string(data), "actions") {
t.Fatalf("summary JSON includes raw distributor report payload:\n%s", string(data))
}
if strings.Contains(string(data), "preflightPath") || strings.Contains(string(data), "generatedTextResultPath") || !strings.Contains(string(data), "preparationPath") || !strings.Contains(string(data), "executionPath") {
t.Fatalf("summary JSON does not use prompt artifact path names:\n%s", string(data))
for _, forbidden := range []string{"reportPath", "metadataPath", "dataPackagePath", "preparationPath", "executionPath", "generatedTextRawPath", "generatedTextPath", "renderContextPath"} {
if strings.Contains(string(data), forbidden) {
t.Fatalf("summary includes %q: %s", forbidden, data)
}
}
}
func TestNewGenerateSummaryOmitsNotificationWhenNotAttempted(t *testing.T) {
generatedAt := time.Date(2026, 5, 29, 13, 30, 0, 0, time.UTC)
result := &app.ReportResult{
DataPackagePath: "/runs/daily/data_package.yaml",
PreparationPath: "/runs/daily/preparation.json",
ReportPath: "/runs/daily/report.md",
OutputPath: "/copies/daily.md",
MetadataPath: "/runs/daily/metadata.json",
Metadata: state.Metadata{
ReportID: report.Daily,
PromptID: "weather.daily_generated_text",
RunID: "20260529T133000Z_daily",
GeneratedAt: generatedAt,
ValidPeriod: testSummaryPeriod(generatedAt),
func TestComparisonSummaryUsesLockedOrderAndSafeFields(t *testing.T) {
started := time.Date(2026, 5, 29, 13, 30, 0, 0, time.UTC)
profileFailure := comparison.NewSafeError("generation", "execute prompt failed")
result := &app.ComparisonResult{
ComparisonID: "comparison_run-123", ReportID: report.Daily, ReportName: "Daily Report",
PromptID: "weather.daily_generated_text", PromptVersion: "2.0.0", PromptHash: strings.Repeat("a", 64),
StartedAt: started, FinishedAt: started.Add(time.Minute), Timezone: "America/Chicago",
ValidPeriod: timeutil.Period{Start: started, End: started.Add(24 * time.Hour)}, OutputDirectory: "/reports/comparison-daily",
ManifestPath: "/reports/comparison-daily/comparison.json", DataPackagePath: "/reports/comparison-daily/data-package.yml",
Total: 2, Succeeded: 1, Failed: 1,
Results: []app.ComparisonProfileResult{
{Position: 1, ProfileID: "weather-light", BackendID: "local", ModelName: "light", Status: comparison.StatusSucceeded, ValidationStatus: promptexec.ValidationPassed, ReportPath: "/reports/comparison-daily/01-weather-light.md"},
{Position: 2, ProfileID: "weather-deep", BackendID: "cloud", ModelName: "deep", Status: comparison.StatusFailed, Error: &profileFailure},
},
}
summary := newGenerateSummary(result, nil)
if summary.ReportID != report.Daily || summary.ReportName != "Daily Report" || summary.Status != "succeeded" {
t.Fatalf("summary = %#v, want successful daily summary", summary)
}
if summary.Notification != nil || summary.NotificationPath != "" {
t.Fatalf("notification summary/path = %#v/%q, want omitted", summary.Notification, summary.NotificationPath)
summary := newComparisonSummary(result, errors.New("comparison completed with 1 failed profiles"))
if summary.Command != commandCompare || summary.Status != summaryStatusFailed || summary.Error == nil || summary.Error.Message != "comparison completed with 1 failed profiles" || len(summary.Results) != 2 || summary.Results[0].ReportPath == "" || summary.Results[1].Error == nil {
t.Fatalf("summary = %#v", summary)
}
data, err := json.Marshal(summary)
if err != nil {
t.Fatalf("Marshal() error = %v", err)
t.Fatal(err)
}
for _, omitted := range []string{"notification"} {
if strings.Contains(string(data), omitted) {
t.Fatalf("summary JSON contains %q, want omitted:\n%s", omitted, string(data))
previous := -1
for _, field := range []string{"command", "comparisonId", "reportId", "reportName", "promptId", "promptVersion", "promptHash", "status", "startedAt", "finishedAt", "timezone", "validPeriod", "outputDirectory", "manifestPath", "dataPackagePath", "total", "succeeded", "failed", "results", "error"} {
position := strings.Index(string(data), `"`+field+`":`)
if field == "error" {
position = strings.LastIndex(string(data), `"`+field+`":`)
}
if position <= previous {
t.Fatalf("field order for %q in %s", field, data)
}
previous = position
}
for _, unsafe := range []string{"provider response", "rawOutput", "renderedPrompt", "endpoint"} {
if strings.Contains(string(data), unsafe) {
t.Fatalf("summary includes unsafe content %q: %s", unsafe, data)
}
}
}
func TestNewGenerateSummaryOmitsUnreachedArtifactPaths(t *testing.T) {
result := &app.ReportResult{
DataPackagePath: "/runs/daily/data_package.yaml",
PreparationPath: "/runs/daily/preparation.json",
Metadata: state.Metadata{
ReportID: report.Daily,
RunID: "20260529T133000Z_daily",
},
func TestComparisonSummaryOmitsUnpublishedArtifactsAndBoundsErrors(t *testing.T) {
result := &app.ComparisonResult{ComparisonID: "comparison_run-123", ReportID: report.Daily, Results: []app.ComparisonProfileResult{{Position: 1, ProfileID: "weather-light", ModelName: "light", Status: comparison.StatusFailed}}}
summary := newComparisonSummary(result, context.Canceled)
if summary.Status != summaryStatusFailed || summary.Error == nil || summary.Error.Message != "comparison canceled" || summary.Results == nil {
t.Fatalf("summary = %#v", summary)
}
data, err := json.Marshal(newGenerateSummary(result, errors.New("metadata write failed")))
data, err := json.Marshal(summary)
if err != nil {
t.Fatalf("Marshal() error = %v", err)
t.Fatal(err)
}
text := string(data)
for _, omitted := range []string{"executionPath", "reportPath", "outputPath", "metadataPath", "generatedTextRawPath", "generatedTextPath", "renderContextPath", "notificationPath"} {
if strings.Contains(text, omitted) {
t.Fatalf("partial summary includes unreached field %q:\n%s", omitted, text)
var fields map[string]json.RawMessage
if err := json.Unmarshal(data, &fields); err != nil {
t.Fatal(err)
}
for _, omitted := range []string{"manifestPath", "dataPackagePath"} {
if _, exists := fields[omitted]; exists {
t.Fatalf("summary includes unpublished %s: %s", omitted, data)
}
}
if !strings.Contains(text, "dataPackagePath") || !strings.Contains(text, "preparationPath") {
t.Fatalf("partial summary omits reached paths:\n%s", text)
}
}
func TestNewGenerateSummaryForNotificationFailure(t *testing.T) {
generatedAt := time.Date(2026, 5, 29, 13, 30, 0, 0, time.UTC)
result := &app.ReportResult{
DataPackagePath: "/runs/hourly/data_package.yaml",
PreparationPath: "/runs/hourly/preparation.json",
ReportPath: "/runs/hourly/report.md",
OutputPath: "/copies/hourly.md",
MetadataPath: "/runs/hourly/metadata.json",
NotificationPath: "/runs/hourly/notification.json",
Metadata: state.Metadata{
ReportID: report.Hourly,
PromptID: "weather.hourly_generated_text",
RunID: "20260529T133000Z_hourly",
GeneratedAt: generatedAt,
ValidPeriod: testSummaryPeriod(generatedAt),
publicationResult := &app.ComparisonResult{
ComparisonID: "comparison_run-123", ReportID: report.Daily, OutputDirectory: "/reports/comparison-daily", Total: 2, Succeeded: 2,
Results: []app.ComparisonProfileResult{
{Position: 1, ProfileID: "weather-light", ModelName: "light", Status: comparison.StatusSucceeded},
{Position: 2, ProfileID: "weather-deep", ModelName: "deep", Status: comparison.StatusSucceeded},
},
}
err := errors.New(`notify report "hourly" run "20260529T133000Z_hourly": upload rejected`)
summary := newGenerateSummary(result, err)
if summary.Status != "failed" || summary.Error != err.Error() {
t.Fatalf("status/error = %q/%q, want failed notification error", summary.Status, summary.Error)
}
if summary.NotificationPath != "/runs/hourly/notification.json" || summary.ReportPath == "" || summary.MetadataPath == "" {
t.Fatalf("artifact paths = report %q metadata %q notification %q, want inspectable paths", summary.ReportPath, summary.MetadataPath, summary.NotificationPath)
unsafe := errors.New("comparison completed with 1 failed profiles; provider response contains sensitive material")
publicationSummary := newComparisonSummary(publicationResult, unsafe)
if publicationSummary.Status != summaryStatusFailed || publicationSummary.Error == nil || publicationSummary.Error.Message != "comparison did not complete" || strings.Contains(publicationSummary.Error.Message, "sensitive") || publicationSummary.ManifestPath != "" || publicationSummary.DataPackagePath != "" {
safe := publicationSummary.Error
t.Fatalf("safe error = %#v", safe)
}
}
func TestNewBatchSummaryStatusDerivation(t *testing.T) {
startedAt := time.Date(2026, 5, 29, 12, 0, 0, 0, time.UTC)
finishedAt := startedAt.Add(2 * time.Minute)
tests := []struct {
name string
result *app.BatchResult
wantStatus string
wantError string
func TestComparisonSummaryClassifiesCompleteAndAllFailedResults(t *testing.T) {
started := time.Date(2026, 5, 29, 13, 30, 0, 0, time.UTC)
failure := comparison.NewSafeError("generation", "execute prompt failed")
complete := &app.ComparisonResult{
ComparisonID: "comparison_run-123", ReportID: report.Daily, PromptID: "weather.daily_generated_text", PromptVersion: "2.0.0", PromptHash: strings.Repeat("a", 64),
StartedAt: started, FinishedAt: started, Timezone: "America/Chicago", ValidPeriod: timeutil.Period{Start: started, End: started.Add(time.Hour)},
OutputDirectory: "/reports/comparison-daily", ManifestPath: "/reports/comparison-daily/comparison.json", DataPackagePath: "/reports/comparison-daily/data-package.yml",
Total: 2, Succeeded: 2,
Results: []app.ComparisonProfileResult{
{Position: 1, ProfileID: "weather-light", ModelName: "light", Status: comparison.StatusSucceeded, ValidationStatus: promptexec.ValidationPassed, ReportPath: "/reports/comparison-daily/01-weather-light.md"},
{Position: 2, ProfileID: "weather-deep", ModelName: "deep", Status: comparison.StatusSucceeded, ValidationStatus: promptexec.ValidationPassed, ReportPath: "/reports/comparison-daily/02-weather-deep.md"},
},
}
if summary := newComparisonSummary(complete, nil); summary.Status != summaryStatusSucceeded || summary.Error != nil {
t.Fatalf("complete summary = %#v", summary)
}
allFailed := *complete
allFailed.Succeeded, allFailed.Failed = 0, 2
allFailed.Results = []app.ComparisonProfileResult{
{Position: 1, ProfileID: "weather-light", ModelName: "light", Status: comparison.StatusFailed, Error: &failure},
{Position: 2, ProfileID: "weather-deep", ModelName: "deep", Status: comparison.StatusFailed, Error: &failure},
}
summary := newComparisonSummary(&allFailed, errors.New("comparison completed with 2 failed profiles"))
if summary.Status != summaryStatusFailed || summary.Error == nil || summary.Error.Message != "comparison completed with 2 failed profiles" || summary.Results[0].ReportPath != "" || summary.Results[1].Error == nil {
t.Fatalf("all-failed summary = %#v", summary)
}
}
func TestSafeComparisonSummaryErrorClassifiesWrappedFailures(t *testing.T) {
unsafeDetail := "unsafe filesystem and provider detail"
unsafeCause := errors.New(unsafeDetail)
for _, test := range []struct {
name string
err error
category string
message string
}{
{
name: "success",
result: &app.BatchResult{
Batch: app.BatchMorning,
StartedAt: startedAt,
FinishedAt: finishedAt,
Total: 1,
Succeeded: 1,
Reports: []app.BatchReportResult{{ReportID: report.Today, Status: "succeeded"}},
},
wantStatus: "succeeded",
name: "aggregate profile failure",
err: fmt.Errorf("outer wrapper: %w", errors.New("comparison completed with 2 failed profiles")),
category: "application",
message: "comparison completed with 2 failed profiles",
},
{
name: "report failure",
result: &app.BatchResult{
Batch: app.BatchMorning,
Total: 2,
Succeeded: 1,
Failed: 1,
Reports: []app.BatchReportResult{
{ReportID: report.Today, Status: "succeeded"},
{ReportID: report.Tomorrow, Status: "failed", Error: "render failed"},
},
},
wantStatus: "failed",
wantError: "batch morning failed: 1 of 2 reports failed",
name: "canceled",
err: fmt.Errorf("outer wrapper: %w", context.Canceled),
category: "canceled",
message: "comparison canceled",
},
{
name: "skipped notification",
result: &app.BatchResult{
Batch: app.BatchEvening,
Total: 2,
Succeeded: 1,
Failed: 1,
Reports: []app.BatchReportResult{{ReportID: report.Tomorrow, Status: "failed"}},
Notification: &app.BatchNotificationResult{
Status: "skipped",
Reason: "one or more reports failed",
},
},
wantStatus: "failed",
wantError: "batch evening failed: 1 of 2 reports failed",
name: "deadline exceeded",
err: fmt.Errorf("outer wrapper: %w", context.DeadlineExceeded),
category: "deadline_exceeded",
message: "comparison deadline exceeded",
},
{
name: "failed notification",
result: &app.BatchResult{
Batch: app.BatchEvening,
Total: 1,
Succeeded: 1,
Reports: []app.BatchReportResult{{ReportID: report.Tomorrow, Status: "succeeded"}},
Notification: &app.BatchNotificationResult{
Status: "failed",
Error: "notify batch evening: upload rejected",
},
},
wantStatus: "failed",
wantError: "batch evening notification failed: notify batch evening: upload rejected",
name: "prompt operation",
err: fmt.Errorf("outer wrapper: %w", promptexec.NewError(promptexec.Generation, "unsafe prompt detail", fmt.Errorf("%w: %s", context.Canceled, unsafeDetail))),
category: string(promptexec.Generation),
message: "comparison prompt operation failed",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
summary := newBatchSummary(tt.result)
if summary.Command != "run" || summary.Status != tt.wantStatus {
t.Fatalf("command/status = %q/%q, want run/%s", summary.Command, summary.Status, tt.wantStatus)
{
name: "destination preflight",
err: fmt.Errorf("outer wrapper: %w", &comparison.DestinationError{
Kind: comparison.DestinationNotEmpty, Target: "/tmp/" + unsafeDetail, Err: fmt.Errorf("%w: %s", context.Canceled, unsafeDetail),
}),
category: "destination_not_empty",
message: "comparison destination preflight failed",
},
{
name: "publication cleanup",
err: fmt.Errorf("outer wrapper: %w", &comparison.PublicationCleanupError{
RetainedBackupPath: "/tmp/" + unsafeDetail, Err: fmt.Errorf("%w: %s", context.Canceled, unsafeDetail),
}),
category: "publication_cleanup",
message: "comparison published but cleanup did not complete",
},
{
name: "unknown",
err: fmt.Errorf("outer wrapper: %w", unsafeCause),
category: "application",
message: "comparison did not complete",
},
} {
t.Run(test.name, func(t *testing.T) {
result := &app.ComparisonResult{ComparisonID: "comparison_test", ReportID: report.Daily}
summary := newComparisonSummary(result, test.err)
if summary.Error == nil || summary.Error.Category != test.category || summary.Error.Message != test.message {
t.Fatalf("summary error = %#v, want %q/%q", summary.Error, test.category, test.message)
}
if summary.Error != tt.wantError {
t.Fatalf("error = %q, want %q", summary.Error, tt.wantError)
data, err := json.Marshal(summary)
if err != nil {
t.Fatal(err)
}
if len(summary.Reports) != len(tt.result.Reports) {
t.Fatalf("reports = %#v, want copied report list", summary.Reports)
if strings.Contains(string(data), unsafeDetail) {
t.Fatalf("summary includes unsafe detail: %s", data)
}
})
}
}
func testSummaryPeriod(start time.Time) timeutil.Period {
return timeutil.Period{
Start: start,
End: start.Add(6 * time.Hour),
}
}

View File

@@ -5,9 +5,12 @@ import (
"flag"
"fmt"
"io"
"os"
"path/filepath"
"gitea.maximumdirect.net/eric/weatherreporter/internal/app"
"gitea.maximumdirect.net/eric/weatherreporter/internal/buildinfo"
"gitea.maximumdirect.net/eric/weatherreporter/internal/comparison"
"gitea.maximumdirect.net/eric/weatherreporter/internal/config"
"gitea.maximumdirect.net/eric/weatherreporter/internal/report"
"gitea.maximumdirect.net/eric/weatherreporter/internal/timeutil"
@@ -24,12 +27,7 @@ Usage:
weatherreporter generate hourly [--config PATH] [--units VALUE] [--tz NAME] [--out PATH] [--llm-debug-dir PATH] [--quiet]
weatherreporter run morning [--config PATH] [--units VALUE] [--tz NAME] [--out-dir PATH] [--llm-debug-dir PATH] [--quiet]
weatherreporter run evening [--config PATH] [--units VALUE] [--tz NAME] [--out-dir PATH] [--llm-debug-dir PATH] [--quiet]
weatherreporter inspect reports [--config PATH] [--limit N]
weatherreporter inspect metadata [--config PATH] RUN_ID
weatherreporter inspect modules [--config PATH] RUN_ID
weatherreporter inspect data-package [--config PATH] RUN_ID
weatherreporter inspect prior [--config PATH] RUN_ID
weatherreporter inspect sources [--config PATH] RUN_ID
weatherreporter compare REPORT --profile PROFILE --profile PROFILE [--config PATH] [--units VALUE] [--tz NAME] [--date YYYY-MM-DD] [--out-dir PATH] [--replace] [--llm-debug-dir PATH] [--quiet]
Options:
-h, --help Show this help message.
@@ -37,16 +35,21 @@ Options:
--config PATH Load configuration from PATH instead of /usr/local/etc/weatherreporter/config.yml.
--units VALUE Override weather API units.
--tz NAME Override weather API timezone.
--out PATH Write an extra Markdown report copy where supported by the generate command.
--llm-debug-dir PATH Write sensitive prompt debug artifacts outside the managed workspace.
--out-dir PATH Write extra Markdown report copies for run commands.
--quiet Suppress successful generate and run output.
--out PATH Write the generated Markdown report to PATH.
--llm-debug-dir PATH Write sensitive prompt debug artifacts under PATH.
--profile PROFILE Select a prompt profile for compare; repeat for every profile.
--out-dir PATH Write generated Markdown reports beneath PATH for run commands, or select the exact comparison directory.
--replace Authorize replacement of a recognized comparison bundle.
--quiet Suppress successful action output.
`
type Runner struct {
Clock timeutil.Clock
ExecutorFactory ExecutorFactory
Version string
Clock timeutil.Clock
ExecutorFactory ExecutorFactory
Version string
WorkingDir string
runBatchDetailed func(context.Context, app.BatchRequest) (*app.BatchResult, error)
compareDetailed func(context.Context, app.ComparisonRequest) (*app.ComparisonResult, error)
}
func Run(ctx context.Context, args []string, stdout io.Writer, stderr io.Writer) error {
@@ -61,6 +64,10 @@ func (r Runner) Run(ctx context.Context, args []string, stdout io.Writer, stderr
_, err := fmt.Fprint(stdout, helpText)
return err
}
if len(args) == 2 && args[0] == "compare" && (args[1] == "--help" || args[1] == "-h") {
_, err := fmt.Fprint(stdout, helpText)
return err
}
if args[0] == "--version" {
if len(args) != 1 {
return fmt.Errorf("--version does not accept arguments")
@@ -92,7 +99,11 @@ func (r Runner) Run(ctx context.Context, args []string, stdout io.Writer, stderr
if err != nil {
return err
}
result, err := app.RunBatchDetailed(ctx, req)
runBatchDetailed := r.runBatchDetailed
if runBatchDetailed == nil {
runBatchDetailed = app.RunBatchDetailed
}
result, err := runBatchDetailed(ctx, req)
if result != nil {
summary := newBatchSummary(result)
if encodeErr := writeActionResult(stdout, stderr, summary, outputOptions{Quiet: opts.Quiet}, func(w io.Writer) {
@@ -105,8 +116,23 @@ func (r Runner) Run(ctx context.Context, args []string, stdout io.Writer, stderr
}
}
return err
case "inspect":
return r.runInspect(ctx, args[1:], stdout)
case "compare":
req, opts, err := r.resolveComparisonAction(args[1:])
if err != nil {
return err
}
compareDetailed := r.compareDetailed
if compareDetailed == nil {
compareDetailed = app.CompareDetailed
}
result, err := compareDetailed(ctx, req)
if result != nil {
summary := newComparisonSummary(result, err)
if encodeErr := writeActionResult(stdout, stderr, summary, outputOptions{Quiet: opts.Quiet}, nil); encodeErr != nil {
return encodeErr
}
}
return err
default:
return fmt.Errorf("unknown command %q", args[0])
}
@@ -127,79 +153,22 @@ type generateOptions struct {
Date string
}
type inspectOptions struct {
ConfigPath string
Limit int
RunID string
type comparisonOptions struct {
commonOptions
Date string
ProfileIDs profileValues
Replace bool
}
type inspectRunCommand struct {
Name string
Inspect func(context.Context, app.InspectRunRequest) (any, error)
type profileValues []string
func (values *profileValues) String() string {
return ""
}
var inspectRunCommands = []inspectRunCommand{
{Name: "metadata", Inspect: func(ctx context.Context, req app.InspectRunRequest) (any, error) {
return app.InspectMetadata(ctx, req)
}},
{Name: "modules", Inspect: func(ctx context.Context, req app.InspectRunRequest) (any, error) {
return app.InspectModules(ctx, req)
}},
{Name: "data-package", Inspect: func(ctx context.Context, req app.InspectRunRequest) (any, error) {
return app.InspectDataPackage(ctx, req)
}},
{Name: "prior", Inspect: func(ctx context.Context, req app.InspectRunRequest) (any, error) {
return app.InspectPriorSnapshot(ctx, req)
}},
{Name: "sources", Inspect: func(ctx context.Context, req app.InspectRunRequest) (any, error) {
return app.InspectSources(ctx, req)
}},
}
func (r Runner) runInspect(ctx context.Context, args []string, stdout io.Writer) error {
if len(args) == 0 {
return fmt.Errorf("inspect requires a command")
}
command := args[0]
switch command {
case "reports":
opts, err := parseInspectReportsFlags(args[1:])
if err != nil {
return err
}
cfg, err := config.Load(config.LoadOptions{Path: opts.ConfigPath})
if err != nil {
return err
}
records, err := app.InspectReports(ctx, app.InspectReportsRequest{Config: cfg, Limit: opts.Limit})
if err != nil {
return err
}
return writeJSON(stdout, records)
default:
for _, candidate := range inspectRunCommands {
if candidate.Name == command {
return runInspectRunCommand(ctx, stdout, candidate, args[1:])
}
}
return fmt.Errorf("unknown inspect command %q", command)
}
}
func runInspectRunCommand(ctx context.Context, stdout io.Writer, command inspectRunCommand, args []string) error {
opts, err := parseInspectRunFlags(command.Name, args)
if err != nil {
return err
}
cfg, err := config.Load(config.LoadOptions{Path: opts.ConfigPath})
if err != nil {
return err
}
value, err := command.Inspect(ctx, app.InspectRunRequest{Config: cfg, RunID: opts.RunID})
if err != nil {
return err
}
return writeJSON(stdout, value)
func (values *profileValues) Set(value string) error {
*values = append(*values, value)
return nil
}
func (r Runner) resolveGenerate(args []string) (app.GenerateRequest, error) {
@@ -240,10 +209,20 @@ func (r Runner) resolveGenerateAction(args []string) (app.GenerateRequest, commo
return app.GenerateRequest{}, commonOptions{}, err
}
workingDir, err := r.workingDir()
if err != nil {
return app.GenerateRequest{}, commonOptions{}, err
}
outputPath, err := resolveOutputOverride(workingDir, opts.Output)
if err != nil {
return app.GenerateRequest{}, commonOptions{}, err
}
req := app.GenerateRequest{
Config: cfg,
Report: reportKind,
OutputPath: opts.Output,
WorkingDir: workingDir,
OutputPath: outputPath,
LLMDebugDir: opts.LLMDebugDir,
Now: r.Clock.Now(),
Executor: executor,
@@ -277,6 +256,76 @@ func (r Runner) resolveRun(args []string) (app.BatchRequest, error) {
return req, err
}
func (r Runner) resolveComparisonAction(args []string) (app.ComparisonRequest, commonOptions, error) {
if r.Clock == nil {
r.Clock = timeutil.SystemClock{}
}
if len(args) == 0 {
return app.ComparisonRequest{}, commonOptions{}, fmt.Errorf("compare requires a report name")
}
if _, err := report.IDForCommandName(args[0]); err != nil {
return app.ComparisonRequest{}, commonOptions{}, fmt.Errorf("unknown compare report %q", args[0])
}
reportKind := app.ReportKind(args[0])
opts, err := parseComparisonFlags(reportKind, args[1:])
if err != nil {
return app.ComparisonRequest{}, commonOptions{}, err
}
if err := comparison.ValidateProfileIDs(opts.ProfileIDs); err != nil {
return app.ComparisonRequest{}, commonOptions{}, err
}
cfg, err := config.Load(config.LoadOptions{
Path: opts.ConfigPath,
Units: opts.Units,
Timezone: opts.Timezone,
})
if err != nil {
return app.ComparisonRequest{}, commonOptions{}, err
}
location, err := timeutil.LoadLocation(cfg.WeatherAPI.Timezone)
if err != nil {
return app.ComparisonRequest{}, commonOptions{}, err
}
workingDir, err := r.workingDir()
if err != nil {
return app.ComparisonRequest{}, commonOptions{}, err
}
outputDir, err := resolveOutputOverride(workingDir, opts.OutputDir)
if err != nil {
return app.ComparisonRequest{}, commonOptions{}, err
}
req := app.ComparisonRequest{
Config: cfg, Report: reportKind, ProfileIDs: append([]string(nil), opts.ProfileIDs...),
WorkingDir: workingDir, OutputDir: outputDir, Replace: opts.Replace, LLMDebugDir: opts.LLMDebugDir, Clock: r.Clock,
}
switch reportKind {
case app.ReportDaily:
if opts.Date == "" {
return app.ComparisonRequest{}, commonOptions{}, fmt.Errorf("compare daily requires --date YYYY-MM-DD")
}
req.Date, err = timeutil.ParseLocalDate(opts.Date, location)
case app.ReportToday:
if opts.Date == "" {
req.Date = timeutil.LocalDate(r.Clock.Now(), location)
} else {
req.Date, err = timeutil.ParseLocalDate(opts.Date, location)
}
}
if err != nil {
return app.ComparisonRequest{}, commonOptions{}, err
}
promptkitConfig := cfg.Promptkit
promptkitConfig.Profile = ""
executor, err := r.promptExecutor(promptkitConfig)
if err != nil {
return app.ComparisonRequest{}, commonOptions{}, err
}
req.Executor = executor
return req, opts.commonOptions, nil
}
func (r Runner) resolveRunAction(args []string) (app.BatchRequest, commonOptions, error) {
if r.Clock == nil {
r.Clock = timeutil.SystemClock{}
@@ -304,7 +353,15 @@ func (r Runner) resolveRunAction(args []string) (app.BatchRequest, commonOptions
if err != nil {
return app.BatchRequest{}, commonOptions{}, err
}
return app.BatchRequest{Config: cfg, Batch: batch, Now: r.Clock.Now(), OutputDir: opts.OutputDir, LLMDebugDir: opts.LLMDebugDir, Executor: executor}, opts, nil
workingDir, err := r.workingDir()
if err != nil {
return app.BatchRequest{}, commonOptions{}, err
}
outputDir, err := resolveOutputOverride(workingDir, opts.OutputDir)
if err != nil {
return app.BatchRequest{}, commonOptions{}, err
}
return app.BatchRequest{Config: cfg, Batch: batch, Now: r.Clock.Now(), WorkingDir: workingDir, OutputDir: outputDir, LLMDebugDir: opts.LLMDebugDir, Executor: executor}, opts, nil
}
func resolveRun(args []string) (app.BatchRequest, error) {
@@ -334,7 +391,7 @@ func parseRunFlags(args []string) (commonOptions, error) {
fs.SetOutput(io.Discard)
opts := commonOptions{}
addCommonFlags(fs, &opts, false)
fs.StringVar(&opts.OutputDir, "out-dir", "", "extra Markdown report copy directory")
fs.StringVar(&opts.OutputDir, "out-dir", "", "generated Markdown report directory")
fs.BoolVar(&opts.Quiet, "quiet", false, "suppress successful action output")
if err := fs.Parse(args); err != nil {
return commonOptions{}, err
@@ -345,36 +402,24 @@ func parseRunFlags(args []string) (commonOptions, error) {
return opts, nil
}
func parseInspectReportsFlags(args []string) (inspectOptions, error) {
fs := flag.NewFlagSet("inspect reports", flag.ContinueOnError)
func parseComparisonFlags(report app.ReportKind, args []string) (comparisonOptions, error) {
fs := flag.NewFlagSet("compare "+string(report), flag.ContinueOnError)
fs.SetOutput(io.Discard)
opts := inspectOptions{Limit: 20}
fs.StringVar(&opts.ConfigPath, "config", "", "configuration file path")
fs.IntVar(&opts.Limit, "limit", 20, "maximum reports to list")
opts := comparisonOptions{}
addCommonFlags(fs, &opts.commonOptions, false)
fs.StringVar(&opts.OutputDir, "out-dir", "", "comparison bundle directory")
fs.BoolVar(&opts.Replace, "replace", false, "replace a recognized comparison bundle")
fs.BoolVar(&opts.Quiet, "quiet", false, "suppress successful action output")
fs.Var(&opts.ProfileIDs, "profile", "prompt profile ID")
if report == app.ReportDaily || report == app.ReportToday {
fs.StringVar(&opts.Date, "date", "", "report date in YYYY-MM-DD")
}
if err := fs.Parse(args); err != nil {
return inspectOptions{}, err
return comparisonOptions{}, err
}
if fs.NArg() > 0 {
return inspectOptions{}, fmt.Errorf("unexpected argument %q", fs.Arg(0))
return comparisonOptions{}, fmt.Errorf("unexpected argument %q", fs.Arg(0))
}
if opts.Limit < 0 {
return inspectOptions{}, fmt.Errorf("limit must be zero or greater")
}
return opts, nil
}
func parseInspectRunFlags(command string, args []string) (inspectOptions, error) {
fs := flag.NewFlagSet("inspect "+command, flag.ContinueOnError)
fs.SetOutput(io.Discard)
opts := inspectOptions{}
fs.StringVar(&opts.ConfigPath, "config", "", "configuration file path")
if err := fs.Parse(args); err != nil {
return inspectOptions{}, err
}
if fs.NArg() != 1 {
return inspectOptions{}, fmt.Errorf("inspect %s requires a run id", command)
}
opts.RunID = fs.Arg(0)
return opts, nil
}
@@ -384,6 +429,31 @@ func addCommonFlags(fs *flag.FlagSet, opts *commonOptions, includeOutput bool) {
fs.StringVar(&opts.Timezone, "tz", "", "weather API timezone")
fs.StringVar(&opts.LLMDebugDir, "llm-debug-dir", "", "write sensitive prompt debug artifacts under PATH")
if includeOutput {
fs.StringVar(&opts.Output, "out", "", "extra Markdown report copy path")
fs.StringVar(&opts.Output, "out", "", "generated Markdown report path")
}
}
func (r Runner) workingDir() (string, error) {
workingDir := r.WorkingDir
if workingDir == "" {
var err error
workingDir, err = os.Getwd()
if err != nil {
return "", fmt.Errorf("get working directory: %w", err)
}
}
if !filepath.IsAbs(workingDir) {
return "", fmt.Errorf("working directory %q must be absolute", workingDir)
}
return filepath.Clean(workingDir), nil
}
func resolveOutputOverride(workingDir, value string) (string, error) {
if value == "" {
return "", nil
}
if !filepath.IsAbs(value) {
value = filepath.Join(workingDir, value)
}
return filepath.Clean(value), nil
}

View File

@@ -1,619 +0,0 @@
package cli
import (
"bytes"
"context"
"encoding/json"
"errors"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"testing"
"time"
"gitea.maximumdirect.net/eric/weatherreporter/internal/app"
"gitea.maximumdirect.net/eric/weatherreporter/internal/module"
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec"
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptinput"
"gitea.maximumdirect.net/eric/weatherreporter/internal/report"
"gitea.maximumdirect.net/eric/weatherreporter/internal/state"
"gitea.maximumdirect.net/eric/weatherreporter/internal/timeutil"
)
const (
testRenderedPrompt = "PRIVATE RENDERED PROMPT"
testSchemaBody = `{"private":"schema"}`
testDataBody = "PRIVATE DATA PACKAGE"
testGeneratedBody = "PRIVATE GENERATED BODY"
testEndpoint = "https://user:credential@example.invalid/v1?token=credential"
testParameters = `{"temperature":0.2,"private":"parameter"}`
testCredential = "cli-secret-credential"
)
type commandOutput struct {
stdout string
stderr string
}
type cliExecutor struct {
fail bool
failPrompt string
}
func (e cliExecutor) InspectPrompt(_ context.Context, id, version string) (promptexec.PromptInspection, error) {
name := strings.TrimSuffix(strings.TrimPrefix(id, "weather."), "_generated_text")
return promptexec.PromptInspection{
PromptID: id, PromptVersion: version, PromptHash: "prompt-hash", DefaultProfileID: "offline-profile",
Inputs: []promptexec.InputDefinition{{Name: "data_package", Required: true, ContentType: "application/yaml"}},
Output: promptexec.OutputContract{Format: "json", ValidationMode: "json_schema", SchemaPath: name + ".generated_text.schema.json"},
}, nil
}
func (e cliExecutor) InspectProfile(_ context.Context, id string) (promptexec.ProfileInspection, error) {
return promptexec.ProfileInspection{ProfileID: id, BackendID: "offline", ModelName: "offline-model"}, nil
}
func (e cliExecutor) Execute(_ context.Context, req promptexec.ExecuteRequest, callback promptexec.PreparationCallback) (*promptexec.Execution, error) {
now := time.Date(2026, 5, 29, 12, 1, 0, 0, time.UTC)
preparation := promptexec.Preparation{
PromptID: req.PromptID, PromptVersion: req.PromptVersion, PromptHash: "prompt-hash", RenderedPromptHash: "rendered-hash",
ProfileID: req.ProfileID, BackendID: "offline", ModelName: "offline-model", DataPackagePath: req.DataPackagePath,
StartedAt: now, EndedAt: now,
}
if err := callback(preparation, nil); err != nil {
return nil, err
}
if e.fail || req.PromptID == e.failPrompt {
return nil, errors.New(strings.Join([]string{
"provider failed", testEndpoint, testCredential, testRenderedPrompt, testSchemaBody, testDataBody, testGeneratedBody, testParameters,
}, " "))
}
raw := []byte(`{"summary":"Showers are possible.","forecast_discussion":["Rain chances continue."],"precipitation_timing":"Rain is most likely this afternoon."}`)
if req.PromptID == "weather.hourly_generated_text" {
raw = []byte(`{"summary":"Storm chances increase.","forecast_discussion":"A front keeps the area unsettled.","precipitation_timing":"Rain is most likely late this morning."}`)
}
return &promptexec.Execution{
RunID: "offline-run", PromptID: req.PromptID, PromptVersion: req.PromptVersion, PromptHash: "prompt-hash", RenderedPromptHash: "rendered-hash",
ProfileID: req.ProfileID, BackendID: "offline", ModelName: "offline-model", GeneratedHash: "generated-hash",
StartedAt: now, EndedAt: now, DataPackagePath: req.DataPackagePath, RawOutput: raw,
Validation: promptexec.NewValidation(promptexec.ValidationPassed, "json_schema", "generated_text.schema.json", nil),
}, nil
}
func TestRunnerHelpListsOnlySupportedCommands(t *testing.T) {
output, err := runCLICommand(Runner{}, "--help")
if err != nil {
t.Fatalf("Run(--help) error = %v", err)
}
for _, command := range []string{
"--version",
"generate daily", "generate today", "generate tomorrow", "generate hourly", "run morning", "run evening",
"inspect reports", "inspect metadata", "inspect modules", "inspect data-package", "inspect prior", "inspect sources",
} {
if !strings.Contains(output.stdout, command) {
t.Fatalf("help missing %q:\n%s", command, output.stdout)
}
}
for _, retired := range []string{"near-term", "three-day", "weekend", "storm"} {
if strings.Contains(output.stdout, retired) {
t.Fatalf("help contains retired command %q:\n%s", retired, output.stdout)
}
}
}
func TestRunnerVersion(t *testing.T) {
for _, test := range []struct {
name string
runner Runner
version string
}{
{name: "development default", runner: Runner{}, version: "development"},
{name: "injected release", runner: Runner{Version: "v0.9.0-test"}, version: "v0.9.0-test"},
} {
t.Run(test.name, func(t *testing.T) {
output, err := runCLICommand(test.runner, "--version")
if err != nil {
t.Fatalf("Run(--version) error = %v", err)
}
if output.stdout != "weatherreporter "+test.version+"\n" || output.stderr != "" {
t.Fatalf("Run(--version) output = stdout %q stderr %q", output.stdout, output.stderr)
}
})
}
if _, err := runCLICommand(Runner{Version: "v0.9.0-test"}, "--version", "extra"); err == nil {
t.Fatal("Run(--version extra) error = nil")
}
}
func TestResolveSupportedCommandsAndFlags(t *testing.T) {
configPath := writeCLIConfig(t, t.TempDir(), "")
runner, constructions := countingRunner(cliExecutor{})
for _, tt := range []struct {
name string
args []string
want app.ReportKind
}{
{name: "daily", args: []string{"daily", "--date", "2026-05-29"}, want: app.ReportDaily},
{name: "today", args: []string{"today"}, want: app.ReportToday},
{name: "tomorrow", args: []string{"tomorrow"}, want: app.ReportTomorrow},
{name: "hourly", args: []string{"hourly"}, want: app.ReportHourly},
} {
t.Run(tt.name, func(t *testing.T) {
args := append(tt.args, "--config", configPath)
req, err := runner.resolveGenerate(args)
if err != nil {
t.Fatalf("resolveGenerate() error = %v", err)
}
if req.Report != tt.want || req.Executor == nil {
t.Fatalf("request = %#v, want report %q with executor", req, tt.want)
}
if tt.want == app.ReportDaily || tt.want == app.ReportToday {
if got := req.Date.Format(timeutil.DateLayout); got != "2026-05-29" {
t.Fatalf("resolved date = %q, want 2026-05-29", got)
}
} else if !req.Date.IsZero() {
t.Fatalf("resolved date = %s, want unset", req.Date)
}
})
}
for _, tt := range []struct {
name string
want app.BatchKind
}{
{name: "morning", want: app.BatchMorning},
{name: "evening", want: app.BatchEvening},
} {
t.Run(tt.name, func(t *testing.T) {
req, err := runner.resolveRun([]string{tt.name, "--config", configPath})
if err != nil {
t.Fatalf("resolveRun() error = %v", err)
}
if req.Batch != tt.want || req.Executor == nil {
t.Fatalf("request = %#v, want batch %q with executor", req, tt.want)
}
})
}
if *constructions != 6 {
t.Fatalf("executor constructions = %d, want one per resolved action", *constructions)
}
}
func TestResolveGenerateAndRunApplySharedActionFlags(t *testing.T) {
configPath := writeCLIConfig(t, t.TempDir(), "")
runner, _ := countingRunner(cliExecutor{})
generate, generateOpts, err := runner.resolveGenerateAction([]string{
"daily", "--config", configPath, "--date", "2026-05-30", "--units", "metric", "--tz", "UTC",
"--out", "daily.md", "--llm-debug-dir", "/safe/debug", "--quiet",
})
if err != nil {
t.Fatalf("resolveGenerateAction() error = %v", err)
}
if generate.Config.WeatherAPI.Units != "metric" || generate.Config.WeatherAPI.Timezone != "UTC" || generate.OutputPath != "daily.md" || generate.LLMDebugDir != "/safe/debug" || !generateOpts.Quiet {
t.Fatalf("generate request/options = %#v/%#v", generate, generateOpts)
}
if got := generate.Date.Format(timeutil.DateLayout); got != "2026-05-30" {
t.Fatalf("generate date = %q, want 2026-05-30", got)
}
batch, batchOpts, err := runner.resolveRunAction([]string{
"evening", "--config", configPath, "--units", "metric", "--tz", "UTC", "--out-dir", "reports",
"--llm-debug-dir", "/safe/debug", "--quiet",
})
if err != nil {
t.Fatalf("resolveRunAction() error = %v", err)
}
if batch.Config.WeatherAPI.Units != "metric" || batch.Config.WeatherAPI.Timezone != "UTC" || batch.OutputDir != "reports" || batch.LLMDebugDir != "/safe/debug" || !batchOpts.Quiet {
t.Fatalf("batch request/options = %#v/%#v", batch, batchOpts)
}
}
func TestGenerateFlagContracts(t *testing.T) {
for _, kind := range []app.ReportKind{app.ReportDaily, app.ReportToday, app.ReportTomorrow, app.ReportHourly} {
t.Run(string(kind), func(t *testing.T) {
opts, err := parseGenerateFlags(kind, []string{"--llm-debug-dir", "/safe/debug", "--quiet", "--out", "report.md"})
if err != nil || opts.LLMDebugDir != "/safe/debug" || !opts.Quiet || opts.Output != "report.md" {
t.Fatalf("parseGenerateFlags() = %#v, %v", opts, err)
}
})
}
runner, _ := countingRunner(cliExecutor{})
for _, tt := range []struct {
name string
args []string
want string
}{
{name: "daily requires date", args: []string{"daily"}, want: "requires --date"},
{name: "malformed daily date", args: []string{"daily", "--date", "bad-date"}, want: "YYYY-MM-DD"},
{name: "malformed today date", args: []string{"today", "--date", "bad-date"}, want: "YYYY-MM-DD"},
{name: "tomorrow rejects date", args: []string{"tomorrow", "--date", "2026-05-29"}},
{name: "hourly rejects date", args: []string{"hourly", "--date", "2026-05-29"}},
{name: "hourly rejects hours", args: []string{"hourly", "--hours", "6"}},
{name: "hourly rejects duration", args: []string{"hourly", "--duration", "6h"}},
{name: "batch rejects output", args: []string{"run", "--out", "report.md"}},
} {
t.Run(tt.name, func(t *testing.T) {
var err error
if tt.args[0] == "run" {
_, err = runner.resolveRun(append([]string{"morning"}, tt.args[1:]...))
} else {
_, err = runner.resolveGenerate(tt.args)
}
if err == nil || (tt.want != "" && !strings.Contains(err.Error(), tt.want)) {
t.Fatalf("error = %v, want rejection containing %q", err, tt.want)
}
})
}
}
func TestResolversRejectRetiredAndUnknownNames(t *testing.T) {
runner, _ := countingRunner(cliExecutor{})
for _, name := range []string{"near-term", "three-day", "weekend", "storm"} {
if _, err := runner.resolveGenerate([]string{name}); err == nil || !strings.Contains(err.Error(), "unknown generate report") {
t.Fatalf("resolveGenerate(%q) error = %v", name, err)
}
}
for _, name := range []string{"daily", "weekend", "storm"} {
if _, err := runner.resolveRun([]string{name}); err == nil || !strings.Contains(err.Error(), "unknown run batch") {
t.Fatalf("resolveRun(%q) error = %v", name, err)
}
}
}
func TestRunnerSuccessfulSingleAndBatchActions(t *testing.T) {
for _, tt := range []struct {
name string
args func(string, string) []string
}{
{name: "single", args: func(configPath, outputPath string) []string {
return []string{"generate", "today", "--config", configPath, "--out", outputPath}
}},
{name: "batch", args: func(configPath, outputPath string) []string {
return []string{"run", "evening", "--config", configPath, "--out-dir", outputPath}
}},
} {
t.Run(tt.name, func(t *testing.T) {
fixture := newCLIFixture(t)
runner, constructions := countingRunner(cliExecutor{})
output, err := runCLICommand(runner, tt.args(fixture.configPath, fixture.path("copies"))...)
if err != nil {
t.Fatalf("Run() error = %v", err)
}
if *constructions != 1 {
t.Fatalf("executor constructions = %d, want 1", *constructions)
}
assertRoutineOutputSafe(t, output)
if tt.name == "single" {
summary := decodeGenerateSummary(t, output.stdout)
if summary.Status != summaryStatusSucceeded || summary.ReportID != report.Today || summary.ReportPath == "" || summary.OutputPath == "" || summary.MetadataPath == "" || summary.DataPackagePath == "" || summary.PreparationPath == "" || summary.ExecutionPath == "" {
t.Fatalf("single summary = %#v", summary)
}
if strings.Contains(output.stdout, `"notification"`) || strings.Contains(output.stdout, `"llmDebugPath"`) {
t.Fatalf("single summary contains absent optional fields:\n%s", output.stdout)
}
} else {
summary := decodeBatchSummary(t, output.stdout)
if summary.Status != summaryStatusSucceeded || summary.Batch != app.BatchEvening || summary.Total != 1 || len(summary.Reports) != 1 || summary.Reports[0].OutputPath == "" {
t.Fatalf("batch summary = %#v", summary)
}
if !strings.Contains(output.stderr, "report=tomorrow status=succeeded") || !strings.Contains(output.stderr, "batch=evening total=1 succeeded=1 failed=0") {
t.Fatalf("batch status = %q", output.stderr)
}
if strings.Contains(output.stdout, `"notification"`) || strings.Contains(output.stdout, `"llmDebugPath"`) {
t.Fatalf("batch summary contains absent optional fields:\n%s", output.stdout)
}
}
})
}
}
func TestRunnerPreRunFailureAndQuietMode(t *testing.T) {
runner, constructions := countingRunner(cliExecutor{})
output, err := runCLICommand(runner, "generate", "daily")
if err == nil || output.stdout != "" || output.stderr != "" {
t.Fatalf("pre-run output/error = %#v/%v, want error without summary", output, err)
}
if *constructions != 1 {
t.Fatalf("executor constructions = %d, want one action-scoped construction", *constructions)
}
fixture := newCLIFixture(t)
runner, _ = countingRunner(cliExecutor{})
output, err = runCLICommand(runner, "generate", "today", "--config", fixture.configPath, "--quiet")
if err != nil || output.stdout != "" || output.stderr != "" {
t.Fatalf("quiet output/error = %#v/%v", output, err)
}
}
func TestRunnerFailedActionReportsSafePartialSummary(t *testing.T) {
fixture := newCLIFixture(t)
runner, constructions := countingRunner(cliExecutor{fail: true})
output, err := runCLICommand(runner, "generate", "today", "--config", fixture.configPath)
if err == nil {
t.Fatal("Run() error = nil, want execution failure")
}
if *constructions != 1 {
t.Fatalf("executor constructions = %d, want 1", *constructions)
}
summary := decodeGenerateSummary(t, output.stdout)
if summary.Status != summaryStatusFailed || summary.RunID == "" || summary.MetadataPath == "" || summary.DataPackagePath == "" || summary.PreparationPath == "" || summary.ExecutionPath == "" || summary.ReportPath != "" {
t.Fatalf("failed summary paths = %#v", summary)
}
if !strings.Contains(summary.Error, "prompt execution failed") {
t.Fatalf("failed summary error = %q", summary.Error)
}
assertRoutineOutputSafe(t, output)
for _, command := range []string{"metadata", "modules", "data-package", "sources"} {
inspected, inspectErr := runCLICommand(runner, "inspect", command, "--config", fixture.configPath, summary.RunID)
if inspectErr != nil {
t.Fatalf("inspect %s error = %v", command, inspectErr)
}
if !strings.Contains(inspected.stdout, summary.RunID) {
t.Fatalf("inspect %s missing failed run id:\n%s", command, inspected.stdout)
}
assertRoutineOutputSafe(t, inspected)
}
}
func TestRunnerMixedBatchReportsSafePartialFailure(t *testing.T) {
fixture := newCLIFixture(t)
runner, constructions := countingRunner(cliExecutor{failPrompt: "weather.tomorrow_generated_text"})
output, err := runCLICommand(runner, "run", "morning", "--config", fixture.configPath)
if err == nil {
t.Fatal("Run() error = nil, want aggregate batch failure")
}
if *constructions != 1 {
t.Fatalf("executor constructions = %d, want 1", *constructions)
}
summary := decodeBatchSummary(t, output.stdout)
if summary.Status != summaryStatusFailed || summary.Total != 2 || summary.Succeeded != 1 || summary.Failed != 1 || len(summary.Reports) != 2 {
t.Fatalf("failed batch summary = %#v", summary)
}
var succeeded, failed *app.BatchReportResult
for index := range summary.Reports {
item := &summary.Reports[index]
if item.Status == summaryStatusSucceeded {
succeeded = item
} else if item.Status == summaryStatusFailed {
failed = item
}
}
if succeeded == nil || succeeded.ReportPath == "" || succeeded.MetadataPath == "" || succeeded.ExecutionPath == "" {
t.Fatalf("successful batch item paths = %#v", succeeded)
}
if failed == nil || failed.ReportPath != "" || failed.MetadataPath == "" || failed.DataPackagePath == "" || failed.PreparationPath == "" || failed.ExecutionPath == "" {
t.Fatalf("failed batch item paths = %#v", failed)
}
if !strings.Contains(output.stderr, "status=succeeded") || !strings.Contains(output.stderr, "status=failed") || !strings.Contains(output.stderr, "batch=morning total=2 succeeded=1 failed=1") {
t.Fatalf("partial batch status = %q", output.stderr)
}
assertRoutineOutputSafe(t, output)
}
func TestRunnerInspectsReportsAndCurrentArtifacts(t *testing.T) {
fixture := newCLIFixture(t)
runner, _ := countingRunner(cliExecutor{})
first := runSuccessfulGenerate(t, runner, fixture.configPath, time.Date(2026, 5, 29, 11, 0, 0, 0, time.UTC))
second := runSuccessfulGenerate(t, runner, fixture.configPath, time.Date(2026, 5, 29, 12, 0, 0, 0, time.UTC))
listed, err := runCLICommand(runner, "inspect", "reports", "--config", fixture.configPath, "--limit", "2")
if err != nil || !strings.Contains(listed.stdout, first.RunID) || !strings.Contains(listed.stdout, second.RunID) {
t.Fatalf("inspect reports output/error = %s/%v", listed.stdout, err)
}
for _, command := range []string{"metadata", "modules", "data-package", "sources"} {
output, inspectErr := runCLICommand(runner, "inspect", command, "--config", fixture.configPath, second.RunID)
if inspectErr != nil || !strings.Contains(output.stdout, second.RunID) {
t.Fatalf("inspect %s output/error = %s/%v", command, output.stdout, inspectErr)
}
assertRoutineOutputSafe(t, output)
}
prior, err := runCLICommand(runner, "inspect", "prior", "--config", fixture.configPath, second.RunID)
if err != nil || !strings.Contains(prior.stdout, first.RunID) {
t.Fatalf("inspect prior output/error = %s/%v", prior.stdout, err)
}
}
func TestRunnerInspectsHistoricalMetadataAndArtifacts(t *testing.T) {
fixture := newCLIFixture(t)
runIDs := []string{
writeHistoricalInspectionFixture(t, fixture.workspaceRoot, report.ID("three_day"), time.Date(2026, 5, 20, 12, 0, 0, 0, time.UTC)),
writeHistoricalInspectionFixture(t, fixture.workspaceRoot, report.ID("weekend"), time.Date(2026, 5, 21, 12, 0, 0, 0, time.UTC)),
writeHistoricalInspectionFixture(t, fixture.workspaceRoot, report.ID("storm"), time.Date(2026, 5, 22, 12, 0, 0, 0, time.UTC)),
}
runID := runIDs[0]
runner, _ := countingRunner(cliExecutor{})
listed, err := runCLICommand(runner, "inspect", "reports", "--config", fixture.configPath)
if err != nil {
t.Fatalf("inspect historical reports output/error = %s/%v", listed.stdout, err)
}
for _, want := range []string{runIDs[0], runIDs[1], runIDs[2], `"reportId": "three_day"`, `"reportId": "weekend"`, `"reportId": "storm"`} {
if !strings.Contains(listed.stdout, want) {
t.Fatalf("inspect historical reports missing %q:\n%s", want, listed.stdout)
}
}
for _, command := range []string{"metadata", "modules", "data-package", "sources"} {
output, inspectErr := runCLICommand(runner, "inspect", command, "--config", fixture.configPath, runID)
if inspectErr != nil || !strings.Contains(output.stdout, runID) {
t.Fatalf("inspect historical %s output/error = %s/%v", command, output.stdout, inspectErr)
}
}
metadata, err := runCLICommand(runner, "inspect", "metadata", "--config", fixture.configPath, runID)
if err != nil || !strings.Contains(metadata.stdout, `"schemaVersion": "weatherreporter.metadata.v1"`) || !strings.Contains(metadata.stdout, `"preflightPath"`) || strings.Contains(metadata.stdout, `"preparationPath"`) {
t.Fatalf("historical metadata aliases/output = %s/%v", metadata.stdout, err)
}
}
func countingRunner(executor promptexec.Executor) (Runner, *int) {
count := new(int)
return Runner{
Clock: timeutil.FixedClock{Time: time.Date(2026, 5, 29, 12, 0, 0, 0, time.UTC)},
ExecutorFactory: func(PromptExecutorConfig) (promptexec.Executor, error) {
*count++
return executor, nil
},
}, count
}
func runCLICommand(runner Runner, args ...string) (commandOutput, error) {
var stdout bytes.Buffer
var stderr bytes.Buffer
err := runner.Run(context.Background(), args, &stdout, &stderr)
return commandOutput{stdout: stdout.String(), stderr: stderr.String()}, err
}
func runSuccessfulGenerate(t *testing.T, base Runner, configPath string, now time.Time) generateSummary {
t.Helper()
base.Clock = timeutil.FixedClock{Time: now}
output, err := runCLICommand(base, "generate", "today", "--config", configPath)
if err != nil {
t.Fatalf("generate current report: %v", err)
}
return decodeGenerateSummary(t, output.stdout)
}
func decodeGenerateSummary(t *testing.T, text string) generateSummary {
t.Helper()
var summary generateSummary
if err := json.Unmarshal([]byte(text), &summary); err != nil {
t.Fatalf("decode generate summary: %v\n%s", err, text)
}
return summary
}
func decodeBatchSummary(t *testing.T, text string) batchSummary {
t.Helper()
var summary batchSummary
if err := json.Unmarshal([]byte(text), &summary); err != nil {
t.Fatalf("decode batch summary: %v\n%s", err, text)
}
return summary
}
func assertRoutineOutputSafe(t *testing.T, output commandOutput) {
t.Helper()
combined := output.stdout + output.stderr
for _, forbidden := range []string{testRenderedPrompt, testSchemaBody, testDataBody, testGeneratedBody, testEndpoint, testParameters, testCredential, "credential@example.invalid"} {
if strings.Contains(combined, forbidden) {
t.Fatalf("routine output contains sensitive value %q:\n%s", forbidden, combined)
}
}
}
type cliFixture struct {
tempDir string
workspaceRoot string
configPath string
}
func newCLIFixture(t *testing.T) cliFixture {
t.Helper()
tempDir := t.TempDir()
workspaceRoot := filepath.Join(tempDir, "workspace")
server := weatherServer(t)
return cliFixture{tempDir: tempDir, workspaceRoot: workspaceRoot, configPath: writeCLIConfig(t, workspaceRoot, server.URL+"/")}
}
func (f cliFixture) path(name string) string { return filepath.Join(f.tempDir, name) }
func writeCLIConfig(t *testing.T, workspaceRoot, baseURL string) string {
t.Helper()
configPath := filepath.Join(t.TempDir(), "config.yml")
body := "weather_api:\n timezone: America/Chicago\n"
if baseURL != "" {
body += " base_url: " + baseURL + "\n"
}
body += "workspace:\n root: " + workspaceRoot + "\n"
if err := os.WriteFile(configPath, []byte(body), 0o600); err != nil {
t.Fatalf("write config: %v", err)
}
return configPath
}
func weatherServer(t *testing.T) *httptest.Server {
t.Helper()
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch r.URL.Path {
case "/observations":
_, _ = w.Write([]byte(`{"data":{"timestamp":"2026-05-29T14:00:00Z","conditionCode":3}}`))
case "/conditions/current":
_, _ = w.Write([]byte(`{"data":{"conditionText":"Clear"}}`))
case "/forecast/hourly":
_, _ = w.Write([]byte(`{"data":{"locationId":"test-grid","locationName":"Testville","issuedAt":"2026-05-29T10:30:00-05:00","product":"hourly","periods":[{"startTime":"2026-05-29T06:00:00-05:00","endTime":"2026-05-29T07:00:00-05:00","textDescription":"Showers","temperatureF":66,"probabilityOfPrecipitationPercent":80},{"startTime":"2026-05-30T06:00:00-05:00","endTime":"2026-05-30T07:00:00-05:00","textDescription":"Showers","temperatureF":67,"probabilityOfPrecipitationPercent":70}]}}`))
case "/forecast/narrative":
_, _ = w.Write([]byte(`{"data":{"issuedAt":"2026-05-29T10:30:00-05:00","product":"narrative","periods":[{"startTime":"2026-05-29T06:00:00-05:00","endTime":"2026-05-29T18:00:00-05:00","textDescription":"Morning showers."},{"startTime":"2026-05-30T06:00:00-05:00","endTime":"2026-05-30T18:00:00-05:00","textDescription":"Tomorrow starts showery."}]}}`))
case "/alerts/active":
_, _ = w.Write([]byte(`{"data":{"alerts":[]}}`))
case "/discussion":
_, _ = w.Write([]byte(`{"data":{"product":"discussion","issuedAt":"2026-05-29T09:25:00-05:00","keyMessages":["Showers remain possible."]}}`))
case "/weatherstories/latest":
_, _ = w.Write([]byte(`{"data":null}`))
case "/outlooks/convective":
_, _ = w.Write([]byte(`{"data":{"asOf":"2026-05-29T16:00:00Z","outlooks":[],"discussions":[]}}`))
default:
http.NotFound(w, r)
}
}))
t.Cleanup(server.Close)
return server
}
func writeHistoricalInspectionFixture(t *testing.T, workspaceRoot string, reportID report.ID, generatedAt time.Time) string {
t.Helper()
runID := "historical-" + string(reportID)
date := generatedAt.Format(timeutil.DateLayout)
dir := filepath.Join(workspaceRoot, "snapshots", string(reportID), date)
modulePath := filepath.Join(dir, "modules."+runID+".json")
dataPath := filepath.Join(workspaceRoot, "data-packages", string(reportID), date, "data_package."+runID+".yaml")
metadataPath := filepath.Join(dir, "metadata."+runID+".json")
if err := os.MkdirAll(filepath.Dir(dataPath), 0o755); err != nil {
t.Fatalf("create historical fixture directory: %v", err)
}
if err := os.MkdirAll(dir, 0o755); err != nil {
t.Fatalf("create historical metadata directory: %v", err)
}
snapshot, err := module.NewSnapshot([]module.Output{{ID: module.Metadata, StanzaName: "metadata", Value: map[string]any{"run_id": runID}}})
if err != nil {
t.Fatalf("build historical module snapshot: %v", err)
}
moduleData, err := json.Marshal(snapshot)
if err != nil {
t.Fatalf("marshal historical module snapshot: %v", err)
}
if err := os.WriteFile(modulePath, moduleData, 0o600); err != nil {
t.Fatalf("write historical module snapshot: %v", err)
}
period := timeutil.Period{Start: generatedAt, End: generatedAt.Add(24 * time.Hour)}
pkg, err := promptinput.Build(promptinput.BuildRequest{Metadata: promptinput.Metadata{
RunID: runID, ReportID: reportID, PromptID: "weather." + string(reportID), GeneratedAt: generatedAt, Timezone: "UTC", ValidPeriod: period,
}, Modules: snapshot})
if err != nil {
t.Fatalf("build historical data package: %v", err)
}
data, err := promptinput.MarshalYAML(pkg)
if err != nil {
t.Fatalf("marshal historical data package: %v", err)
}
if err := os.WriteFile(dataPath, data, 0o600); err != nil {
t.Fatalf("write historical data package: %v", err)
}
metadata := state.Metadata{
SchemaVersion: state.MetadataSchemaVersionV1, RunID: runID, ReportID: reportID, PromptID: "weather." + string(reportID),
GeneratedAt: generatedAt, Timezone: "UTC", ValidPeriod: period, SourceLocation: "historical archive",
ModuleSnapshotPath: modulePath, DataPackagePath: dataPath, PreflightPath: "/archive/preflight.json", GeneratedTextResultPath: "/archive/result.json",
}
metadataData, err := json.Marshal(metadata)
if err != nil {
t.Fatalf("marshal historical metadata: %v", err)
}
if err := os.WriteFile(metadataPath, metadataData, 0o600); err != nil {
t.Fatalf("write historical metadata: %v", err)
}
return runID
}

View File

@@ -1,11 +1,18 @@
package cli
import (
"bytes"
"context"
"encoding/json"
"errors"
"os"
"path/filepath"
"strconv"
"strings"
"testing"
"time"
"gitea.maximumdirect.net/eric/weatherreporter/internal/app"
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec"
"gitea.maximumdirect.net/eric/weatherreporter/internal/timeutil"
)
@@ -21,7 +28,7 @@ func TestResolveRunActionConstructsOneExecutor(t *testing.T) {
for _, command := range []string{"morning", "evening"} {
t.Run(command, func(t *testing.T) {
configPath := filepath.Join(t.TempDir(), "config.yml")
if err := os.WriteFile(configPath, []byte("workspace:\n root: "+filepath.Join(t.TempDir(), "workspace")+"\n"), 0o600); err != nil {
if err := os.WriteFile(configPath, []byte("weather_api:\n base_url: https://weather.api.example.com/\n"), 0o600); err != nil {
t.Fatal(err)
}
calls := 0
@@ -40,3 +47,162 @@ func TestResolveRunActionConstructsOneExecutor(t *testing.T) {
})
}
}
func TestResolveGenerateActionUsesInjectedWorkingDirectoryForOutputOverrides(t *testing.T) {
workingDir := t.TempDir()
configPath := filepath.Join(t.TempDir(), "config.yml")
const configuredDirectory = "configured/../reports"
if err := os.WriteFile(configPath, []byte("weather_api:\n base_url: https://weather.api.example.com/\noutput:\n directory: "+configuredDirectory+"\n"), 0o600); err != nil {
t.Fatal(err)
}
absoluteOutput := filepath.Join(t.TempDir(), "daily.md")
for _, scenario := range []struct {
name string
out string
want string
}{
{name: "default", want: ""},
{name: "relative", out: "reports/daily.md", want: filepath.Join(workingDir, "reports", "daily.md")},
{name: "absolute", out: absoluteOutput, want: absoluteOutput},
} {
t.Run(scenario.name, func(t *testing.T) {
runner := Runner{
Clock: timeutil.FixedClock{Time: time.Date(2026, 5, 29, 8, 0, 0, 0, time.UTC)},
WorkingDir: workingDir,
ExecutorFactory: func(PromptExecutorConfig) (promptexec.Executor, error) {
return &factoryExecutor{}, nil
},
}
args := []string{"daily", "--date", "2026-05-29", "--config", configPath}
if scenario.out != "" {
args = append(args, "--out", scenario.out)
}
req, _, err := runner.resolveGenerateAction(args)
if err != nil || req.WorkingDir != workingDir || req.OutputPath != scenario.want || req.Config.Output.Directory != configuredDirectory {
t.Fatalf("resolveGenerateAction() request/error = %#v/%v", req, err)
}
})
}
}
func TestResolveActionsPreserveConfiguredOutputDirectoryWithoutAnOverride(t *testing.T) {
workingDir := t.TempDir()
absoluteDirectory := filepath.Join(t.TempDir(), "reports")
for _, configuredDirectory := range []string{"configured/../reports", absoluteDirectory} {
t.Run(configuredDirectory, func(t *testing.T) {
configPath := filepath.Join(t.TempDir(), "config.yml")
contents := "weather_api:\n base_url: https://weather.api.example.com/\noutput:\n directory: " + strconv.Quote(configuredDirectory) + "\n"
if err := os.WriteFile(configPath, []byte(contents), 0o600); err != nil {
t.Fatal(err)
}
runner := Runner{
Clock: timeutil.FixedClock{Time: time.Date(2026, 5, 29, 8, 0, 0, 0, time.UTC)},
WorkingDir: workingDir,
ExecutorFactory: func(PromptExecutorConfig) (promptexec.Executor, error) {
return &factoryExecutor{}, nil
},
}
generateReq, _, generateErr := runner.resolveGenerateAction([]string{"daily", "--date", "2026-05-29", "--config", configPath})
if generateErr != nil || generateReq.Config.Output.Directory != configuredDirectory || generateReq.OutputPath != "" {
t.Fatalf("resolveGenerateAction() request/error = %#v/%v", generateReq, generateErr)
}
batchReq, _, batchErr := runner.resolveRunAction([]string{"morning", "--config", configPath})
if batchErr != nil || batchReq.Config.Output.Directory != configuredDirectory || batchReq.OutputDir != "" {
t.Fatalf("resolveRunAction() request/error = %#v/%v", batchReq, batchErr)
}
})
}
}
func TestResolveRunActionUsesInjectedWorkingDirectoryForOutputOverrides(t *testing.T) {
workingDir := t.TempDir()
configPath := filepath.Join(t.TempDir(), "config.yml")
const configuredDirectory = "configured/../reports"
if err := os.WriteFile(configPath, []byte("weather_api:\n base_url: https://weather.api.example.com/\noutput:\n directory: "+configuredDirectory+"\n"), 0o600); err != nil {
t.Fatal(err)
}
absoluteOutput := filepath.Join(t.TempDir(), "reports")
for _, scenario := range []struct {
name string
out string
want string
}{
{name: "relative", out: "reports/../published", want: filepath.Join(workingDir, "published")},
{name: "absolute", out: absoluteOutput, want: absoluteOutput},
} {
t.Run(scenario.name, func(t *testing.T) {
runner := Runner{
Clock: timeutil.FixedClock{Time: time.Date(2026, 5, 29, 8, 0, 0, 0, time.UTC)},
WorkingDir: workingDir,
ExecutorFactory: func(PromptExecutorConfig) (promptexec.Executor, error) {
return &factoryExecutor{}, nil
},
}
req, _, err := runner.resolveRunAction([]string{"morning", "--config", configPath, "--out-dir", scenario.out})
if err != nil || req.WorkingDir != workingDir || req.OutputDir != scenario.want || req.Config.Output.Directory != configuredDirectory {
t.Fatalf("resolveRunAction() request/error = %#v/%v", req, err)
}
})
}
}
func TestRunActionReturnsFailureForBatchNotificationFailure(t *testing.T) {
configPath := filepath.Join(t.TempDir(), "config.yml")
if err := os.WriteFile(configPath, []byte("weather_api:\n base_url: https://weather.api.example.com/\n"), 0o600); err != nil {
t.Fatal(err)
}
result := &app.BatchResult{
Batch: app.BatchMorning, Total: 2, Succeeded: 2,
Reports: []app.BatchReportResult{
{ReportID: "today", Status: "succeeded", OutputPath: "/reports/today.md"},
{ReportID: "tomorrow", Status: "succeeded", OutputPath: "/reports/tomorrow.md"},
},
Notification: &app.BatchNotificationResult{Status: "failed", Error: "distributor unavailable"},
}
var stdout, stderr bytes.Buffer
runner := Runner{
Clock: timeutil.FixedClock{Time: time.Date(2026, 5, 29, 8, 0, 0, 0, time.UTC)},
ExecutorFactory: func(PromptExecutorConfig) (promptexec.Executor, error) {
return &factoryExecutor{}, nil
},
runBatchDetailed: func(context.Context, app.BatchRequest) (*app.BatchResult, error) {
return result, nil
},
}
err := runner.Run(context.Background(), []string{"run", "morning", "--config", configPath}, &stdout, &stderr)
var batchErr app.BatchError
if !errors.As(err, &batchErr) || !strings.Contains(err.Error(), "notification failed") {
t.Fatalf("Run() error = %v", err)
}
var summary batchSummary
if err := json.Unmarshal(stdout.Bytes(), &summary); err != nil {
t.Fatalf("decode summary: %v", err)
}
if summary.Status != summaryStatusFailed || summary.Total != 2 || summary.Succeeded != 2 || summary.Failed != 0 || summary.Notification == nil || summary.Notification.Status != "failed" {
t.Fatalf("summary = %#v", summary)
}
if !strings.Contains(stderr.String(), `batchNotification status="failed"`) {
t.Fatalf("stdout/stderr = %q/%q", stdout.String(), stderr.String())
}
for _, field := range []string{"notificationStatus", "notificationRunId", "notificationPipelineId", "notificationError"} {
if strings.Contains(stdout.String(), field) || strings.Contains(stderr.String(), field) {
t.Fatalf("stdout/stderr includes removed field %q: %q/%q", field, stdout.String(), stderr.String())
}
}
}
func TestInspectCommandIsUnknownAndAbsentFromHelp(t *testing.T) {
var stdout, stderr bytes.Buffer
err := (Runner{}).Run(context.Background(), []string{"inspect", "reports"}, &stdout, &stderr)
if err == nil || err.Error() != `unknown command "inspect"` {
t.Fatalf("Run(inspect) error = %v", err)
}
if err := (Runner{}).Run(context.Background(), []string{"--help"}, &stdout, &stderr); err != nil {
t.Fatalf("Run(--help) error = %v", err)
}
if strings.Contains(stdout.String(), "inspect") {
t.Fatalf("help contains removed inspect command:\n%s", stdout.String())
}
}

View File

@@ -0,0 +1,385 @@
// Package comparison owns logical profile-comparison bundle contracts and
// guarded filesystem destination planning and publication.
package comparison
import (
"crypto/sha256"
"encoding/hex"
"encoding/json"
"fmt"
"path"
"strings"
"time"
"unicode/utf8"
)
const (
// SchemaVersion identifies the supported comparison manifest schema.
SchemaVersion = "weatherreporter.comparison.v1"
// ManifestFilename is the canonical name of a comparison manifest.
ManifestFilename = "comparison.json"
// DataPackageFilename is the canonical name of the shared prompt input.
DataPackageFilename = "data-package.yml"
// StatusSucceeded identifies a profile that produced a validated report.
StatusSucceeded = "succeeded"
// StatusFailed identifies a profile that did not produce a report.
StatusFailed = "failed"
maxProfileSlugBytes = 64
maxErrorMessageBytes = 1024
)
// Manifest is the versioned, authoritative index of a comparison bundle.
// Field declaration order is the JSON field order.
type Manifest struct {
SchemaVersion string `json:"schemaVersion"`
ComparisonID string `json:"comparisonId"`
StartedAt time.Time `json:"startedAt"`
FinishedAt time.Time `json:"finishedAt"`
ReportID string `json:"reportId"`
ValidPeriod ValidPeriod `json:"validPeriod"`
Timezone string `json:"timezone"`
PromptID string `json:"promptId"`
PromptVersion string `json:"promptVersion"`
PromptHash string `json:"promptHash"`
DataPackage DataPackageReference `json:"dataPackage"`
Total int `json:"total"`
Succeeded int `json:"succeeded"`
Failed int `json:"failed"`
Results []Result `json:"results"`
}
// ValidPeriod records the report's resolved half-open period.
type ValidPeriod struct {
Start time.Time `json:"start"`
End time.Time `json:"end"`
}
// DataPackageReference identifies and verifies the shared prompt input.
type DataPackageReference struct {
Path string `json:"path"`
SHA256 string `json:"sha256"`
}
// Result records one explicitly selected profile in selection order.
type Result struct {
Position int `json:"position"`
ProfileID string `json:"profileId"`
BackendID string `json:"backendId,omitempty"`
ModelName string `json:"modelName"`
Status string `json:"status"`
ValidationStatus string `json:"validationStatus,omitempty"`
ReportPath string `json:"reportPath,omitempty"`
Error *SafeError `json:"error,omitempty"`
}
// SafeError contains bounded, operator-safe failure information only.
type SafeError struct {
Category string `json:"category"`
Message string `json:"message"`
}
// LogicalBundle contains every byte required to publish a comparison without
// coupling the comparison contract to a filesystem implementation.
type LogicalBundle struct {
Manifest Manifest
DataPackage []byte
Reports []BundleReport
}
// BundleReport is one rendered Markdown document in a logical bundle.
type BundleReport struct {
Position int
Path string
Markdown []byte
}
// ValidateProfileIDs requires at least two distinct, nonblank profile IDs. It
// intentionally preserves accepted IDs unchanged and treats case distinctly.
func ValidateProfileIDs(profileIDs []string) error {
if len(profileIDs) < 2 {
return fmt.Errorf("comparison requires at least two profile IDs")
}
seen := make(map[string]struct{}, len(profileIDs))
for _, profileID := range profileIDs {
if strings.TrimSpace(profileID) == "" {
return fmt.Errorf("profile ID must not be blank")
}
if _, duplicate := seen[profileID]; duplicate {
return fmt.Errorf("duplicate profile ID %q", profileID)
}
seen[profileID] = struct{}{}
}
return nil
}
// ProfileSlug returns a deterministic, filesystem-safe representation of a
// logical profile ID. The logical ID remains authoritative in the manifest.
func ProfileSlug(profileID string) string {
var builder strings.Builder
lastReplacement := false
for _, r := range profileID {
if isSlugRune(r) {
builder.WriteRune(r)
lastReplacement = false
continue
}
if !lastReplacement {
builder.WriteByte('-')
lastReplacement = true
}
}
slug := strings.Trim(builder.String(), "-_")
if len(slug) > maxProfileSlugBytes {
slug = strings.Trim(slug[:maxProfileSlugBytes], "-_")
}
if slug == "" {
return "profile"
}
return slug
}
func isSlugRune(r rune) bool {
return r >= 'a' && r <= 'z' ||
r >= 'A' && r <= 'Z' ||
r >= '0' && r <= '9' ||
r == '-' || r == '_'
}
// OrdinalWidth returns the zero-padding width for a comparison of count
// profiles.
func OrdinalWidth(count int) int {
width := 2
for value := count; value >= 100; value /= 10 {
width++
}
return width
}
// ReportFilename derives a report's deterministic bundle filename.
func ReportFilename(position, profileCount int, profileID string) (string, error) {
if profileCount < 1 {
return "", fmt.Errorf("profile count must be positive")
}
if position < 1 || position > profileCount {
return "", fmt.Errorf("profile position %d is outside 1..%d", position, profileCount)
}
return fmt.Sprintf("%0*d-%s.md", OrdinalWidth(profileCount), position, ProfileSlug(profileID)), nil
}
// BuildComparisonID derives the stable comparison identity for a resolved
// report run.
func BuildComparisonID(reportRunID string) (string, error) {
if strings.TrimSpace(reportRunID) == "" {
return "", fmt.Errorf("report run ID must not be blank")
}
return "comparison_" + reportRunID, nil
}
// DefaultDirectoryName derives the bundle directory name for a resolved report
// output filename.
func DefaultDirectoryName(reportOutputName string) (string, error) {
if !strings.HasSuffix(reportOutputName, ".md") {
return "", fmt.Errorf("report output name %q must end in .md", reportOutputName)
}
if !isArtifactBasename(reportOutputName) {
return "", fmt.Errorf("report output name %q must be a basename", reportOutputName)
}
stem := strings.TrimSuffix(reportOutputName, ".md")
if stem == "" {
return "", fmt.Errorf("report output name %q has an empty stem", reportOutputName)
}
return "comparison-" + stem, nil
}
// SHA256 returns a lowercase hexadecimal SHA-256 digest.
func SHA256(content []byte) string {
digest := sha256.Sum256(content)
return hex.EncodeToString(digest[:])
}
// NewSafeError returns bounded, valid UTF-8 error information suitable for a
// comparison manifest.
func NewSafeError(category, message string) SafeError {
return SafeError{Category: category, Message: TruncateErrorMessage(message)}
}
// TruncateErrorMessage returns a valid UTF-8 message of at most 1,024 bytes.
func TruncateErrorMessage(message string) string {
message = strings.ToValidUTF8(message, "\uFFFD")
if len(message) <= maxErrorMessageBytes {
return message
}
end := maxErrorMessageBytes
for end > 0 && !utf8.RuneStart(message[end]) {
end--
}
return message[:end]
}
// Validate checks every invariant required for a current comparison manifest.
func (manifest Manifest) Validate() error {
if manifest.SchemaVersion != SchemaVersion {
return fmt.Errorf("unsupported comparison schema version %q", manifest.SchemaVersion)
}
if strings.TrimSpace(manifest.ComparisonID) == "" {
return fmt.Errorf("comparison ID must not be blank")
}
if manifest.StartedAt.IsZero() || manifest.FinishedAt.IsZero() {
return fmt.Errorf("comparison timestamps must be nonzero")
}
if manifest.StartedAt.Location() != time.UTC || manifest.FinishedAt.Location() != time.UTC {
return fmt.Errorf("comparison timestamps must use UTC")
}
if manifest.FinishedAt.Before(manifest.StartedAt) {
return fmt.Errorf("comparison finish time precedes start time")
}
if manifest.ValidPeriod.Start.IsZero() || manifest.ValidPeriod.End.IsZero() || !manifest.ValidPeriod.End.After(manifest.ValidPeriod.Start) {
return fmt.Errorf("valid period must have a nonempty increasing range")
}
if strings.TrimSpace(manifest.ReportID) == "" || strings.TrimSpace(manifest.Timezone) == "" {
return fmt.Errorf("report ID and timezone must not be blank")
}
if strings.TrimSpace(manifest.PromptID) == "" || strings.TrimSpace(manifest.PromptVersion) == "" || !isSHA256(manifest.PromptHash) {
return fmt.Errorf("prompt identity is invalid")
}
if manifest.DataPackage.Path != DataPackageFilename || !isSHA256(manifest.DataPackage.SHA256) {
return fmt.Errorf("data package reference is invalid")
}
if manifest.Total < 2 || manifest.Total != len(manifest.Results) {
return fmt.Errorf("comparison result count is invalid")
}
if manifest.Succeeded < 0 || manifest.Failed < 0 || manifest.Succeeded+manifest.Failed != manifest.Total {
return fmt.Errorf("comparison result totals are inconsistent")
}
profiles := make(map[string]struct{}, len(manifest.Results))
reportPaths := make(map[string]struct{}, len(manifest.Results))
succeeded, failed := 0, 0
for index, result := range manifest.Results {
if result.Position != index+1 {
return fmt.Errorf("result position %d is not ordered", result.Position)
}
if strings.TrimSpace(result.ProfileID) == "" {
return fmt.Errorf("result %d has a blank profile ID", result.Position)
}
if _, duplicate := profiles[result.ProfileID]; duplicate {
return fmt.Errorf("result %d duplicates profile ID %q", result.Position, result.ProfileID)
}
profiles[result.ProfileID] = struct{}{}
if strings.TrimSpace(result.ModelName) == "" {
return fmt.Errorf("result %d has a blank model name", result.Position)
}
switch result.Status {
case StatusSucceeded:
succeeded++
if result.ValidationStatus != "passed" {
return fmt.Errorf("successful result %d did not pass validation", result.Position)
}
if result.Error != nil || !isReportPath(result.ReportPath) {
return fmt.Errorf("successful result %d has invalid report details", result.Position)
}
if _, duplicate := reportPaths[result.ReportPath]; duplicate {
return fmt.Errorf("result %d duplicates report path %q", result.Position, result.ReportPath)
}
reportPaths[result.ReportPath] = struct{}{}
case StatusFailed:
failed++
if result.ReportPath != "" || result.Error == nil || !isValidationStatus(result.ValidationStatus) {
return fmt.Errorf("failed result %d has invalid failure details", result.Position)
}
if err := result.Error.validate(); err != nil {
return fmt.Errorf("failed result %d: %w", result.Position, err)
}
default:
return fmt.Errorf("result %d has unsupported status %q", result.Position, result.Status)
}
}
if succeeded != manifest.Succeeded || failed != manifest.Failed {
return fmt.Errorf("comparison status counts do not match results")
}
return nil
}
func (safeError SafeError) validate() error {
if strings.TrimSpace(safeError.Category) == "" || strings.TrimSpace(safeError.Message) == "" {
return fmt.Errorf("safe error category and message must not be blank")
}
if !utf8.ValidString(safeError.Message) || len(safeError.Message) > maxErrorMessageBytes {
return fmt.Errorf("safe error message is not bounded UTF-8")
}
return nil
}
func isValidationStatus(status string) bool {
return status == "" || status == "failed" || status == "skipped"
}
func isSHA256(value string) bool {
if len(value) != sha256.Size*2 {
return false
}
for _, r := range value {
if (r < '0' || r > '9') && (r < 'a' || r > 'f') {
return false
}
}
return true
}
func isReportPath(value string) bool {
return strings.HasSuffix(value, ".md") && isArtifactBasename(value) && value != ManifestFilename && value != DataPackageFilename
}
func isArtifactBasename(value string) bool {
return value != "" && value != "." && value != ".." &&
!strings.ContainsAny(value, "/\\") && path.Base(value) == value
}
// Validate checks that the manifest and in-memory bundle payloads agree.
func (bundle LogicalBundle) Validate() error {
if err := bundle.Manifest.Validate(); err != nil {
return err
}
if SHA256(bundle.DataPackage) != bundle.Manifest.DataPackage.SHA256 {
return fmt.Errorf("data package digest does not match manifest")
}
if len(bundle.Reports) != bundle.Manifest.Succeeded {
return fmt.Errorf("bundle report count does not match manifest")
}
reportIndex := 0
for _, result := range bundle.Manifest.Results {
if result.Status != StatusSucceeded {
continue
}
report := bundle.Reports[reportIndex]
if report.Position != result.Position || report.Path != result.ReportPath || !isReportPath(report.Path) {
return fmt.Errorf("bundle report for result %d does not match manifest", result.Position)
}
reportIndex++
}
return nil
}
// EncodeManifest validates and deterministically encodes a manifest as the
// canonical two-space-indented JSON document with one trailing newline.
func EncodeManifest(manifest Manifest) ([]byte, error) {
if err := manifest.Validate(); err != nil {
return nil, err
}
encoded, err := json.MarshalIndent(manifest, "", " ")
if err != nil {
return nil, fmt.Errorf("encode comparison manifest: %w", err)
}
return append(encoded, '\n'), nil
}

View File

@@ -0,0 +1,314 @@
package comparison
import (
"encoding/json"
"strings"
"testing"
"time"
"unicode/utf8"
)
func TestValidateProfileIDs(t *testing.T) {
t.Parallel()
tests := []struct {
name string
profileIDs []string
wantErr bool
}{
{name: "accepts exact case distinct IDs", profileIDs: []string{"weather-light", "Weather-Light"}},
{name: "preserves surrounding whitespace", profileIDs: []string{" weather-light", "weather-deep "}},
{name: "rejects one ID", profileIDs: []string{"weather-light"}, wantErr: true},
{name: "rejects blank ID", profileIDs: []string{"weather-light", " \t\n"}, wantErr: true},
{name: "rejects exact duplicate", profileIDs: []string{"weather-light", "weather-light"}, wantErr: true},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
t.Parallel()
err := ValidateProfileIDs(test.profileIDs)
if (err != nil) != test.wantErr {
t.Fatalf("ValidateProfileIDs(%q) error = %v, want error %t", test.profileIDs, err, test.wantErr)
}
})
}
}
func TestProfileSlug(t *testing.T) {
t.Parallel()
long := strings.Repeat("a", 62) + "-_more"
tests := []struct {
profileID string
want string
}{
{profileID: "weather-light", want: "weather-light"},
{profileID: "model.v1", want: "model-v1"},
{profileID: "nested/path\\name", want: "nested-path-name"},
{profileID: " weather\tlight\n", want: "weather-light"},
{profileID: "\x00\x01", want: "profile"},
{profileID: "météo-東京", want: "m-t-o"},
{profileID: "___", want: "profile"},
{profileID: long, want: strings.Repeat("a", 62)},
}
for _, test := range tests {
t.Run(test.profileID, func(t *testing.T) {
t.Parallel()
if got := ProfileSlug(test.profileID); got != test.want {
t.Fatalf("ProfileSlug(%q) = %q, want %q", test.profileID, got, test.want)
}
})
}
}
func TestReportNaming(t *testing.T) {
t.Parallel()
if got := OrdinalWidth(99); got != 2 {
t.Fatalf("OrdinalWidth(99) = %d, want 2", got)
}
if got := OrdinalWidth(100); got != 3 {
t.Fatalf("OrdinalWidth(100) = %d, want 3", got)
}
if got, err := ReportFilename(1, 3, "weather.light"); err != nil || got != "01-weather-light.md" {
t.Fatalf("ReportFilename() = %q, %v, want %q, nil", got, err, "01-weather-light.md")
}
if got, err := ReportFilename(100, 100, "weather-light"); err != nil || got != "100-weather-light.md" {
t.Fatalf("ReportFilename() = %q, %v, want %q, nil", got, err, "100-weather-light.md")
}
first, err := ReportFilename(1, 2, "model.v1")
if err != nil {
t.Fatalf("first ReportFilename() error = %v", err)
}
second, err := ReportFilename(2, 2, "model/v1")
if err != nil {
t.Fatalf("second ReportFilename() error = %v", err)
}
if first == second {
t.Fatalf("normalized profile collisions produced the same filename %q", first)
}
if _, err := ReportFilename(0, 2, "weather-light"); err == nil {
t.Fatal("ReportFilename accepted position zero")
}
if _, err := ReportFilename(1, 0, "weather-light"); err == nil {
t.Fatal("ReportFilename accepted zero profile count")
}
if got, err := BuildComparisonID("daily-2026-08-24"); err != nil || got != "comparison_daily-2026-08-24" {
t.Fatalf("BuildComparisonID() = %q, %v", got, err)
}
if _, err := BuildComparisonID(" \t"); err == nil {
t.Fatal("BuildComparisonID accepted blank run ID")
}
if got, err := DefaultDirectoryName("daily-2026-08-24.md"); err != nil || got != "comparison-daily-2026-08-24" {
t.Fatalf("DefaultDirectoryName() = %q, %v", got, err)
}
for _, name := range []string{"daily.txt", "nested/daily.md", ".md"} {
if _, err := DefaultDirectoryName(name); err == nil {
t.Fatalf("DefaultDirectoryName(%q) accepted invalid name", name)
}
}
}
func TestManifestEncodingAndRoundTrip(t *testing.T) {
t.Parallel()
manifest := validManifest()
encoded, err := EncodeManifest(manifest)
if err != nil {
t.Fatalf("EncodeManifest() error = %v", err)
}
want := "{\n" +
" \"schemaVersion\": \"weatherreporter.comparison.v1\",\n" +
" \"comparisonId\": \"comparison_daily-2026-08-24\",\n" +
" \"startedAt\": \"2026-08-24T12:00:00Z\",\n" +
" \"finishedAt\": \"2026-08-24T12:01:00Z\",\n" +
" \"reportId\": \"daily\",\n" +
" \"validPeriod\": {\n" +
" \"start\": \"2026-08-24T00:00:00-04:00\",\n" +
" \"end\": \"2026-08-25T00:00:00-04:00\"\n" +
" },\n" +
" \"timezone\": \"America/New_York\",\n" +
" \"promptId\": \"daily-report\",\n" +
" \"promptVersion\": \"2026-08-01\",\n" +
" \"promptHash\": \"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\",\n" +
" \"dataPackage\": {\n" +
" \"path\": \"data-package.yml\",\n" +
" \"sha256\": \"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\"\n" +
" },\n" +
" \"total\": 2,\n" +
" \"succeeded\": 1,\n" +
" \"failed\": 1,\n" +
" \"results\": [\n" +
" {\n" +
" \"position\": 1,\n" +
" \"profileId\": \"weather-light\",\n" +
" \"backendId\": \"openai\",\n" +
" \"modelName\": \"gpt-5-mini\",\n" +
" \"status\": \"succeeded\",\n" +
" \"validationStatus\": \"passed\",\n" +
" \"reportPath\": \"01-weather-light.md\"\n" +
" },\n" +
" {\n" +
" \"position\": 2,\n" +
" \"profileId\": \"weather-deep\",\n" +
" \"modelName\": \"gpt-5\",\n" +
" \"status\": \"failed\",\n" +
" \"error\": {\n" +
" \"category\": \"application\",\n" +
" \"message\": \"generated text was rejected\"\n" +
" }\n" +
" }\n" +
" ]\n" +
"}\n"
if string(encoded) != want {
t.Fatalf("EncodeManifest() =\n%s\nwant\n%s", encoded, want)
}
var decoded Manifest
if err := json.Unmarshal(encoded, &decoded); err != nil {
t.Fatalf("json.Unmarshal() error = %v", err)
}
if err := decoded.Validate(); err != nil {
t.Fatalf("decoded manifest validation error = %v", err)
}
}
func TestManifestValidateRejectsInvariants(t *testing.T) {
t.Parallel()
tests := []struct {
name string
mutate func(*Manifest)
}{
{name: "schema version", mutate: func(manifest *Manifest) { manifest.SchemaVersion = "v0" }},
{name: "non UTC timestamp", mutate: func(manifest *Manifest) { manifest.StartedAt = manifest.StartedAt.In(time.FixedZone("UTC", 0)) }},
{name: "reversed timestamps", mutate: func(manifest *Manifest) { manifest.FinishedAt = manifest.StartedAt.Add(-time.Second) }},
{name: "empty valid period", mutate: func(manifest *Manifest) { manifest.ValidPeriod.End = manifest.ValidPeriod.Start }},
{name: "bad prompt hash", mutate: func(manifest *Manifest) { manifest.PromptHash = "ABC" }},
{name: "bad data package path", mutate: func(manifest *Manifest) { manifest.DataPackage.Path = "nested/data-package.yml" }},
{name: "inconsistent totals", mutate: func(manifest *Manifest) { manifest.Succeeded = 2 }},
{name: "unordered position", mutate: func(manifest *Manifest) { manifest.Results[1].Position = 3 }},
{name: "duplicate profile", mutate: func(manifest *Manifest) { manifest.Results[1].ProfileID = manifest.Results[0].ProfileID }},
{name: "unsupported status", mutate: func(manifest *Manifest) { manifest.Results[1].Status = "skipped" }},
{name: "successful result without report", mutate: func(manifest *Manifest) { manifest.Results[0].ReportPath = "" }},
{name: "successful result with error", mutate: func(manifest *Manifest) {
manifest.Results[0].Error = &SafeError{Category: "application", Message: "bad"}
}},
{name: "successful result without passed validation", mutate: func(manifest *Manifest) { manifest.Results[0].ValidationStatus = "failed" }},
{name: "failed result with report", mutate: func(manifest *Manifest) { manifest.Results[1].ReportPath = "02-weather-deep.md" }},
{name: "failed result without error", mutate: func(manifest *Manifest) { manifest.Results[1].Error = nil }},
{name: "traversal report path", mutate: func(manifest *Manifest) { manifest.Results[0].ReportPath = "../report.md" }},
{name: "duplicate report path", mutate: func(manifest *Manifest) {
manifest.Results[1] = Result{Position: 2, ProfileID: "weather-deep", ModelName: "gpt-5", Status: StatusSucceeded, ValidationStatus: "passed", ReportPath: manifest.Results[0].ReportPath}
manifest.Succeeded = 2
manifest.Failed = 0
}},
{name: "oversized error", mutate: func(manifest *Manifest) { manifest.Results[1].Error.Message = strings.Repeat("x", 1025) }},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
t.Parallel()
manifest := validManifest()
test.mutate(&manifest)
if err := manifest.Validate(); err == nil {
t.Fatal("Manifest.Validate() succeeded for invalid manifest")
}
})
}
}
func TestLogicalBundleValidate(t *testing.T) {
t.Parallel()
dataPackage := []byte("report: daily\n")
manifest := validManifest()
manifest.DataPackage.SHA256 = SHA256(dataPackage)
bundle := LogicalBundle{
Manifest: manifest,
DataPackage: dataPackage,
Reports: []BundleReport{{
Position: 1,
Path: "01-weather-light.md",
Markdown: []byte("# Daily\n"),
}},
}
if err := bundle.Validate(); err != nil {
t.Fatalf("LogicalBundle.Validate() error = %v", err)
}
bundle.Reports[0].Path = "other.md"
if err := bundle.Validate(); err == nil {
t.Fatal("LogicalBundle.Validate() accepted mismatched report path")
}
bundle.Reports[0].Path = "01-weather-light.md"
bundle.Reports[0].Position = 2
if err := bundle.Validate(); err == nil {
t.Fatal("LogicalBundle.Validate() accepted unordered report position")
}
}
func TestSHA256AndTruncateErrorMessage(t *testing.T) {
t.Parallel()
if got, want := SHA256([]byte("weather")), "e5e72beb4e3c6926d3dc9e3e2ef7833ba50cd919c2460a782b244fd071e920de"; got != want {
t.Fatalf("SHA256() = %q, want %q", got, want)
}
message := strings.Repeat("€", 400)
got := TruncateErrorMessage(message)
if len(got) > 1024 || !utf8.ValidString(got) {
t.Fatalf("TruncateErrorMessage() returned %d bytes of valid UTF-8 = %t", len(got), utf8.ValidString(got))
}
if want := strings.Repeat("€", 341); got != want {
t.Fatalf("TruncateErrorMessage() = %q, want %q", got, want)
}
invalid := string([]byte{'x', 0xff, 'y'})
if got := TruncateErrorMessage(invalid); !utf8.ValidString(got) {
t.Fatal("TruncateErrorMessage() retained invalid UTF-8")
}
}
func validManifest() Manifest {
newYork := time.FixedZone("-0400", -4*60*60)
return Manifest{
SchemaVersion: SchemaVersion,
ComparisonID: "comparison_daily-2026-08-24",
StartedAt: time.Date(2026, time.August, 24, 12, 0, 0, 0, time.UTC),
FinishedAt: time.Date(2026, time.August, 24, 12, 1, 0, 0, time.UTC),
ReportID: "daily",
ValidPeriod: ValidPeriod{
Start: time.Date(2026, time.August, 24, 0, 0, 0, 0, newYork),
End: time.Date(2026, time.August, 25, 0, 0, 0, 0, newYork),
},
Timezone: "America/New_York",
PromptID: "daily-report",
PromptVersion: "2026-08-01",
PromptHash: strings.Repeat("a", 64),
DataPackage: DataPackageReference{
Path: DataPackageFilename,
SHA256: strings.Repeat("a", 64),
},
Total: 2,
Succeeded: 1,
Failed: 1,
Results: []Result{
{
Position: 1,
ProfileID: "weather-light",
BackendID: "openai",
ModelName: "gpt-5-mini",
Status: StatusSucceeded,
ValidationStatus: "passed",
ReportPath: "01-weather-light.md",
},
{
Position: 2,
ProfileID: "weather-deep",
ModelName: "gpt-5",
Status: StatusFailed,
Error: &SafeError{Category: "application", Message: "generated text was rejected"},
},
},
}
}

View File

@@ -0,0 +1,460 @@
package comparison
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"os"
"path/filepath"
"strings"
)
// DestinationErrorKind identifies a comparison destination safety failure.
type DestinationErrorKind string
const (
DestinationInvalidPath DestinationErrorKind = "invalid_path"
DestinationFilesystemRoot DestinationErrorKind = "filesystem_root"
DestinationWorkingDirectory DestinationErrorKind = "working_directory"
DestinationSymlink DestinationErrorKind = "symlink"
DestinationNotDirectory DestinationErrorKind = "not_directory"
DestinationNotEmpty DestinationErrorKind = "not_empty"
DestinationUnrecognized DestinationErrorKind = "unrecognized"
DestinationInspection DestinationErrorKind = "inspection"
)
// DestinationError provides inspectable context without making filesystem
// implementation details part of a manifest or command-result schema.
type DestinationError struct {
Kind DestinationErrorKind
Target string
Err error
}
func (err *DestinationError) Error() string {
if err.Err == nil {
return fmt.Sprintf("comparison destination %q is %s", err.Target, err.Kind)
}
return fmt.Sprintf("comparison destination %q is %s: %v", err.Target, err.Kind, err.Err)
}
func (err *DestinationError) Unwrap() error {
return err.Err
}
// DestinationPlan records a preflighted, exact bundle directory. Callers must
// pass it to Publish rather than recreating destination policy themselves.
type DestinationPlan struct {
WorkingDirectory string
Target string
Replace bool
state destinationState
}
type destinationState uint8
const (
destinationAbsent destinationState = iota
destinationEmpty
destinationBundle
)
// ErrUnrecognizedBundle marks a directory that is not a valid current-schema
// Weatherreporter comparison bundle.
var ErrUnrecognizedBundle = errors.New("unrecognized comparison bundle")
// PlanDestination performs the read-only comparison destination preflight.
func PlanDestination(workingDirectory, target string, replace bool) (DestinationPlan, error) {
workingDirectory, err := absoluteCleanPath(workingDirectory)
if err != nil {
return DestinationPlan{}, newDestinationError(DestinationInvalidPath, workingDirectory, err)
}
target, err = absoluteCleanPath(target)
if err != nil {
return DestinationPlan{}, newDestinationError(DestinationInvalidPath, target, err)
}
if filepath.Dir(target) == target {
return DestinationPlan{}, newDestinationError(DestinationFilesystemRoot, target, nil)
}
if target == workingDirectory {
return DestinationPlan{}, newDestinationError(DestinationWorkingDirectory, target, nil)
}
plan := DestinationPlan{WorkingDirectory: workingDirectory, Target: target, Replace: replace}
info, err := os.Lstat(target)
if err != nil {
if !errors.Is(err, os.ErrNotExist) {
return DestinationPlan{}, newDestinationError(DestinationInspection, target, err)
}
if err := inspectMissingDestinationParent(target); err != nil {
return DestinationPlan{}, err
}
return plan, nil
}
if info.Mode()&os.ModeSymlink != 0 {
return DestinationPlan{}, newDestinationError(DestinationSymlink, target, nil)
}
if !info.IsDir() {
return DestinationPlan{}, newDestinationError(DestinationNotDirectory, target, nil)
}
entries, err := os.ReadDir(target)
if err != nil {
return DestinationPlan{}, newDestinationError(DestinationInspection, target, err)
}
if len(entries) == 0 {
plan.state = destinationEmpty
return plan, nil
}
if !replace {
return DestinationPlan{}, newDestinationError(DestinationNotEmpty, target, nil)
}
if _, err := RecognizeBundle(target); err != nil {
return DestinationPlan{}, newDestinationError(DestinationUnrecognized, target, err)
}
plan.state = destinationBundle
return plan, nil
}
func absoluteCleanPath(value string) (string, error) {
if strings.TrimSpace(value) == "" {
return "", fmt.Errorf("path is required")
}
if !filepath.IsAbs(value) {
return "", fmt.Errorf("path %q must be absolute", value)
}
return filepath.Clean(value), nil
}
func inspectMissingDestinationParent(target string) error {
for parent := filepath.Dir(target); ; parent = filepath.Dir(parent) {
info, err := os.Lstat(parent)
if err == nil {
if info.Mode()&os.ModeSymlink != 0 {
resolved, statErr := os.Stat(parent)
if statErr != nil {
return newDestinationError(DestinationInspection, target, statErr)
}
if !resolved.IsDir() {
return newDestinationError(DestinationNotDirectory, target, nil)
}
return nil
}
if !info.IsDir() {
return newDestinationError(DestinationNotDirectory, target, nil)
}
return nil
}
if !errors.Is(err, os.ErrNotExist) {
return newDestinationError(DestinationInspection, target, err)
}
if filepath.Dir(parent) == parent {
return newDestinationError(DestinationInspection, target, fmt.Errorf("no existing directory ancestor"))
}
}
}
func newDestinationError(kind DestinationErrorKind, target string, err error) error {
return &DestinationError{Kind: kind, Target: target, Err: err}
}
// RecognizeBundle verifies that directory contains exactly one valid current
// comparison bundle. It never follows bundle entries through symlinks.
func RecognizeBundle(directory string) (Manifest, error) {
info, err := os.Lstat(directory)
if err != nil {
return Manifest{}, unrecognizedBundleError("inspect directory", err)
}
if info.Mode()&os.ModeSymlink != 0 || !info.IsDir() {
return Manifest{}, unrecognizedBundleError("directory is not a real directory", nil)
}
entries, err := os.ReadDir(directory)
if err != nil {
return Manifest{}, unrecognizedBundleError("read directory", err)
}
manifestData, err := readBundleFile(directory, ManifestFilename)
if err != nil {
return Manifest{}, err
}
manifest, err := decodeManifest(manifestData)
if err != nil {
return Manifest{}, err
}
if err := manifest.Validate(); err != nil {
return Manifest{}, unrecognizedBundleError("validate manifest", err)
}
expected := map[string]struct{}{
ManifestFilename: {},
DataPackageFilename: {},
}
for _, result := range manifest.Results {
if result.Status == StatusSucceeded {
expected[result.ReportPath] = struct{}{}
}
}
if len(entries) != len(expected) {
return Manifest{}, unrecognizedBundleError("directory entries do not match manifest", nil)
}
for _, entry := range entries {
if _, ok := expected[entry.Name()]; !ok {
return Manifest{}, unrecognizedBundleError("directory has an undeclared entry", nil)
}
if _, err := readBundleFile(directory, entry.Name()); err != nil {
return Manifest{}, err
}
}
dataPackage, err := readBundleFile(directory, DataPackageFilename)
if err != nil {
return Manifest{}, err
}
if SHA256(dataPackage) != manifest.DataPackage.SHA256 {
return Manifest{}, unrecognizedBundleError("data package digest does not match manifest", nil)
}
return manifest, nil
}
func readBundleFile(directory, name string) ([]byte, error) {
if !isArtifactBasename(name) {
return nil, unrecognizedBundleError("bundle entry name is unsafe", nil)
}
filePath := filepath.Join(directory, name)
info, err := os.Lstat(filePath)
if err != nil {
return nil, unrecognizedBundleError("inspect bundle entry", err)
}
if info.Mode()&os.ModeSymlink != 0 || !info.Mode().IsRegular() {
return nil, unrecognizedBundleError("bundle entry is not a regular file", nil)
}
data, err := os.ReadFile(filePath)
if err != nil {
return nil, unrecognizedBundleError("read bundle entry", err)
}
return data, nil
}
func decodeManifest(data []byte) (Manifest, error) {
decoder := json.NewDecoder(bytes.NewReader(data))
decoder.DisallowUnknownFields()
var manifest Manifest
if err := decoder.Decode(&manifest); err != nil {
return Manifest{}, unrecognizedBundleError("decode manifest", err)
}
var trailing any
if err := decoder.Decode(&trailing); err != io.EOF {
if err == nil {
return Manifest{}, unrecognizedBundleError("manifest has multiple JSON values", nil)
}
return Manifest{}, unrecognizedBundleError("decode manifest", err)
}
return manifest, nil
}
func unrecognizedBundleError(action string, err error) error {
if err == nil {
return fmt.Errorf("%w: %s", ErrUnrecognizedBundle, action)
}
return fmt.Errorf("%w: %s: %v", ErrUnrecognizedBundle, action, err)
}
// PublicationResult describes the durable state of a publication attempt.
type PublicationResult struct {
Committed bool
RetainedBackupPath string
}
// PublicationCleanupError reports that a committed bundle could not remove its
// prior sibling backup. The new bundle remains installed and the backup path
// is retained for operator recovery.
type PublicationCleanupError struct {
RetainedBackupPath string
Err error
}
func (err *PublicationCleanupError) Error() string {
return fmt.Sprintf("remove comparison backup %q: %v", err.RetainedBackupPath, err.Err)
}
func (err *PublicationCleanupError) Unwrap() error {
return err.Err
}
// Publish writes a complete logical bundle through a private sibling directory
// and atomically installs it at a preflighted destination.
func Publish(ctx context.Context, plan DestinationPlan, bundle LogicalBundle) (PublicationResult, error) {
return publish(ctx, plan, bundle, publishOperations{rename: os.Rename, removeAll: os.RemoveAll})
}
type publishOperations struct {
rename func(string, string) error
removeAll func(string) error
beforeCommit func()
}
func publish(ctx context.Context, plan DestinationPlan, bundle LogicalBundle, operations publishOperations) (PublicationResult, error) {
if operations.rename == nil {
operations.rename = os.Rename
}
if operations.removeAll == nil {
operations.removeAll = os.RemoveAll
}
if err := bundle.Validate(); err != nil {
return PublicationResult{}, fmt.Errorf("validate comparison bundle: %w", err)
}
manifestData, err := EncodeManifest(bundle.Manifest)
if err != nil {
return PublicationResult{}, err
}
if err := ctx.Err(); err != nil {
return PublicationResult{}, err
}
plan, err = PlanDestination(plan.WorkingDirectory, plan.Target, plan.Replace)
if err != nil {
return PublicationResult{}, err
}
if err := os.MkdirAll(filepath.Dir(plan.Target), 0o755); err != nil {
return PublicationResult{}, fmt.Errorf("create comparison destination parent %q: %w", filepath.Dir(plan.Target), err)
}
temporaryDirectory, err := os.MkdirTemp(filepath.Dir(plan.Target), "."+filepath.Base(plan.Target)+".staging-")
if err != nil {
return PublicationResult{}, fmt.Errorf("create comparison staging directory: %w", err)
}
if err := os.Chmod(temporaryDirectory, 0o700); err != nil {
os.RemoveAll(temporaryDirectory)
return PublicationResult{}, fmt.Errorf("secure comparison staging directory: %w", err)
}
defer func() {
if temporaryDirectory != "" {
_ = os.RemoveAll(temporaryDirectory)
}
}()
if err := writeLogicalBundle(ctx, temporaryDirectory, bundle, manifestData); err != nil {
return PublicationResult{}, err
}
currentPlan, err := PlanDestination(plan.WorkingDirectory, plan.Target, plan.Replace)
if err != nil {
return PublicationResult{}, err
}
if operations.beforeCommit != nil {
operations.beforeCommit()
}
if err := ctx.Err(); err != nil {
return PublicationResult{}, err
}
if currentPlan.state == destinationAbsent {
if err := operations.rename(temporaryDirectory, currentPlan.Target); err != nil {
return PublicationResult{}, fmt.Errorf("publish comparison bundle to %q: %w", currentPlan.Target, err)
}
temporaryDirectory = ""
return PublicationResult{Committed: true}, nil
}
backupDirectory, err := uniqueSiblingPath(filepath.Dir(currentPlan.Target), "."+filepath.Base(currentPlan.Target)+".backup-")
if err != nil {
return PublicationResult{}, err
}
if err := operations.rename(currentPlan.Target, backupDirectory); err != nil {
return PublicationResult{}, fmt.Errorf("back up comparison destination %q: %w", currentPlan.Target, err)
}
if err := authorizeMovedDestination(currentPlan, backupDirectory); err != nil {
return PublicationResult{}, restoreMovedDestination(operations, backupDirectory, currentPlan.Target, err)
}
if err := operations.rename(temporaryDirectory, currentPlan.Target); err != nil {
return PublicationResult{}, restoreMovedDestination(operations, backupDirectory, currentPlan.Target, fmt.Errorf("replace comparison destination %q: %w", currentPlan.Target, err))
}
temporaryDirectory = ""
if err := operations.removeAll(backupDirectory); err != nil {
cleanupErr := &PublicationCleanupError{RetainedBackupPath: backupDirectory, Err: err}
return PublicationResult{Committed: true, RetainedBackupPath: backupDirectory}, cleanupErr
}
return PublicationResult{Committed: true}, nil
}
func authorizeMovedDestination(plan DestinationPlan, backupDirectory string) error {
backupPlan, err := PlanDestination(plan.WorkingDirectory, backupDirectory, plan.Replace)
if err != nil {
return fmt.Errorf("authorize moved comparison destination %q: %w", backupDirectory, err)
}
if backupPlan.state != destinationEmpty && backupPlan.state != destinationBundle {
return fmt.Errorf("authorize moved comparison destination %q: destination disappeared", backupDirectory)
}
return nil
}
func restoreMovedDestination(operations publishOperations, backupDirectory, target string, cause error) error {
if _, err := os.Lstat(target); err == nil {
return errors.Join(
cause,
fmt.Errorf("restore prior comparison destination from %q: destination %q reappeared", backupDirectory, target),
)
} else if !errors.Is(err, os.ErrNotExist) {
return errors.Join(
cause,
fmt.Errorf("inspect comparison destination %q before restoring from %q: %w", target, backupDirectory, err),
)
}
if restoreErr := operations.rename(backupDirectory, target); restoreErr != nil {
return errors.Join(
cause,
fmt.Errorf("restore prior comparison destination from %q: %w", backupDirectory, restoreErr),
)
}
return cause
}
func writeLogicalBundle(ctx context.Context, directory string, bundle LogicalBundle, manifestData []byte) error {
if err := writeBundleFile(ctx, filepath.Join(directory, DataPackageFilename), bundle.DataPackage); err != nil {
return err
}
for _, report := range bundle.Reports {
if err := writeBundleFile(ctx, filepath.Join(directory, report.Path), report.Markdown); err != nil {
return err
}
}
if err := writeBundleFile(ctx, filepath.Join(directory, ManifestFilename), manifestData); err != nil {
return err
}
return nil
}
func writeBundleFile(ctx context.Context, filePath string, content []byte) error {
if err := ctx.Err(); err != nil {
return err
}
file, err := os.OpenFile(filePath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600)
if err != nil {
return fmt.Errorf("create comparison bundle file %q: %w", filePath, err)
}
if _, err := file.Write(content); err != nil {
file.Close()
return fmt.Errorf("write comparison bundle file %q: %w", filePath, err)
}
if err := file.Close(); err != nil {
return fmt.Errorf("close comparison bundle file %q: %w", filePath, err)
}
return nil
}
func uniqueSiblingPath(parent, prefix string) (string, error) {
file, err := os.CreateTemp(parent, prefix)
if err != nil {
return "", fmt.Errorf("reserve comparison backup path: %w", err)
}
path := file.Name()
if err := file.Close(); err != nil {
os.Remove(path)
return "", fmt.Errorf("close comparison backup reservation: %w", err)
}
if err := os.Remove(path); err != nil {
return "", fmt.Errorf("release comparison backup reservation: %w", err)
}
return path, nil
}

View File

@@ -0,0 +1,754 @@
package comparison
import (
"context"
"encoding/json"
"errors"
"os"
"path/filepath"
"runtime"
"strings"
"testing"
)
func TestPlanDestination(t *testing.T) {
workingDirectory := t.TempDir()
absent := filepath.Join(workingDirectory, "absent")
plan, err := PlanDestination(workingDirectory, absent, false)
if err != nil {
t.Fatalf("PlanDestination() error = %v", err)
}
if plan.Target != absent || plan.state != destinationAbsent {
t.Fatalf("PlanDestination() = %#v, want absent target", plan)
}
nested := filepath.Join(workingDirectory, "missing-parent", "comparison")
if _, err := PlanDestination(workingDirectory, nested, false); err != nil {
t.Fatalf("PlanDestination(nested) error = %v", err)
}
if _, err := os.Stat(filepath.Dir(nested)); !errors.Is(err, os.ErrNotExist) {
t.Fatalf("read-only plan created parent: stat error = %v", err)
}
empty := filepath.Join(workingDirectory, "empty")
if err := os.Mkdir(empty, 0o755); err != nil {
t.Fatal(err)
}
plan, err = PlanDestination(workingDirectory, empty, false)
if err != nil || plan.state != destinationEmpty {
t.Fatalf("PlanDestination(empty) = %#v, %v", plan, err)
}
file := filepath.Join(workingDirectory, "file")
if err := os.WriteFile(file, []byte("not a directory"), 0o600); err != nil {
t.Fatal(err)
}
assertDestinationErrorKind(t, workingDirectory, file, false, DestinationNotDirectory)
assertDestinationErrorKind(t, workingDirectory, workingDirectory, false, DestinationWorkingDirectory)
assertDestinationErrorKind(t, workingDirectory, string(os.PathSeparator), false, DestinationFilesystemRoot)
assertDestinationErrorKind(t, workingDirectory, "relative", false, DestinationInvalidPath)
link := filepath.Join(workingDirectory, "link")
if err := os.Symlink(empty, link); err != nil {
t.Fatal(err)
}
assertDestinationErrorKind(t, workingDirectory, link, false, DestinationSymlink)
dangling := filepath.Join(workingDirectory, "dangling")
if err := os.Symlink(filepath.Join(workingDirectory, "missing"), dangling); err != nil {
t.Fatal(err)
}
assertDestinationErrorKind(t, workingDirectory, dangling, false, DestinationSymlink)
assertDestinationErrorKind(t, workingDirectory, filepath.Join(dangling, "child"), false, DestinationInspection)
nonempty := filepath.Join(workingDirectory, "nonempty")
if err := os.Mkdir(nonempty, 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(nonempty, "extra"), []byte("x"), 0o600); err != nil {
t.Fatal(err)
}
assertDestinationErrorKind(t, workingDirectory, nonempty, false, DestinationNotEmpty)
assertDestinationErrorKind(t, workingDirectory, nonempty, true, DestinationUnrecognized)
}
func TestPlanDestinationRejectsUnreadableDirectory(t *testing.T) {
if os.Geteuid() == 0 {
t.Skip("root can inspect directories regardless of mode")
}
workingDirectory := t.TempDir()
target := filepath.Join(workingDirectory, "unreadable")
if err := os.Mkdir(target, 0o700); err != nil {
t.Fatal(err)
}
if err := os.Chmod(target, 0); err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = os.Chmod(target, 0o700) })
assertDestinationErrorKind(t, workingDirectory, target, false, DestinationInspection)
}
func TestRecognizeBundleRejectsUnsafeAndMalformedContents(t *testing.T) {
tests := []struct {
name string
mutate func(t *testing.T, directory string)
}{
{name: "extra file", mutate: func(t *testing.T, directory string) {
t.Helper()
if err := os.WriteFile(filepath.Join(directory, "extra.txt"), []byte("extra"), 0o600); err != nil {
t.Fatal(err)
}
}},
{name: "subdirectory", mutate: func(t *testing.T, directory string) {
t.Helper()
if err := os.Mkdir(filepath.Join(directory, "nested"), 0o700); err != nil {
t.Fatal(err)
}
}},
{name: "missing report", mutate: func(t *testing.T, directory string) {
t.Helper()
if err := os.Remove(filepath.Join(directory, "01-weather-light.md")); err != nil {
t.Fatal(err)
}
}},
{name: "symlink report", mutate: func(t *testing.T, directory string) {
t.Helper()
path := filepath.Join(directory, "01-weather-light.md")
if err := os.Remove(path); err != nil {
t.Fatal(err)
}
if err := os.Symlink(filepath.Join(directory, DataPackageFilename), path); err != nil {
t.Fatal(err)
}
}},
{name: "digest mismatch", mutate: func(t *testing.T, directory string) {
t.Helper()
if err := os.WriteFile(filepath.Join(directory, DataPackageFilename), []byte("altered"), 0o600); err != nil {
t.Fatal(err)
}
}},
{name: "unknown manifest field", mutate: func(t *testing.T, directory string) {
t.Helper()
path := filepath.Join(directory, ManifestFilename)
data, err := os.ReadFile(path)
if err != nil {
t.Fatal(err)
}
data = append(data[:len(data)-2], []byte(",\n \"unknown\": true\n}\n")...)
if err := os.WriteFile(path, data, 0o600); err != nil {
t.Fatal(err)
}
}},
{name: "traversal report path", mutate: func(t *testing.T, directory string) {
t.Helper()
path := filepath.Join(directory, ManifestFilename)
data, err := os.ReadFile(path)
if err != nil {
t.Fatal(err)
}
var manifest Manifest
if err := json.Unmarshal(data, &manifest); err != nil {
t.Fatal(err)
}
manifest.Results[0].ReportPath = "../outside.md"
data, err = json.Marshal(manifest)
if err != nil {
t.Fatal(err)
}
if err := os.WriteFile(path, data, 0o600); err != nil {
t.Fatal(err)
}
}},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
directory := publishTestBundle(t, testBundle())
test.mutate(t, directory)
if _, err := RecognizeBundle(directory); !errors.Is(err, ErrUnrecognizedBundle) {
t.Fatalf("RecognizeBundle() error = %v, want ErrUnrecognizedBundle", err)
}
})
}
}
func TestPlanDestinationAcceptsRecognizedReplacement(t *testing.T) {
directory := publishTestBundle(t, testBundle())
plan, err := PlanDestination(filepath.Dir(directory), directory, true)
if err != nil {
t.Fatalf("PlanDestination() error = %v", err)
}
if plan.state != destinationBundle {
t.Fatal("PlanDestination() did not record recognized existing destination")
}
}
func TestPublishReauthorizesMovedDestination(t *testing.T) {
tests := []struct {
name string
replace bool
requiresSymlink bool
prepare func(t *testing.T, workingDirectory, target string)
mutate func(t *testing.T, target string)
verify func(t *testing.T, target string)
}{
{
name: "regular file after absent preflight",
mutate: func(t *testing.T, target string) {
t.Helper()
if err := os.WriteFile(target, []byte("unrelated file"), 0o600); err != nil {
t.Fatal(err)
}
},
verify: func(t *testing.T, target string) {
t.Helper()
if data := readFile(t, target); string(data) != "unrelated file" {
t.Fatalf("unrelated file data = %q", data)
}
},
},
{
name: "nonempty directory after empty preflight",
prepare: func(t *testing.T, _, target string) {
t.Helper()
if err := os.Mkdir(target, 0o700); err != nil {
t.Fatal(err)
}
},
mutate: writeUnrecognizedDirectory,
verify: func(t *testing.T, target string) {
t.Helper()
if data := readFile(t, filepath.Join(target, "unrelated")); string(data) != "unrelated" {
t.Fatalf("unrelated data = %q", data)
}
},
},
{
name: "regular file after empty preflight",
prepare: func(t *testing.T, _, target string) {
t.Helper()
if err := os.Mkdir(target, 0o700); err != nil {
t.Fatal(err)
}
},
mutate: func(t *testing.T, target string) {
t.Helper()
if err := os.WriteFile(target, []byte("unrelated file"), 0o600); err != nil {
t.Fatal(err)
}
},
verify: func(t *testing.T, target string) {
t.Helper()
if data := readFile(t, target); string(data) != "unrelated file" {
t.Fatalf("unrelated file data = %q", data)
}
},
},
{
name: "symlink after empty preflight",
requiresSymlink: true,
prepare: func(t *testing.T, _, target string) {
t.Helper()
if err := os.Mkdir(target, 0o700); err != nil {
t.Fatal(err)
}
},
mutate: func(t *testing.T, target string) {
t.Helper()
if err := os.Symlink("unrelated-target", target); err != nil {
t.Fatal(err)
}
},
verify: func(t *testing.T, target string) {
t.Helper()
info, err := os.Lstat(target)
if err != nil || info.Mode()&os.ModeSymlink == 0 {
t.Fatalf("symlink stat = %v, %v", info, err)
}
},
},
{
name: "unrecognized directory after bundle preflight",
replace: true,
prepare: func(t *testing.T, workingDirectory, target string) {
t.Helper()
plan, err := PlanDestination(workingDirectory, target, false)
if err != nil {
t.Fatal(err)
}
if _, err := Publish(context.Background(), plan, testBundle()); err != nil {
t.Fatal(err)
}
},
mutate: writeUnrecognizedDirectory,
verify: func(t *testing.T, target string) {
t.Helper()
if data := readFile(t, filepath.Join(target, "unrelated")); string(data) != "unrelated" {
t.Fatalf("unrelated data = %q", data)
}
},
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
if test.requiresSymlink && runtime.GOOS == "windows" {
t.Skip("symlink replacement coverage requires Unix symlink semantics")
}
workingDirectory := t.TempDir()
target := filepath.Join(workingDirectory, "comparison-daily")
if test.prepare != nil {
test.prepare(t, workingDirectory, target)
}
plan, err := PlanDestination(workingDirectory, target, test.replace)
if err != nil {
t.Fatal(err)
}
_, err = publish(context.Background(), plan, testBundle(), publishOperations{
rename: os.Rename,
beforeCommit: func() {
if err := os.RemoveAll(target); err != nil {
t.Fatal(err)
}
test.mutate(t, target)
},
})
if err == nil {
t.Fatal("publish() succeeded despite unauthorized replacement")
}
test.verify(t, target)
assertOnlyDestinationEntry(t, workingDirectory, filepath.Base(target))
})
}
}
func TestPublishRetainsUnauthorizedMovedDestinationWhenRestoreFails(t *testing.T) {
workingDirectory := t.TempDir()
target := filepath.Join(workingDirectory, "comparison-daily")
if err := os.Mkdir(target, 0o700); err != nil {
t.Fatal(err)
}
plan, err := PlanDestination(workingDirectory, target, false)
if err != nil {
t.Fatal(err)
}
var backupPath string
calls := 0
_, err = publish(context.Background(), plan, testBundle(), publishOperations{
rename: func(oldPath, newPath string) error {
calls++
if calls == 1 {
backupPath = newPath
}
if calls == 2 {
return errors.New("restore failed")
}
return os.Rename(oldPath, newPath)
},
beforeCommit: func() {
if err := os.RemoveAll(target); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(target, []byte("unrelated file"), 0o600); err != nil {
t.Fatal(err)
}
},
})
if err == nil || !strings.Contains(err.Error(), backupPath) {
t.Fatalf("publish() error = %v, want retained backup path %q", err, backupPath)
}
if data := readFile(t, backupPath); string(data) != "unrelated file" {
t.Fatalf("retained backup data = %q", data)
}
if _, err := os.Lstat(target); !errors.Is(err, os.ErrNotExist) {
t.Fatalf("target stat error = %v, want not exist", err)
}
assertOnlyDestinationEntry(t, workingDirectory, filepath.Base(backupPath))
}
func TestPublishRetainsMovedDestinationWhenTargetReappears(t *testing.T) {
workingDirectory := t.TempDir()
target := filepath.Join(workingDirectory, "comparison-daily")
if err := os.Mkdir(target, 0o700); err != nil {
t.Fatal(err)
}
plan, err := PlanDestination(workingDirectory, target, false)
if err != nil {
t.Fatal(err)
}
var backupPath string
_, err = publish(context.Background(), plan, testBundle(), publishOperations{
rename: func(oldPath, newPath string) error {
if backupPath != "" {
return os.Rename(oldPath, newPath)
}
backupPath = newPath
if err := os.Rename(oldPath, newPath); err != nil {
return err
}
if err := os.WriteFile(filepath.Join(backupPath, "unrelated"), []byte("unrelated"), 0o600); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(target, []byte("reappeared"), 0o600); err != nil {
t.Fatal(err)
}
return nil
},
})
if err == nil || !strings.Contains(err.Error(), "reappeared") || !strings.Contains(err.Error(), backupPath) {
t.Fatalf("publish() error = %v, want reappeared target and retained backup path %q", err, backupPath)
}
if data := readFile(t, target); string(data) != "reappeared" {
t.Fatalf("reappeared target data = %q", data)
}
if data := readFile(t, filepath.Join(backupPath, "unrelated")); string(data) != "unrelated" {
t.Fatalf("retained backup data = %q", data)
}
assertDestinationEntries(t, workingDirectory, filepath.Base(target), filepath.Base(backupPath))
}
func TestPublishWritesAndReplacesBundle(t *testing.T) {
workingDirectory := t.TempDir()
target := filepath.Join(workingDirectory, "comparison-daily")
plan, err := PlanDestination(workingDirectory, target, false)
if err != nil {
t.Fatal(err)
}
if _, err := Publish(context.Background(), plan, testBundle()); err != nil {
t.Fatalf("Publish() error = %v", err)
}
if _, err := RecognizeBundle(target); err != nil {
t.Fatalf("RecognizeBundle() error = %v", err)
}
assertMode(t, target, 0o700)
for _, name := range []string{ManifestFilename, DataPackageFilename, "01-weather-light.md"} {
assertMode(t, filepath.Join(target, name), 0o600)
}
next := testBundle()
next.Manifest.Results[0].ProfileID = "weather-balanced"
next.Manifest.Results[0].ReportPath = "01-weather-balanced.md"
next.Reports[0].Path = "01-weather-balanced.md"
next.Reports[0].Markdown = []byte("# Replacement\n")
plan, err = PlanDestination(workingDirectory, target, true)
if err != nil {
t.Fatal(err)
}
if _, err := Publish(context.Background(), plan, next); err != nil {
t.Fatalf("Publish(replace) error = %v", err)
}
if _, err := os.Stat(filepath.Join(target, "01-weather-light.md")); !errors.Is(err, os.ErrNotExist) {
t.Fatalf("stale report stat error = %v, want not exist", err)
}
if data, err := os.ReadFile(filepath.Join(target, "01-weather-balanced.md")); err != nil || string(data) != "# Replacement\n" {
t.Fatalf("replacement report = %q, %v", data, err)
}
}
func TestPublishReplacesEmptyDirectory(t *testing.T) {
workingDirectory := t.TempDir()
target := filepath.Join(workingDirectory, "comparison-daily")
if err := os.Mkdir(target, 0o755); err != nil {
t.Fatal(err)
}
plan, err := PlanDestination(workingDirectory, target, false)
if err != nil {
t.Fatal(err)
}
if _, err := Publish(context.Background(), plan, testBundle()); err != nil {
t.Fatalf("Publish() error = %v", err)
}
if _, err := RecognizeBundle(target); err != nil {
t.Fatalf("RecognizeBundle() error = %v", err)
}
}
func TestPublishReportsCommittedBundleWhenBackupCleanupFails(t *testing.T) {
workingDirectory := t.TempDir()
target := filepath.Join(workingDirectory, "comparison-daily")
initialPlan, err := PlanDestination(workingDirectory, target, false)
if err != nil {
t.Fatal(err)
}
if _, err := Publish(context.Background(), initialPlan, testBundle()); err != nil {
t.Fatal(err)
}
previous := directorySnapshot(t, target)
next := testBundle()
next.Reports[0].Markdown = []byte("# Next\n")
plan, err := PlanDestination(workingDirectory, target, true)
if err != nil {
t.Fatal(err)
}
cleanupCause := errors.New("backup removal failed")
var backupPath string
result, err := publish(context.Background(), plan, next, publishOperations{
rename: os.Rename,
removeAll: func(path string) error {
backupPath = path
return cleanupCause
},
})
var cleanupErr *PublicationCleanupError
if !result.Committed || result.RetainedBackupPath != backupPath || !filepath.IsAbs(backupPath) || !errors.As(err, &cleanupErr) || cleanupErr.RetainedBackupPath != backupPath || !errors.Is(err, cleanupCause) {
t.Fatalf("publish() result/error = %#v/%v", result, err)
}
if _, err := RecognizeBundle(target); err != nil {
t.Fatalf("new bundle recognition error = %v", err)
}
if retained := directorySnapshot(t, backupPath); !equalSnapshots(previous, retained) {
t.Fatalf("retained backup = %#v, want %#v", retained, previous)
}
manifestData, err := os.ReadFile(filepath.Join(target, ManifestFilename))
if err != nil || strings.Contains(string(manifestData), backupPath) {
t.Fatalf("manifest/backup path = %q/%q, error = %v", manifestData, backupPath, err)
}
}
func TestPublishRestoresExistingBundleAfterReplacementFailure(t *testing.T) {
workingDirectory := t.TempDir()
target := filepath.Join(workingDirectory, "comparison-daily")
initial := testBundle()
initialPlan, err := PlanDestination(workingDirectory, target, false)
if err != nil {
t.Fatal(err)
}
if _, err := Publish(context.Background(), initialPlan, initial); err != nil {
t.Fatal(err)
}
before := directorySnapshot(t, target)
next := testBundle()
next.Reports[0].Markdown = []byte("# New\n")
plan, err := PlanDestination(workingDirectory, target, true)
if err != nil {
t.Fatal(err)
}
calls := 0
publication, err := publish(context.Background(), plan, next, publishOperations{rename: func(oldPath, newPath string) error {
calls++
if calls == 2 {
return errors.New("replace failed")
}
return os.Rename(oldPath, newPath)
}})
if publication.Committed || err == nil {
t.Fatal("publish() succeeded despite replacement failure")
}
after := directorySnapshot(t, target)
if !equalSnapshots(before, after) {
t.Fatal("failed publication changed the prior bundle")
}
entries, err := os.ReadDir(workingDirectory)
if err != nil {
t.Fatal(err)
}
if len(entries) != 1 || entries[0].Name() != filepath.Base(target) {
t.Fatalf("failed publication left sibling artifacts: %#v", entries)
}
}
func TestPublishRetainsBackupWhenRestorationFails(t *testing.T) {
workingDirectory := t.TempDir()
target := filepath.Join(workingDirectory, "comparison-daily")
initialPlan, err := PlanDestination(workingDirectory, target, false)
if err != nil {
t.Fatal(err)
}
if _, err := Publish(context.Background(), initialPlan, testBundle()); err != nil {
t.Fatal(err)
}
plan, err := PlanDestination(workingDirectory, target, true)
if err != nil {
t.Fatal(err)
}
var backupPath string
calls := 0
_, err = publish(context.Background(), plan, testBundle(), publishOperations{rename: func(oldPath, newPath string) error {
calls++
if calls == 1 {
backupPath = newPath
}
if calls == 2 || calls == 3 {
return errors.New("rename failed")
}
return os.Rename(oldPath, newPath)
}})
if err == nil || !strings.Contains(err.Error(), backupPath) {
t.Fatalf("publish() error = %v, want retained backup path %q", err, backupPath)
}
if info, statErr := os.Stat(backupPath); statErr != nil || !info.IsDir() {
t.Fatalf("backup stat = %v, %v, want retained directory", info, statErr)
}
}
func TestPublishCancellationBeforeCommitLeavesNoDestination(t *testing.T) {
workingDirectory := t.TempDir()
target := filepath.Join(workingDirectory, "comparison-daily")
plan, err := PlanDestination(workingDirectory, target, false)
if err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithCancel(context.Background())
if publication, err := publish(ctx, plan, testBundle(), publishOperations{rename: os.Rename, beforeCommit: cancel}); publication.Committed || !errors.Is(err, context.Canceled) {
t.Fatalf("Publish() error = %v, want context cancellation", err)
}
if _, err := os.Lstat(target); !errors.Is(err, os.ErrNotExist) {
t.Fatalf("destination stat error = %v, want not exist", err)
}
}
func assertDestinationErrorKind(t *testing.T, workingDirectory, target string, replace bool, want DestinationErrorKind) {
t.Helper()
_, err := PlanDestination(workingDirectory, target, replace)
var destinationError *DestinationError
if !errors.As(err, &destinationError) || destinationError.Kind != want {
t.Fatalf("PlanDestination(%q) error = %v, want destination error %q", target, err, want)
}
}
func publishTestBundle(t *testing.T, bundle LogicalBundle) string {
t.Helper()
workingDirectory := t.TempDir()
target := filepath.Join(workingDirectory, "comparison-daily")
plan, err := PlanDestination(workingDirectory, target, false)
if err != nil {
t.Fatal(err)
}
if _, err := Publish(context.Background(), plan, bundle); err != nil {
t.Fatal(err)
}
return target
}
func writeUnrecognizedDirectory(t *testing.T, target string) {
t.Helper()
if err := os.Mkdir(target, 0o700); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(target, "unrelated"), []byte("unrelated"), 0o600); err != nil {
t.Fatal(err)
}
}
func readFile(t *testing.T, path string) []byte {
t.Helper()
data, err := os.ReadFile(path)
if err != nil {
t.Fatal(err)
}
return data
}
func assertOnlyDestinationEntry(t *testing.T, directory, want string) {
t.Helper()
assertDestinationEntries(t, directory, want)
}
func assertDestinationEntries(t *testing.T, directory string, wants ...string) {
t.Helper()
entries, err := os.ReadDir(directory)
if err != nil {
t.Fatal(err)
}
if len(entries) != len(wants) {
t.Fatalf("directory entries = %#v, want %#v", entries, wants)
}
for _, want := range wants {
found := false
for _, entry := range entries {
if entry.Name() == want {
found = true
break
}
}
if !found {
t.Fatalf("directory entries = %#v, missing %q", entries, want)
}
}
}
func testBundle() LogicalBundle {
dataPackage := []byte("report: daily\n")
manifest := validManifest()
manifest.DataPackage.SHA256 = SHA256(dataPackage)
return LogicalBundle{
Manifest: manifest,
DataPackage: dataPackage,
Reports: []BundleReport{{
Position: 1,
Path: "01-weather-light.md",
Markdown: []byte("# Daily\n"),
}},
}
}
func assertMode(t *testing.T, path string, want os.FileMode) {
t.Helper()
info, err := os.Stat(path)
if err != nil {
t.Fatal(err)
}
if got := info.Mode().Perm(); got != want {
t.Fatalf("mode for %q = %#o, want %#o", path, got, want)
}
}
func directorySnapshot(t *testing.T, directory string) map[string]string {
t.Helper()
entries, err := os.ReadDir(directory)
if err != nil {
t.Fatal(err)
}
snapshot := make(map[string]string, len(entries))
for _, entry := range entries {
data, err := os.ReadFile(filepath.Join(directory, entry.Name()))
if err != nil {
t.Fatal(err)
}
snapshot[entry.Name()] = string(data)
}
return snapshot
}
func equalSnapshots(left, right map[string]string) bool {
if len(left) != len(right) {
return false
}
for name, value := range left {
if right[name] != value {
return false
}
}
return true
}
func TestRecognizeBundleRejectsTrailingJSONValue(t *testing.T) {
directory := publishTestBundle(t, testBundle())
path := filepath.Join(directory, ManifestFilename)
data, err := os.ReadFile(path)
if err != nil {
t.Fatal(err)
}
if err := os.WriteFile(path, append(data, []byte("{}")...), 0o600); err != nil {
t.Fatal(err)
}
if _, err := RecognizeBundle(directory); !errors.Is(err, ErrUnrecognizedBundle) {
t.Fatalf("RecognizeBundle() error = %v, want ErrUnrecognizedBundle", err)
}
}
func TestPlanDestinationRejectsWhitespaceTarget(t *testing.T) {
workingDirectory := t.TempDir()
assertDestinationErrorKind(t, workingDirectory, " \t", false, DestinationInvalidPath)
}
func TestRecognizeBundleDoesNotAcceptNonRegularManifest(t *testing.T) {
directory := t.TempDir()
if err := os.Mkdir(filepath.Join(directory, ManifestFilename), 0o700); err != nil {
t.Fatal(err)
}
if _, err := RecognizeBundle(directory); !strings.Contains(err.Error(), "regular file") {
t.Fatalf("RecognizeBundle() error = %v, want regular file rejection", err)
}
}

View File

@@ -25,12 +25,11 @@ type Config struct {
WeatherAPI WeatherAPIConfig `yaml:"weather_api"`
Location LocationConfig `yaml:"location"`
Secrets SecretsConfig `yaml:"secrets"`
Output OutputConfig `yaml:"output"`
Notify NotifyConfig `yaml:"notify"`
MissingSource MissingSourceConfig `yaml:"missing_source"`
Promptkit PromptkitConfig `yaml:"promptkit"`
Workspace WorkspaceConfig `yaml:"workspace"`
Dayparts []DaypartConfig `yaml:"dayparts"`
RecentChange RecentChangeConfig `yaml:"recent_change"`
Reports map[string]ReportConfig `yaml:"reports"`
}
@@ -53,6 +52,10 @@ type SecretsConfig struct {
Directory string `yaml:"directory"`
}
type OutputConfig struct {
Directory string `yaml:"directory"`
}
type NotifyConfig struct {
Distributor DistributorNotifyConfig `yaml:"distributor"`
}
@@ -94,28 +97,12 @@ type PromptkitLocalConfig struct {
ConcurrencyLimit int `yaml:"concurrency_limit"`
}
type WorkspaceConfig struct {
Root string `yaml:"root"`
SnapshotsDir string `yaml:"snapshots_dir"`
ReportsDir string `yaml:"reports_dir"`
DataPackagesDir string `yaml:"data_packages_dir"`
PreflightDir string `yaml:"preflight_dir"`
NotificationsDir string `yaml:"notifications_dir"`
}
type DaypartConfig struct {
Name string `yaml:"name"`
Start string `yaml:"start"`
End string `yaml:"end"`
}
type RecentChangeConfig struct {
TemperatureDegrees float64 `yaml:"temperature_degrees"`
PrecipProbabilityPoints int `yaml:"precip_probability_points"`
WindGustMilesPerHour int `yaml:"wind_gust_miles_per_hour"`
PrecipTimingShiftMinutes int `yaml:"precip_timing_shift_minutes"`
}
type ReportConfig struct {
DeterministicModules []ModuleConfigItem `yaml:"deterministic_modules"`
Distributor ReportDistributorConfig `yaml:"distributor"`

View File

@@ -4,6 +4,7 @@ import (
"os"
"path/filepath"
"reflect"
"strconv"
"strings"
"testing"
"time"
@@ -37,6 +38,9 @@ func TestDefaults(t *testing.T) {
if cfg.Secrets.Directory != "" {
t.Fatalf("Secrets.Directory = %q, want empty", cfg.Secrets.Directory)
}
if cfg.Output.Directory != "" {
t.Fatalf("Output.Directory = %q, want empty", cfg.Output.Directory)
}
if cfg.Notify.Distributor.Enabled {
t.Fatalf("Notify.Distributor.Enabled = true, want false")
}
@@ -78,6 +82,120 @@ func TestDefaults(t *testing.T) {
}
}
func TestOutputDirectoryLoading(t *testing.T) {
tests := []struct {
name string
yaml string
wantValue string
wantErr string
}{
{
name: "omitted",
yaml: "{}\n",
wantValue: "",
},
{
name: "explicit empty",
yaml: `
output:
directory: ""
`,
wantValue: "",
},
{
name: "absolute path",
yaml: `
output:
directory: /var/lib/weatherreporter/reports
`,
wantValue: "/var/lib/weatherreporter/reports",
},
{
name: "relative path",
yaml: `
output:
directory: reports/../published
`,
wantValue: "reports/../published",
},
{
name: "whitespace only",
yaml: `
output:
directory: " \t "
`,
wantErr: "output.directory",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
cfg, err := LoadFile(writeConfig(t, tt.yaml))
if tt.wantErr != "" {
if err == nil || !strings.Contains(err.Error(), tt.wantErr) {
t.Fatalf("LoadFile() error = %v, want %q", err, tt.wantErr)
}
return
}
if err != nil {
t.Fatalf("LoadFile() error = %v", err)
}
if cfg.Output.Directory != tt.wantValue {
t.Fatalf("Output.Directory = %q, want %q", cfg.Output.Directory, tt.wantValue)
}
})
}
}
func TestOutputDirectoryValidationIsConsistentForLoadedAndConstructedConfigs(t *testing.T) {
tests := []struct {
name string
directory string
wantErr string
}{
{name: "empty"},
{name: "absolute path", directory: "/var/lib/weatherreporter/reports"},
{name: "relative path", directory: "reports/../published"},
{name: "whitespace only", directory: " \t ", wantErr: "output.directory"},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
yaml := "output:\n directory: " + strconv.Quote(tt.directory) + "\n"
_, loadErr := LoadFile(writeConfig(t, yaml))
cfg := Defaults()
cfg.Output.Directory = tt.directory
validateErr := Validate(cfg)
if (loadErr == nil) != (validateErr == nil) {
t.Fatalf("LoadFile() error = %v, Validate() error = %v", loadErr, validateErr)
}
if tt.wantErr != "" {
if loadErr == nil || !strings.Contains(loadErr.Error(), tt.wantErr) {
t.Fatalf("LoadFile() error = %v, want %q", loadErr, tt.wantErr)
}
if validateErr == nil || !strings.Contains(validateErr.Error(), tt.wantErr) {
t.Fatalf("Validate() error = %v, want %q", validateErr, tt.wantErr)
}
}
})
}
}
func TestOutputDirectoryRejectsUnknownFields(t *testing.T) {
_, err := LoadFile(writeConfig(t, `
output:
location: reports
`))
if err == nil {
t.Fatal("LoadFile() error = nil, want strict decoding error")
}
if !strings.Contains(err.Error(), "field location not found") {
t.Fatalf("LoadFile() error = %q, want output field rejection", err.Error())
}
}
func TestLoadExampleConfig(t *testing.T) {
cfg, err := LoadFile(filepath.Join("..", "..", "examples", "config.yml"))
if err != nil {
@@ -96,6 +214,9 @@ func TestLoadExampleConfig(t *testing.T) {
if cfg.Location.ID != "home" || cfg.Location.Name != "Brentwood" || cfg.Location.Region != "St. Louis Metro" {
t.Fatalf("Location = %#v, want example location", cfg.Location)
}
if cfg.Output.Directory != "/var/lib/weatherreporter/reports" {
t.Fatalf("Output.Directory = %q, want maintained example value", cfg.Output.Directory)
}
if cfg.Notify.Distributor.PipelineIDTemplate != "weatherreporter.{report_id}" {
t.Fatalf("PipelineIDTemplate = %q, want example pipeline template", cfg.Notify.Distributor.PipelineIDTemplate)
}
@@ -147,12 +268,6 @@ func TestLoadMinimalExampleConfig(t *testing.T) {
if cfg.Promptkit.Timeout != 2*time.Minute || cfg.Promptkit.Local.ConcurrencyLimit != 1 {
t.Fatalf("Promptkit defaults = %#v", cfg.Promptkit)
}
if cfg.Workspace.Root != "workspace" {
t.Fatalf("Workspace.Root = %q, want default workspace", cfg.Workspace.Root)
}
if cfg.Workspace.NotificationsDir != "notifications" {
t.Fatalf("Workspace.NotificationsDir = %q, want notifications", cfg.Workspace.NotificationsDir)
}
if cfg.Location.Name != "Brentwood" {
t.Fatalf("Location.Name = %q, want default Brentwood", cfg.Location.Name)
}
@@ -165,6 +280,20 @@ func TestLoadRejectsRetiredExecutionConfiguration(t *testing.T) {
}
}
func TestLoadRejectsRemovedRecentChangeConfiguration(t *testing.T) {
_, err := LoadFile(writeConfig(t, "recent_change:\n temperature_degrees: 5\n"))
if err == nil || !strings.Contains(err.Error(), "recent_change") {
t.Fatalf("LoadFile() error = %v, want removed configuration rejection", err)
}
}
func TestLoadRejectsRemovedWorkspaceConfiguration(t *testing.T) {
_, err := LoadFile(writeConfig(t, "workspace:\n root: workspace\n"))
if err == nil || !strings.Contains(err.Error(), "workspace") {
t.Fatalf("LoadFile() error = %v, want removed configuration rejection", err)
}
}
func TestLoadReportModuleOverrides(t *testing.T) {
path := writeConfig(t, `
reports:
@@ -1666,6 +1795,9 @@ func TestLoadSecretsRejectsInvalidDirectoryEntries(t *testing.T) {
t.Cleanup(func() {
_ = os.Chmod(path, 0o600)
})
if _, err := os.ReadFile(path); err == nil {
t.Skip("test process can read files without permission bits")
}
},
wantErr: "read secret file",
},

View File

@@ -21,6 +21,9 @@ func Defaults() Config {
Secrets: SecretsConfig{
Directory: "",
},
Output: OutputConfig{
Directory: "",
},
Notify: NotifyConfig{
Distributor: DistributorNotifyConfig{
Enabled: false,
@@ -49,14 +52,6 @@ func Defaults() Config {
ConcurrencyLimit: 1,
},
},
Workspace: WorkspaceConfig{
Root: "workspace",
SnapshotsDir: "snapshots",
ReportsDir: "reports",
DataPackagesDir: "data-packages",
PreflightDir: "preflight",
NotificationsDir: "notifications",
},
Dayparts: []DaypartConfig{
{Name: "overnight", Start: "00:00", End: "06:00"},
{Name: "morning", Start: "06:00", End: "10:00"},
@@ -64,12 +59,6 @@ func Defaults() Config {
{Name: "afternoon", Start: "15:00", End: "17:00"},
{Name: "evening", Start: "17:00", End: "24:00"},
},
RecentChange: RecentChangeConfig{
TemperatureDegrees: 5,
PrecipProbabilityPoints: 20,
WindGustMilesPerHour: 10,
PrecipTimingShiftMinutes: 120,
},
Reports: map[string]ReportConfig{},
}
}

View File

@@ -1,6 +1,7 @@
package config
import (
"bytes"
"errors"
"fmt"
"os"
@@ -62,7 +63,9 @@ func mergeFile(cfg *Config, path string) error {
if err := rejectRetiredExecutionConfig(data); err != nil {
return fmt.Errorf("parse config %q: %w", path, err)
}
if err := yaml.Unmarshal(data, cfg); err != nil {
decoder := yaml.NewDecoder(bytes.NewReader(data))
decoder.KnownFields(true)
if err := decoder.Decode(cfg); err != nil {
return fmt.Errorf("parse config %q: %w", path, err)
}
if cfg.MissingSource.Sources == nil {

View File

@@ -15,6 +15,9 @@ func Validate(cfg Config) error {
if err := validateReportDistributorPathOverrides(cfg); err != nil {
return err
}
if cfg.Output.Directory != "" && strings.TrimSpace(cfg.Output.Directory) == "" {
return fmt.Errorf("output.directory must not be blank when configured")
}
if cfg.WeatherAPI.BaseURL != "" {
parsed, err := url.Parse(cfg.WeatherAPI.BaseURL)
if err != nil || parsed.Scheme == "" || parsed.Host == "" {
@@ -62,9 +65,6 @@ func Validate(cfg Config) error {
if err := validatePromptkit(cfg.Promptkit); err != nil {
return err
}
if cfg.Workspace.Root == "" {
return fmt.Errorf("workspace.root is required")
}
if len(cfg.Dayparts) == 0 {
return fmt.Errorf("dayparts must contain at least one entry")
}

View File

@@ -1,4 +1,4 @@
// Package fileutil provides narrow filesystem helpers for durable artifacts.
// Package fileutil provides narrow filesystem helpers for operator-owned outputs.
package fileutil
import (
@@ -38,11 +38,3 @@ func WriteJSONAtomic(path string, value any) error {
}
return WriteFileAtomic(path, data)
}
func CopyFileAtomic(source string, target string) error {
data, err := os.ReadFile(source)
if err != nil {
return fmt.Errorf("read %q: %w", source, err)
}
return WriteFileAtomic(target, data)
}

View File

@@ -80,24 +80,3 @@ func TestWriteJSONAtomic(t *testing.T) {
t.Fatalf("json = %q, want indented object", data)
}
}
func TestCopyFileAtomic(t *testing.T) {
dir := t.TempDir()
source := filepath.Join(dir, "source.txt")
target := filepath.Join(dir, "nested", "target.txt")
if err := os.WriteFile(source, []byte("copied"), 0o600); err != nil {
t.Fatalf("WriteFile() error = %v", err)
}
if err := CopyFileAtomic(source, target); err != nil {
t.Fatalf("CopyFileAtomic() error = %v", err)
}
data, err := os.ReadFile(target)
if err != nil {
t.Fatalf("ReadFile() error = %v", err)
}
if string(data) != "copied" {
t.Fatalf("data = %q, want copied", data)
}
}

View File

@@ -147,7 +147,8 @@ func TestCatalogValidationDispatchSupportsKnownSchemas(t *testing.T) {
}
hourly, normalized, err := hourlyHandler.Validate([]byte(`{
"summary": " Storm chances increase. ",
"forecast_discussion": " A front will keep the region unsettled. "
"forecast_discussion": " A front will keep the region unsettled. ",
"precipitation_timing": ""
}`))
if err != nil {
t.Fatalf("Validate(hourly) error = %v", err)
@@ -165,7 +166,8 @@ func TestCatalogValidationDispatchSupportsKnownSchemas(t *testing.T) {
}
tomorrow, normalized, err := tomorrowHandler.Validate([]byte(`{
"summary": " Storms become more likely tomorrow. ",
"forecast_discussion": [" A front will keep showers in the forecast. ", ""]
"forecast_discussion": [" A front will keep showers in the forecast. ", ""],
"precipitation_timing": ""
}`))
if err != nil {
t.Fatalf("Validate(tomorrow) error = %v", err)
@@ -187,7 +189,8 @@ func TestCatalogValidationDispatchSupportsKnownSchemas(t *testing.T) {
}
today, normalized, err := todayHandler.Validate([]byte(`{
"summary": " Showers are likely today. ",
"forecast_discussion": [" A front will keep rain chances elevated. ", ""]
"forecast_discussion": [" A front will keep rain chances elevated. ", ""],
"precipitation_timing": ""
}`))
if err != nil {
t.Fatalf("Validate(today) error = %v", err)
@@ -209,7 +212,8 @@ func TestCatalogValidationDispatchSupportsKnownSchemas(t *testing.T) {
}
daily, normalized, err := dailyHandler.Validate([]byte(`{
"summary": " Showers are possible during the selected day. ",
"forecast_discussion": [" A front will keep rain chances in the forecast. ", ""]
"forecast_discussion": [" A front will keep rain chances in the forecast. ", ""],
"precipitation_timing": ""
}`))
if err != nil {
t.Fatalf("Validate(daily) error = %v", err)

View File

@@ -3,8 +3,7 @@ package generatedtext
type Daily struct {
Summary string `json:"summary"`
ForecastDiscussion []string `json:"forecast_discussion"`
PrecipitationTiming string `json:"precipitation_timing,omitempty"`
Confidence string `json:"confidence,omitempty"`
PrecipitationTiming string `json:"precipitation_timing"`
}
func ValidateDaily(data []byte) (Daily, []byte, error) {
@@ -16,7 +15,6 @@ func (d *Daily) dayStyleFields() dayStyleFields {
Summary: d.Summary,
ForecastDiscussion: d.ForecastDiscussion,
PrecipitationTiming: d.PrecipitationTiming,
Confidence: d.Confidence,
}
}
@@ -24,5 +22,4 @@ func (d *Daily) setDayStyleFields(fields dayStyleFields) {
d.Summary = fields.Summary
d.ForecastDiscussion = fields.ForecastDiscussion
d.PrecipitationTiming = fields.PrecipitationTiming
d.Confidence = fields.Confidence
}

View File

@@ -13,8 +13,7 @@ func TestValidateDailyNormalizesJSON(t *testing.T) {
"",
" Temperatures stay seasonable by afternoon. "
],
"precipitation_timing": " Rain is most likely during the afternoon. ",
"confidence": " Medium "
"precipitation_timing": " Rain is most likely during the afternoon. "
}`))
if err != nil {
t.Fatalf("ValidateDaily() error = %v", err)
@@ -28,23 +27,22 @@ func TestValidateDailyNormalizesJSON(t *testing.T) {
if value.PrecipitationTiming != "Rain is most likely during the afternoon." {
t.Fatalf("PrecipitationTiming = %q, want trimmed precipitation timing", value.PrecipitationTiming)
}
want := `{"summary":"Showers are possible during the selected day.","forecast_discussion":["A front will keep rain chances in the forecast.","Temperatures stay seasonable by afternoon."],"precipitation_timing":"Rain is most likely during the afternoon.","confidence":"Medium"}`
want := `{"summary":"Showers are possible during the selected day.","forecast_discussion":["A front will keep rain chances in the forecast.","Temperatures stay seasonable by afternoon."],"precipitation_timing":"Rain is most likely during the afternoon."}`
if string(normalized) != want {
t.Fatalf("normalized = %s, want %s", normalized, want)
}
}
func TestValidateDailyOmitsEmptyOptionalFields(t *testing.T) {
func TestValidateDailyPreservesRequiredEmptyPrecipitationTiming(t *testing.T) {
_, normalized, err := ValidateDaily([]byte(`{
"summary": "Showers are possible during the selected day.",
"forecast_discussion": ["A front will keep rain chances in the forecast."],
"precipitation_timing": " ",
"confidence": " "
"precipitation_timing": " "
}`))
if err != nil {
t.Fatalf("ValidateDaily() error = %v", err)
}
want := `{"summary":"Showers are possible during the selected day.","forecast_discussion":["A front will keep rain chances in the forecast."]}`
want := `{"summary":"Showers are possible during the selected day.","forecast_discussion":["A front will keep rain chances in the forecast."],"precipitation_timing":""}`
if string(normalized) != want {
t.Fatalf("normalized = %s, want %s", normalized, want)
}

View File

@@ -9,7 +9,6 @@ type dayStyleFields struct {
Summary string
ForecastDiscussion []string
PrecipitationTiming string
Confidence string
}
type dayStyleGeneratedText interface {
@@ -31,7 +30,6 @@ func validateDayStyleGeneratedText[T any, PT interface {
fields := pointer.dayStyleFields()
fields.Summary = strings.TrimSpace(fields.Summary)
fields.PrecipitationTiming = strings.TrimSpace(fields.PrecipitationTiming)
fields.Confidence = strings.TrimSpace(fields.Confidence)
fields.ForecastDiscussion = trimNonEmpty(fields.ForecastDiscussion)
if fields.Summary == "" {
var zero T
@@ -41,6 +39,10 @@ func validateDayStyleGeneratedText[T any, PT interface {
var zero T
return zero, nil, fmt.Errorf("%s generated text forecast discussion is required", name)
}
if err := requireGeneratedTextStringField(data, name, "precipitation_timing"); err != nil {
var zero T
return zero, nil, err
}
pointer.setDayStyleFields(fields)
normalized, err := normalizeGeneratedText(value, name)

View File

@@ -60,8 +60,7 @@ func TestValidateDayStyleGeneratedTextSharedBehavior(t *testing.T) {
"",
" Second paragraph. "
],
"precipitation_timing": " Afternoon. ",
"confidence": " Medium "
"precipitation_timing": " Afternoon. "
}`))
if err != nil {
t.Fatalf("validate() error = %v", err)
@@ -76,31 +75,35 @@ func TestValidateDayStyleGeneratedTextSharedBehavior(t *testing.T) {
if fields.PrecipitationTiming != "Afternoon." {
t.Fatalf("PrecipitationTiming = %q, want trimmed precipitation timing", fields.PrecipitationTiming)
}
if fields.Confidence != "Medium" {
t.Fatalf("Confidence = %q, want trimmed confidence", fields.Confidence)
}
want := `{"summary":"Shared summary.","forecast_discussion":["First paragraph.","Second paragraph."],"precipitation_timing":"Afternoon.","confidence":"Medium"}`
want := `{"summary":"Shared summary.","forecast_discussion":["First paragraph.","Second paragraph."],"precipitation_timing":"Afternoon."}`
if string(normalized) != want {
t.Fatalf("normalized = %s, want %s", normalized, want)
}
})
t.Run("omits empty optional fields", func(t *testing.T) {
t.Run("preserves required empty precipitation timing", func(t *testing.T) {
_, normalized, err := report.validate([]byte(`{
"summary": "Shared summary.",
"forecast_discussion": ["First paragraph."],
"precipitation_timing": " ",
"confidence": " "
"precipitation_timing": " "
}`))
if err != nil {
t.Fatalf("validate() error = %v", err)
}
want := `{"summary":"Shared summary.","forecast_discussion":["First paragraph."]}`
want := `{"summary":"Shared summary.","forecast_discussion":["First paragraph."],"precipitation_timing":""}`
if string(normalized) != want {
t.Fatalf("normalized = %s, want %s", normalized, want)
}
})
t.Run("requires precipitation timing field", func(t *testing.T) {
_, _, err := report.validate([]byte(`{"summary":"Shared summary.","forecast_discussion":["First paragraph."]}`))
want := fmt.Sprintf("%s generated text precipitation timing is required", report.name)
if err == nil || err.Error() != want {
t.Fatalf("validate() error = %v, want %q", err, want)
}
})
t.Run("rejects unknown fields", func(t *testing.T) {
_, _, err := report.validate([]byte(`{"summary":"Shared summary.","forecast_discussion":["First paragraph."],"extra":"value"}`))
if err == nil {
@@ -110,6 +113,13 @@ func TestValidateDayStyleGeneratedTextSharedBehavior(t *testing.T) {
t.Fatalf("validate() error = %v, want unknown field error", err)
}
})
t.Run("rejects retired confidence field", func(t *testing.T) {
_, _, err := report.validate([]byte(`{"summary":"Shared summary.","forecast_discussion":["First paragraph."],"precipitation_timing":"","confidence":"Medium"}`))
if err == nil || !strings.Contains(err.Error(), `unknown field "confidence"`) {
t.Fatalf("validate() error = %v, want retired confidence field rejection", err)
}
})
})
}
}

View File

@@ -9,8 +9,7 @@ import (
type Hourly struct {
Summary string `json:"summary"`
ForecastDiscussion string `json:"forecast_discussion"`
PrecipitationTiming string `json:"precipitation_timing,omitempty"`
Confidence string `json:"confidence,omitempty"`
PrecipitationTiming string `json:"precipitation_timing"`
}
func ValidateHourly(data []byte) (Hourly, []byte, error) {
@@ -22,13 +21,15 @@ func ValidateHourly(data []byte) (Hourly, []byte, error) {
value.Summary = strings.TrimSpace(value.Summary)
value.ForecastDiscussion = strings.TrimSpace(value.ForecastDiscussion)
value.PrecipitationTiming = strings.TrimSpace(value.PrecipitationTiming)
value.Confidence = strings.TrimSpace(value.Confidence)
if value.Summary == "" {
return Hourly{}, nil, fmt.Errorf("hourly generated text summary is required")
}
if value.ForecastDiscussion == "" {
return Hourly{}, nil, fmt.Errorf("hourly generated text forecast discussion is required")
}
if err := requireGeneratedTextStringField(data, "hourly", "precipitation_timing"); err != nil {
return Hourly{}, nil, err
}
normalized, err := normalizeGeneratedText(value, "hourly")
if err != nil {

View File

@@ -9,8 +9,7 @@ func TestValidateHourlyNormalizesJSON(t *testing.T) {
value, normalized, err := ValidateHourly([]byte(`{
"summary": " Storm chances increase. ",
"forecast_discussion": " A front will keep the region unsettled. ",
"precipitation_timing": " Showers are most likely early this afternoon. ",
"confidence": " Medium "
"precipitation_timing": " Showers are most likely early this afternoon. "
}`))
if err != nil {
t.Fatalf("ValidateHourly() error = %v", err)
@@ -24,23 +23,22 @@ func TestValidateHourlyNormalizesJSON(t *testing.T) {
if value.PrecipitationTiming != "Showers are most likely early this afternoon." {
t.Fatalf("PrecipitationTiming = %q, want trimmed precipitation timing", value.PrecipitationTiming)
}
want := `{"summary":"Storm chances increase.","forecast_discussion":"A front will keep the region unsettled.","precipitation_timing":"Showers are most likely early this afternoon.","confidence":"Medium"}`
want := `{"summary":"Storm chances increase.","forecast_discussion":"A front will keep the region unsettled.","precipitation_timing":"Showers are most likely early this afternoon."}`
if string(normalized) != want {
t.Fatalf("normalized = %s, want %s", normalized, want)
}
}
func TestValidateHourlyOmitsEmptyConfidence(t *testing.T) {
func TestValidateHourlyPreservesRequiredEmptyPrecipitationTiming(t *testing.T) {
_, normalized, err := ValidateHourly([]byte(`{
"summary": "Storm chances increase.",
"forecast_discussion": "A front will keep the region unsettled.",
"precipitation_timing": " ",
"confidence": " "
"precipitation_timing": " "
}`))
if err != nil {
t.Fatalf("ValidateHourly() error = %v", err)
}
want := `{"summary":"Storm chances increase.","forecast_discussion":"A front will keep the region unsettled."}`
want := `{"summary":"Storm chances increase.","forecast_discussion":"A front will keep the region unsettled.","precipitation_timing":""}`
if string(normalized) != want {
t.Fatalf("normalized = %s, want %s", normalized, want)
}
@@ -72,6 +70,21 @@ func TestValidateHourlyRejectsInvalidInput(t *testing.T) {
in: `{"summary":"Storm chances increase.","forecast_discussion":" "}`,
want: "forecast discussion is required",
},
{
name: "missing precipitation timing",
in: `{"summary":"Storm chances increase.","forecast_discussion":"A front will keep the region unsettled."}`,
want: "precipitation timing is required",
},
{
name: "null precipitation timing",
in: `{"summary":"Storm chances increase.","forecast_discussion":"A front will keep the region unsettled.","precipitation_timing":null}`,
want: "precipitation timing must be a string",
},
{
name: "retired confidence field rejected",
in: `{"summary":"Storm chances increase.","forecast_discussion":"A front will keep the region unsettled.","precipitation_timing":"","confidence":"Medium"}`,
want: `unknown field "confidence"`,
},
{
name: "old timing field rejected",
in: `{"summary":"Storm chances increase.","forecast_discussion":"A front will keep the region unsettled.","timing":"Late morning."}`,

Some files were not shown because too many files have changed in this diff Show More