Compare commits
25 Commits
v0.10.0
...
2b1fb26e7d
| Author | SHA1 | Date | |
|---|---|---|---|
| 2b1fb26e7d | |||
| 3c7383e2ce | |||
| eed47b4f68 | |||
| 6c185b8d0e | |||
| faf547e4a8 | |||
| acb476a142 | |||
| 606b4423f1 | |||
| 1716702c99 | |||
| e0229d9c90 | |||
| ccf6b66880 | |||
| b489c56a48 | |||
| d39e42de30 | |||
| d642791c10 | |||
| 236e3d16c4 | |||
| 4fa873983d | |||
| de1ae896b3 | |||
| 6173e50d25 | |||
| 3bca2f41f7 | |||
| af9cb0c0dc | |||
| 20c82776dc | |||
| f364ce773d | |||
| 0dc6a06cd3 | |||
| 2af6a5cfd2 | |||
| 0c4c575eea | |||
| 114f7f5f85 |
12
README.md
12
README.md
@@ -4,7 +4,9 @@ Weatherreporter is a Go CLI that turns normalized weather data into
|
||||
human-facing Markdown reports.
|
||||
|
||||
It produces a Markdown report at an operator-owned destination and can upload
|
||||
the completed output through Distributor.
|
||||
the completed output through Distributor. It can also compare explicitly
|
||||
selected Promptkit profiles against one shared prepared report and publish a
|
||||
local comparison bundle.
|
||||
|
||||
## Quickstart
|
||||
|
||||
@@ -14,14 +16,16 @@ weatherreporter generate today
|
||||
|
||||
Configure a Weather API endpoint first; see the
|
||||
[configuration reference](docs/config.md). The report is written to
|
||||
`today.md` in the current directory; use `--out` to choose another destination.
|
||||
See the [CLI reference](docs/cli.md) and [operations guide](docs/operations.md)
|
||||
for command and operating details.
|
||||
`today.md` in the current directory when `output.directory` is not configured.
|
||||
Set that configuration value for an ordinary publication directory, or use
|
||||
`--out` for one command. See the [CLI reference](docs/cli.md) and [operations
|
||||
guide](docs/operations.md) for command and operating details.
|
||||
|
||||
## Documentation
|
||||
|
||||
- [CLI reference](docs/cli.md)
|
||||
- [Configuration reference](docs/config.md)
|
||||
- [Operations guide](docs/operations.md)
|
||||
- [Comparison bundle contract](docs/integrations/comparison-bundle.md)
|
||||
- [Development guide](docs/development.md)
|
||||
- [Architecture policy](docs/policy/architecture.md)
|
||||
|
||||
103
docs/cli.md
103
docs/cli.md
@@ -1,7 +1,8 @@
|
||||
# Weatherreporter CLI
|
||||
|
||||
`weatherreporter` generates Markdown weather reports and runs report batches.
|
||||
It has no command for inspecting prior runs or application-owned state.
|
||||
`weatherreporter` generates Markdown weather reports, runs report batches, and
|
||||
compares explicitly selected Promptkit profiles against one prepared report. It
|
||||
has no command for inspecting prior runs or application-owned state.
|
||||
|
||||
## Shortest Useful Command
|
||||
|
||||
@@ -9,9 +10,10 @@ It has no command for inspecting prior runs or application-owned state.
|
||||
weatherreporter generate today
|
||||
```
|
||||
|
||||
The command uses the configured Weather API and writes an atomically replaced
|
||||
`today.md` in the current directory. See the [configuration reference](config.md)
|
||||
to supply the required Weather API endpoint.
|
||||
The command uses the configured Weather API and atomically writes `today.md`.
|
||||
With no configured output directory, it writes in the current directory. See
|
||||
the [configuration reference](config.md) to supply the required Weather API
|
||||
endpoint and choose an ordinary output directory.
|
||||
|
||||
## Commands And Usage
|
||||
|
||||
@@ -24,6 +26,7 @@ weatherreporter generate tomorrow [--config PATH] [--units VALUE] [--tz NAME] [-
|
||||
weatherreporter generate hourly [--config PATH] [--units VALUE] [--tz NAME] [--out PATH] [--llm-debug-dir PATH] [--quiet]
|
||||
weatherreporter run morning [--config PATH] [--units VALUE] [--tz NAME] [--out-dir PATH] [--llm-debug-dir PATH] [--quiet]
|
||||
weatherreporter run evening [--config PATH] [--units VALUE] [--tz NAME] [--out-dir PATH] [--llm-debug-dir PATH] [--quiet]
|
||||
weatherreporter compare REPORT --profile PROFILE --profile PROFILE [--config PATH] [--units VALUE] [--tz NAME] [--date YYYY-MM-DD] [--out-dir PATH] [--replace] [--llm-debug-dir PATH] [--quiet]
|
||||
```
|
||||
|
||||
`weatherreporter --version` prints the version embedded in the executable.
|
||||
@@ -36,32 +39,41 @@ builds report `development`.
|
||||
| `generate today` | Accepts an optional `--date YYYY-MM-DD`; without it, the current local date in the effective report timezone is used. Its default filename is `today.md`. |
|
||||
| `generate tomorrow` | Uses the next local civil day and writes `tomorrow.md` by default. |
|
||||
| `generate hourly` | Covers the next six hours in the effective report timezone and writes `hourly.md` by default. It does not accept `--date`, `--hours`, or `--duration`. |
|
||||
| `run morning` and `run evening` | Run their defined report batches and write each selected report beneath the current directory unless `--out-dir` selects another directory. `--out` is not accepted. |
|
||||
| `run morning` and `run evening` | Run their defined report batches beneath the configured output directory, or the current directory when none is configured. `--out-dir` selects another directory. `--out` is not accepted. |
|
||||
| `compare REPORT` | Accepts `daily`, `today`, `tomorrow`, or `hourly`. It requires at least two distinct, nonblank `--profile` values in their supplied order. Daily requires `--date`; Today accepts it optionally; Tomorrow and Hourly do not accept it. |
|
||||
|
||||
`generate` accepts the four report command names shown above. `run` accepts
|
||||
only `morning` and `evening`. Batch membership and notification ordering are
|
||||
described in the [operations guide](operations.md).
|
||||
only `morning` and `evening`. `compare` always requires explicit profile
|
||||
selection: `promptkit.profile` is not used as a comparison default. Batch
|
||||
membership and notification ordering are described in the
|
||||
[operations guide](operations.md).
|
||||
|
||||
## Output, Errors, And Quiet Mode
|
||||
|
||||
For `generate`, the default output is the report's filename in the current
|
||||
directory. `--out PATH` selects one output file instead. A relative path is
|
||||
resolved from the current directory; an absolute path is used as given. For a
|
||||
batch, the equivalent default is the current directory and `--out-dir PATH`
|
||||
selects its output directory. Successful summaries always report the resulting
|
||||
absolute `outputPath` values.
|
||||
For `generate`, the report's default filename is placed beneath
|
||||
`output.directory` when configured, otherwise the current directory. `--out
|
||||
PATH` selects one complete output file instead. A relative path is resolved
|
||||
from the current directory; an absolute path is used as given. For a batch,
|
||||
the configured directory has the same role and `--out-dir PATH` selects its
|
||||
output directory instead. For `compare`, `--out-dir PATH` selects one exact
|
||||
bundle directory; otherwise the report-derived comparison directory is placed
|
||||
beneath the configured directory or current directory. `--replace` is required
|
||||
to replace an existing nonempty recognized comparison bundle. See the
|
||||
[configuration reference](config.md) for the field's validation and path rules
|
||||
and the [comparison bundle contract](integrations/comparison-bundle.md) for the
|
||||
bundle format.
|
||||
|
||||
Outputs are written atomically. A generation, rendering, write, or cancellation
|
||||
failure before publication leaves an existing destination unchanged. A
|
||||
notification failure occurs after publication, so the newly written output
|
||||
remains available.
|
||||
|
||||
Action commands (`generate` and `run`) write a JSON summary to stdout unless
|
||||
`--quiet` is set. `run` also writes compact per-report and batch status lines
|
||||
to stderr. A pre-run error, such as an invalid flag, missing required argument,
|
||||
or configuration-load failure, produces no partial JSON summary. When an
|
||||
action fails after it has produced a result, its summary has `"status": "failed"`
|
||||
and an `error` field.
|
||||
Action commands (`generate`, `run`, and `compare`) write a JSON summary to
|
||||
stdout unless `--quiet` is set. `run` also writes compact per-report and batch
|
||||
status lines to stderr. A pre-run error, such as an invalid flag, missing
|
||||
required argument, or configuration-load failure, produces no partial JSON
|
||||
summary. When an action fails after it has produced a result, its summary has
|
||||
`"status": "failed"` and an `error` field.
|
||||
|
||||
`--quiet` is supported by action commands only. It suppresses action summaries
|
||||
and routine batch status output; it does not suppress command errors.
|
||||
@@ -110,19 +122,53 @@ report=today status=succeeded output="/srv/weather/reports/today.md"
|
||||
batch=morning total=2 succeeded=2 failed=0
|
||||
```
|
||||
|
||||
### Compare Summary
|
||||
|
||||
A comparison summary contains these fields in this order: `command`,
|
||||
`comparisonId`, `reportId`, `reportName`, `promptId`, `promptVersion`,
|
||||
`promptHash`, `status`, `startedAt`, `finishedAt`, `timezone`, `validPeriod`,
|
||||
`outputDirectory`, `manifestPath`, `dataPackagePath`, `total`, `succeeded`,
|
||||
`failed`, `results`, and optional `error`. Published artifact paths and each
|
||||
successful `results[].reportPath` are absolute. `results` preserves the
|
||||
supplied profile order and each item contains `position`, `profileId`, optional
|
||||
`backendId`, `modelName`, `status`, optional `validationStatus`, optional
|
||||
`reportPath`, optional `llmDebugPath`, and optional safe `error`.
|
||||
|
||||
The comparison status is `succeeded` only when every selected profile succeeds
|
||||
and the bundle is published. Individual profile failures still publish a
|
||||
complete partial bundle and return a failed command result. Cancellation or a
|
||||
failure before publication omits the artifact paths and returns a safe
|
||||
top-level error; the resolved `outputDirectory` and finalized timestamp remain
|
||||
when available. The safe error includes only a category and message: aggregate
|
||||
and unclassified application failures use `application`; cancellation uses
|
||||
`canceled`; deadlines use `deadline_exceeded`; prompt execution uses its
|
||||
published Promptkit category; destination failures use `destination_<kind>`;
|
||||
and committed cleanup failures use `publication_cleanup`. It does not expose
|
||||
provider diagnostics, filesystem causes, or recovery paths. See the
|
||||
[comparison bundle contract](integrations/comparison-bundle.md) for durable
|
||||
artifact fields and failure invariants.
|
||||
|
||||
If the bundle is published but cleanup of its replaced prior bundle fails, the
|
||||
summary still includes the published artifact paths and has status `failed`.
|
||||
Its JSON error is `publication_cleanup` with the message `comparison published
|
||||
but cleanup did not complete`; the returned command error identifies the
|
||||
retained backup path for operator recovery.
|
||||
|
||||
## Flag Reference
|
||||
|
||||
| Flag | Accepted by | Meaning |
|
||||
| --- | --- | --- |
|
||||
| `-h`, `--help` | top level | Show help. |
|
||||
| `-h`, `--help` | top level, `compare` | Show help without loading configuration or contacting a provider. |
|
||||
| `--config PATH` | all commands | Load `PATH` instead of `/usr/local/etc/weatherreporter/config.yml`. |
|
||||
| `--units VALUE` | `generate`, `run` | Override `weather_api.units` for this command. |
|
||||
| `--tz NAME` | `generate`, `run` | Override `weather_api.timezone` for this command. |
|
||||
| `--out PATH` | every `generate` command | Write the report to this file instead of its current-directory default. |
|
||||
| `--llm-debug-dir PATH` | every `generate` and `run` command | Write requested sensitive prompt diagnostics under this absolute path. |
|
||||
| `--out-dir PATH` | `run morning`, `run evening` | Write batch reports beneath this directory instead of the current directory. |
|
||||
| `--quiet` | `generate`, `run` | Suppress action summaries and routine batch status output. |
|
||||
| `--date YYYY-MM-DD` | `generate daily`, `generate today` | Required for Daily; optional for Today. |
|
||||
| `--units VALUE` | `generate`, `run`, `compare` | Override `weather_api.units` for this command. |
|
||||
| `--tz NAME` | `generate`, `run`, `compare` | Override `weather_api.timezone` for this command. |
|
||||
| `--out PATH` | every `generate` command | Write the report to this complete file destination instead of the configured or current-directory default. |
|
||||
| `--llm-debug-dir PATH` | every `generate`, `run`, and `compare` command | Write requested sensitive prompt diagnostics under this absolute path. |
|
||||
| `--profile PROFILE` | `compare` | Select one explicit profile. Repeat at least twice with distinct, nonblank IDs. |
|
||||
| `--out-dir PATH` | `run morning`, `run evening`, `compare` | Write batch reports beneath this directory, or select the exact comparison directory. |
|
||||
| `--replace` | `compare` | Authorize replacement of a recognized nonempty comparison bundle. |
|
||||
| `--quiet` | `generate`, `run`, `compare` | Suppress successful action output and routine batch status output. |
|
||||
| `--date YYYY-MM-DD` | `generate daily`, `generate today`, `compare daily`, `compare today` | Required for Daily; optional for Today. |
|
||||
|
||||
Distributor notification is configured through `notify.distributor`; there are
|
||||
no Distributor-specific CLI flags. See the [configuration reference](config.md).
|
||||
@@ -135,4 +181,5 @@ weatherreporter generate today --out ./reports/today.md
|
||||
weatherreporter generate hourly --out /srv/weather/hourly.md
|
||||
weatherreporter generate today --llm-debug-dir /var/tmp/weatherreporter-debug
|
||||
weatherreporter run morning --out-dir ./reports --llm-debug-dir /var/tmp/weatherreporter-debug
|
||||
weatherreporter compare daily --date 2026-05-29 --profile weather-light --profile weather-balanced --out-dir ./comparison-daily-2026-05-29
|
||||
```
|
||||
|
||||
@@ -81,6 +81,32 @@ Missing directories, unreadable files, subdirectories, symlinks, non-regular
|
||||
files, and invalid names fail configuration loading. Put only secret values in
|
||||
this directory, never in the YAML file.
|
||||
|
||||
### `output`
|
||||
|
||||
`output.directory` selects the ordinary operator-owned publication directory
|
||||
for individual reports, batches, and the default parent of comparison bundles.
|
||||
|
||||
| Field | Default | Rules |
|
||||
| --- | --- | --- |
|
||||
| `directory` | empty | An omitted or empty value uses the invocation working directory. A nonempty value must contain at least one non-whitespace character. |
|
||||
|
||||
The configured value is preserved while configuration loads: it is not cleaned,
|
||||
made absolute, inspected, created, or expanded through environment variables or
|
||||
a home-directory shortcut. At execution, an absolute directory is used as
|
||||
given; a relative directory resolves from the invocation working directory, not
|
||||
from the configuration file's location. A missing directory is created when a
|
||||
report is successfully published. An existing non-directory or an uninspectable
|
||||
path fails output preflight before prompt inspection, weather collection, or
|
||||
publication.
|
||||
|
||||
For one `generate` command, `--out` is a complete file destination and takes
|
||||
precedence over `output.directory`. For `run`, `--out-dir` takes precedence.
|
||||
For `compare`, `--out-dir` selects its exact bundle directory; without it, the
|
||||
comparison's report-derived directory is placed beneath `output.directory`.
|
||||
Those explicit flags do not inspect or rebase beneath the configured directory.
|
||||
See the [CLI reference](cli.md) for command selection and the [operations
|
||||
guide](operations.md) for publication and failure handling.
|
||||
|
||||
### `notify.distributor`
|
||||
|
||||
Distributor notification is disabled by default. Its fields are:
|
||||
@@ -142,11 +168,11 @@ source keys are `observations`, `current`, `narrative`, `alerts`, `discussion`,
|
||||
### `promptkit`
|
||||
|
||||
Promptkit configuration selects the executor and prompt/profile checks for
|
||||
every `generate` and `run` command. A top-level `scriptorium:` configuration
|
||||
every `generate`, `run`, and `compare` command. A top-level `scriptorium:` configuration
|
||||
key is rejected with a migration error; it is not translated or ignored.
|
||||
|
||||
Prompt debug capture has no YAML setting. Use `--llm-debug-dir PATH` on an
|
||||
individual `generate` or `run` command when explicitly needed.
|
||||
individual `generate`, `run`, or `compare` command when explicitly needed.
|
||||
|
||||
| Field | Default | Rules |
|
||||
| --- | --- | --- |
|
||||
|
||||
@@ -21,14 +21,15 @@ boundaries and invariants.
|
||||
| Adding, changing, reviewing, or deleting tests | [Testing policy](policy/testing.md) and focused package tests | The policy defines risk-based sufficiency, durable test boundaries, doubles, and test-maintenance criteria. |
|
||||
| CLI commands, flags, output, quiet mode, or command wiring | [CLI reference](cli.md) and [CLI internals](internal/cli.md) | The reference owns the user contract; the internal guide owns command composition and output flow. |
|
||||
| Configuration fields, defaults, loading, overrides, validation, or secrets | [Configuration reference](config.md), [architecture policy](policy/architecture.md), and tests under `internal/config` | These separate the user-visible contract, architectural rules, and executable behavior. |
|
||||
| Top-level generation, batch, collection, output publication, or notification workflow | [App orchestration internals](internal/app-orchestration.md) | It owns workflow ordering, output publication, failure propagation, and orchestration invariants. |
|
||||
| Top-level generation, batch, comparison, collection, output publication, or notification workflow | [App orchestration internals](internal/app-orchestration.md), [comparison execution internals](internal/comparison-execution.md), and [comparison publication internals](internal/comparison-publication.md) | They own workflow ordering, concurrent profile execution, output publication, failure propagation, and orchestration invariants. |
|
||||
| Weather API transport, source envelopes, source warnings, or collection | [Weather API integration](integrations/weatherapi.md), [weather-data internals](internal/weather-data.md), and [collection internals](internal/collect.md) | These separate the external contract, normalized source facts, and app-facing collection behavior. |
|
||||
| Forecast periods, weather derivation, collected facts, or derived facts | [Forecast derivation internals](internal/forecast-derivation.md) and [fact contracts](internal/facts.md) | They own deterministic derivation and the fact boundaries used by reports. |
|
||||
| Report definitions, valid periods, report IDs, output naming, or batch composition | [Report registry internals](internal/report-registry.md) and [app orchestration internals](internal/app-orchestration.md) | Report definitions own selection and period rules; orchestration owns execution. |
|
||||
| Module IDs, module composition, briefing values, or prompt-facing exports | [Module contract internals](internal/module.md), [module builder internals](internal/briefing.md), and [prompt-input internals](internal/prompt-input.md) | These own module contracts, value construction, and the curated prompt-package boundary. |
|
||||
| Prompt execution, profiles, prompt inputs, or result handling | `internal/promptexec`, the Promptkit adapter, and [prompt-input internals](internal/prompt-input.md) | These separate the executor contract and input construction. |
|
||||
| Prompt execution, profiles, prepared report inputs, or result handling | `internal/promptexec`, the Promptkit adapter, [prepared report internals](internal/prepared-report.md), and [prompt-input internals](internal/prompt-input.md) | These separate the executor contract, immutable preparation, and input construction. |
|
||||
| Durable comparison bundles or their compatibility | [Comparison bundle contract](integrations/comparison-bundle.md) and [comparison publication internals](internal/comparison-publication.md) | The integration document owns the external schema; internals own how it is published. |
|
||||
| Generated-text schemas, validation, render contexts, templates, or Markdown rendering | [Generated-text internals](internal/generatedtext.md), [report-template internals](internal/reporttemplate.md), and [report template guide](templates.md) | These own structured text, renderer implementation, and the maintainer-facing template surface. |
|
||||
| Output destinations, atomic publication, prompt diagnosis, or legacy cleanup | [Operations guide](operations.md) and [App orchestration internals](internal/app-orchestration.md) | Operations owns operator workflows; app internals owns the implementation boundary. |
|
||||
| Output destinations, atomic publication, prompt diagnosis, or legacy cleanup | [Operations guide](operations.md), [App orchestration internals](internal/app-orchestration.md), and [comparison publication internals](internal/comparison-publication.md) | Operations owns operator workflows; internals own implementation boundaries. |
|
||||
| Distributor bundles, uploads, notification results, or failures | [Distributor adapter internals](internal/distributor-adapter.md), [Distributor integration contracts](integrations/distributor/), and [operations guide](operations.md) | These separate adapter behavior, external contracts, and operational lifecycle. |
|
||||
| Maintained example configuration | [Configuration reference](config.md) and files under `examples/` | The reference owns field meaning; examples own complete copyable files. |
|
||||
| Release preparation, tagging, publication, or verification | [Release procedure](release.md) | It owns version selection, release-note preparation, candidate validation, tag publication, CI behavior, and post-publication checks. |
|
||||
@@ -44,7 +45,8 @@ present before introducing a new package or abstraction.
|
||||
| --- | --- |
|
||||
| `cmd/weatherreporter` | Binary entry point. |
|
||||
| `internal/cli` | Command parsing, flags, help, output, and command wiring. |
|
||||
| `internal/app` | Stateless generation, batches, collection coordination, output publication, and notification. |
|
||||
| `internal/app` | Stateless generation, batches, comparisons, collection coordination, output publication, and notification. |
|
||||
| `internal/comparison` | Comparison identities, logical bundles, guarded destinations, and atomic bundle publication. |
|
||||
| `internal/config` | Configuration defaults, loading, precedence, secrets, and validation. |
|
||||
| `internal/adapters` | Weather API, Promptkit, and Distributor boundaries. |
|
||||
| `internal/weatherdata`, `internal/forecast`, `internal/facts` | Normalized source facts and deterministic derivation. |
|
||||
|
||||
92
docs/integrations/comparison-bundle.md
Normal file
92
docs/integrations/comparison-bundle.md
Normal file
@@ -0,0 +1,92 @@
|
||||
# Comparison Bundle Contract
|
||||
|
||||
A comparison bundle is the durable, flat artifact produced when one report is
|
||||
executed with multiple explicit Promptkit profiles. This document is the
|
||||
canonical contract for consumers of those bundles. Command invocation and JSON
|
||||
action summaries belong to the [CLI reference](../cli.md); destination handling
|
||||
and retention belong to the [operations guide](../operations.md).
|
||||
|
||||
## Version And Layout
|
||||
|
||||
The current and only supported manifest schema version is
|
||||
`weatherreporter.comparison.v1`. A bundle directory contains exactly these
|
||||
regular, non-symlinked files:
|
||||
|
||||
```text
|
||||
comparison.json
|
||||
data-package.yml
|
||||
NN-profile-slug.md
|
||||
```
|
||||
|
||||
`comparison.json` is the manifest and `data-package.yml` is the exact YAML
|
||||
input supplied to every selected profile. There is one Markdown file for each
|
||||
successful result and none for failed results. `NN` is the one-based selected
|
||||
profile position, zero padded to at least two digits (and widened only when
|
||||
needed for 100 or more profiles). The profile slug preserves ASCII letters,
|
||||
digits, `-`, and `_`; each run of other characters becomes one `-`; edge `-`
|
||||
and `_` characters are removed; the value is capped at 64 bytes; and an empty
|
||||
slug becomes `profile`. Logical profile IDs remain authoritative in the
|
||||
manifest.
|
||||
|
||||
All manifest paths are basenames relative to the bundle root. They never use
|
||||
path separators, `.` or `..`. The CLI reports absolute paths only after a
|
||||
bundle has been published.
|
||||
|
||||
## Manifest Schema
|
||||
|
||||
The manifest is UTF-8 JSON, encoded as two-space-indented JSON with one
|
||||
trailing newline. Its fields appear in this order:
|
||||
|
||||
```text
|
||||
schemaVersion, comparisonId, startedAt, finishedAt, reportId, validPeriod,
|
||||
timezone, promptId, promptVersion, promptHash, dataPackage, total, succeeded,
|
||||
failed, results
|
||||
```
|
||||
|
||||
`validPeriod` contains `start` and `end`; it is a nonempty half-open period.
|
||||
`dataPackage` contains `path` (always `data-package.yml`) and `sha256` (the
|
||||
lowercase, 64-character SHA-256 digest of that file's exact bytes). `results`
|
||||
is in the explicit profile-selection order. Its result-object fields appear in
|
||||
this order:
|
||||
|
||||
```text
|
||||
position, profileId, backendId, modelName, status, validationStatus,
|
||||
reportPath, error
|
||||
```
|
||||
|
||||
`startedAt` and `finishedAt` are nonzero UTC timestamps, and the latter is not
|
||||
earlier than the former. `validPeriod` retains its resolved time offset.
|
||||
`reportId`, `timezone`, prompt identity, model name, and comparison ID are
|
||||
nonblank. `promptHash` and `dataPackage.sha256` are lowercase SHA-256 digests.
|
||||
|
||||
## Result Invariants
|
||||
|
||||
`total` is at least two and equals the number of results. Positions are
|
||||
contiguous from one, profile IDs are distinct and nonblank, and
|
||||
`succeeded + failed == total`.
|
||||
|
||||
A successful result has `status: "succeeded"`, `validationStatus: "passed"`,
|
||||
a unique Markdown `reportPath`, and no `error`. A failed result has
|
||||
`status: "failed"`, no `reportPath`, and an `error` object with nonblank
|
||||
`category` and `message`. Its validation status is absent, `failed`, or
|
||||
`skipped`. Error messages are valid UTF-8 and no longer than 1,024 bytes.
|
||||
`backendId` and `validationStatus` are omitted when unavailable.
|
||||
|
||||
Every successful Markdown file is declared by exactly one successful result.
|
||||
The directory contains no extra entries. Consumers can therefore verify the
|
||||
data-package digest and the full manifest-to-file mapping without scanning a
|
||||
larger workspace.
|
||||
|
||||
## Compatibility And Sensitivity
|
||||
|
||||
Weatherreporter recognizes a replaceable bundle only when it exactly satisfies
|
||||
the current version, schema, file set, file types, relative-path rules, and
|
||||
data-package digest. It rejects unknown manifest fields, multiple JSON values,
|
||||
extra entries, symlinks, and future or otherwise unsupported versions. Treat a
|
||||
bundle that fails recognition as an ordinary directory, not as a compatible
|
||||
bundle.
|
||||
|
||||
The manifest contains safe operational provenance, but `data-package.yml` and
|
||||
the generated Markdown can contain sensitive weather or location context. Do
|
||||
not assume these artifacts are safe for public distribution. Handle retention,
|
||||
access, and deletion according to the [operations guide](../operations.md).
|
||||
@@ -29,6 +29,22 @@ Profiles that require a direct API key are unsupported; a profile that reports `
|
||||
|
||||
Promptkit receives the YAML data package as an inline input and returns structured JSON that Weatherreporter validates before rendering its own Markdown template. Safe active provenance remains in memory. Content-rich diagnostics are opt-in through `--llm-debug-dir`; see [operations](../operations.md) for retention and permissions.
|
||||
|
||||
## Comparison Execution
|
||||
|
||||
For `compare`, Weatherreporter validates one exact prompt and every explicitly
|
||||
selected profile before weather collection. It prepares one deterministic YAML
|
||||
data package, retains immutable copies of the report inputs, and executes every
|
||||
profile against the same exact data-package bytes. Each profile remains an
|
||||
independent Promptkit execution: one provider or validation failure does not
|
||||
stop its peers, while caller cancellation applies to every in-flight execution.
|
||||
|
||||
Weatherreporter starts selected profile executions concurrently and does not
|
||||
add an application-level concurrency limit. Promptkit owns backend capacity and
|
||||
any profile or backend concurrency policy. The durable comparison output and
|
||||
its compatibility rules are defined by the
|
||||
[comparison bundle contract](comparison-bundle.md); the user-facing command
|
||||
contract is in the [CLI reference](../cli.md).
|
||||
|
||||
The generated-text schemas require `summary`, `forecast_discussion`, and `precipitation_timing`, and reject additional properties. Prompts return an empty string for `precipitation_timing` when the deterministic package contains no precipitation windows.
|
||||
|
||||
Prompt/profile configuration and the maintained local override example are owned by the [configuration reference](../config.md). Adapter construction and mapping are documented in the [Promptkit adapter internals](../internal/promptkit-adapter.md).
|
||||
|
||||
@@ -1,10 +1,13 @@
|
||||
# Application Orchestration Internals
|
||||
|
||||
`internal/app` owns stateless report generation, batch execution, atomic output publication, and notification coordination after `internal/cli` has parsed arguments and loaded configuration. The user contract is owned by the [CLI reference](../cli.md) and [operations guide](../operations.md).
|
||||
`internal/app` owns stateless report generation, batch execution, comparison
|
||||
orchestration, atomic output publication, and notification coordination after
|
||||
`internal/cli` has parsed arguments and loaded configuration. The user contract
|
||||
is owned by the [CLI reference](../cli.md) and [operations guide](../operations.md).
|
||||
|
||||
## Single-Report Flow
|
||||
|
||||
`GenerateDetailed` resolves the requested report and output destination, then initializes an optional explicit debug writer. It validates the exact Promptkit prompt and selected profile before collecting weather data. The resolved profile, backend, and model are carried in the active result.
|
||||
`GenerateDetailed` resolves the requested report and output destination before initializing an optional explicit debug writer. An explicit output file wins; otherwise the configured output directory is used, falling back to the captured working directory. It validates the exact Promptkit prompt and selected profile before collecting weather data. The resolved profile, backend, and model are carried in the active result.
|
||||
|
||||
The workflow builds facts, a module snapshot, briefing metadata, and the YAML prompt package in memory. It executes Promptkit, validates the returned generated text, builds a render context, and renders Markdown. `fileutil` atomically writes the completed Markdown to the selected output path. Only after that write succeeds does single-report notification run.
|
||||
|
||||
@@ -12,10 +15,40 @@ Failures return an active partial result with safe identity, profile, warning, v
|
||||
|
||||
## Batches
|
||||
|
||||
`RunBatchDetailed` captures one output directory, creates at most one explicit debug writer, and uses one executor. Before collection it validates the prompt and profile candidates for the selected batch. It collects once, calculates the data-dependent plan, then validates and retains the final output path for every planned report before invoking the same generation core sequentially.
|
||||
`RunBatchDetailed` selects an explicit output directory first, otherwise the configured directory and then the captured working directory. It does this before creating at most one explicit debug writer or validating prompt and profile candidates for the selected batch. It collects once, calculates the data-dependent plan, then validates and retains the final output path for every planned report before invoking the same generation core sequentially.
|
||||
|
||||
Each item has an independent result. A failed item does not stop later items; successful items retain their published output paths. Per-report notification is suppressed during a batch. Batch notification runs only after every planned report has published successfully. It is skipped when any item failed. Batch result counters count report items only; a batch notification failure is represented by the top-level notification result and still produces a failed batch outcome.
|
||||
|
||||
## Comparisons
|
||||
|
||||
`CompareDetailed` validates ordered explicit profile IDs, resolves the report,
|
||||
and preflights the exact bundle destination before initializing optional prompt
|
||||
debugging, prompt inspection, or collection. It then inspects the one prompt
|
||||
and every selected profile, collects once, and delegates shared report
|
||||
construction to the prepared-report flow. It does not accept a notifier.
|
||||
|
||||
Once the destination is resolved, the partial result retains its absolute
|
||||
output directory even when later preflight, debug initialization, inspection,
|
||||
collection, or preparation fails. Every initialized result is finalized with a
|
||||
finished timestamp. If prompt inspection succeeds before a later profile
|
||||
inspection fails, the partial result retains the resolved prompt ID, version,
|
||||
and hash. Artifact paths are added only after publication commits.
|
||||
|
||||
The comparison execution core starts each inspected profile independently,
|
||||
keeps results in selection order, and waits for all started work. Independent
|
||||
profile failures are recorded and do not stop peers. Context cancellation marks
|
||||
unfinished work and prevents publication. Details of prepared values, execution
|
||||
and debugging, and publication are documented in [prepared report
|
||||
internals](prepared-report.md), [comparison execution
|
||||
internals](comparison-execution.md), and [comparison publication
|
||||
internals](comparison-publication.md).
|
||||
|
||||
When publication has committed its new bundle, application results contain the
|
||||
absolute manifest, data-package, and successful report paths even if removal of
|
||||
the previous sibling backup then fails. That cleanup failure is still returned
|
||||
as an operational error rather than treating the new bundle as unpublished;
|
||||
the returned error retains the recovery path and underlying filesystem cause.
|
||||
|
||||
## Boundaries And Verification
|
||||
|
||||
The package does not parse flags, load YAML, implement transport, construct provider SDKs, or define report-period policy. Prompt, profile, weather, and Distributor implementations remain behind project-owned contracts.
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
|
||||
The root `--version` flag reports the build version supplied by `internal/buildinfo`. Tagged release builds replace its development default at link time.
|
||||
|
||||
For each `generate` or `run` action, `Runner` constructs one project-owned Promptkit executor after configuration loads. It captures an absolute working directory, resolves a relative output override against it, and passes the working directory, resolved override, and any `--llm-debug-dir` request to the app. With no override, the app derives the report filename in that working directory. `run` uses the same resolution rule for `--out-dir`.
|
||||
For each `generate` or `run` action, `Runner` constructs one project-owned Promptkit executor after configuration loads. It captures an absolute working directory, resolves only a relative explicit output override against it, and passes the working directory, loaded configuration, resolved override, and any `--llm-debug-dir` request to the app. The raw configured fallback remains in the configuration for app-owned destination selection. `run` uses the same explicit-resolution rule for `--out-dir`.
|
||||
|
||||
The CLI dispatches only generation and batch actions. It has no persisted-run or inspection dispatch. Summaries include report identity, status, output path, effective profile/backend/model, source warnings, validation, requested debug path, and notification result when available. They intentionally exclude prompt input, raw generated text, render context, endpoints, credentials, and full Distributor payloads. A failed action with a partial result still emits its safe summary before its error is returned.
|
||||
|
||||
|
||||
27
docs/internal/comparison-execution.md
Normal file
27
docs/internal/comparison-execution.md
Normal file
@@ -0,0 +1,27 @@
|
||||
# Comparison Execution Internals
|
||||
|
||||
The comparison execution core receives an already prepared report and an
|
||||
already inspected, ordered profile list. It initializes an outcome for every
|
||||
selected profile, launches each started profile in its own goroutine, and
|
||||
waits for every started goroutine before returning. Results retain the supplied
|
||||
selection order even though execution completes in an arbitrary order.
|
||||
|
||||
Every profile uses the exact inspected prompt identity and a private copy of
|
||||
the same prepared data package. Provider, generated-text validation, rendering,
|
||||
or debug-write failure becomes that profile's safe failed outcome and does not
|
||||
cancel its peers. The application deliberately imposes no additional semaphore:
|
||||
Promptkit owns backend capacity. Cancellation or a deadline marks unfinished
|
||||
outcomes as skipped or failed, joins work, and prevents bundle publication.
|
||||
|
||||
When debugging is enabled, each execution receives a deterministic reference
|
||||
derived from the comparison identity, ordered profile position, and safe
|
||||
profile slug. This keeps concurrent captures separate. The debug writer itself
|
||||
owns secure-root validation and file permissions. It safely creates shared
|
||||
missing ancestors during concurrent writes, then rejects symlink and non-
|
||||
directory components. Operational retention and sensitivity are documented in
|
||||
the [operations guide](../operations.md).
|
||||
|
||||
The output result and its safe errors are converted into the durable contract
|
||||
only by comparison publication. See [comparison publication
|
||||
internals](comparison-publication.md) and the external [comparison bundle
|
||||
contract](../integrations/comparison-bundle.md).
|
||||
30
docs/internal/comparison-publication.md
Normal file
30
docs/internal/comparison-publication.md
Normal file
@@ -0,0 +1,30 @@
|
||||
# Comparison Publication Internals
|
||||
|
||||
`internal/comparison` separates the logical bundle from filesystem mechanics.
|
||||
The application builds a validated manifest, exact shared data-package bytes,
|
||||
and only the Markdown files for successful profiles. The durable layout,
|
||||
schema, and compatibility rules are owned by the [comparison bundle
|
||||
contract](../integrations/comparison-bundle.md).
|
||||
|
||||
Destination planning is read-only. It requires an exact absolute target that
|
||||
is neither the filesystem root nor the working directory, rejects unsafe
|
||||
symlinks and non-directories, accepts a missing or empty directory, and permits
|
||||
replacement only for a recognized current bundle. Publication rechecks that
|
||||
authorization immediately before it writes a private sibling staging directory.
|
||||
For replacement, it moves the prior bundle to a private sibling backup,
|
||||
reauthorizes that moved entry, and restores it if installing the new bundle
|
||||
fails.
|
||||
|
||||
The new bundle is committed only after the staged directory has been installed
|
||||
at the target. From that point its artifact paths are authoritative: a failure
|
||||
to remove the retained sibling backup does not roll back the new bundle.
|
||||
Publication returns an inspectable cleanup error with the absolute backup path
|
||||
and underlying filesystem cause so an operator can recover or remove that
|
||||
backup manually.
|
||||
|
||||
The application preflights before prompt inspection and collection, then
|
||||
preflights again before publication. A cancellation or any failure before the
|
||||
commit leaves the prior destination untouched. Completed bundles include
|
||||
partial profile results; comparison publication never coordinates Distributor
|
||||
notification. Operator-facing lifecycle and cleanup are in the
|
||||
[operations guide](../operations.md).
|
||||
20
docs/internal/prepared-report.md
Normal file
20
docs/internal/prepared-report.md
Normal file
@@ -0,0 +1,20 @@
|
||||
# Prepared Report Internals
|
||||
|
||||
`internal/app` builds a `preparedReport` after collection and before profile
|
||||
execution. This is the immutable boundary shared by ordinary report generation
|
||||
and profile comparison; it is not a durable artifact.
|
||||
|
||||
Preparation builds report facts, the configured module snapshot, briefing
|
||||
metadata, the curated prompt-input package, serialized YAML, and the
|
||||
generated-text definition. It deep-copies mutable facts, snapshots, metadata,
|
||||
and data-package bytes before returning them. Consumers receive independent
|
||||
copies so one execution cannot change another's input or rendering context.
|
||||
|
||||
Single-report generation executes one prepared profile and publishes its
|
||||
Markdown. Comparison prepares once, gives every selected profile the same YAML
|
||||
bytes, and only then assembles the resulting logical bundle. The prompt-input
|
||||
shape is owned by [prompt-input internals](prompt-input.md); profile execution
|
||||
semantics are owned by [Promptkit integration](../integrations/promptkit.md).
|
||||
|
||||
Preparation failure has no publication side effects. Tests for this boundary
|
||||
cover mutation isolation, byte equality, and reuse by both execution paths.
|
||||
@@ -13,12 +13,20 @@ After configuring a Weather API endpoint, generate one report:
|
||||
weatherreporter generate today
|
||||
```
|
||||
|
||||
The command writes `today.md` in the current directory. Choose a different
|
||||
operator-owned file with `--out`; a relative path is resolved from the current
|
||||
directory and an absolute path is used directly. Weatherreporter renders in
|
||||
memory and atomically replaces the selected destination only after generation
|
||||
and rendering succeed. It does not create a default workspace, metadata,
|
||||
receipts, or intermediate output files.
|
||||
With no configured output directory, the command writes `today.md` in the
|
||||
current directory. Set `output.directory` to use one ordinary publication
|
||||
directory for reports, or choose a one-command operator-owned file with
|
||||
`--out`; a relative path is resolved from the current directory and an absolute
|
||||
path is used directly. The explicit flag takes precedence over the configured
|
||||
directory. Weatherreporter renders in memory and atomically replaces the
|
||||
selected destination only after generation and rendering succeed. It does not
|
||||
create a default workspace, metadata, receipts, or intermediate output files.
|
||||
|
||||
A missing configured directory is created only as part of successful report
|
||||
publication. If its existing path is not a directory or cannot be inspected,
|
||||
the command stops before prompt inspection or weather collection, leaving any
|
||||
existing report unchanged. See the [configuration reference](config.md) for the
|
||||
field definition and validation rules.
|
||||
|
||||
Before a destination is published, provider, validation, rendering, write, and
|
||||
cancellation failures leave an existing report unchanged. A notification
|
||||
@@ -36,7 +44,9 @@ Run a scheduled batch with an explicit output directory when appropriate:
|
||||
weatherreporter run morning --out-dir ./reports
|
||||
```
|
||||
|
||||
Without `--out-dir`, batch reports are written beneath the current directory.
|
||||
Without `--out-dir`, batch reports are written beneath `output.directory` when
|
||||
configured, otherwise the current directory. The explicit directory applies
|
||||
only to that command and takes precedence over the configured fallback.
|
||||
Morning runs Today, Tomorrow, and every eligible dated Daily Report; evening
|
||||
runs Tomorrow and the same eligible Daily Reports. Eligible Daily dates begin
|
||||
after tomorrow and require complete hourly coverage for their local civil day.
|
||||
@@ -62,6 +72,38 @@ For a single report, Distributor notification follows the atomic output write.
|
||||
See the [configuration reference](config.md) for pipeline, bundle,
|
||||
idempotency-key, and per-report path templates.
|
||||
|
||||
## Comparison Bundles
|
||||
|
||||
Use `compare` when an operator needs to evaluate explicit Promptkit profiles
|
||||
against the same report input. The command writes one flat, operator-owned
|
||||
bundle directory and never sends a Distributor notification. Command syntax,
|
||||
profile validation, JSON output, and exit behavior belong to the
|
||||
[CLI reference](cli.md); the durable file contract belongs to the
|
||||
[comparison bundle contract](integrations/comparison-bundle.md).
|
||||
|
||||
The output destination follows the normal `output.directory` fallback. An
|
||||
explicit `--out-dir` takes precedence and names the exact bundle directory,
|
||||
not a parent to be combined with another name. The standard names are derived
|
||||
from the report output name, such as `comparison-today` and
|
||||
`comparison-daily-2026-05-29`; see the [configuration reference](config.md)
|
||||
for output-directory resolution.
|
||||
|
||||
A comparison bundle contains the shared data package, a manifest, and one
|
||||
Markdown file for every successful profile. Treat all of these files as
|
||||
potentially sensitive: the data package and generated reports can contain
|
||||
location or forecast context. Weatherreporter creates no application-owned
|
||||
history, retention store, or cleanup job. Retain, archive, or remove only the
|
||||
specific bundle directories your operating policy permits.
|
||||
|
||||
The destination is preflighted before prompt inspection and collection, then
|
||||
rechecked immediately before an atomic publish. A missing or empty directory
|
||||
is usable. A nonempty directory can be replaced only when `--replace` is given
|
||||
and it is recognized as a current Weatherreporter comparison bundle; ordinary
|
||||
directories, symlinks, and unsafe destinations are rejected. Cancellation and
|
||||
all failures before publication preserve an existing bundle. Profile failures
|
||||
are different: the command publishes a complete partial bundle, with failed
|
||||
profiles represented in the manifest and no Markdown file for those profiles.
|
||||
|
||||
## Local Prompt Profile Override
|
||||
|
||||
Hourly normally selects the embedded `weather-light` profile. To use a local
|
||||
@@ -89,9 +131,11 @@ weatherreporter generate today --llm-debug-dir /var/tmp/weatherreporter-debug
|
||||
The directory must be absolute. Requested captures are written with restrictive
|
||||
permissions beneath the supplied directory, organized by report and run. They
|
||||
can contain rendered prompts and generated output, so limit access to trusted
|
||||
operators and remove the captures when they are no longer needed. Normal output,
|
||||
summaries, and routine logs omit that sensitive content. Debug capture is never
|
||||
created for an ordinary command without `--llm-debug-dir`.
|
||||
operators and remove the captures when they are no longer needed. Comparison
|
||||
captures additionally identify each selected profile so concurrent executions
|
||||
remain distinct. Normal output, summaries, and routine logs omit that sensitive
|
||||
content. Debug capture is never created for an ordinary command without
|
||||
`--llm-debug-dir`.
|
||||
|
||||
If capture creation or writing fails, the affected run fails rather than
|
||||
silently continuing without the requested diagnostics.
|
||||
@@ -101,8 +145,20 @@ silently continuing without the requested diagnostics.
|
||||
Start with the command error and JSON summary. For a report generation failure,
|
||||
the selected destination was not replaced; for a notification failure, inspect
|
||||
the completed destination and the notification result. For a batch failure,
|
||||
use the per-report statuses and retain successful output files. Enable explicit
|
||||
debug capture only when content-rich Promptkit diagnostics are necessary.
|
||||
use the per-report statuses and retain successful output files. For a comparison
|
||||
failure, inspect the published manifest when its path is present: individual
|
||||
profile failures retain their safe result and successful Markdown files, while
|
||||
cancellation and pre-publication errors leave the prior destination unchanged.
|
||||
|
||||
If a replacement commits but cleanup of its prior sibling backup fails, the new
|
||||
bundle remains valid and its artifact paths appear in the failed command
|
||||
summary. The summary records a safe `publication_cleanup` error, while the
|
||||
returned command error reports the retained backup path. Preserve that backup
|
||||
until it has been inspected and cleaned up manually; do not remove the new
|
||||
bundle to retry that cleanup.
|
||||
|
||||
Enable explicit debug capture only when content-rich Promptkit diagnostics are
|
||||
necessary.
|
||||
|
||||
Weatherreporter does not retain runs for later inspection, resume failed work,
|
||||
or provide automatic cleanup, archival, remote state, daemon operation, or
|
||||
|
||||
@@ -15,6 +15,11 @@ and renders repository-owned Markdown in memory. Completed Markdown is
|
||||
atomically published to an operator-owned output destination and may then be
|
||||
uploaded through Distributor.
|
||||
|
||||
An explicit profile comparison prepares one report input once, executes the
|
||||
same exact prompt and data package across selected profiles concurrently, and
|
||||
atomically publishes one operator-owned comparison bundle. It remains local:
|
||||
it does not create application state or send a Distributor notification.
|
||||
|
||||
The supported report products are Daily, Today, Tomorrow, and Hourly. A batch
|
||||
collects once, validates its complete candidate prompt/profile set before
|
||||
collection, then determines and validates every planned output destination
|
||||
@@ -29,6 +34,8 @@ planned report succeeds.
|
||||
- `internal/config` owns defaults, loading, validation, and secret loading.
|
||||
- `internal/app` owns in-memory workflow order, partial results, atomic output
|
||||
publication, and notification coordination through project-owned contracts.
|
||||
- `internal/comparison` owns comparison identity, durable logical bundle
|
||||
validation, safe destination recognition, and atomic bundle publication.
|
||||
- Deterministic domain packages own weather derivation, report periods, modules,
|
||||
generated-text validation, and template contexts.
|
||||
- `internal/adapters/weatherapi`, `internal/adapters/promptkit`, and
|
||||
@@ -47,6 +54,9 @@ directly.
|
||||
before collection.
|
||||
- Prompt and profile validation completes before weather collection. Raw output
|
||||
is validated before template rendering.
|
||||
- Comparison validates every explicit profile before collection, prepares one
|
||||
immutable report input, and delegates backend capacity to Promptkit rather
|
||||
than adding an application-wide execution limit.
|
||||
- Generated text fills defined prose slots only. Deterministic facts remain
|
||||
authoritative and repository-owned templates produce all Markdown output.
|
||||
- Sensitive rendered prompts, schemas, input bodies, provider endpoints, and
|
||||
@@ -62,11 +72,18 @@ directly.
|
||||
pre-publication failure, including cancellation observed immediately before
|
||||
publication, does not replace an existing destination; a notification failure
|
||||
does not remove a newly published output.
|
||||
- Configuration or explicit CLI input selects that operator-owned destination;
|
||||
it does not create an application-owned state boundary.
|
||||
- Comparison bundles are flat, versioned operator outputs. Their guarded
|
||||
replacement accepts only a recognized current bundle; cancellation and every
|
||||
pre-publication failure preserve a prior bundle, while individual profile
|
||||
failures can publish a complete partial bundle.
|
||||
- Distributor uploads use only the published Markdown output, never a scan of
|
||||
local files. Single notification follows publication; batch notification
|
||||
follows publication of every selected report. Batch counters describe report
|
||||
outcomes only; a failed batch notification is represented separately at the
|
||||
batch level.
|
||||
- Comparison never invokes Distributor notification.
|
||||
- Default tests are deterministic, offline, and use Promptkit/provider fakes
|
||||
rather than live provider calls. See the [testing policy](testing.md).
|
||||
|
||||
|
||||
33
docs/releases/v0.10.1.md
Normal file
33
docs/releases/v0.10.1.md
Normal file
@@ -0,0 +1,33 @@
|
||||
# Weatherreporter v0.10.1
|
||||
|
||||
This release repairs release validation after the `v0.10.0` pipeline failed in
|
||||
its privileged build container. Application behavior is unchanged from
|
||||
`v0.10.0`.
|
||||
|
||||
## Summary
|
||||
|
||||
The unreadable-secret configuration test now verifies that its process is
|
||||
actually subject to file permission bits before asserting that a mode-`000`
|
||||
file cannot be read. This keeps the test meaningful for ordinary users while
|
||||
allowing the release suite to run correctly in privileged containers.
|
||||
|
||||
## Compatibility
|
||||
|
||||
This patch release makes no changes to Weatherreporter's CLI, configuration,
|
||||
report output, integrations, prompts, profiles, or operating behavior. It is
|
||||
fully compatible with `v0.10.0`.
|
||||
|
||||
## Upgrade
|
||||
|
||||
No special operator action is required. Use `v0.10.1` in place of `v0.10.0`;
|
||||
the `v0.10.0` tag remains immutable, but its failed pipeline did not publish
|
||||
release binaries.
|
||||
|
||||
## Changes
|
||||
|
||||
- Made the unreadable-secret test capability-aware when the test process can
|
||||
bypass filesystem permission bits.
|
||||
- Preserved the production contract that genuinely unreadable secret files
|
||||
fail configuration loading.
|
||||
- Restored portable release validation in Woodpecker's privileged Go
|
||||
container.
|
||||
37
docs/releases/v0.11.0.md
Normal file
37
docs/releases/v0.11.0.md
Normal file
@@ -0,0 +1,37 @@
|
||||
# Weatherreporter v0.11.0
|
||||
|
||||
This release adds a configurable default publication directory for generated
|
||||
weather reports.
|
||||
|
||||
## Summary
|
||||
|
||||
Operators can now set `output.directory` once for both individual reports and
|
||||
scheduled batches. Explicit `--out` and `--out-dir` destinations continue to
|
||||
take precedence, while installations that omit the setting retain the existing
|
||||
current-directory behavior.
|
||||
|
||||
## Compatibility
|
||||
|
||||
This release is additive and compatible with `v0.10.1`. Existing configuration
|
||||
files, commands, report filenames, Promptkit behavior, and Distributor
|
||||
notification behavior remain valid and unchanged.
|
||||
|
||||
## Upgrade
|
||||
|
||||
No special action is required. To use the new default destination, configure
|
||||
`output.directory` as described in the [configuration
|
||||
reference](../config.md). Existing deployments may continue using the current
|
||||
working directory or explicit CLI output flags.
|
||||
|
||||
## Changes
|
||||
|
||||
- Added strict configuration loading and validation for the optional
|
||||
`output.directory` field.
|
||||
- Applied the configured directory consistently to `generate` and `run`, with
|
||||
explicit CLI destinations retaining highest precedence.
|
||||
- Preserved relative-path handling, absolute result paths, atomic publication,
|
||||
cancellation safety, and Distributor notification ordering.
|
||||
- Strengthened output preflight so existing non-directory paths, uninspectable
|
||||
paths, and dangling symlink components fail before expensive report work.
|
||||
- Updated the [CLI reference](../cli.md) and [operations
|
||||
guide](../operations.md) for the new destination-selection behavior.
|
||||
398
docs/roadmap/implementation.md
Normal file
398
docs/roadmap/implementation.md
Normal file
@@ -0,0 +1,398 @@
|
||||
# LLM Profile Comparison Implementation Plan
|
||||
|
||||
Status: Complete.
|
||||
|
||||
## Purpose And Authority
|
||||
|
||||
This document is the ordered implementation plan for the accepted [LLM
|
||||
Profile Comparison Roadmap](profile-comparison.md). The roadmap owns the
|
||||
feature purpose, policy, scope, and desired end state. This plan records the
|
||||
completed implementation and records the corrective work completed during
|
||||
post-implementation review.
|
||||
|
||||
All implementation work in this plan is complete. The recorded work leaves the
|
||||
repository compiling, tested, documented to its implemented boundary, and
|
||||
internally coherent.
|
||||
|
||||
## Implementation Rules
|
||||
|
||||
The following rules governed every implementation stage:
|
||||
|
||||
- Read `docs/development.md`, the task-specific documents it identifies, all
|
||||
files under `docs/policy/`, and the feature roadmap before changing code.
|
||||
- Preserve the existing `generate`, `run`, and `compare` command contracts
|
||||
except for the explicit comparison corrections defined below.
|
||||
- Keep Promptkit types and calls behind `internal/adapters/promptkit` and the
|
||||
dependency-neutral `internal/promptexec` interface.
|
||||
- Keep comparison artifacts operator-owned and explicit. They are not durable
|
||||
application state and must never be discovered or consumed implicitly by a
|
||||
later invocation.
|
||||
- Preserve comparison's prepare-once, execute-concurrently, order-results-by-
|
||||
selection, publish-on-profile-failure, and never-notify invariants.
|
||||
- Do not add a Weatherreporter concurrency limit. Promptkit owns backend
|
||||
capacity.
|
||||
- Never expose provider bodies, prompts, schemas, model output, endpoints,
|
||||
credentials, or arbitrary wrapped error text in normal JSON summaries or
|
||||
manifests.
|
||||
- Use deterministic, offline, credential-free tests. Test filesystem safety,
|
||||
concurrency, and recovery through the narrowest stable behavioral boundary;
|
||||
do not rely on timing-only sleeps or host permission behavior.
|
||||
- Run `gofmt` on changed Go files and `git diff --check` in every stage. Run
|
||||
focused tests while developing and `GOWORK=off go test -count=1 ./...` before
|
||||
completing each stage. Stages involving concurrency or filesystem mutation
|
||||
must also run affected packages with `-race`.
|
||||
- Do not commit, tag, push, or prepare a release unless the implementing prompt
|
||||
separately requests it.
|
||||
|
||||
## Completed Stages
|
||||
|
||||
### Stage 1: Comparison Artifact And Naming Contracts
|
||||
|
||||
Added the dependency-neutral comparison model, schema version, manifest
|
||||
validation and encoding, safe errors, deterministic profile filenames,
|
||||
comparison identities, default directory names, and content hashing.
|
||||
|
||||
### Stage 2: Destination Recognition And Transactional Publication
|
||||
|
||||
Added read-only destination planning, strict recognition of current comparison
|
||||
bundles, private sibling staging, guarded replacement, rollback, and atomic
|
||||
directory publication.
|
||||
|
||||
### Stage 3: Ordered Multi-Profile Preflight
|
||||
|
||||
Added exact prompt inspection followed by sequential profile inspection before
|
||||
weather collection, including effective backend, model, and credential checks.
|
||||
|
||||
### Stage 4: Immutable Shared Report Preparation
|
||||
|
||||
Extracted one immutable prepared-report value so comparison collection,
|
||||
derivation, module construction, and data-package serialization happen once.
|
||||
|
||||
### Stage 5: Profile Execution And In-Memory Rendering
|
||||
|
||||
Separated profile-specific Promptkit execution, generated-text validation, and
|
||||
Markdown rendering from output publication while preserving ordinary report
|
||||
generation behavior.
|
||||
|
||||
### Stage 6: Concurrent Ordered Profile Execution
|
||||
|
||||
Added one goroutine per selected profile using one shared executor and one
|
||||
prepared input, deterministic debug identities, isolated profile failures,
|
||||
joined cancellation, and selection-ordered results.
|
||||
|
||||
### Stage 7: Application-Level Comparison
|
||||
|
||||
Added `app.CompareDetailed`, coherent complete and partial bundle construction,
|
||||
aggregate profile-failure behavior, absolute published paths, and the
|
||||
application-level guarantee that comparison never notifies Distributor.
|
||||
|
||||
### Stage 8: Compare Command Parsing
|
||||
|
||||
Added the `compare` command request path, repeatable ordered `--profile`, exact
|
||||
`--out-dir`, guarded `--replace`, applicable common flags, validation, and one
|
||||
executor construction per invocation.
|
||||
|
||||
### Stage 9: CLI Results And Exit Behavior
|
||||
|
||||
Added structured success and failure summaries, quiet-mode suppression,
|
||||
ordered per-profile results, safe bounded errors, and nonzero exit behavior for
|
||||
partial or command-level failure.
|
||||
|
||||
### Stage 10: Canonical Documentation And Initial Validation
|
||||
|
||||
Documented the implemented CLI, operations, Promptkit integration, comparison
|
||||
bundle, application orchestration, execution, publication, architecture, and
|
||||
development contracts, then passed the original repository-wide validation
|
||||
gate.
|
||||
|
||||
## Stage 11: Make Concurrent Prompt Debug Creation Race-Safe
|
||||
|
||||
### Goal
|
||||
|
||||
Ensure concurrent comparison profiles can create their distinct debug runs
|
||||
under one new report/date directory without spuriously failing or leaving a
|
||||
test goroutine blocked.
|
||||
|
||||
### Work
|
||||
|
||||
1. Update `internal/promptdebug.ensureSecureDirectory` so concurrent creation
|
||||
of the same missing directory is idempotent. If `os.Mkdir` reports that the
|
||||
path already exists, inspect the path with `Lstat` and accept it only when it
|
||||
is the expected real directory. Continue to reject symlinks, non-directories,
|
||||
unsafe modes, and every unrelated filesystem error.
|
||||
2. Preserve the existing absolute-path, containment, `0700` directory, `0600`
|
||||
file, and no-symlink guarantees. Do not weaken debug-root validation or make
|
||||
all `EEXIST` errors successful.
|
||||
3. Add a focused prompt-debug concurrency regression that starts multiple
|
||||
writers beneath a shared missing ancestor, joins every goroutine, and
|
||||
verifies every expected artifact and permission invariant.
|
||||
4. Make comparison execution test barriers time-bounded and failure-aware. A
|
||||
callback failure before executor entry must fail the test promptly rather
|
||||
than leave `waitForProfileStarts` waiting forever.
|
||||
5. Retain distinct deterministic debug references and profile-local debug
|
||||
failure behavior.
|
||||
|
||||
### Tests And Exit Criteria
|
||||
|
||||
- The focused prompt-debug concurrency test passes repeatedly and with the race
|
||||
detector.
|
||||
- `TestExecuteComparisonProfilesUsesDistinctDeterministicDebugReferences`
|
||||
cannot hang when a profile fails before reaching the fake executor.
|
||||
- Run, at minimum:
|
||||
|
||||
```sh
|
||||
GOWORK=off go test -count=100 ./internal/promptdebug
|
||||
GOWORK=off go test -count=100 -run TestExecuteComparisonProfilesUsesDistinctDeterministicDebugReferences ./internal/app
|
||||
GOWORK=off go test -race -count=1 ./internal/promptdebug ./internal/app
|
||||
GOWORK=off go test -count=1 ./...
|
||||
```
|
||||
|
||||
## Stage 12: Make Replacement Authorization Commit-Safe
|
||||
|
||||
### Goal
|
||||
|
||||
Prevent a destination changed after the final read-only preflight from being
|
||||
treated as the previously authorized empty directory or recognized bundle and
|
||||
then deleted during replacement.
|
||||
|
||||
### Work
|
||||
|
||||
1. Replace `DestinationPlan.Exists` as the publication decision with an
|
||||
explicit destination-state classification: absent, empty real directory, or
|
||||
recognized current-schema bundle. Keep `Replace` in the plan so publication
|
||||
can apply the same authorization policy at commit time.
|
||||
2. Continue to call `PlanDestination` immediately before publication. For an
|
||||
absent target, install staging with one rename; a concurrently created
|
||||
target must cause that rename to fail without modifying the new target.
|
||||
3. For an existing target, rename that exact filesystem entry to the unique
|
||||
sibling backup before deleting or installing anything. Classify the moved
|
||||
backup while it is at its stable backup path and authorize it under the
|
||||
original replacement policy:
|
||||
|
||||
- an empty real directory is allowed with or without `Replace`;
|
||||
- a recognized current-schema comparison bundle is allowed only with
|
||||
`Replace`; and
|
||||
- a file, symlink, unrecognized/nonempty directory, unreadable entry, or
|
||||
other classification failure is not allowed.
|
||||
|
||||
4. Treat this post-move classification as the destructive-action
|
||||
authorization point. If it fails, restore the moved entry to the target and
|
||||
return an error without installing staging. If the target has concurrently
|
||||
reappeared or restoration otherwise fails, retain the backup and return an
|
||||
actionable joined or typed error that identifies its recovery path; never
|
||||
delete either entry to force restoration.
|
||||
5. Install staging only after the moved target has passed authorization. Never
|
||||
remove a backup that did not pass post-move authorization.
|
||||
6. Preserve the existing final cancellation linearization rule: cancellation
|
||||
observed before the rename transaction prevents replacement; after the
|
||||
transaction starts, finish commit or rollback rather than abandoning it.
|
||||
7. Add a package-private filesystem-operation seam only if needed for
|
||||
deterministic tests. Keep the public destination and publication APIs free
|
||||
of test-only hooks.
|
||||
|
||||
### Tests And Exit Criteria
|
||||
|
||||
- Deterministically replace an initially accepted destination after final
|
||||
preflight but before its move with each consequential unauthorized type:
|
||||
unrelated nonempty directory, regular file, and symlink. Publication must
|
||||
fail, staging must not become the target, and the moved entry must be restored
|
||||
or retained at a reported recovery path.
|
||||
- Cover an initially empty directory whose contents change before its move and
|
||||
a recognized bundle swapped for an unrecognized directory.
|
||||
- Retain coverage for absent targets, empty directories, recognized bundle
|
||||
replacement, cancellation before commit, install failure, successful
|
||||
rollback, failed rollback, and cleanup of ordinary staging failures.
|
||||
- Run `GOWORK=off go test -race -count=1 ./internal/comparison` and the
|
||||
repository-wide standard test command.
|
||||
|
||||
## Stage 13: Represent Committed Publication Cleanup Failures Accurately
|
||||
|
||||
### Goal
|
||||
|
||||
Keep application and CLI results truthful when the new comparison bundle has
|
||||
been committed but removal of the old sibling backup fails.
|
||||
|
||||
### Work
|
||||
|
||||
1. Change comparison publication to return a dependency-neutral result as well
|
||||
as an error:
|
||||
|
||||
```go
|
||||
type PublicationResult struct {
|
||||
Committed bool
|
||||
RetainedBackupPath string
|
||||
}
|
||||
|
||||
func Publish(
|
||||
ctx context.Context,
|
||||
plan DestinationPlan,
|
||||
bundle LogicalBundle,
|
||||
) (PublicationResult, error)
|
||||
```
|
||||
|
||||
2. Define `Committed` as meaning the complete staged bundle is now installed
|
||||
at the target. Pre-commit, staging, authorization, install, and successful-
|
||||
rollback failures return `Committed == false`. A successful install returns
|
||||
`Committed == true` even if later backup cleanup fails.
|
||||
3. Add a typed post-commit cleanup error that unwraps its filesystem cause and
|
||||
records the retained backup path for operator recovery. On this error,
|
||||
return `Committed == true` and the absolute retained backup path. Do not
|
||||
roll back or remove the newly committed valid bundle merely because old
|
||||
backup cleanup failed.
|
||||
4. In `app.CompareDetailed`, populate `ManifestPath`, `DataPackagePath`, and
|
||||
successful profile `ReportPath` values whenever publication reports
|
||||
`Committed == true`, before returning any cleanup error.
|
||||
5. Treat post-commit cleanup failure as a command-level operational failure:
|
||||
return the non-nil structured result plus an error, produce status `failed`,
|
||||
and exit nonzero even though the published artifact paths are present. The
|
||||
ordinary safe JSON error must not contain the raw filesystem cause or backup
|
||||
path; the wrapped diagnostic returned on stderr may identify the retained
|
||||
backup for recovery.
|
||||
6. Keep `RetainedBackupPath` out of the versioned comparison manifest. It
|
||||
describes an incomplete local transaction cleanup, not the logical bundle.
|
||||
|
||||
### Tests And Exit Criteria
|
||||
|
||||
- Inject a deterministic backup-removal failure after successful installation
|
||||
and assert the target is the new recognized bundle, the old bundle remains
|
||||
at the reported backup, `Committed` is true, and the error is inspectable by
|
||||
type.
|
||||
- At the application boundary, assert all committed artifact paths are
|
||||
absolute and populated while the method still returns an error.
|
||||
- At the CLI boundary, assert status `failed`, nonzero return, present artifact
|
||||
paths, and a bounded generic safe error with no raw filesystem detail.
|
||||
- Retain tests showing every pre-commit or rolled-back failure omits published
|
||||
artifact paths.
|
||||
- Run comparison, application, and CLI tests with `-race`, then the
|
||||
repository-wide standard test command.
|
||||
|
||||
## Stage 14: Complete Structured Failure Metadata And Classification
|
||||
|
||||
### Goal
|
||||
|
||||
Make every non-nil comparison result a reliable description of the attempted
|
||||
run and preserve useful safe error categories in the top-level CLI summary.
|
||||
|
||||
### Work
|
||||
|
||||
1. In `app.CompareDetailed`, assign the absolute resolved
|
||||
`OutputDirectory` immediately after output-directory resolution and before
|
||||
destination preflight. Do not wait for `PlanDestination` to succeed.
|
||||
2. Once the initial `ComparisonResult` exists, guarantee that every return path
|
||||
sets a nonzero UTC `FinishedAt` that is not before `StartedAt`. Use one
|
||||
centralized finalization path or a defer; do not scatter timestamp writes
|
||||
across individual failures.
|
||||
3. Continue to omit manifest, data-package, and report paths until publication
|
||||
commits. Preserve whatever prompt identity fields have actually been
|
||||
resolved; never invent a hash or profile result for a phase that did not
|
||||
run.
|
||||
4. Update `safeComparisonSummaryError` to use this stable mapping, always
|
||||
passing messages through `comparison.NewSafeError`:
|
||||
|
||||
| Error | Category | Safe message |
|
||||
| --- | --- | --- |
|
||||
| aggregate profile failure | `application` | existing bounded aggregate message |
|
||||
| `context.Canceled` | `canceled` | `comparison canceled` |
|
||||
| `context.DeadlineExceeded` | `deadline_exceeded` | `comparison deadline exceeded` |
|
||||
| categorized `promptexec` error | exact `promptexec.CategoryOf` value | `comparison prompt operation failed` |
|
||||
| `comparison.DestinationError` | `destination_<kind>` | `comparison destination preflight failed` |
|
||||
| post-commit cleanup error | `publication_cleanup` | `comparison published but cleanup did not complete` |
|
||||
| any unknown error | `application` | `comparison did not complete` |
|
||||
|
||||
5. Apply the most specific mapping before a more general wrapped match. In
|
||||
particular, detect the post-commit cleanup and destination types before
|
||||
falling back to a nested filesystem or context cause.
|
||||
6. Do not copy `DestinationError.Target`, wrapped causes, or arbitrary
|
||||
`error.Error()` text into normal JSON. Detailed returned errors remain
|
||||
available on stderr and through Go error inspection.
|
||||
|
||||
### Tests And Exit Criteria
|
||||
|
||||
- Add application tests for destination-preflight, debug initialization,
|
||||
prompt-preflight, collection, and preparation failures. Whenever a non-nil
|
||||
result is returned, assert an absolute output directory, nonzero ordered UTC
|
||||
timestamps, and omission of unpublished artifact paths.
|
||||
- Add table-driven CLI tests for every mapping row, including wrapped errors,
|
||||
and assert that unsafe sentinel text cannot enter serialized output.
|
||||
- Preserve existing ordered profile-level categories and safe messages.
|
||||
- Run application and CLI tests with `-race`, then the repository-wide standard
|
||||
test command.
|
||||
|
||||
## Stage 15: Remove Temporary Seams, Reconcile Documentation, And Validate
|
||||
|
||||
### Goal
|
||||
|
||||
Remove review-discovered maintenance debt, document the corrected implemented
|
||||
behavior in its canonical owners, and complete the release-equivalent gate.
|
||||
|
||||
### Work
|
||||
|
||||
1. Remove the unused `Runner.resolveComparison` wrapper. Remove the
|
||||
test-oriented `Runner.executeComparison` seam if it has no production
|
||||
caller, and rewrite its remaining coverage through `Runner.Run`,
|
||||
`resolveComparisonAction`, or another stable behavioral boundary.
|
||||
2. Remove `comparisonProfileOutcome.err` if production code still does not use
|
||||
it. Keep raw failures in returned/wrapped errors or explicit internal error
|
||||
types; do not retain an otherwise dead field solely for private test
|
||||
assertions.
|
||||
3. Correct the `internal/comparison` package comment so it describes the
|
||||
package's actual ownership of both logical comparison contracts and
|
||||
filesystem destination/publication behavior.
|
||||
4. Update only the canonical current-state documents affected by Stages 11
|
||||
through 14:
|
||||
|
||||
- `docs/internal/comparison-publication.md` owns post-move authorization,
|
||||
commit state, rollback, retained backups, and cleanup mechanics;
|
||||
- `docs/internal/comparison-execution.md` owns concurrency and debug-write
|
||||
behavior;
|
||||
- `docs/internal/app-orchestration.md` owns partial results and committed
|
||||
publication error handling;
|
||||
- `docs/cli.md` owns structured status, safe category, path, and exit
|
||||
behavior; and
|
||||
- `docs/operations.md` owns operator recovery for a retained sibling backup.
|
||||
|
||||
Link rather than duplicating complete contracts, and update architecture or
|
||||
integration documentation only if its existing invariant is inaccurate.
|
||||
5. Mark this plan `Complete` and restore the feature roadmap's implemented
|
||||
status after every exit criterion below passes. Retain or remove the two
|
||||
roadmap documents only according to a later maintainer-directed roadmap
|
||||
cleanup; do not archive them as a second current-state reference in this
|
||||
stage.
|
||||
|
||||
### Tests And Exit Criteria
|
||||
|
||||
- Confirm no production-only helper or field remains solely to support tests,
|
||||
and no test loses meaningful behavioral coverage during cleanup.
|
||||
- Verify changed relative links and fenced examples. Search current-state docs
|
||||
for stale claims about comparison publication, debug behavior, results, or
|
||||
recovery.
|
||||
- Run the release-equivalent local gate:
|
||||
|
||||
```sh
|
||||
set -eu
|
||||
test -z "$(git ls-files go.work go.work.sum)"
|
||||
test ! -e vendor
|
||||
if grep -Eq '^[[:space:]]*replace([[:space:]]|\()' go.mod; then
|
||||
printf '%s\n' 'go.mod contains a replacement' >&2
|
||||
exit 1
|
||||
fi
|
||||
GOWORK=off go test -count=1 ./...
|
||||
GOWORK=off go test -race -count=1 ./...
|
||||
GOWORK=off go vet ./...
|
||||
GOWORK=off go build ./...
|
||||
GOWORK=off go mod tidy -diff
|
||||
unformatted="$(git ls-files '*.go' | while IFS= read -r file; do gofmt -l "$file"; done)"
|
||||
test -z "$unformatted"
|
||||
git diff --check
|
||||
```
|
||||
|
||||
- Run `GOWORK=off go run ./cmd/weatherreporter --help` and
|
||||
`GOWORK=off go run ./cmd/weatherreporter compare --help` without credentials
|
||||
or network access, and confirm that help agrees with `docs/cli.md`.
|
||||
- Inspect the final diff for accidental generated artifacts, secrets,
|
||||
workspaces, vendored dependencies, release notes, or unrelated changes.
|
||||
|
||||
## Open Questions
|
||||
|
||||
None. The roadmap and the contracts in Stages 11 through 15 define the
|
||||
remaining decisions needed to complete the corrective work.
|
||||
398
docs/roadmap/profile-comparison.md
Normal file
398
docs/roadmap/profile-comparison.md
Normal file
@@ -0,0 +1,398 @@
|
||||
# LLM Profile Comparison Roadmap
|
||||
|
||||
Status: Implemented; retained as the feature roadmap.
|
||||
|
||||
## Purpose
|
||||
|
||||
Prompt development currently requires separate Weatherreporter invocations to
|
||||
compare several LLM profiles. Those invocations may collect different weather
|
||||
snapshots or rebuild inputs at different times, making model output harder to
|
||||
compare and slowing prompt iteration.
|
||||
|
||||
Weatherreporter should provide a first-class `compare` command that resolves
|
||||
one report, prepares one exact data package, executes the same prompt and data
|
||||
package concurrently through several explicitly selected Promptkit profiles,
|
||||
and publishes a self-contained local comparison bundle.
|
||||
|
||||
An illustrative invocation is:
|
||||
|
||||
```sh
|
||||
weatherreporter compare daily \
|
||||
--date 2026-08-24 \
|
||||
--profile weather-light \
|
||||
--profile weather-balanced \
|
||||
--profile weather-deep
|
||||
```
|
||||
|
||||
This is a prompt-development workflow, not an automated model evaluator. Its
|
||||
output gives a maintainer consistent evidence for human comparison without
|
||||
assigning scores or selecting a winner.
|
||||
|
||||
## Prerequisite
|
||||
|
||||
Configurable output directories are implemented. Profile comparison must reuse
|
||||
the current [configuration reference](../config.md) and [operations
|
||||
guide](../operations.md) rather than introduce a second destination policy.
|
||||
|
||||
## User Intent
|
||||
|
||||
The command is intended for deliberate evaluation of multiple profiles,
|
||||
including sets of eight to twelve candidate models. Concurrency is part of the
|
||||
feature, not a future optimization. Promptkit should retain ownership of
|
||||
backend-specific capacity, while Weatherreporter owns comparison-wide
|
||||
coordination, cancellation, deterministic results, and artifact publication.
|
||||
|
||||
Every profile must receive byte-for-byte identical prompt input. Weather data,
|
||||
derived facts, modules, prompt metadata, and serialized YAML must not be
|
||||
recollected or rebuilt separately for individual profiles.
|
||||
|
||||
Comparison bundles are explicitly requested, operator-owned development
|
||||
outputs. They are not Weatherreporter state, are never read implicitly by a
|
||||
later run, and do not weaken the ordinary stateless execution model.
|
||||
|
||||
## Command Contract
|
||||
|
||||
The command form is:
|
||||
|
||||
```text
|
||||
weatherreporter compare REPORT [options]
|
||||
```
|
||||
|
||||
`REPORT` accepts the implemented generated-text reports: `daily`, `today`,
|
||||
`tomorrow`, and `hourly`. Report-date behavior matches `generate`: `daily`
|
||||
requires `--date`, `today` may accept an explicit date or use the current local
|
||||
date, and the remaining report types retain their existing period policies.
|
||||
|
||||
The command accepts the applicable common generation options, including
|
||||
`--config`, `--units`, `--tz`, `--date`, `--llm-debug-dir`, and `--quiet`, plus:
|
||||
|
||||
- repeatable `--profile PROFILE_ID` selections;
|
||||
- `--out-dir PATH` for the exact comparison-bundle directory; and
|
||||
- `--replace` to authorize guarded replacement of a recognized existing
|
||||
comparison bundle.
|
||||
|
||||
At least two distinct, nonblank profile IDs are required. Their command-line
|
||||
order is significant and is preserved in filenames, summaries, and
|
||||
`comparison.json`. Duplicate profile IDs are rejected rather than silently
|
||||
deduplicated or executed twice.
|
||||
|
||||
Profiles are always explicit for this command. `promptkit.profile` does not add
|
||||
or replace a comparison selection, but all other effective Promptkit settings,
|
||||
profile-source precedence, local backend configuration, credential lookup, and
|
||||
profile overrides remain in force.
|
||||
|
||||
The initial feature has no Weatherreporter-specific concurrency flag or
|
||||
artificial profile-count ceiling. The explicit profile list bounds the
|
||||
comparison, and Promptkit owns capacity enforcement for each selected backend.
|
||||
|
||||
## Preparation And Execution Invariants
|
||||
|
||||
A comparison has this logical lifecycle:
|
||||
|
||||
1. Parse and validate the report, date, profile list, configuration, output
|
||||
destination, and replacement authorization.
|
||||
2. Resolve the report definition, valid period, prompt identity, and default
|
||||
output name once.
|
||||
3. Inspect the exact prompt once and preflight every selected profile,
|
||||
including its effective backend, model, and required credential
|
||||
availability, before weather collection.
|
||||
4. Collect weather data exactly once.
|
||||
5. Build collected and derived facts, the module snapshot, briefing metadata,
|
||||
and the prompt data package exactly once.
|
||||
6. Marshal the data package to one immutable YAML byte sequence exactly once.
|
||||
7. Execute the exact prompt version concurrently for every selected profile,
|
||||
passing the same immutable YAML bytes to every execution.
|
||||
8. Validate and render each profile result independently from the shared
|
||||
deterministic inputs.
|
||||
9. Assemble results in requested-profile order and publish one coherent
|
||||
comparison bundle.
|
||||
|
||||
This lifecycle describes the required end-state behavior rather than an
|
||||
implementation-stage sequence.
|
||||
|
||||
No profile execution may cause recollection, report re-resolution, module
|
||||
rebuilding, or data-package remarshalling. Prompt execution may perform
|
||||
Promptkit-owned validation or repair behavior, but Weatherreporter does not
|
||||
retry a failed comparison execution independently.
|
||||
|
||||
## Concurrency And Cancellation
|
||||
|
||||
Weatherreporter starts one execution for each preflighted profile and permits
|
||||
them to run concurrently through one shared, concurrency-safe Promptkit
|
||||
executor. Promptkit's engine-local backend pools remain authoritative for
|
||||
backend concurrency and waiting capacity. Profiles routed to a limited local
|
||||
backend therefore respect its configured limit, while profiles routed to
|
||||
other backends may proceed independently.
|
||||
|
||||
Weatherreporter must not add a second semaphore that obscures or overrides
|
||||
Promptkit's backend policy. It must safely coordinate goroutine lifecycles,
|
||||
result collection, debug callbacks, and output assembly without data races.
|
||||
|
||||
One profile failure does not cancel its peers. Provider, capacity, validation,
|
||||
and rendering failures are recorded for that profile while other executions
|
||||
continue. Cancellation or deadline expiration of the comparison command is
|
||||
propagated to every outstanding execution, prevents new publication, and is
|
||||
joined without leaking goroutines.
|
||||
|
||||
Completion order must not affect filenames, manifest order, CLI summaries, or
|
||||
error aggregation. Those outputs always follow the original `--profile`
|
||||
order.
|
||||
|
||||
## Output Destination
|
||||
|
||||
Without `--out-dir`, Weatherreporter derives a comparison directory from the
|
||||
resolved report's existing default Markdown filename by removing `.md` and
|
||||
prefixing `comparison-`:
|
||||
|
||||
| Report output | Comparison directory |
|
||||
| --- | --- |
|
||||
| `today.md` | `comparison-today/` |
|
||||
| `tomorrow.md` | `comparison-tomorrow/` |
|
||||
| `hourly.md` | `comparison-hourly/` |
|
||||
| `daily-2026-08-24.md` | `comparison-daily-2026-08-24/` |
|
||||
|
||||
The derived directory is created beneath `output.directory` when configured,
|
||||
or beneath the present working directory otherwise. An explicit `--out-dir`
|
||||
is the exact bundle directory, resolves relative to the present working
|
||||
directory when necessary, and overrides `output.directory` completely.
|
||||
|
||||
All destination selection and validation completes before weather collection.
|
||||
The resolved comparison directory is returned in the command's structured
|
||||
result.
|
||||
|
||||
## Comparison Bundle
|
||||
|
||||
A successful three-profile comparison has a flat layout:
|
||||
|
||||
```text
|
||||
comparison-daily-2026-08-24/
|
||||
├── comparison.json
|
||||
├── data-package.yml
|
||||
├── 01-weather-light.md
|
||||
├── 02-weather-balanced.md
|
||||
└── 03-weather-deep.md
|
||||
```
|
||||
|
||||
`data-package.yml` contains the exact YAML bytes passed to every Promptkit
|
||||
execution. It is written once and its SHA-256 digest is recorded in the
|
||||
manifest.
|
||||
|
||||
Each report filename begins with its one-based, zero-padded selection position
|
||||
and a filesystem-safe representation of the requested logical profile ID. The
|
||||
safe representation must not permit absolute paths, traversal, separators, or
|
||||
control characters. The manifest retains the exact case-sensitive profile ID,
|
||||
so filename normalization never becomes the authority for profile identity.
|
||||
|
||||
`comparison.json` is the authoritative index for the bundle. It uses an
|
||||
explicit schema version and records safe comparison information including:
|
||||
|
||||
- comparison identity and start and finish timestamps;
|
||||
- report ID, resolved valid period, and effective timezone;
|
||||
- prompt ID, version, and inspected prompt hash;
|
||||
- the relative data-package filename and SHA-256 digest;
|
||||
- total, succeeded, and failed profile counts; and
|
||||
- one ordered result per requested profile containing the exact profile ID,
|
||||
resolved backend and model, relative report filename when present,
|
||||
execution and validation status, and safe error information when failed.
|
||||
|
||||
The manifest and normal command summary must not contain credentials, provider
|
||||
request bodies, raw model output, rendered prompts, schemas, provider
|
||||
endpoints, or other content-rich diagnostics. The explicit data package and
|
||||
generated reports contain the development material the user requested and
|
||||
must be handled as operator-owned potentially sensitive output.
|
||||
|
||||
## Failure And Publication Policy
|
||||
|
||||
Failure before concurrent execution, including invalid profiles, missing
|
||||
credentials, collection failure, preparation failure, or unsafe destination,
|
||||
publishes no comparison bundle and performs no model calls where the failure
|
||||
is discoverable during preflight.
|
||||
|
||||
After execution begins, Weatherreporter waits for every non-cancelled profile.
|
||||
If one or more profiles fail, it still publishes a coherent partial bundle
|
||||
containing `data-package.yml`, every successfully rendered report, and a
|
||||
manifest describing all successes and failures. It then returns a non-zero
|
||||
exit status. A failed profile has no report file unless a future contract
|
||||
explicitly introduces a separately named diagnostic artifact.
|
||||
|
||||
Bundle contents are staged outside the destination and published only after
|
||||
the manifest is complete. Ordinary publication accepts only an absent or empty
|
||||
target directory. A nonempty existing directory fails without modification
|
||||
unless `--replace` is present.
|
||||
|
||||
`--replace` may replace only the exact resolved target and must reject broad or
|
||||
unsafe targets such as a filesystem root, the present working directory, a
|
||||
symlink, or an unrecognized nonempty directory. A recognized prior bundle must
|
||||
contain a valid Weatherreporter comparison manifest. Replacement publishes the
|
||||
new complete or coherent partial bundle as a unit, prevents stale reports from
|
||||
the prior comparison from surviving, and preserves or restores the prior
|
||||
bundle if the final replacement operation fails.
|
||||
|
||||
An interrupted or cancelled comparison does not replace an existing bundle.
|
||||
Temporary staging artifacts are cleaned up on ordinary failure and
|
||||
cancellation without scanning or modifying unrelated directories.
|
||||
|
||||
## Prompt Debugging
|
||||
|
||||
The existing `--llm-debug-dir` mechanism remains available. Concurrent
|
||||
comparison executions require distinct, deterministic debug identities that
|
||||
include the comparison and exact profile selection so callbacks cannot collide
|
||||
or overwrite another profile's artifacts.
|
||||
|
||||
Debug writing must be concurrency-safe and retain the existing permission,
|
||||
redaction, explicit-opt-in, and path-containment guarantees. Debug artifacts
|
||||
remain separate from the comparison bundle; the bundle does not implicitly
|
||||
enable full Promptkit diagnostics.
|
||||
|
||||
## Notification Policy
|
||||
|
||||
Profile comparisons never invoke Distributor notification, even when
|
||||
notification is enabled in the effective configuration. Comparison reports
|
||||
are local development artifacts rather than ordinary report publications.
|
||||
|
||||
Adding comparison publication or upload behavior would require a separate
|
||||
accepted feature scope and explicit operator authorization.
|
||||
|
||||
## Architectural End State
|
||||
|
||||
Application orchestration exposes a reusable prepared-report boundary that
|
||||
contains the resolved report, shared collected and derived facts, module
|
||||
snapshot, briefing metadata, generated-text handler, render inputs, and exact
|
||||
serialized data package. That boundary is immutable during concurrent profile
|
||||
execution.
|
||||
|
||||
Ordinary `generate` behavior continues to prepare once and execute once.
|
||||
`compare` prepares once and executes many without duplicating the generation
|
||||
workflow or calling `GenerateDetailed` in a loop. Shared preparation,
|
||||
profile-specific Promptkit execution, structured-output validation, rendering,
|
||||
and artifact publication remain distinct responsibilities.
|
||||
|
||||
The Promptkit adapter remains the only owner of dependency-specific types and
|
||||
engine calls. The CLI owns parsing and user-facing summaries. The configuration
|
||||
package owns configuration. Application orchestration owns comparison order,
|
||||
concurrency lifecycle, failure aggregation, and bundle publication. Domain,
|
||||
prompt-input, generated-text, and template packages retain their existing
|
||||
deterministic contracts.
|
||||
|
||||
## Scope
|
||||
|
||||
The completed feature includes:
|
||||
|
||||
- the `compare` CLI command for every implemented generated-text report;
|
||||
- repeatable explicit profile selection and validation;
|
||||
- configured and CLI output-directory integration through the implemented
|
||||
destination policy;
|
||||
- one-time report resolution, collection, deterministic preparation, and YAML
|
||||
serialization;
|
||||
- concurrent execution through one Promptkit executor with backend capacity
|
||||
respected;
|
||||
- independent validation and rendering with deterministic ordered results;
|
||||
- the flat, versioned comparison-bundle contract;
|
||||
- safe filename derivation and data-package hashing;
|
||||
- coherent partial-result publication and non-zero failure behavior;
|
||||
- guarded whole-bundle replacement through `--replace`;
|
||||
- comparison-aware, concurrency-safe optional prompt debugging;
|
||||
- explicit suppression of Distributor notification;
|
||||
- structured normal and quiet-mode CLI behavior consistent with existing
|
||||
commands;
|
||||
- focused race-safe tests across configuration, CLI, application,
|
||||
Promptkit-adapter, rendering, and filesystem boundaries; and
|
||||
- updates to every affected canonical user, operator, architecture,
|
||||
integration, and internal document.
|
||||
|
||||
## Compatibility
|
||||
|
||||
The feature is additive. Existing `generate` and `run` commands, report
|
||||
definitions, profile defaults, configuration, output filenames, notification
|
||||
behavior, and exit contracts remain unchanged.
|
||||
|
||||
The comparison manifest and bundle layout begin as versioned contracts. They
|
||||
do not become inputs accepted by Weatherreporter, and no backward-compatible
|
||||
replay or long-term archive guarantee is implied beyond identifying the schema
|
||||
used to interpret a produced bundle.
|
||||
|
||||
## Testing Expectations
|
||||
|
||||
Tests should provide durable coverage for:
|
||||
|
||||
- report and date parsing consistent with `generate`;
|
||||
- rejection of fewer than two profiles, blanks, and duplicates;
|
||||
- inspection of the exact prompt and every profile before collection;
|
||||
- no collection or model execution after a preflight failure;
|
||||
- exactly one weather collection and one preparation for several profiles;
|
||||
- byte-for-byte identical data-package input in every execution;
|
||||
- observable concurrent execution through a concurrency-safe fake executor;
|
||||
- respect for Promptkit-owned backend capacity in an assembled adapter test
|
||||
where that integration adds distinct confidence;
|
||||
- deterministic filenames, manifest order, summaries, and errors under varied
|
||||
completion order;
|
||||
- continuation and coherent partial publication after one profile fails;
|
||||
- cancellation propagation, goroutine completion, and preservation of an
|
||||
existing destination;
|
||||
- destination precedence and each derived default directory;
|
||||
- safe filename handling for unusual valid profile IDs;
|
||||
- absent, empty, occupied, symlinked, unsafe, recognized, and unrecognized
|
||||
replacement targets;
|
||||
- removal of stale prior report files during authorized whole-bundle
|
||||
replacement;
|
||||
- exact package digest and manifest/result consistency;
|
||||
- concurrency-safe, non-colliding opt-in debug artifacts; and
|
||||
- absence of Distributor calls for complete and partial comparisons.
|
||||
|
||||
Concurrency and replacement behavior require race-enabled and consequential
|
||||
failure-path coverage. Tests must remain deterministic, offline, credential
|
||||
free, and independent of real Promptkit providers or machine-specific paths.
|
||||
|
||||
## Documentation End State
|
||||
|
||||
Once implemented, the [CLI reference](../cli.md) owns command syntax, flags,
|
||||
summary, and exit behavior. The [operations guide](../operations.md) owns the
|
||||
bundle lifecycle, replacement procedure, sensitivity guidance, and practical
|
||||
prompt-comparison workflow. The [architecture policy](../policy/architecture.md)
|
||||
owns the statelessness, concurrency, notification, and publication invariants.
|
||||
|
||||
The [Promptkit integration guide](../integrations/promptkit.md) should describe
|
||||
the consumer-visible multi-profile execution boundary without duplicating
|
||||
Promptkit's backend-capacity reference. The versioned manifest and flat bundle
|
||||
format belong in a focused contract under `docs/integrations/`. App
|
||||
orchestration, prompt input, generated text, prompt debugging, and bundle
|
||||
publication mechanics belong in focused documents under `docs/internal/`.
|
||||
|
||||
Current-state documentation must not describe profile comparison as available
|
||||
until the implementation lands.
|
||||
|
||||
## Non-Goals
|
||||
|
||||
This roadmap does not introduce:
|
||||
|
||||
- automatic model scoring, ranking, recommendation, or winner selection;
|
||||
- semantic or textual diff generation between reports;
|
||||
- repeated sampling of one profile or statistical evaluation;
|
||||
- prompt or profile editing through Weatherreporter;
|
||||
- replaying a saved data package as command input;
|
||||
- comparing several report types in one command;
|
||||
- Weatherreporter-owned backend concurrency or queue configuration;
|
||||
- automatic retries beyond Promptkit's existing execution contract;
|
||||
- Distributor upload or other external publication;
|
||||
- comparison history, indexing, retention, cleanup schedules, or implicit
|
||||
discovery of prior bundles; or
|
||||
- changes to ordinary report content or normal generation behavior.
|
||||
|
||||
Any later automated evaluation, replay, sampling, or publication feature
|
||||
requires a separate accepted roadmap.
|
||||
|
||||
## Completion Criteria
|
||||
|
||||
The feature is complete when a maintainer can select several Promptkit
|
||||
profiles, have them execute concurrently against one exact prepared report
|
||||
package, and receive a safe, flat, deterministic comparison bundle whose
|
||||
manifest accurately describes every success and failure. Configured and
|
||||
explicit destinations must follow the accepted output policy, replacement must
|
||||
never mix or silently destroy unrelated contents, cancellation and partial
|
||||
failure must be race-safe, ordinary notification must remain disabled, and all
|
||||
affected canonical documentation must describe the implemented behavior.
|
||||
|
||||
## Open Questions
|
||||
|
||||
None. The scope, prerequisites, user intent, and target behavior are defined
|
||||
above.
|
||||
@@ -14,6 +14,9 @@ location:
|
||||
secrets:
|
||||
directory: ""
|
||||
|
||||
output:
|
||||
directory: /var/lib/weatherreporter/reports
|
||||
|
||||
notify:
|
||||
distributor:
|
||||
enabled: false
|
||||
|
||||
@@ -4,9 +4,7 @@ package app
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
distributoradapter "gitea.maximumdirect.net/eric/weatherreporter/internal/adapters/distributor"
|
||||
@@ -251,7 +249,7 @@ func GenerateDetailed(ctx context.Context, req GenerateRequest) (*ReportResult,
|
||||
return nil, err
|
||||
}
|
||||
result := initialReportResult(req, resolved, PromptInspectionResult{})
|
||||
outputPath, err := resolveReportOutputPath(req.WorkingDir, req.OutputPath, resolved)
|
||||
outputPath, err := resolveReportOutputPath(req.WorkingDir, req.OutputPath, req.Config.Output.Directory, resolved)
|
||||
if err != nil {
|
||||
return result, err
|
||||
}
|
||||
@@ -302,7 +300,7 @@ func RunBatchDetailed(ctx context.Context, req BatchRequest) (*BatchResult, erro
|
||||
if _, err := report.BatchForCommandName(string(req.Batch)); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
outputDir, err := resolveOutputDir(req.WorkingDir, req.OutputDir)
|
||||
outputDir, err := resolveOutputDirWithConfigured(req.WorkingDir, req.OutputDir, req.Config.Output.Directory)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -431,104 +429,6 @@ func batchReportResult(planned plannedBatchReport) BatchReportResult {
|
||||
}
|
||||
}
|
||||
|
||||
func plannedBatchOutputPath(outputDir string, planned plannedBatchReport) (string, error) {
|
||||
outputName, err := planned.Resolved.OutputName()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return validateOutputPath(filepath.Join(outputDir, outputName))
|
||||
}
|
||||
|
||||
func prepareBatchOutputs(outputDir string, plannedReports []plannedBatchReport) error {
|
||||
for index := range plannedReports {
|
||||
outputPath, err := plannedBatchOutputPath(outputDir, plannedReports[index])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
plannedReports[index].OutputPath = outputPath
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func resolveReportOutputPath(workingDir, override string, resolved report.Resolved) (string, error) {
|
||||
outputName, err := resolved.OutputName()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return resolveOutputPath(workingDir, override, outputName)
|
||||
}
|
||||
|
||||
func resolveOutputDir(workingDir, override string) (string, error) {
|
||||
workingDir, err := validateWorkingDir(workingDir)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if override == "" {
|
||||
return workingDir, nil
|
||||
}
|
||||
if strings.TrimSpace(override) == "" {
|
||||
return "", fmt.Errorf("output directory is required")
|
||||
}
|
||||
directory := override
|
||||
if !filepath.IsAbs(directory) {
|
||||
directory = filepath.Join(workingDir, directory)
|
||||
}
|
||||
directory = filepath.Clean(directory)
|
||||
if info, err := os.Stat(directory); err == nil && !info.IsDir() {
|
||||
return "", fmt.Errorf("output directory %q is not a directory", directory)
|
||||
} else if err != nil && !os.IsNotExist(err) {
|
||||
return "", fmt.Errorf("inspect output directory %q: %w", directory, err)
|
||||
}
|
||||
return directory, nil
|
||||
}
|
||||
|
||||
func resolveOutputPath(workingDir, override, defaultName string) (string, error) {
|
||||
workingDir, err := validateWorkingDir(workingDir)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
path := override
|
||||
if path == "" {
|
||||
path = defaultName
|
||||
}
|
||||
if strings.TrimSpace(path) == "" {
|
||||
return "", fmt.Errorf("final output path is required")
|
||||
}
|
||||
if !filepath.IsAbs(path) {
|
||||
path = filepath.Join(workingDir, path)
|
||||
}
|
||||
return validateOutputPath(path)
|
||||
}
|
||||
|
||||
func validateWorkingDir(workingDir string) (string, error) {
|
||||
if strings.TrimSpace(workingDir) == "" {
|
||||
return "", fmt.Errorf("working directory is required")
|
||||
}
|
||||
if !filepath.IsAbs(workingDir) {
|
||||
return "", fmt.Errorf("working directory %q must be absolute", workingDir)
|
||||
}
|
||||
return filepath.Clean(workingDir), nil
|
||||
}
|
||||
|
||||
func validateOutputPath(path string) (string, error) {
|
||||
if strings.TrimSpace(path) == "" {
|
||||
return "", fmt.Errorf("final output path is required")
|
||||
}
|
||||
path = filepath.Clean(path)
|
||||
if !filepath.IsAbs(path) {
|
||||
return "", fmt.Errorf("final output path %q must be absolute", path)
|
||||
}
|
||||
if filepath.Dir(path) == path {
|
||||
return "", fmt.Errorf("final output path %q must not be a filesystem root", path)
|
||||
}
|
||||
if info, err := os.Stat(path); err == nil && info.IsDir() {
|
||||
return "", fmt.Errorf("final output path %q is a directory", path)
|
||||
} else if err != nil && !os.IsNotExist(err) {
|
||||
return "", fmt.Errorf("inspect final output path %q: %w", path, err)
|
||||
}
|
||||
return path, nil
|
||||
}
|
||||
|
||||
func ResolveGenerate(req GenerateRequest, now time.Time) (report.Resolved, error) {
|
||||
location, err := timeutil.LoadLocation(req.Config.WeatherAPI.Timezone)
|
||||
if err != nil {
|
||||
|
||||
@@ -61,6 +61,100 @@ func TestRunBatchDetailedNotifiesOnlyAfterAllOutputsExist(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunBatchDetailedUsesDefaultAndConfiguredOutputDirectories(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
directory func(t *testing.T, workingDir string) string
|
||||
wantDir func(t *testing.T, workingDir string, configuredDir string) string
|
||||
}{
|
||||
{
|
||||
name: "working directory default",
|
||||
directory: func(_ *testing.T, _ string) string {
|
||||
return ""
|
||||
},
|
||||
wantDir: func(_ *testing.T, workingDir string, _ string) string {
|
||||
return workingDir
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "absolute directory",
|
||||
directory: func(t *testing.T, _ string) string {
|
||||
return filepath.Join(t.TempDir(), "reports")
|
||||
},
|
||||
wantDir: func(_ *testing.T, _ string, configuredDir string) string {
|
||||
return configuredDir
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "relative directory",
|
||||
directory: func(_ *testing.T, _ string) string {
|
||||
return "configured/../reports"
|
||||
},
|
||||
wantDir: func(_ *testing.T, workingDir string, _ string) string {
|
||||
return filepath.Join(workingDir, "reports")
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
workingDir := t.TempDir()
|
||||
configuredDir := tt.directory(t, workingDir)
|
||||
cfg := generationDistributorConfig()
|
||||
cfg.Output.Directory = configuredDir
|
||||
bundle := generationBundle(t)
|
||||
bundle.Hourly.Periods = bundle.Hourly.Periods[:1]
|
||||
notifier := &generationNotifier{}
|
||||
|
||||
result, err := RunBatchDetailed(context.Background(), BatchRequest{
|
||||
Config: cfg, Batch: BatchMorning,
|
||||
Now: generationTime("2026-05-29T08:30:00-05:00"), WorkingDir: workingDir,
|
||||
Collector: &generationCollector{bundle: &bundle}, Executor: &generationExecutor{}, Notifier: notifier,
|
||||
})
|
||||
wantDir := tt.wantDir(t, workingDir, configuredDir)
|
||||
if err != nil || result == nil || result.Succeeded != len(result.Reports) || notifier.batchCalls != 1 {
|
||||
t.Fatalf("RunBatchDetailed() result/error/notifier = %#v/%v/%#v", result, err, notifier)
|
||||
}
|
||||
for _, item := range result.Reports {
|
||||
if filepath.Dir(item.OutputPath) != wantDir {
|
||||
t.Fatalf("report output %q, want directory %q", item.OutputPath, wantDir)
|
||||
}
|
||||
}
|
||||
for _, file := range notifier.batchRequest.Files {
|
||||
if filepath.Dir(file.SourcePath) != wantDir {
|
||||
t.Fatalf("notification source %q, want directory %q", file.SourcePath, wantDir)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunBatchDetailedExplicitOutputDirectoryIgnoresConfiguredDirectory(t *testing.T) {
|
||||
configuredPath := filepath.Join(t.TempDir(), "not-a-directory")
|
||||
if err := os.WriteFile(configuredPath, []byte("not a directory"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
explicitDir := t.TempDir()
|
||||
cfg := generationDistributorConfig()
|
||||
cfg.Output.Directory = configuredPath
|
||||
bundle := generationBundle(t)
|
||||
bundle.Hourly.Periods = bundle.Hourly.Periods[:1]
|
||||
|
||||
result, err := RunBatchDetailed(context.Background(), BatchRequest{
|
||||
Config: cfg, Batch: BatchMorning,
|
||||
Now: generationTime("2026-05-29T08:30:00-05:00"), WorkingDir: t.TempDir(), OutputDir: explicitDir,
|
||||
Collector: &generationCollector{bundle: &bundle}, Executor: &generationExecutor{}, Notifier: &generationNotifier{},
|
||||
})
|
||||
if err != nil || result == nil || result.Succeeded != len(result.Reports) {
|
||||
t.Fatalf("RunBatchDetailed() result/error = %#v/%v", result, err)
|
||||
}
|
||||
for _, item := range result.Reports {
|
||||
if filepath.Dir(item.OutputPath) != explicitDir {
|
||||
t.Fatalf("report output %q, want directory %q", item.OutputPath, explicitDir)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunBatchDetailedPreflightsAllOutputPaths(t *testing.T) {
|
||||
bundle := generationBundle(t)
|
||||
bundle.Hourly.Periods = bundle.Hourly.Periods[:1]
|
||||
|
||||
240
internal/app/comparison.go
Normal file
240
internal/app/comparison.go
Normal file
@@ -0,0 +1,240 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/comparison"
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/config"
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptdebug"
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec"
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/report"
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/timeutil"
|
||||
)
|
||||
|
||||
// ComparisonRequest describes one explicit, multi-profile report comparison.
|
||||
// It deliberately does not accept a notifier: comparison publication is local.
|
||||
type ComparisonRequest struct {
|
||||
Config config.Config
|
||||
Report ReportKind
|
||||
ProfileIDs []string
|
||||
WorkingDir string
|
||||
OutputDir string
|
||||
Replace bool
|
||||
LLMDebugDir string
|
||||
Date time.Time
|
||||
Clock timeutil.Clock
|
||||
Collector Collector
|
||||
Executor promptexec.Executor
|
||||
}
|
||||
|
||||
// ComparisonResult records the resolved comparison and profile outcomes.
|
||||
type ComparisonResult struct {
|
||||
ComparisonID string
|
||||
ReportID report.ID
|
||||
ReportName string
|
||||
PromptID string
|
||||
PromptVersion string
|
||||
PromptHash string
|
||||
StartedAt time.Time
|
||||
FinishedAt time.Time
|
||||
Timezone string
|
||||
ValidPeriod timeutil.Period
|
||||
OutputDirectory string
|
||||
ManifestPath string
|
||||
DataPackagePath string
|
||||
Total int
|
||||
Succeeded int
|
||||
Failed int
|
||||
Results []ComparisonProfileResult
|
||||
}
|
||||
|
||||
// ComparisonProfileResult records one explicitly selected profile.
|
||||
type ComparisonProfileResult struct {
|
||||
Position int
|
||||
ProfileID string
|
||||
BackendID string
|
||||
ModelName string
|
||||
Status string
|
||||
ValidationStatus promptexec.ValidationStatus
|
||||
ReportPath string
|
||||
LLMDebugPath string
|
||||
Error *comparison.SafeError
|
||||
}
|
||||
|
||||
type comparisonPublisher func(context.Context, comparison.DestinationPlan, comparison.LogicalBundle) (comparison.PublicationResult, error)
|
||||
|
||||
// CompareDetailed assembles, executes, and atomically publishes a comparison
|
||||
// bundle. Profile failures publish a complete partial bundle. Failures before
|
||||
// commit leave the destination untouched; a post-commit cleanup failure leaves
|
||||
// the new bundle installed and returns its artifact paths with an error.
|
||||
func CompareDetailed(ctx context.Context, req ComparisonRequest) (*ComparisonResult, error) {
|
||||
return compareDetailed(ctx, req, comparison.Publish)
|
||||
}
|
||||
|
||||
func compareDetailed(ctx context.Context, req ComparisonRequest, publish comparisonPublisher) (*ComparisonResult, error) {
|
||||
if err := comparison.ValidateProfileIDs(req.ProfileIDs); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
clock := req.Clock
|
||||
if clock == nil {
|
||||
clock = timeutil.SystemClock{}
|
||||
}
|
||||
now := clock.Now()
|
||||
resolved, err := ResolveGenerate(GenerateRequest{Config: req.Config, Report: req.Report, Date: req.Date}, now)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
metadata := resolved.Metadata()
|
||||
comparisonID, err := comparison.BuildComparisonID(metadata.RunID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("build comparison identity: %w", err)
|
||||
}
|
||||
result := initialComparisonResult(req, resolved, comparisonID, now.UTC())
|
||||
defer func() {
|
||||
if result.FinishedAt.IsZero() {
|
||||
finalizeComparisonResult(result, clock)
|
||||
}
|
||||
}()
|
||||
|
||||
outputName, err := resolved.OutputName()
|
||||
if err != nil {
|
||||
return result, fmt.Errorf("resolve comparison output name: %w", err)
|
||||
}
|
||||
outputDirectory, err := resolveComparisonOutputDirectory(req.WorkingDir, req.OutputDir, req.Config.Output.Directory, outputName)
|
||||
if err != nil {
|
||||
return result, err
|
||||
}
|
||||
result.OutputDirectory = outputDirectory
|
||||
_, err = comparison.PlanDestination(req.WorkingDir, outputDirectory, req.Replace)
|
||||
if err != nil {
|
||||
return result, fmt.Errorf("preflight comparison destination: %w", err)
|
||||
}
|
||||
|
||||
debugWriter, err := promptdebug.NewPromptDebugWriter(req.LLMDebugDir)
|
||||
if err != nil {
|
||||
return result, promptexec.NewError(promptexec.InvalidConfiguration, "initialize prompt debug", err)
|
||||
}
|
||||
inspection, err := InspectComparisonExecution(ctx, ComparisonInspectionRequest{
|
||||
Resolved: resolved, ProfileIDs: req.ProfileIDs, Executor: req.Executor, LookupEnv: os.LookupEnv,
|
||||
})
|
||||
result.PromptID, result.PromptVersion, result.PromptHash = inspection.PromptID, inspection.PromptVersion, inspection.PromptHash
|
||||
if err != nil {
|
||||
return result, err
|
||||
}
|
||||
|
||||
collection, err := collectWeather(ctx, req.Config, req.Collector)
|
||||
if err != nil {
|
||||
return result, err
|
||||
}
|
||||
prepared, err := prepareReport(prepareReportRequest{Config: req.Config, Resolved: resolved, Collection: *collection})
|
||||
if err != nil {
|
||||
return result, fmt.Errorf("prepare comparison report: %w", err)
|
||||
}
|
||||
|
||||
executed := executeComparisonProfiles(ctx, comparisonExecutionRequest{
|
||||
Prepared: prepared, Inspection: inspection, ComparisonID: comparisonID, DebugWriter: debugWriter, Executor: req.Executor,
|
||||
})
|
||||
finalizeComparisonResult(result, clock)
|
||||
copyComparisonOutcomes(result, executed.Outcomes, false)
|
||||
if executed.Canceled {
|
||||
return result, fmt.Errorf("comparison execution: %w", ctx.Err())
|
||||
}
|
||||
|
||||
bundle := comparisonBundle(result, prepared.dataPackageCopy(), executed.Outcomes)
|
||||
if err := bundle.Validate(); err != nil {
|
||||
return result, fmt.Errorf("build comparison bundle: %w", err)
|
||||
}
|
||||
publicationPlan, err := comparison.PlanDestination(req.WorkingDir, outputDirectory, req.Replace)
|
||||
if err != nil {
|
||||
return result, fmt.Errorf("re-preflight comparison destination: %w", err)
|
||||
}
|
||||
publication, err := publish(ctx, publicationPlan, bundle)
|
||||
if publication.Committed {
|
||||
result.OutputDirectory = publicationPlan.Target
|
||||
result.ManifestPath = filepath.Join(publicationPlan.Target, comparison.ManifestFilename)
|
||||
result.DataPackagePath = filepath.Join(publicationPlan.Target, comparison.DataPackageFilename)
|
||||
copyComparisonOutcomes(result, executed.Outcomes, true)
|
||||
}
|
||||
if err != nil {
|
||||
return result, fmt.Errorf("publish comparison bundle: %w", err)
|
||||
}
|
||||
|
||||
if result.Failed > 0 {
|
||||
return result, fmt.Errorf("comparison completed with %d failed profiles", result.Failed)
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func finalizeComparisonResult(result *ComparisonResult, clock timeutil.Clock) {
|
||||
finishedAt := clock.Now().UTC()
|
||||
if finishedAt.IsZero() {
|
||||
finishedAt = time.Unix(0, 1).UTC()
|
||||
}
|
||||
if finishedAt.Before(result.StartedAt) {
|
||||
finishedAt = result.StartedAt
|
||||
}
|
||||
result.FinishedAt = finishedAt
|
||||
}
|
||||
|
||||
func initialComparisonResult(req ComparisonRequest, resolved report.Resolved, comparisonID string, startedAt time.Time) *ComparisonResult {
|
||||
metadata := resolved.Metadata()
|
||||
return &ComparisonResult{
|
||||
ComparisonID: comparisonID,
|
||||
ReportID: resolved.Definition.ID,
|
||||
ReportName: resolved.Definition.Name,
|
||||
StartedAt: startedAt,
|
||||
Timezone: req.Config.WeatherAPI.Timezone,
|
||||
ValidPeriod: metadata.ValidPeriod,
|
||||
}
|
||||
}
|
||||
|
||||
func copyComparisonOutcomes(result *ComparisonResult, outcomes []comparisonProfileOutcome, published bool) {
|
||||
result.Results = make([]ComparisonProfileResult, len(outcomes))
|
||||
result.Total, result.Succeeded, result.Failed = len(outcomes), 0, 0
|
||||
for index, outcome := range outcomes {
|
||||
profile := ComparisonProfileResult{
|
||||
Position: outcome.Position, ProfileID: outcome.ProfileID, BackendID: outcome.BackendID, ModelName: outcome.ModelName,
|
||||
Status: outcome.Status, ValidationStatus: outcome.ValidationStatus, LLMDebugPath: outcome.LLMDebugPath, Error: outcome.Error,
|
||||
}
|
||||
if published && outcome.Status == comparison.StatusSucceeded {
|
||||
profile.ReportPath = filepath.Join(result.OutputDirectory, outcome.ReportPath)
|
||||
}
|
||||
result.Results[index] = profile
|
||||
if outcome.Status == comparison.StatusSucceeded {
|
||||
result.Succeeded++
|
||||
} else {
|
||||
result.Failed++
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func comparisonBundle(result *ComparisonResult, dataPackage []byte, outcomes []comparisonProfileOutcome) comparison.LogicalBundle {
|
||||
manifest := comparison.Manifest{
|
||||
SchemaVersion: comparison.SchemaVersion, ComparisonID: result.ComparisonID,
|
||||
StartedAt: result.StartedAt.UTC(), FinishedAt: result.FinishedAt.UTC(),
|
||||
ReportID: string(result.ReportID), Timezone: result.Timezone,
|
||||
ValidPeriod: comparison.ValidPeriod{Start: result.ValidPeriod.Start, End: result.ValidPeriod.End},
|
||||
PromptID: result.PromptID, PromptVersion: result.PromptVersion, PromptHash: result.PromptHash,
|
||||
DataPackage: comparison.DataPackageReference{Path: comparison.DataPackageFilename, SHA256: comparison.SHA256(dataPackage)},
|
||||
Total: result.Total, Succeeded: result.Succeeded, Failed: result.Failed,
|
||||
Results: make([]comparison.Result, len(outcomes)),
|
||||
}
|
||||
bundle := comparison.LogicalBundle{Manifest: manifest, DataPackage: dataPackage}
|
||||
for index, outcome := range outcomes {
|
||||
manifestResult := comparison.Result{
|
||||
Position: outcome.Position, ProfileID: outcome.ProfileID, BackendID: outcome.BackendID, ModelName: outcome.ModelName,
|
||||
Status: outcome.Status, ValidationStatus: string(outcome.ValidationStatus), Error: outcome.Error,
|
||||
}
|
||||
if outcome.Status == comparison.StatusSucceeded {
|
||||
manifestResult.ReportPath = outcome.ReportPath
|
||||
bundle.Reports = append(bundle.Reports, comparison.BundleReport{Position: outcome.Position, Path: outcome.ReportPath, Markdown: outcome.Markdown})
|
||||
}
|
||||
bundle.Manifest.Results[index] = manifestResult
|
||||
}
|
||||
return bundle
|
||||
}
|
||||
165
internal/app/comparison_execution.go
Normal file
165
internal/app/comparison_execution.go
Normal file
@@ -0,0 +1,165 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sync"
|
||||
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/comparison"
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptdebug"
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec"
|
||||
)
|
||||
|
||||
type comparisonExecutionRequest struct {
|
||||
Prepared preparedReport
|
||||
Inspection ComparisonInspectionResult
|
||||
ComparisonID string
|
||||
DebugWriter *promptdebug.PromptDebugWriter
|
||||
Executor promptexec.Executor
|
||||
}
|
||||
|
||||
type comparisonExecutionResult struct {
|
||||
Outcomes []comparisonProfileOutcome
|
||||
Canceled bool
|
||||
}
|
||||
|
||||
type comparisonProfileOutcome struct {
|
||||
Position int
|
||||
ProfileID string
|
||||
BackendID string
|
||||
ModelName string
|
||||
Status string
|
||||
ValidationStatus promptexec.ValidationStatus
|
||||
ReportPath string
|
||||
Markdown []byte
|
||||
LLMDebugPath string
|
||||
Error *comparison.SafeError
|
||||
}
|
||||
|
||||
func executeComparisonProfiles(ctx context.Context, req comparisonExecutionRequest) comparisonExecutionResult {
|
||||
profiles := req.Inspection.Profiles
|
||||
result := comparisonExecutionResult{Outcomes: make([]comparisonProfileOutcome, len(profiles))}
|
||||
for index, profile := range profiles {
|
||||
result.Outcomes[index] = comparisonProfileOutcome{
|
||||
Position: index + 1,
|
||||
ProfileID: profile.ProfileID,
|
||||
BackendID: profile.BackendID,
|
||||
ModelName: profile.ModelName,
|
||||
Status: comparison.StatusFailed,
|
||||
}
|
||||
}
|
||||
|
||||
var waitGroup sync.WaitGroup
|
||||
for index, profile := range profiles {
|
||||
if err := ctx.Err(); err != nil {
|
||||
result.Canceled = true
|
||||
markUnstartedComparisonOutcomes(result.Outcomes[index:], err)
|
||||
break
|
||||
}
|
||||
index, profile := index, profile
|
||||
waitGroup.Add(1)
|
||||
go func() {
|
||||
defer waitGroup.Done()
|
||||
result.Outcomes[index] = executeComparisonProfile(ctx, req, index, profile)
|
||||
}()
|
||||
}
|
||||
waitGroup.Wait()
|
||||
if err := ctx.Err(); err != nil {
|
||||
result.Canceled = true
|
||||
for index := range result.Outcomes {
|
||||
if result.Outcomes[index].Status != comparison.StatusSucceeded {
|
||||
markCanceledComparisonOutcome(&result.Outcomes[index], err)
|
||||
}
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func executeComparisonProfile(ctx context.Context, req comparisonExecutionRequest, index int, profile ComparisonProfileInspection) comparisonProfileOutcome {
|
||||
position := index + 1
|
||||
outcome := comparisonProfileOutcome{
|
||||
Position: position, ProfileID: profile.ProfileID, BackendID: profile.BackendID, ModelName: profile.ModelName,
|
||||
Status: comparison.StatusFailed,
|
||||
}
|
||||
debugRef := promptdebug.PromptDebugRef{
|
||||
ReportID: req.Prepared.resolved.Definition.ID,
|
||||
ValidDate: req.Prepared.resolved.ValidPeriod.Start.Format("2006-01-02"),
|
||||
RunID: comparisonDebugRunID(req.ComparisonID, position, len(req.Inspection.Profiles), profile.ProfileID),
|
||||
}
|
||||
execution, markdown, err := executePreparedProfile(ctx, profileExecutionRequest{
|
||||
Prepared: req.Prepared,
|
||||
Prompt: PromptInspectionResult{
|
||||
PromptID: req.Inspection.PromptID, PromptVersion: req.Inspection.PromptVersion, PromptHash: req.Inspection.PromptHash,
|
||||
},
|
||||
Profile: promptexec.ProfileInspection{ProfileID: profile.ProfileID, BackendID: profile.BackendID, ModelName: profile.ModelName},
|
||||
Executor: req.Executor, DebugWriter: req.DebugWriter, DebugRef: &debugRef,
|
||||
})
|
||||
outcome.ProfileID, outcome.BackendID, outcome.ModelName = execution.ProfileID, execution.BackendID, execution.ModelName
|
||||
outcome.ValidationStatus = execution.ValidationStatus
|
||||
outcome.LLMDebugPath = execution.LLMDebugPath
|
||||
if err != nil {
|
||||
safe := comparisonSafeExecutionError(err)
|
||||
outcome.Error = &safe
|
||||
return outcome
|
||||
}
|
||||
reportPath, err := comparison.ReportFilename(position, len(req.Inspection.Profiles), profile.ProfileID)
|
||||
if err != nil {
|
||||
safe := comparison.NewSafeError("application", "derive comparison report filename failed")
|
||||
outcome.Error = &safe
|
||||
return outcome
|
||||
}
|
||||
outcome.Status = comparison.StatusSucceeded
|
||||
outcome.ReportPath = reportPath
|
||||
outcome.Markdown = append([]byte(nil), markdown...)
|
||||
return outcome
|
||||
}
|
||||
|
||||
func comparisonDebugRunID(comparisonID string, position, profileCount int, profileID string) string {
|
||||
return fmt.Sprintf("%s_%0*d-%s", comparisonID, comparison.OrdinalWidth(profileCount), position, comparison.ProfileSlug(profileID))
|
||||
}
|
||||
|
||||
func markUnstartedComparisonOutcomes(outcomes []comparisonProfileOutcome, err error) {
|
||||
for index := range outcomes {
|
||||
markCanceledComparisonOutcome(&outcomes[index], err)
|
||||
}
|
||||
}
|
||||
|
||||
func markCanceledComparisonOutcome(outcome *comparisonProfileOutcome, err error) {
|
||||
outcome.Status = comparison.StatusFailed
|
||||
outcome.ValidationStatus = promptexec.ValidationSkipped
|
||||
outcome.ReportPath = ""
|
||||
outcome.Markdown = nil
|
||||
safe := comparisonSafeExecutionError(err)
|
||||
outcome.Error = &safe
|
||||
}
|
||||
|
||||
func comparisonSafeExecutionError(err error) comparison.SafeError {
|
||||
category := promptexec.CategoryOf(err)
|
||||
if category == "" {
|
||||
switch {
|
||||
case errors.Is(err, context.Canceled):
|
||||
category = promptexec.Canceled
|
||||
case errors.Is(err, context.DeadlineExceeded):
|
||||
category = promptexec.DeadlineExceeded
|
||||
}
|
||||
}
|
||||
if category == "" {
|
||||
return comparison.NewSafeError("application", comparisonExecutionMessage(err))
|
||||
}
|
||||
return comparison.NewSafeError(string(category), comparisonExecutionMessage(err))
|
||||
}
|
||||
|
||||
func comparisonExecutionMessage(err error) string {
|
||||
if errors.Is(err, context.Canceled) {
|
||||
return "profile execution canceled"
|
||||
}
|
||||
if errors.Is(err, context.DeadlineExceeded) {
|
||||
return "profile execution deadline exceeded"
|
||||
}
|
||||
var execution *profileExecutionError
|
||||
if errors.As(err, &execution) {
|
||||
return comparison.TruncateErrorMessage(execution.operation + " failed")
|
||||
}
|
||||
return "profile execution failed"
|
||||
}
|
||||
300
internal/app/comparison_execution_test.go
Normal file
300
internal/app/comparison_execution_test.go
Normal file
@@ -0,0 +1,300 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/comparison"
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptdebug"
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec"
|
||||
)
|
||||
|
||||
func TestExecuteComparisonProfilesRunsOrderedProfilesConcurrently(t *testing.T) {
|
||||
prepared, prompt := preparedDailyProfile(t)
|
||||
profiles := comparisonProfiles(10)
|
||||
executor := newBarrierExecutor(profiles)
|
||||
results := make(chan comparisonExecutionResult, 1)
|
||||
go func() {
|
||||
results <- executeComparisonProfiles(context.Background(), comparisonExecutionRequest{
|
||||
Prepared: prepared, Inspection: comparisonInspection(prompt, profiles), ComparisonID: "comparison_daily", Executor: executor,
|
||||
})
|
||||
}()
|
||||
waitForProfileStarts(t, executor, profiles, results)
|
||||
if executor.maximumInFlight() < 2 {
|
||||
t.Fatalf("maximum in-flight executions = %d, want overlap", executor.maximumInFlight())
|
||||
}
|
||||
for index := len(profiles) - 1; index >= 0; index-- {
|
||||
executor.release(profiles[index].ProfileID)
|
||||
}
|
||||
result := <-results
|
||||
if result.Canceled || len(result.Outcomes) != len(profiles) {
|
||||
t.Fatalf("result = %#v", result)
|
||||
}
|
||||
for index, profile := range profiles {
|
||||
outcome := result.Outcomes[index]
|
||||
wantPath, err := comparison.ReportFilename(index+1, len(profiles), profile.ProfileID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if outcome.Position != index+1 || outcome.ProfileID != profile.ProfileID || outcome.Status != comparison.StatusSucceeded || outcome.ValidationStatus != promptexec.ValidationPassed || outcome.ReportPath != wantPath || len(outcome.Markdown) == 0 || outcome.Error != nil {
|
||||
t.Fatalf("outcome[%d] = %#v", index, outcome)
|
||||
}
|
||||
request, ok := executor.request(profile.ProfileID)
|
||||
if !ok || request.PromptVersion != prompt.PromptVersion || !bytesEqual(request.DataPackage, prepared.dataPackage) {
|
||||
t.Fatalf("request for %q = %#v, want prompt version %q and shared data package", profile.ProfileID, request, prompt.PromptVersion)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteComparisonProfilesContinuesAfterProfileFailure(t *testing.T) {
|
||||
prepared, prompt := preparedDailyProfile(t)
|
||||
profiles := comparisonProfiles(3)
|
||||
executor := newBarrierExecutor(profiles)
|
||||
executor.setError(profiles[1].ProfileID, errors.New("provider response body must not escape"))
|
||||
results := make(chan comparisonExecutionResult, 1)
|
||||
go func() {
|
||||
results <- executeComparisonProfiles(context.Background(), comparisonExecutionRequest{
|
||||
Prepared: prepared, Inspection: comparisonInspection(prompt, profiles), ComparisonID: "comparison_daily", Executor: executor,
|
||||
})
|
||||
}()
|
||||
waitForProfileStarts(t, executor, profiles, results)
|
||||
for _, profile := range profiles {
|
||||
executor.release(profile.ProfileID)
|
||||
}
|
||||
result := <-results
|
||||
if result.Canceled || result.Outcomes[0].Status != comparison.StatusSucceeded || result.Outcomes[1].Status != comparison.StatusFailed || result.Outcomes[2].Status != comparison.StatusSucceeded {
|
||||
t.Fatalf("outcomes = %#v", result.Outcomes)
|
||||
}
|
||||
failure := result.Outcomes[1]
|
||||
if failure.Error == nil || failure.Error.Category != string(promptexec.Generation) || failure.Error.Message != "execute prompt failed" || failure.ReportPath != "" || len(failure.Markdown) != 0 {
|
||||
t.Fatalf("failure outcome = %#v", failure)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteComparisonProfilesPropagatesCancellationAndJoins(t *testing.T) {
|
||||
prepared, prompt := preparedDailyProfile(t)
|
||||
profiles := comparisonProfiles(4)
|
||||
executor := newBarrierExecutor(profiles)
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
results := make(chan comparisonExecutionResult, 1)
|
||||
go func() {
|
||||
results <- executeComparisonProfiles(ctx, comparisonExecutionRequest{
|
||||
Prepared: prepared, Inspection: comparisonInspection(prompt, profiles), ComparisonID: "comparison_daily", Executor: executor,
|
||||
})
|
||||
}()
|
||||
waitForProfileStarts(t, executor, profiles, results)
|
||||
cancel()
|
||||
result := <-results
|
||||
if !result.Canceled || executor.inFlightCount() != 0 {
|
||||
t.Fatalf("result/in-flight = %#v/%d", result, executor.inFlightCount())
|
||||
}
|
||||
for _, outcome := range result.Outcomes {
|
||||
if outcome.Status != comparison.StatusFailed || outcome.Error == nil || outcome.Error.Category != string(promptexec.Canceled) || outcome.ValidationStatus != promptexec.ValidationSkipped || outcome.ReportPath != "" || len(outcome.Markdown) != 0 {
|
||||
t.Fatalf("canceled outcome = %#v", outcome)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteComparisonProfilesUsesDistinctDeterministicDebugReferences(t *testing.T) {
|
||||
prepared, prompt := preparedDailyProfile(t)
|
||||
profiles := []ComparisonProfileInspection{
|
||||
{ProfileID: "light.one", BackendID: "local", ModelName: "light"},
|
||||
{ProfileID: "deep/two", BackendID: "cloud", ModelName: "deep"},
|
||||
}
|
||||
debugWriter, err := promptdebug.NewPromptDebugWriter(t.TempDir())
|
||||
if err != nil {
|
||||
t.Fatalf("NewPromptDebugWriter() error = %v", err)
|
||||
}
|
||||
executor := newBarrierExecutor(profiles)
|
||||
results := make(chan comparisonExecutionResult, 1)
|
||||
go func() {
|
||||
results <- executeComparisonProfiles(context.Background(), comparisonExecutionRequest{
|
||||
Prepared: prepared, Inspection: comparisonInspection(prompt, profiles), ComparisonID: "comparison_daily", DebugWriter: debugWriter, Executor: executor,
|
||||
})
|
||||
}()
|
||||
waitForProfileStarts(t, executor, profiles, results)
|
||||
for _, profile := range profiles {
|
||||
executor.release(profile.ProfileID)
|
||||
}
|
||||
result := <-results
|
||||
paths := map[string]struct{}{}
|
||||
for index, outcome := range result.Outcomes {
|
||||
wantName := fmt.Sprintf("comparison_daily_%0*d-%s", comparison.OrdinalWidth(len(profiles)), index+1, comparison.ProfileSlug(outcome.ProfileID))
|
||||
if filepath.Base(outcome.LLMDebugPath) != wantName {
|
||||
t.Fatalf("debug path = %q, want base %q", outcome.LLMDebugPath, wantName)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(outcome.LLMDebugPath, "preparation.json")); err != nil {
|
||||
t.Fatalf("preparation artifact %q: %v", outcome.LLMDebugPath, err)
|
||||
}
|
||||
paths[outcome.LLMDebugPath] = struct{}{}
|
||||
}
|
||||
if len(paths) != len(profiles) {
|
||||
t.Fatalf("debug paths = %#v", paths)
|
||||
}
|
||||
}
|
||||
|
||||
type barrierExecutor struct {
|
||||
mu sync.Mutex
|
||||
started chan string
|
||||
callbackFailures chan error
|
||||
releases map[string]chan struct{}
|
||||
requests map[string]promptexec.ExecuteRequest
|
||||
errors map[string]error
|
||||
inFlight int
|
||||
maximum int
|
||||
}
|
||||
|
||||
func newBarrierExecutor(profiles []ComparisonProfileInspection) *barrierExecutor {
|
||||
releases := make(map[string]chan struct{}, len(profiles))
|
||||
for _, profile := range profiles {
|
||||
releases[profile.ProfileID] = make(chan struct{})
|
||||
}
|
||||
return &barrierExecutor{
|
||||
started: make(chan string, len(profiles)), callbackFailures: make(chan error, len(profiles)), releases: releases,
|
||||
requests: make(map[string]promptexec.ExecuteRequest, len(profiles)), errors: map[string]error{},
|
||||
}
|
||||
}
|
||||
|
||||
func (e *barrierExecutor) InspectPrompt(context.Context, string, string) (promptexec.PromptInspection, error) {
|
||||
return promptexec.PromptInspection{}, errors.New("unexpected prompt inspection")
|
||||
}
|
||||
|
||||
func (e *barrierExecutor) InspectProfile(context.Context, string) (promptexec.ProfileInspection, error) {
|
||||
return promptexec.ProfileInspection{}, errors.New("unexpected profile inspection")
|
||||
}
|
||||
|
||||
func (e *barrierExecutor) Execute(ctx context.Context, req promptexec.ExecuteRequest, callback promptexec.PreparationCallback) (*promptexec.Execution, error) {
|
||||
stamp := time.Date(2026, 5, 29, 15, 0, 0, 0, time.UTC)
|
||||
if err := callback(promptexec.Preparation{PromptID: req.PromptID, PromptVersion: req.PromptVersion, PromptHash: "prompt-hash", ProfileID: req.ProfileID, BackendID: "backend-" + req.ProfileID, ModelName: "model-" + req.ProfileID, StartedAt: stamp, EndedAt: stamp}, nil); err != nil {
|
||||
e.callbackFailures <- err
|
||||
return nil, err
|
||||
}
|
||||
e.mu.Lock()
|
||||
e.requests[req.ProfileID] = promptexec.ExecuteRequest{PromptID: req.PromptID, PromptVersion: req.PromptVersion, ProfileID: req.ProfileID, DataPackage: append([]byte(nil), req.DataPackage...), CaptureDebug: req.CaptureDebug}
|
||||
e.inFlight++
|
||||
if e.inFlight > e.maximum {
|
||||
e.maximum = e.inFlight
|
||||
}
|
||||
release := e.releases[req.ProfileID]
|
||||
e.mu.Unlock()
|
||||
e.started <- req.ProfileID
|
||||
select {
|
||||
case <-release:
|
||||
case <-ctx.Done():
|
||||
e.mu.Lock()
|
||||
e.inFlight--
|
||||
e.mu.Unlock()
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
e.mu.Lock()
|
||||
e.inFlight--
|
||||
err := e.errors[req.ProfileID]
|
||||
e.mu.Unlock()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &promptexec.Execution{
|
||||
PromptID: req.PromptID, PromptVersion: req.PromptVersion, PromptHash: "prompt-hash",
|
||||
ProfileID: req.ProfileID, BackendID: "backend-" + req.ProfileID, ModelName: "model-" + req.ProfileID,
|
||||
StartedAt: stamp, EndedAt: stamp, RawOutput: comparisonRawOutput(),
|
||||
Validation: promptexec.NewValidation(promptexec.ValidationPassed, "json_schema", generationDefinitionForPrompt(req.PromptID).GeneratedTextSchemaID+".generated_text.schema.json", nil),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (e *barrierExecutor) request(profileID string) (promptexec.ExecuteRequest, bool) {
|
||||
e.mu.Lock()
|
||||
defer e.mu.Unlock()
|
||||
request, ok := e.requests[profileID]
|
||||
return request, ok
|
||||
}
|
||||
|
||||
func (e *barrierExecutor) setError(profileID string, err error) {
|
||||
e.mu.Lock()
|
||||
defer e.mu.Unlock()
|
||||
e.errors[profileID] = err
|
||||
}
|
||||
|
||||
func (e *barrierExecutor) release(profileID string) {
|
||||
close(e.releases[profileID])
|
||||
}
|
||||
|
||||
func (e *barrierExecutor) releaseAll() {
|
||||
for _, release := range e.releases {
|
||||
select {
|
||||
case <-release:
|
||||
default:
|
||||
close(release)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (e *barrierExecutor) maximumInFlight() int {
|
||||
e.mu.Lock()
|
||||
defer e.mu.Unlock()
|
||||
return e.maximum
|
||||
}
|
||||
|
||||
func (e *barrierExecutor) inFlightCount() int {
|
||||
e.mu.Lock()
|
||||
defer e.mu.Unlock()
|
||||
return e.inFlight
|
||||
}
|
||||
|
||||
func waitForProfileStarts(t *testing.T, executor *barrierExecutor, profiles []ComparisonProfileInspection, results <-chan comparisonExecutionResult) {
|
||||
t.Helper()
|
||||
timeout := time.NewTimer(5 * time.Second)
|
||||
defer timeout.Stop()
|
||||
seen := map[string]struct{}{}
|
||||
for range profiles {
|
||||
var profileID string
|
||||
select {
|
||||
case profileID = <-executor.started:
|
||||
case err := <-executor.callbackFailures:
|
||||
executor.releaseAll()
|
||||
select {
|
||||
case result := <-results:
|
||||
t.Fatalf("comparison profile preparation failed before executor entry: %v; result: %#v", err, result)
|
||||
case <-timeout.C:
|
||||
t.Fatalf("comparison profile preparation failed before executor entry: %v; comparison did not finish", err)
|
||||
}
|
||||
case result := <-results:
|
||||
t.Fatalf("comparison completed before all profiles started: %#v", result)
|
||||
case <-timeout.C:
|
||||
t.Fatal("timed out waiting for comparison profile starts")
|
||||
}
|
||||
if _, duplicate := seen[profileID]; duplicate {
|
||||
t.Fatalf("duplicate execution start for %q", profileID)
|
||||
}
|
||||
seen[profileID] = struct{}{}
|
||||
}
|
||||
}
|
||||
|
||||
func comparisonProfiles(count int) []ComparisonProfileInspection {
|
||||
profiles := make([]ComparisonProfileInspection, 0, count)
|
||||
for index := 1; index <= count; index++ {
|
||||
profiles = append(profiles, ComparisonProfileInspection{ProfileID: fmt.Sprintf("profile.%02d", index), BackendID: "backend", ModelName: "model"})
|
||||
}
|
||||
return profiles
|
||||
}
|
||||
|
||||
func comparisonInspection(prompt PromptInspectionResult, profiles []ComparisonProfileInspection) ComparisonInspectionResult {
|
||||
return ComparisonInspectionResult{PromptID: prompt.PromptID, PromptVersion: prompt.PromptVersion, PromptHash: prompt.PromptHash, Profiles: profiles}
|
||||
}
|
||||
|
||||
func comparisonRawOutput() []byte {
|
||||
return []byte(`{"summary":"Showers are possible during the selected day.","forecast_discussion":["A front will keep rain chances in the forecast."],"precipitation_timing":"Rain is most likely during the afternoon."}`)
|
||||
}
|
||||
|
||||
func bytesEqual(left, right []byte) bool {
|
||||
return reflect.DeepEqual(left, right)
|
||||
}
|
||||
|
||||
var _ promptexec.Executor = (*barrierExecutor)(nil)
|
||||
332
internal/app/comparison_test.go
Normal file
332
internal/app/comparison_test.go
Normal file
@@ -0,0 +1,332 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/comparison"
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/config"
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec"
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/timeutil"
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/weatherdata"
|
||||
)
|
||||
|
||||
func TestCompareDetailedPublishesOneCoherentBundle(t *testing.T) {
|
||||
cfg := comparisonConfig()
|
||||
bundle := generationBundle(t)
|
||||
workingDir := t.TempDir()
|
||||
executor := &generationExecutor{}
|
||||
inspectedBeforeCollection := false
|
||||
result, err := CompareDetailed(context.Background(), ComparisonRequest{
|
||||
Config: cfg, Report: ReportDaily, ProfileIDs: []string{"weather-light", "weather-deep"},
|
||||
WorkingDir: workingDir, Date: generationTime("2026-05-29T12:00:00-05:00"),
|
||||
Clock: timeutil.FixedClock{Time: generationTime("2026-05-29T08:30:00-05:00")},
|
||||
Collector: &generationCollector{bundle: &bundle, beforeRun: func() {
|
||||
inspectedBeforeCollection = executor.promptInspections == 1 && executor.profileInspections == 2
|
||||
}}, Executor: executor,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("CompareDetailed() error = %v", err)
|
||||
}
|
||||
if result == nil || result.Total != 2 || result.Succeeded != 2 || result.Failed != 0 || executor.promptInspections != 1 || executor.profileInspections != 2 || executor.executeCalls != 2 || !inspectedBeforeCollection || result.ManifestPath == "" || result.DataPackagePath == "" {
|
||||
t.Fatalf("result/executor = %#v/%#v", result, executor)
|
||||
}
|
||||
if result.OutputDirectory != filepath.Dir(result.ManifestPath) || !filepath.IsAbs(result.ManifestPath) || !filepath.IsAbs(result.DataPackagePath) {
|
||||
t.Fatalf("published paths = %#v", result)
|
||||
}
|
||||
for index, profile := range result.Results {
|
||||
if profile.Position != index+1 || profile.Status != comparison.StatusSucceeded || !filepath.IsAbs(profile.ReportPath) || profile.Error != nil {
|
||||
t.Fatalf("profile result = %#v", profile)
|
||||
}
|
||||
}
|
||||
data, readErr := os.ReadFile(result.ManifestPath)
|
||||
if readErr != nil {
|
||||
t.Fatal(readErr)
|
||||
}
|
||||
var manifest comparison.Manifest
|
||||
if err := json.Unmarshal(data, &manifest); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if manifest.ComparisonID != result.ComparisonID || manifest.Total != result.Total || manifest.Succeeded != result.Succeeded || manifest.DataPackage.SHA256 == "" || len(manifest.Results) != 2 {
|
||||
t.Fatalf("manifest = %#v", manifest)
|
||||
}
|
||||
if manifest.Results[0].ReportPath != filepath.Base(result.Results[0].ReportPath) || manifest.Results[1].ReportPath != filepath.Base(result.Results[1].ReportPath) {
|
||||
t.Fatalf("manifest report paths = %#v", manifest.Results)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompareDetailedPublishesPartialBundleAndReturnsAggregateError(t *testing.T) {
|
||||
cfg := comparisonConfig()
|
||||
bundle := generationBundle(t)
|
||||
executor := &generationExecutor{executeErrors: map[string]error{"weather-deep": errors.New("provider detail must not escape")}}
|
||||
result, err := CompareDetailed(context.Background(), ComparisonRequest{
|
||||
Config: cfg, Report: ReportDaily, ProfileIDs: []string{"weather-light", "weather-deep", "weather-fallback"},
|
||||
WorkingDir: t.TempDir(), Date: generationTime("2026-05-29T12:00:00-05:00"),
|
||||
Clock: timeutil.FixedClock{Time: generationTime("2026-05-29T08:30:00-05:00")},
|
||||
Collector: &generationCollector{bundle: &bundle}, Executor: executor,
|
||||
})
|
||||
if err == nil || err.Error() != "comparison completed with 1 failed profiles" || result == nil || result.Total != 3 || result.Succeeded != 2 || result.Failed != 1 {
|
||||
t.Fatalf("CompareDetailed() result/error = %#v/%v", result, err)
|
||||
}
|
||||
failure := result.Results[1]
|
||||
if failure.Status != comparison.StatusFailed || failure.ReportPath != "" || failure.Error == nil || strings.Contains(failure.Error.Message, "provider detail") {
|
||||
t.Fatalf("failure = %#v", failure)
|
||||
}
|
||||
if _, statErr := os.Stat(result.ManifestPath); statErr != nil {
|
||||
t.Fatalf("partial manifest: %v", statErr)
|
||||
}
|
||||
if _, statErr := os.Stat(filepath.Join(result.OutputDirectory, filepath.Base(result.Results[0].ReportPath))); statErr != nil {
|
||||
t.Fatalf("successful partial report: %v", statErr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompareDetailedRetainsCommittedPathsWhenBackupCleanupFails(t *testing.T) {
|
||||
bundle := generationBundle(t)
|
||||
backupPath := filepath.Join(t.TempDir(), ".comparison-daily.backup-retained")
|
||||
cleanupCause := errors.New("backup cleanup failed")
|
||||
publish := func(context.Context, comparison.DestinationPlan, comparison.LogicalBundle) (comparison.PublicationResult, error) {
|
||||
return comparison.PublicationResult{Committed: true, RetainedBackupPath: backupPath}, &comparison.PublicationCleanupError{RetainedBackupPath: backupPath, Err: cleanupCause}
|
||||
}
|
||||
|
||||
result, err := compareDetailed(context.Background(), ComparisonRequest{
|
||||
Config: comparisonConfig(), Report: ReportDaily, ProfileIDs: []string{"weather-light", "weather-deep"},
|
||||
WorkingDir: t.TempDir(), Date: generationTime("2026-05-29T12:00:00-05:00"),
|
||||
Clock: timeutil.FixedClock{Time: generationTime("2026-05-29T08:30:00-05:00")},
|
||||
Collector: &generationCollector{bundle: &bundle}, Executor: &generationExecutor{},
|
||||
}, publish)
|
||||
var cleanupErr *comparison.PublicationCleanupError
|
||||
if result == nil || !errors.As(err, &cleanupErr) || !errors.Is(err, cleanupCause) || cleanupErr.RetainedBackupPath != backupPath || !filepath.IsAbs(result.ManifestPath) || !filepath.IsAbs(result.DataPackagePath) {
|
||||
t.Fatalf("CompareDetailed() result/error = %#v/%v", result, err)
|
||||
}
|
||||
for _, profile := range result.Results {
|
||||
if profile.Status == comparison.StatusSucceeded && !filepath.IsAbs(profile.ReportPath) {
|
||||
t.Fatalf("published profile result = %#v", profile)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompareDetailedPreflightsBeforePromptOrCollection(t *testing.T) {
|
||||
invalidDestination := filepath.Join(t.TempDir(), "not-a-directory")
|
||||
if err := os.WriteFile(invalidDestination, []byte("x"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
bundle := generationBundle(t)
|
||||
collector := &generationCollector{bundle: &bundle}
|
||||
executor := &generationExecutor{}
|
||||
result, err := CompareDetailed(context.Background(), ComparisonRequest{
|
||||
Config: comparisonConfig(), Report: ReportDaily, ProfileIDs: []string{"weather-light", "weather-deep"},
|
||||
WorkingDir: t.TempDir(), OutputDir: invalidDestination, Date: generationTime("2026-05-29T12:00:00-05:00"),
|
||||
Clock: timeutil.FixedClock{Time: generationTime("2026-05-29T08:30:00-05:00")}, Collector: collector, Executor: executor,
|
||||
})
|
||||
if err == nil || result == nil || collector.called || executor.promptInspections != 0 || executor.executeCalls != 0 || result.ManifestPath != "" {
|
||||
t.Fatalf("result/error/collector/executor = %#v/%v/%#v/%#v", result, err, collector, executor)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompareDetailedFinalizesUnpublishedFailures(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
prepare func(t *testing.T, outputDirectory string)
|
||||
debugDir string
|
||||
executor *generationExecutor
|
||||
collector *generationCollector
|
||||
wantPrompt bool
|
||||
wantCollection bool
|
||||
}{
|
||||
{
|
||||
name: "destination preflight",
|
||||
prepare: func(t *testing.T, outputDirectory string) {
|
||||
t.Helper()
|
||||
if err := os.WriteFile(outputDirectory, []byte("not a directory"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
},
|
||||
executor: &generationExecutor{},
|
||||
},
|
||||
{
|
||||
name: "debug initialization",
|
||||
debugDir: "relative-debug-directory",
|
||||
executor: &generationExecutor{},
|
||||
collector: &generationCollector{},
|
||||
},
|
||||
{
|
||||
name: "prompt preflight",
|
||||
executor: &generationExecutor{inspectErr: promptexec.NewError(promptexec.PromptLoad, "unsafe prompt detail", errors.New("unsafe cause"))},
|
||||
collector: &generationCollector{},
|
||||
wantPrompt: false,
|
||||
},
|
||||
{
|
||||
name: "profile preflight",
|
||||
executor: &generationExecutor{profileInspectErrors: map[string]error{
|
||||
"weather-deep": promptexec.NewError(promptexec.MissingCredential, "profile credential is unavailable", errors.New("unsafe cause")),
|
||||
}},
|
||||
collector: &generationCollector{},
|
||||
wantPrompt: true,
|
||||
},
|
||||
{
|
||||
name: "collection",
|
||||
executor: &generationExecutor{},
|
||||
collector: &generationCollector{err: errors.New("collection failed")},
|
||||
wantPrompt: true,
|
||||
wantCollection: true,
|
||||
},
|
||||
{
|
||||
name: "preparation",
|
||||
executor: &generationExecutor{},
|
||||
collector: &generationCollector{bundle: &weatherdata.Bundle{}},
|
||||
wantPrompt: true,
|
||||
wantCollection: true,
|
||||
},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
workingDirectory := t.TempDir()
|
||||
outputDirectory := filepath.Join(workingDirectory, "comparison-output")
|
||||
if test.prepare != nil {
|
||||
test.prepare(t, outputDirectory)
|
||||
}
|
||||
collector := test.collector
|
||||
if collector == nil {
|
||||
bundle := generationBundle(t)
|
||||
collector = &generationCollector{bundle: &bundle}
|
||||
}
|
||||
result, err := CompareDetailed(context.Background(), ComparisonRequest{
|
||||
Config: comparisonConfig(), Report: ReportDaily, ProfileIDs: []string{"weather-light", "weather-deep"},
|
||||
WorkingDir: workingDirectory, OutputDir: outputDirectory, LLMDebugDir: test.debugDir,
|
||||
Date: generationTime("2026-05-29T12:00:00-05:00"), Clock: timeutil.FixedClock{Time: generationTime("2026-05-29T08:30:00-05:00")},
|
||||
Collector: collector, Executor: test.executor,
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("CompareDetailed() error = nil")
|
||||
}
|
||||
assertUnpublishedComparisonResult(t, result, outputDirectory)
|
||||
if (result.PromptID != "") != test.wantPrompt || (result.PromptHash != "") != test.wantPrompt {
|
||||
t.Fatalf("prompt identity = %q/%q, want resolved=%t", result.PromptID, result.PromptHash, test.wantPrompt)
|
||||
}
|
||||
if collector.called != test.wantCollection || test.executor.executeCalls != 0 {
|
||||
t.Fatalf("collection/execution = %t/%d, want collection=%t and no execution", collector.called, test.executor.executeCalls, test.wantCollection)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompareDetailedLeavesDestinationWhenCollectionOrPreparationFails(t *testing.T) {
|
||||
collectionErr := errors.New("weather collection failed")
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
collector *generationCollector
|
||||
}{
|
||||
{name: "collection", collector: &generationCollector{err: collectionErr}},
|
||||
{name: "preparation", collector: &generationCollector{bundle: &weatherdata.Bundle{}}},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
workingDir := t.TempDir()
|
||||
executor := &generationExecutor{}
|
||||
result, err := CompareDetailed(context.Background(), ComparisonRequest{
|
||||
Config: comparisonConfig(), Report: ReportDaily, ProfileIDs: []string{"weather-light", "weather-deep"},
|
||||
WorkingDir: workingDir, Date: generationTime("2026-05-29T12:00:00-05:00"),
|
||||
Clock: timeutil.FixedClock{Time: generationTime("2026-05-29T08:30:00-05:00")}, Collector: test.collector, Executor: executor,
|
||||
})
|
||||
if err == nil || result == nil || executor.executeCalls != 0 || result.ManifestPath != "" || result.DataPackagePath != "" {
|
||||
t.Fatalf("CompareDetailed() result/error/executor = %#v/%v/%#v", result, err, executor)
|
||||
}
|
||||
if _, statErr := os.Stat(filepath.Join(workingDir, "comparison-daily-2026-05-29")); !os.IsNotExist(statErr) {
|
||||
t.Fatalf("comparison destination stat error = %v", statErr)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompareDetailedPublishesManifestWhenEveryProfileFails(t *testing.T) {
|
||||
bundle := generationBundle(t)
|
||||
executor := &generationExecutor{executeErrors: map[string]error{
|
||||
"weather-light": errors.New("first provider failure"), "weather-deep": errors.New("second provider failure"),
|
||||
}}
|
||||
result, err := CompareDetailed(context.Background(), ComparisonRequest{
|
||||
Config: comparisonConfig(), Report: ReportDaily, ProfileIDs: []string{"weather-light", "weather-deep"},
|
||||
WorkingDir: t.TempDir(), Date: generationTime("2026-05-29T12:00:00-05:00"),
|
||||
Clock: timeutil.FixedClock{Time: generationTime("2026-05-29T08:30:00-05:00")},
|
||||
Collector: &generationCollector{bundle: &bundle}, Executor: executor,
|
||||
})
|
||||
if err == nil || err.Error() != "comparison completed with 2 failed profiles" || result == nil || result.Succeeded != 0 || result.Failed != 2 || result.ManifestPath == "" {
|
||||
t.Fatalf("CompareDetailed() result/error = %#v/%v", result, err)
|
||||
}
|
||||
for _, profile := range result.Results {
|
||||
if profile.ReportPath != "" || profile.Error == nil {
|
||||
t.Fatalf("failed profile = %#v", profile)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompareDetailedCancellationPreservesPublishedBundle(t *testing.T) {
|
||||
workingDir := t.TempDir()
|
||||
bundle := generationBundle(t)
|
||||
request := ComparisonRequest{
|
||||
Config: comparisonConfig(), Report: ReportDaily, ProfileIDs: []string{"weather-light", "weather-deep"},
|
||||
WorkingDir: workingDir, Date: generationTime("2026-05-29T12:00:00-05:00"),
|
||||
Clock: timeutil.FixedClock{Time: generationTime("2026-05-29T08:30:00-05:00")}, Collector: &generationCollector{bundle: &bundle}, Executor: &generationExecutor{},
|
||||
}
|
||||
previous, err := CompareDetailed(context.Background(), request)
|
||||
if err != nil {
|
||||
t.Fatalf("initial CompareDetailed() error = %v", err)
|
||||
}
|
||||
before, err := os.ReadFile(previous.ManifestPath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
request.Replace = true
|
||||
request.Executor = &generationExecutor{cancelBeforeReturn: cancel}
|
||||
result, err := CompareDetailed(ctx, request)
|
||||
if !errors.Is(err, context.Canceled) || result == nil || result.ManifestPath != "" || result.DataPackagePath != "" {
|
||||
t.Fatalf("canceled CompareDetailed() result/error = %#v/%v", result, err)
|
||||
}
|
||||
after, readErr := os.ReadFile(previous.ManifestPath)
|
||||
if readErr != nil || string(after) != string(before) {
|
||||
t.Fatalf("published manifest changed = %q, error = %v", after, readErr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompareDetailedLeavesExistingBundleWhenPublicationPreflightChanges(t *testing.T) {
|
||||
workingDir := t.TempDir()
|
||||
target := filepath.Join(workingDir, "comparison-output")
|
||||
bundle := generationBundle(t)
|
||||
executor := &generationExecutor{beforeExecute: func(promptexec.ExecuteRequest) {
|
||||
_ = os.WriteFile(target, []byte("changed"), 0o600)
|
||||
}}
|
||||
result, err := CompareDetailed(context.Background(), ComparisonRequest{
|
||||
Config: comparisonConfig(), Report: ReportDaily, ProfileIDs: []string{"weather-light", "weather-deep"},
|
||||
WorkingDir: workingDir, OutputDir: target, Date: generationTime("2026-05-29T12:00:00-05:00"),
|
||||
Clock: timeutil.FixedClock{Time: generationTime("2026-05-29T08:30:00-05:00")}, Collector: &generationCollector{bundle: &bundle}, Executor: executor,
|
||||
})
|
||||
if err == nil || result == nil || result.ManifestPath != "" || result.DataPackagePath != "" || result.Results[0].ReportPath != "" {
|
||||
t.Fatalf("CompareDetailed() result/error = %#v/%v", result, err)
|
||||
}
|
||||
data, readErr := os.ReadFile(target)
|
||||
if readErr != nil || string(data) != "changed" {
|
||||
t.Fatalf("destination = %q, error = %v", data, readErr)
|
||||
}
|
||||
}
|
||||
|
||||
func comparisonConfig() config.Config {
|
||||
cfg := config.Defaults()
|
||||
cfg.WeatherAPI.Timezone, cfg.Location.ID = "America/Chicago", "home"
|
||||
return cfg
|
||||
}
|
||||
|
||||
func assertUnpublishedComparisonResult(t *testing.T, result *ComparisonResult, outputDirectory string) {
|
||||
t.Helper()
|
||||
if result == nil || result.OutputDirectory != outputDirectory || !filepath.IsAbs(result.OutputDirectory) || result.FinishedAt.IsZero() || result.FinishedAt.Location() != time.UTC || result.FinishedAt.Before(result.StartedAt) || result.ManifestPath != "" || result.DataPackagePath != "" {
|
||||
t.Fatalf("unpublished comparison result = %#v", result)
|
||||
}
|
||||
for _, profile := range result.Results {
|
||||
if profile.ReportPath != "" {
|
||||
t.Fatalf("unpublished profile result = %#v", profile)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -20,6 +21,7 @@ type generationCollector struct {
|
||||
bundle *weatherdata.Bundle
|
||||
err error
|
||||
called bool
|
||||
calls int
|
||||
beforeRun func()
|
||||
}
|
||||
|
||||
@@ -28,57 +30,89 @@ func (c *generationCollector) Run(context.Context, collect.Request) (*collect.Re
|
||||
c.beforeRun()
|
||||
}
|
||||
c.called = true
|
||||
c.calls++
|
||||
return &collect.Result{Bundle: c.bundle}, c.err
|
||||
}
|
||||
|
||||
type generationExecutor struct {
|
||||
called bool
|
||||
promptInspections int
|
||||
inspectErr error
|
||||
executeErr error
|
||||
cancelBeforeReturn context.CancelFunc
|
||||
validation promptexec.ValidationStatus
|
||||
rawOutput []byte
|
||||
failedPrompt string
|
||||
called bool
|
||||
executeCalls int
|
||||
promptInspections int
|
||||
profileInspections int
|
||||
inspectErr error
|
||||
profileInspectErrors map[string]error
|
||||
executeErr error
|
||||
executeErrors map[string]error
|
||||
beforeExecute func(promptexec.ExecuteRequest)
|
||||
cancelBeforeReturn context.CancelFunc
|
||||
validation promptexec.ValidationStatus
|
||||
rawOutput []byte
|
||||
failedPrompt string
|
||||
}
|
||||
|
||||
var generationExecutorMu sync.Mutex
|
||||
|
||||
func (e *generationExecutor) InspectPrompt(_ context.Context, id, version string) (promptexec.PromptInspection, error) {
|
||||
generationExecutorMu.Lock()
|
||||
defer generationExecutorMu.Unlock()
|
||||
e.promptInspections++
|
||||
if e.inspectErr != nil {
|
||||
return promptexec.PromptInspection{}, e.inspectErr
|
||||
}
|
||||
definition := generationDefinitionForPrompt(id)
|
||||
return promptexec.PromptInspection{PromptID: id, PromptVersion: version, PromptHash: "prompt-hash", DefaultProfileID: "fixture", Inputs: []promptexec.InputDefinition{{Name: "data_package", Required: true, ContentType: "application/yaml"}}, Output: promptexec.OutputContract{Format: "json", ValidationMode: "json_schema", SchemaPath: definition.GeneratedTextSchemaID + ".generated_text.schema.json"}}, nil
|
||||
return promptexec.PromptInspection{PromptID: id, PromptVersion: version, PromptHash: generationPromptHash, DefaultProfileID: "fixture", Inputs: []promptexec.InputDefinition{{Name: "data_package", Required: true, ContentType: "application/yaml"}}, Output: promptexec.OutputContract{Format: "json", ValidationMode: "json_schema", SchemaPath: definition.GeneratedTextSchemaID + ".generated_text.schema.json"}}, nil
|
||||
}
|
||||
func (*generationExecutor) InspectProfile(_ context.Context, id string) (promptexec.ProfileInspection, error) {
|
||||
func (e *generationExecutor) InspectProfile(_ context.Context, id string) (promptexec.ProfileInspection, error) {
|
||||
generationExecutorMu.Lock()
|
||||
defer generationExecutorMu.Unlock()
|
||||
e.profileInspections++
|
||||
if err := e.profileInspectErrors[id]; err != nil {
|
||||
return promptexec.ProfileInspection{}, err
|
||||
}
|
||||
return promptexec.ProfileInspection{ProfileID: id, BackendID: "fixture", ModelName: "fixture-model"}, nil
|
||||
}
|
||||
func (e *generationExecutor) Execute(_ context.Context, req promptexec.ExecuteRequest, callback promptexec.PreparationCallback) (*promptexec.Execution, error) {
|
||||
stamp := time.Date(2026, 5, 29, 15, 0, 0, 0, time.UTC)
|
||||
if err := callback(promptexec.Preparation{PromptID: req.PromptID, PromptVersion: req.PromptVersion, PromptHash: "prompt-hash", RenderedPromptHash: "rendered-hash", ProfileID: req.ProfileID, BackendID: "fixture", ModelName: "fixture-model", StartedAt: stamp, EndedAt: stamp}, nil); err != nil {
|
||||
if err := callback(promptexec.Preparation{PromptID: req.PromptID, PromptVersion: req.PromptVersion, PromptHash: generationPromptHash, RenderedPromptHash: "rendered-hash", ProfileID: req.ProfileID, BackendID: "fixture", ModelName: "fixture-model", StartedAt: stamp, EndedAt: stamp}, nil); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
generationExecutorMu.Lock()
|
||||
e.called = true
|
||||
if e.executeErr != nil {
|
||||
return nil, e.executeErr
|
||||
}
|
||||
e.executeCalls++
|
||||
beforeExecute := e.beforeExecute
|
||||
profileErr := e.executeErrors[req.ProfileID]
|
||||
executeErr := e.executeErr
|
||||
status := e.validation
|
||||
rawOutput := append([]byte(nil), e.rawOutput...)
|
||||
failedPrompt := e.failedPrompt
|
||||
cancelBeforeReturn := e.cancelBeforeReturn
|
||||
generationExecutorMu.Unlock()
|
||||
if beforeExecute != nil {
|
||||
beforeExecute(req)
|
||||
}
|
||||
if profileErr != nil {
|
||||
return nil, profileErr
|
||||
}
|
||||
if executeErr != nil {
|
||||
return nil, executeErr
|
||||
}
|
||||
if status == "" {
|
||||
status = promptexec.ValidationPassed
|
||||
}
|
||||
if e.failedPrompt == req.PromptID {
|
||||
if failedPrompt == req.PromptID {
|
||||
status = promptexec.ValidationFailed
|
||||
}
|
||||
rawOutput := e.rawOutput
|
||||
if rawOutput == nil {
|
||||
rawOutput = []byte(`{"summary":"Showers are possible during the selected day.","forecast_discussion":["A front will keep rain chances in the forecast."],"precipitation_timing":"Rain is most likely during the afternoon."}`)
|
||||
}
|
||||
if e.cancelBeforeReturn != nil {
|
||||
e.cancelBeforeReturn()
|
||||
if cancelBeforeReturn != nil {
|
||||
cancelBeforeReturn()
|
||||
}
|
||||
return &promptexec.Execution{RunID: "provider-run", PromptID: req.PromptID, PromptVersion: req.PromptVersion, PromptHash: "prompt-hash", RenderedPromptHash: "rendered-hash", ProfileID: req.ProfileID, BackendID: "fixture", ModelName: "fixture-model", StartedAt: stamp, EndedAt: stamp, RawOutput: rawOutput, Validation: promptexec.NewValidation(status, "json_schema", generationDefinitionForPrompt(req.PromptID).GeneratedTextSchemaID+".generated_text.schema.json", nil)}, nil
|
||||
return &promptexec.Execution{RunID: "provider-run", PromptID: req.PromptID, PromptVersion: req.PromptVersion, PromptHash: generationPromptHash, RenderedPromptHash: "rendered-hash", ProfileID: req.ProfileID, BackendID: "fixture", ModelName: "fixture-model", StartedAt: stamp, EndedAt: stamp, RawOutput: rawOutput, Validation: promptexec.NewValidation(status, "json_schema", generationDefinitionForPrompt(req.PromptID).GeneratedTextSchemaID+".generated_text.schema.json", nil)}, nil
|
||||
}
|
||||
|
||||
const generationPromptHash = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"
|
||||
|
||||
func generationDefinitionForPrompt(promptID string) report.Definition {
|
||||
for _, definition := range report.DefaultRegistry().All() {
|
||||
if definition.PromptID == promptID {
|
||||
@@ -93,12 +127,13 @@ func TestGenerateDetailedPublishesOnlySelectedOutput(t *testing.T) {
|
||||
cfg.WeatherAPI.Timezone, cfg.Location.ID = "America/Chicago", "home"
|
||||
bundle := generationBundle(t)
|
||||
executor := &generationExecutor{}
|
||||
collector := &generationCollector{bundle: &bundle}
|
||||
workingDir := t.TempDir()
|
||||
result, err := GenerateDetailed(context.Background(), GenerateRequest{Config: cfg, Report: ReportDaily, Date: generationTime("2026-05-29T12:00:00-05:00"), Now: generationTime("2026-05-29T08:30:00-05:00"), WorkingDir: workingDir, Collector: &generationCollector{bundle: &bundle}, Executor: executor})
|
||||
result, err := GenerateDetailed(context.Background(), GenerateRequest{Config: cfg, Report: ReportDaily, Date: generationTime("2026-05-29T12:00:00-05:00"), Now: generationTime("2026-05-29T08:30:00-05:00"), WorkingDir: workingDir, Collector: collector, Executor: executor})
|
||||
if err != nil {
|
||||
t.Fatalf("GenerateDetailed() error = %v", err)
|
||||
}
|
||||
if !executor.called || result.OutputPath != filepath.Join(workingDir, "daily-2026-05-29.md") || result.ValidationStatus != promptexec.ValidationPassed || result.ProfileID == "" || result.BackendID == "" || result.ModelName == "" {
|
||||
if !executor.called || executor.executeCalls != 1 || collector.calls != 1 || result.OutputPath != filepath.Join(workingDir, "daily-2026-05-29.md") || result.ValidationStatus != promptexec.ValidationPassed || result.ProfileID == "" || result.BackendID == "" || result.ModelName == "" {
|
||||
t.Fatalf("result = %#v", result)
|
||||
}
|
||||
if result.LLMDebugPath != "" {
|
||||
@@ -113,6 +148,108 @@ func TestGenerateDetailedPublishesOnlySelectedOutput(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestGenerateDetailedUsesConfiguredOutputDirectory(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
directory func(t *testing.T, workingDir string) string
|
||||
wantDir func(t *testing.T, workingDir string, configuredDir string) string
|
||||
}{
|
||||
{
|
||||
name: "absolute directory",
|
||||
directory: func(t *testing.T, _ string) string {
|
||||
return filepath.Join(t.TempDir(), "reports")
|
||||
},
|
||||
wantDir: func(_ *testing.T, _ string, configuredDir string) string {
|
||||
return configuredDir
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "relative directory",
|
||||
directory: func(_ *testing.T, _ string) string {
|
||||
return "configured/../reports"
|
||||
},
|
||||
wantDir: func(_ *testing.T, workingDir string, _ string) string {
|
||||
return filepath.Join(workingDir, "reports")
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
workingDir := t.TempDir()
|
||||
configuredDir := tt.directory(t, workingDir)
|
||||
cfg := generationDistributorConfig()
|
||||
cfg.Output.Directory = configuredDir
|
||||
bundle := generationBundle(t)
|
||||
notifier := &generationNotifier{}
|
||||
|
||||
result, err := GenerateDetailed(context.Background(), GenerateRequest{
|
||||
Config: cfg, Report: ReportDaily,
|
||||
Date: generationTime("2026-05-29T12:00:00-05:00"), Now: generationTime("2026-05-29T08:30:00-05:00"),
|
||||
WorkingDir: workingDir, Collector: &generationCollector{bundle: &bundle}, Executor: &generationExecutor{}, Notifier: notifier,
|
||||
})
|
||||
wantPath := filepath.Join(tt.wantDir(t, workingDir, configuredDir), "daily-2026-05-29.md")
|
||||
if err != nil || result == nil || result.OutputPath != wantPath || notifier.request.ReportPath != wantPath {
|
||||
t.Fatalf("GenerateDetailed() result/error/notification = %#v/%v/%#v", result, err, notifier.request)
|
||||
}
|
||||
if info, statErr := os.Stat(filepath.Dir(wantPath)); statErr != nil || !info.IsDir() {
|
||||
t.Fatalf("configured output directory info/error = %#v/%v", info, statErr)
|
||||
}
|
||||
if _, statErr := os.Stat(wantPath); statErr != nil {
|
||||
t.Fatalf("output %q: %v", wantPath, statErr)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestGenerateDetailedExplicitOutputPathIgnoresConfiguredDirectory(t *testing.T) {
|
||||
configuredPath := filepath.Join(t.TempDir(), "not-a-directory")
|
||||
if err := os.WriteFile(configuredPath, []byte("not a directory"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
explicitPath := filepath.Join(t.TempDir(), "explicit.md")
|
||||
cfg := generationConfig()
|
||||
cfg.Output.Directory = configuredPath
|
||||
bundle := generationBundle(t)
|
||||
|
||||
result, err := GenerateDetailed(context.Background(), GenerateRequest{
|
||||
Config: cfg, Report: ReportDaily,
|
||||
Date: generationTime("2026-05-29T12:00:00-05:00"), Now: generationTime("2026-05-29T08:30:00-05:00"),
|
||||
WorkingDir: t.TempDir(), OutputPath: explicitPath, Collector: &generationCollector{bundle: &bundle}, Executor: &generationExecutor{},
|
||||
})
|
||||
if err != nil || result == nil || result.OutputPath != explicitPath {
|
||||
t.Fatalf("GenerateDetailed() result/error = %#v/%v", result, err)
|
||||
}
|
||||
if _, statErr := os.Stat(explicitPath); statErr != nil {
|
||||
t.Fatalf("explicit output %q: %v", explicitPath, statErr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGenerateDetailedRejectsConfiguredNonDirectoryBeforeWork(t *testing.T) {
|
||||
configuredPath := filepath.Join(t.TempDir(), "not-a-directory")
|
||||
if err := os.WriteFile(configuredPath, []byte("not a directory"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cfg := generationDistributorConfig()
|
||||
cfg.Output.Directory = configuredPath
|
||||
bundle := generationBundle(t)
|
||||
collector := &generationCollector{bundle: &bundle}
|
||||
executor := &generationExecutor{}
|
||||
notifier := &generationNotifier{}
|
||||
|
||||
result, err := GenerateDetailed(context.Background(), GenerateRequest{
|
||||
Config: cfg, Report: ReportDaily,
|
||||
Date: generationTime("2026-05-29T12:00:00-05:00"), Now: generationTime("2026-05-29T08:30:00-05:00"),
|
||||
WorkingDir: t.TempDir(), Collector: collector, Executor: executor, Notifier: notifier,
|
||||
})
|
||||
if err == nil || result == nil || collector.called || executor.promptInspections != 0 || executor.called || notifier.calls != 0 {
|
||||
t.Fatalf("GenerateDetailed() result/error/collector/executor/notifier = %#v/%v/%t/%#v/%#v", result, err, collector.called, executor, notifier)
|
||||
}
|
||||
if data, readErr := os.ReadFile(configuredPath); readErr != nil || string(data) != "not a directory" {
|
||||
t.Fatalf("configured path = %q, error = %v", data, readErr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGenerateDetailedReturnsResolvedResultWhenCollectionFails(t *testing.T) {
|
||||
cfg := config.Defaults()
|
||||
cfg.WeatherAPI.Timezone, cfg.Location.ID = "America/Chicago", "home"
|
||||
@@ -230,6 +367,24 @@ func TestGenerateDetailedDoesNotReplaceDirectoryOutput(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestGenerateDetailedWritesRequestedPromptDebugArtifacts(t *testing.T) {
|
||||
bundle := generationBundle(t)
|
||||
debugRoot := t.TempDir()
|
||||
result, err := GenerateDetailed(context.Background(), GenerateRequest{
|
||||
Config: generationConfig(), Report: ReportDaily,
|
||||
Date: generationTime("2026-05-29T12:00:00-05:00"), Now: generationTime("2026-05-29T08:30:00-05:00"),
|
||||
WorkingDir: t.TempDir(), LLMDebugDir: debugRoot, Collector: &generationCollector{bundle: &bundle}, Executor: &generationExecutor{},
|
||||
})
|
||||
if err != nil || result == nil || result.LLMDebugPath == "" {
|
||||
t.Fatalf("GenerateDetailed() result/error = %#v/%v", result, err)
|
||||
}
|
||||
for _, name := range []string{"preparation.json", "execution.json"} {
|
||||
if _, statErr := os.Stat(filepath.Join(result.LLMDebugPath, name)); statErr != nil {
|
||||
t.Fatalf("debug artifact %q: %v", name, statErr)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func generationConfig() config.Config {
|
||||
cfg := config.Defaults()
|
||||
cfg.WeatherAPI.Timezone, cfg.Location.ID = "America/Chicago", "home"
|
||||
@@ -244,14 +399,19 @@ func generationBundlePointer(t *testing.T) *weatherdata.Bundle {
|
||||
type generationNotifier struct {
|
||||
err error
|
||||
batchErr error
|
||||
calls int
|
||||
request NotificationRequest
|
||||
batchRequest batchNotificationRequest
|
||||
batchCalls int
|
||||
}
|
||||
|
||||
func (n *generationNotifier) Notify(_ context.Context, request NotificationRequest) (*NotificationResult, error) {
|
||||
n.calls++
|
||||
n.request = request
|
||||
return nil, n.err
|
||||
if n.err != nil {
|
||||
return nil, n.err
|
||||
}
|
||||
return &NotificationResult{Status: "succeeded"}, nil
|
||||
}
|
||||
|
||||
func (n *generationNotifier) NotifyBatch(_ context.Context, request batchNotificationRequest) (*NotificationResult, error) {
|
||||
|
||||
193
internal/app/output.go
Normal file
193
internal/app/output.go
Normal file
@@ -0,0 +1,193 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/comparison"
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/report"
|
||||
)
|
||||
|
||||
func plannedBatchOutputPath(outputDir string, planned plannedBatchReport) (string, error) {
|
||||
outputName, err := planned.Resolved.OutputName()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return validateOutputPath(filepath.Join(outputDir, outputName))
|
||||
}
|
||||
|
||||
func prepareBatchOutputs(outputDir string, plannedReports []plannedBatchReport) error {
|
||||
for index := range plannedReports {
|
||||
outputPath, err := plannedBatchOutputPath(outputDir, plannedReports[index])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
plannedReports[index].OutputPath = outputPath
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func resolveReportOutputPath(workingDir, override, configuredDir string, resolved report.Resolved) (string, error) {
|
||||
outputName, err := resolved.OutputName()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if override != "" {
|
||||
return resolveOutputPath(workingDir, override, outputName)
|
||||
}
|
||||
outputDir, err := resolveOutputDir(workingDir, configuredDir)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return validateOutputPath(filepath.Join(outputDir, outputName))
|
||||
}
|
||||
|
||||
func resolveOutputDirWithConfigured(workingDir, override, configuredDir string) (string, error) {
|
||||
directory := configuredDir
|
||||
if override != "" {
|
||||
directory = override
|
||||
}
|
||||
return resolveOutputDir(workingDir, directory)
|
||||
}
|
||||
|
||||
func resolveComparisonOutputDirectory(workingDir, override, configuredDir, reportOutputName string) (string, error) {
|
||||
workingDir, err := validateWorkingDir(workingDir)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if override != "" {
|
||||
return resolveComparisonDirectoryPath(workingDir, override)
|
||||
}
|
||||
outputDir, err := resolveOutputDir(workingDir, configuredDir)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
name, err := comparison.DefaultDirectoryName(reportOutputName)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return filepath.Join(outputDir, name), nil
|
||||
}
|
||||
|
||||
func resolveComparisonDirectoryPath(workingDir, directory string) (string, error) {
|
||||
if strings.TrimSpace(directory) == "" {
|
||||
return "", fmt.Errorf("comparison output directory is required")
|
||||
}
|
||||
if !filepath.IsAbs(directory) {
|
||||
directory = filepath.Join(workingDir, directory)
|
||||
}
|
||||
return filepath.Clean(directory), nil
|
||||
}
|
||||
|
||||
func resolveOutputDir(workingDir, override string) (string, error) {
|
||||
workingDir, err := validateWorkingDir(workingDir)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if override == "" {
|
||||
return workingDir, nil
|
||||
}
|
||||
if strings.TrimSpace(override) == "" {
|
||||
return "", fmt.Errorf("output directory is required")
|
||||
}
|
||||
directory := override
|
||||
if !filepath.IsAbs(directory) {
|
||||
directory = filepath.Join(workingDir, directory)
|
||||
}
|
||||
directory = filepath.Clean(directory)
|
||||
if err := preflightOutputDirectory(directory); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return directory, nil
|
||||
}
|
||||
|
||||
func preflightOutputDirectory(directory string) error {
|
||||
info, err := os.Stat(directory)
|
||||
if err == nil {
|
||||
if !info.IsDir() {
|
||||
return fmt.Errorf("output directory %q is not a directory", directory)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if !os.IsNotExist(err) {
|
||||
return fmt.Errorf("inspect output directory %q: %w", directory, err)
|
||||
}
|
||||
|
||||
// A missing directory is valid, but os.Stat also reports ErrNotExist for a
|
||||
// dangling symlink. Walk to the first existing component so invalid links
|
||||
// fail preflight instead of being discovered only during publication.
|
||||
for component := directory; ; component = filepath.Dir(component) {
|
||||
componentInfo, componentErr := os.Lstat(component)
|
||||
if componentErr == nil {
|
||||
if componentInfo.Mode()&os.ModeSymlink != 0 {
|
||||
targetInfo, targetErr := os.Stat(component)
|
||||
if targetErr != nil {
|
||||
return fmt.Errorf("inspect output directory %q at %q: %w", directory, component, targetErr)
|
||||
}
|
||||
if !targetInfo.IsDir() {
|
||||
return fmt.Errorf("output directory %q has non-directory path component %q", directory, component)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if !componentInfo.IsDir() {
|
||||
return fmt.Errorf("output directory %q has non-directory path component %q", directory, component)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if !os.IsNotExist(componentErr) {
|
||||
return fmt.Errorf("inspect output directory %q at %q: %w", directory, component, componentErr)
|
||||
}
|
||||
if filepath.Dir(component) == component {
|
||||
return fmt.Errorf("inspect output directory %q: no existing directory ancestor", directory)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func resolveOutputPath(workingDir, override, defaultName string) (string, error) {
|
||||
workingDir, err := validateWorkingDir(workingDir)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
path := override
|
||||
if path == "" {
|
||||
path = defaultName
|
||||
}
|
||||
if strings.TrimSpace(path) == "" {
|
||||
return "", fmt.Errorf("final output path is required")
|
||||
}
|
||||
if !filepath.IsAbs(path) {
|
||||
path = filepath.Join(workingDir, path)
|
||||
}
|
||||
return validateOutputPath(path)
|
||||
}
|
||||
|
||||
func validateWorkingDir(workingDir string) (string, error) {
|
||||
if strings.TrimSpace(workingDir) == "" {
|
||||
return "", fmt.Errorf("working directory is required")
|
||||
}
|
||||
if !filepath.IsAbs(workingDir) {
|
||||
return "", fmt.Errorf("working directory %q must be absolute", workingDir)
|
||||
}
|
||||
return filepath.Clean(workingDir), nil
|
||||
}
|
||||
|
||||
func validateOutputPath(path string) (string, error) {
|
||||
if strings.TrimSpace(path) == "" {
|
||||
return "", fmt.Errorf("final output path is required")
|
||||
}
|
||||
path = filepath.Clean(path)
|
||||
if !filepath.IsAbs(path) {
|
||||
return "", fmt.Errorf("final output path %q must be absolute", path)
|
||||
}
|
||||
if filepath.Dir(path) == path {
|
||||
return "", fmt.Errorf("final output path %q must not be a filesystem root", path)
|
||||
}
|
||||
if info, err := os.Stat(path); err == nil && info.IsDir() {
|
||||
return "", fmt.Errorf("final output path %q is a directory", path)
|
||||
} else if err != nil && !os.IsNotExist(err) {
|
||||
return "", fmt.Errorf("inspect final output path %q: %w", path, err)
|
||||
}
|
||||
return path, nil
|
||||
}
|
||||
75
internal/app/output_test.go
Normal file
75
internal/app/output_test.go
Normal file
@@ -0,0 +1,75 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestResolveComparisonOutputDirectory(t *testing.T) {
|
||||
workingDir := t.TempDir()
|
||||
configured := filepath.Join(workingDir, "configured")
|
||||
blocked := filepath.Join(workingDir, "not-a-directory")
|
||||
if err := os.WriteFile(blocked, []byte("blocked"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
override string
|
||||
configuredDir string
|
||||
reportOutputName string
|
||||
want string
|
||||
wantErr bool
|
||||
}{
|
||||
{name: "working directory default", reportOutputName: "today.md", want: filepath.Join(workingDir, "comparison-today")},
|
||||
{name: "configured relative directory", configuredDir: "configured", reportOutputName: "tomorrow.md", want: filepath.Join(configured, "comparison-tomorrow")},
|
||||
{name: "configured absolute directory", configuredDir: configured, reportOutputName: "hourly.md", want: filepath.Join(configured, "comparison-hourly")},
|
||||
{name: "relative explicit directory", override: "exact", configuredDir: blocked, reportOutputName: "daily-2026-08-24.md", want: filepath.Join(workingDir, "exact")},
|
||||
{name: "absolute explicit directory", override: filepath.Join(workingDir, "absolute"), reportOutputName: "today.md", want: filepath.Join(workingDir, "absolute")},
|
||||
{name: "invalid report suffix", reportOutputName: "today.txt", wantErr: true},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
got, err := resolveComparisonOutputDirectory(workingDir, test.override, test.configuredDir, test.reportOutputName)
|
||||
if (err != nil) != test.wantErr {
|
||||
t.Fatalf("resolveComparisonOutputDirectory() error = %v, want error %t", err, test.wantErr)
|
||||
}
|
||||
if !test.wantErr && got != test.want {
|
||||
t.Fatalf("resolveComparisonOutputDirectory() = %q, want %q", got, test.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveOutputDirRejectsDanglingSymlinkComponents(t *testing.T) {
|
||||
workingDir := t.TempDir()
|
||||
dangling := filepath.Join(workingDir, "dangling")
|
||||
if err := os.Symlink(filepath.Join(workingDir, "missing"), dangling); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
for _, directory := range []string{dangling, filepath.Join(dangling, "reports")} {
|
||||
t.Run(filepath.Base(directory), func(t *testing.T) {
|
||||
if _, err := resolveOutputDir(workingDir, directory); err == nil {
|
||||
t.Fatalf("resolveOutputDir(%q) error = nil, want dangling symlink error", directory)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveOutputDirAllowsMissingDirectoryBelowValidSymlink(t *testing.T) {
|
||||
workingDir := t.TempDir()
|
||||
target := t.TempDir()
|
||||
link := filepath.Join(workingDir, "linked")
|
||||
if err := os.Symlink(target, link); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
directory := filepath.Join(link, "reports")
|
||||
got, err := resolveOutputDir(workingDir, directory)
|
||||
if err != nil || got != directory {
|
||||
t.Fatalf("resolveOutputDir() = %q, %v, want %q, nil", got, err, directory)
|
||||
}
|
||||
}
|
||||
150
internal/app/prepared_report.go
Normal file
150
internal/app/prepared_report.go
Normal file
@@ -0,0 +1,150 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/briefing"
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/collect"
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/config"
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/generatedtext"
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/module"
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptinput"
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/report"
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/weatherdata"
|
||||
)
|
||||
|
||||
// preparedReport contains the immutable deterministic inputs shared by prompt
|
||||
// executions for one resolved report.
|
||||
type preparedReport struct {
|
||||
resolved report.Resolved
|
||||
reportFacts ReportFacts
|
||||
moduleSnapshot module.Snapshot
|
||||
briefingMetadata briefing.Metadata
|
||||
sourceWarnings []weatherdata.SourceWarning
|
||||
dataPackage []byte
|
||||
handler generatedtext.Handler
|
||||
}
|
||||
|
||||
type prepareReportRequest struct {
|
||||
Config config.Config
|
||||
Resolved report.Resolved
|
||||
Collection collect.Result
|
||||
}
|
||||
|
||||
type preparationError struct {
|
||||
operation string
|
||||
err error
|
||||
}
|
||||
|
||||
func (e *preparationError) Error() string {
|
||||
return e.operation + ": " + e.err.Error()
|
||||
}
|
||||
|
||||
func (e *preparationError) Unwrap() error {
|
||||
return e.err
|
||||
}
|
||||
|
||||
func prepareReport(req prepareReportRequest) (preparedReport, error) {
|
||||
if req.Collection.Bundle == nil {
|
||||
return preparedReport{}, &preparationError{operation: "prepare report", err: fmt.Errorf("collected weather bundle is required")}
|
||||
}
|
||||
|
||||
reportFacts, err := BuildReportFacts(ModuleSnapshotRequest{Config: req.Config, Resolved: req.Resolved}, req.Collection.Bundle)
|
||||
if err != nil {
|
||||
return preparedReport{}, &preparationError{operation: "build report facts", err: err}
|
||||
}
|
||||
moduleSnapshot, err := BuildModuleSnapshotFromFacts(ModuleSnapshotRequest{Config: req.Config, Resolved: req.Resolved}, reportFacts)
|
||||
if err != nil {
|
||||
return preparedReport{}, &preparationError{operation: "build module snapshot", err: err}
|
||||
}
|
||||
metadata := briefing.BuildMetadata(briefingBuildContext(req.Config, req.Resolved, reportFacts.Collected))
|
||||
dataPackage, err := promptinput.Build(promptinput.BuildRequest{Metadata: promptMetadata(metadata), Modules: moduleSnapshot})
|
||||
if err != nil {
|
||||
return preparedReport{}, &preparationError{operation: "build data package", err: err}
|
||||
}
|
||||
serializedDataPackage, err := promptinput.MarshalYAML(dataPackage)
|
||||
if err != nil {
|
||||
return preparedReport{}, &preparationError{operation: "marshal data package", err: err}
|
||||
}
|
||||
handler, err := generatedtext.LookupDefinition(req.Resolved.Definition)
|
||||
if err != nil {
|
||||
return preparedReport{}, &preparationError{operation: "lookup generated text catalog", err: err}
|
||||
}
|
||||
|
||||
clonedFacts, err := clonePreparedValue(reportFacts)
|
||||
if err != nil {
|
||||
return preparedReport{}, &preparationError{operation: "copy prepared report facts", err: err}
|
||||
}
|
||||
clonedSnapshot, err := clonePreparedValue(moduleSnapshot)
|
||||
if err != nil {
|
||||
return preparedReport{}, &preparationError{operation: "copy prepared module snapshot", err: err}
|
||||
}
|
||||
clonedMetadata, err := clonePreparedValue(metadata)
|
||||
if err != nil {
|
||||
return preparedReport{}, &preparationError{operation: "copy prepared briefing metadata", err: err}
|
||||
}
|
||||
prepared := preparedReport{
|
||||
resolved: cloneResolved(req.Resolved),
|
||||
reportFacts: clonedFacts,
|
||||
moduleSnapshot: clonedSnapshot,
|
||||
briefingMetadata: clonedMetadata,
|
||||
sourceWarnings: append([]weatherdata.SourceWarning(nil), clonedMetadata.SourceWarnings...),
|
||||
dataPackage: append([]byte(nil), serializedDataPackage...),
|
||||
handler: handler,
|
||||
}
|
||||
return prepared, nil
|
||||
}
|
||||
|
||||
func cloneResolved(value report.Resolved) report.Resolved {
|
||||
cloned := value
|
||||
cloned.Definition.DistributorPathTemplates = append([]string(nil), value.Definition.DistributorPathTemplates...)
|
||||
cloned.Definition.Modules = make([]module.ConfigItem, len(value.Definition.Modules))
|
||||
for i, item := range value.Definition.Modules {
|
||||
cloned.Definition.Modules[i] = item
|
||||
switch options := item.Options.(type) {
|
||||
case module.AreaForecastDiscussionOptions:
|
||||
options.Sections = append([]string(nil), options.Sections...)
|
||||
cloned.Definition.Modules[i].Options = options
|
||||
}
|
||||
}
|
||||
return cloned
|
||||
}
|
||||
|
||||
func (p preparedReport) dataPackageCopy() []byte {
|
||||
return append([]byte(nil), p.dataPackage...)
|
||||
}
|
||||
|
||||
func (p preparedReport) sourceWarningsCopy() []weatherdata.SourceWarning {
|
||||
return append([]weatherdata.SourceWarning(nil), p.sourceWarnings...)
|
||||
}
|
||||
|
||||
func (p preparedReport) renderInputs() (briefing.Metadata, module.Snapshot, ReportFacts, error) {
|
||||
metadata, err := clonePreparedValue(p.briefingMetadata)
|
||||
if err != nil {
|
||||
return briefing.Metadata{}, module.Snapshot{}, ReportFacts{}, err
|
||||
}
|
||||
snapshot, err := clonePreparedValue(p.moduleSnapshot)
|
||||
if err != nil {
|
||||
return briefing.Metadata{}, module.Snapshot{}, ReportFacts{}, err
|
||||
}
|
||||
reportFacts, err := clonePreparedValue(p.reportFacts)
|
||||
if err != nil {
|
||||
return briefing.Metadata{}, module.Snapshot{}, ReportFacts{}, err
|
||||
}
|
||||
return metadata, snapshot, reportFacts, nil
|
||||
}
|
||||
|
||||
func clonePreparedValue[T any](value T) (T, error) {
|
||||
encoded, err := json.Marshal(value)
|
||||
if err != nil {
|
||||
var zero T
|
||||
return zero, fmt.Errorf("marshal immutable prepared value: %w", err)
|
||||
}
|
||||
var cloned T
|
||||
if err := json.Unmarshal(encoded, &cloned); err != nil {
|
||||
var zero T
|
||||
return zero, fmt.Errorf("unmarshal immutable prepared value: %w", err)
|
||||
}
|
||||
return cloned, nil
|
||||
}
|
||||
64
internal/app/prepared_report_test.go
Normal file
64
internal/app/prepared_report_test.go
Normal file
@@ -0,0 +1,64 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/collect"
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/weatherdata"
|
||||
)
|
||||
|
||||
func TestPrepareReportBuildsImmutableDeterministicInputs(t *testing.T) {
|
||||
cfg := generationConfig()
|
||||
bundle := generationBundle(t)
|
||||
resolved, err := ResolveGenerate(GenerateRequest{
|
||||
Config: cfg, Report: ReportDaily,
|
||||
Date: generationTime("2026-05-29T12:00:00-05:00"), Now: generationTime("2026-05-29T08:30:00-05:00"),
|
||||
}, generationTime("2026-05-29T08:30:00-05:00"))
|
||||
if err != nil {
|
||||
t.Fatalf("ResolveGenerate() error = %v", err)
|
||||
}
|
||||
|
||||
request := prepareReportRequest{Config: cfg, Resolved: resolved, Collection: collect.Result{Bundle: &bundle}}
|
||||
prepared, err := prepareReport(request)
|
||||
if err != nil {
|
||||
t.Fatalf("prepareReport() error = %v", err)
|
||||
}
|
||||
repeated, err := prepareReport(request)
|
||||
if err != nil {
|
||||
t.Fatalf("second prepareReport() error = %v", err)
|
||||
}
|
||||
if len(prepared.dataPackage) == 0 || !bytes.Equal(prepared.dataPackage, repeated.dataPackage) || !reflect.DeepEqual(prepared.briefingMetadata, repeated.briefingMetadata) {
|
||||
t.Fatalf("prepared package/metadata are not deterministic: %q/%#v", prepared.dataPackage, prepared.briefingMetadata)
|
||||
}
|
||||
|
||||
originalDataPackage := append([]byte(nil), prepared.dataPackage...)
|
||||
originalMetadata := prepared.briefingMetadata
|
||||
originalWarnings := append([]weatherdata.SourceWarning(nil), prepared.sourceWarnings...)
|
||||
metadata, snapshot, reportFacts, err := prepared.renderInputs()
|
||||
if err != nil {
|
||||
t.Fatalf("renderInputs() error = %v", err)
|
||||
}
|
||||
metadata.SourceWarnings = append(metadata.SourceWarnings, weatherdata.SourceWarning{Source: "test", Message: "consumer mutation"})
|
||||
snapshot.Outputs = nil
|
||||
reportFacts.Collected.Hourly.Periods[0].TextDescription = "consumer mutation"
|
||||
bundle.Hourly.Periods[0].TextDescription = "mutated after preparation"
|
||||
bundle.Warnings = append(bundle.Warnings, weatherdata.SourceWarning{Source: "test", Message: "mutated warning"})
|
||||
if len(bundle.Sources) > 0 {
|
||||
if bundle.Sources[0].Query == nil {
|
||||
bundle.Sources[0].Query = map[string]string{}
|
||||
}
|
||||
bundle.Sources[0].Query["mutated"] = "true"
|
||||
}
|
||||
|
||||
if !bytes.Equal(prepared.dataPackage, originalDataPackage) || !reflect.DeepEqual(prepared.briefingMetadata, originalMetadata) || !reflect.DeepEqual(prepared.sourceWarnings, originalWarnings) {
|
||||
t.Fatalf("prepared values changed after caller mutation: %#v", prepared)
|
||||
}
|
||||
if prepared.reportFacts.Collected.Hourly.Periods[0].TextDescription == "mutated after preparation" {
|
||||
t.Fatal("prepared report facts retain caller-owned weather data")
|
||||
}
|
||||
if prepared.reportFacts.Collected.Hourly.Periods[0].TextDescription == "consumer mutation" || len(prepared.moduleSnapshot.Outputs) == 0 {
|
||||
t.Fatal("prepared report values retain consumer mutation")
|
||||
}
|
||||
}
|
||||
127
internal/app/profile_execution.go
Normal file
127
internal/app/profile_execution.go
Normal file
@@ -0,0 +1,127 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptdebug"
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec"
|
||||
)
|
||||
|
||||
type profileExecutionRequest struct {
|
||||
Prepared preparedReport
|
||||
Prompt PromptInspectionResult
|
||||
Profile promptexec.ProfileInspection
|
||||
Executor promptexec.Executor
|
||||
DebugWriter *promptdebug.PromptDebugWriter
|
||||
DebugRef *promptdebug.PromptDebugRef
|
||||
}
|
||||
|
||||
type profileExecutionOutcome struct {
|
||||
ProfileID string
|
||||
BackendID string
|
||||
ModelName string
|
||||
ValidationStatus promptexec.ValidationStatus
|
||||
LLMDebugPath string
|
||||
}
|
||||
|
||||
type profileExecutionError struct {
|
||||
operation string
|
||||
err error
|
||||
callbackFailure bool
|
||||
}
|
||||
|
||||
func (e *profileExecutionError) Error() string {
|
||||
return e.operation + ": " + e.err.Error()
|
||||
}
|
||||
|
||||
func (e *profileExecutionError) Unwrap() error {
|
||||
return e.err
|
||||
}
|
||||
|
||||
func executePreparedProfile(ctx context.Context, req profileExecutionRequest) (profileExecutionOutcome, []byte, error) {
|
||||
outcome := profileExecutionOutcome{
|
||||
ProfileID: req.Profile.ProfileID,
|
||||
BackendID: req.Profile.BackendID,
|
||||
ModelName: req.Profile.ModelName,
|
||||
}
|
||||
if req.Executor == nil {
|
||||
return outcome, nil, &profileExecutionError{operation: "execute prompt", err: promptexec.NewError(promptexec.InvalidConfiguration, "prompt executor is required", nil)}
|
||||
}
|
||||
|
||||
callbackFailed := false
|
||||
preparationCallback := func(preparation promptexec.Preparation, debug *promptexec.PreparationDebug) error {
|
||||
outcome.ProfileID, outcome.BackendID, outcome.ModelName = preparation.ProfileID, preparation.BackendID, preparation.ModelName
|
||||
if req.DebugWriter == nil || !req.DebugWriter.Enabled() {
|
||||
return nil
|
||||
}
|
||||
if req.DebugRef == nil {
|
||||
callbackFailed = true
|
||||
return promptDebugWriteError(fmt.Errorf("prompt debug reference is required"))
|
||||
}
|
||||
path, err := req.DebugWriter.WritePreparation(*req.DebugRef, preparation, debug)
|
||||
if err != nil {
|
||||
callbackFailed = true
|
||||
return promptDebugWriteError(err)
|
||||
}
|
||||
outcome.LLMDebugPath = path
|
||||
return nil
|
||||
}
|
||||
|
||||
captureDebug := req.DebugWriter != nil && req.DebugWriter.Enabled()
|
||||
execution, err := req.Executor.Execute(ctx, promptexec.ExecuteRequest{
|
||||
PromptID: req.Prompt.PromptID,
|
||||
PromptVersion: req.Prompt.PromptVersion,
|
||||
ProfileID: req.Profile.ProfileID,
|
||||
DataPackage: req.Prepared.dataPackageCopy(),
|
||||
CaptureDebug: captureDebug,
|
||||
}, preparationCallback)
|
||||
if err != nil {
|
||||
if callbackFailed {
|
||||
return outcome, nil, &profileExecutionError{operation: "execute prompt", err: err, callbackFailure: true}
|
||||
}
|
||||
return outcome, nil, &profileExecutionError{operation: "execute prompt", err: classifiedPromptError("prompt execution failed", err)}
|
||||
}
|
||||
if execution == nil {
|
||||
return outcome, nil, &profileExecutionError{operation: "execute prompt", err: promptexec.NewError(promptexec.Generation, "prompt executor returned no execution", nil)}
|
||||
}
|
||||
|
||||
outcome.ValidationStatus = execution.Validation.Status
|
||||
if req.DebugWriter != nil && req.DebugWriter.Enabled() {
|
||||
if req.DebugRef == nil {
|
||||
return outcome, nil, &profileExecutionError{operation: "write prompt debug", err: promptDebugWriteError(fmt.Errorf("prompt debug reference is required"))}
|
||||
}
|
||||
path, err := req.DebugWriter.WriteExecution(*req.DebugRef, *execution)
|
||||
if err != nil {
|
||||
return outcome, nil, &profileExecutionError{operation: "write prompt debug", err: promptDebugWriteError(err)}
|
||||
}
|
||||
if path != "" {
|
||||
outcome.LLMDebugPath = path
|
||||
}
|
||||
}
|
||||
|
||||
if execution.Validation.Status != promptexec.ValidationPassed && execution.Validation.Status != promptexec.ValidationFailed {
|
||||
return outcome, nil, &profileExecutionError{operation: "validate prompt execution", err: promptexec.NewError(promptexec.OperationalValidation, "prompt execution did not complete validation", nil)}
|
||||
}
|
||||
if execution.Validation.Status == promptexec.ValidationFailed {
|
||||
return outcome, nil, &profileExecutionError{operation: "validate prompt execution", err: promptexec.NewError(promptexec.ValidationRejected, "prompt output did not satisfy its schema", nil)}
|
||||
}
|
||||
|
||||
generatedText, _, err := req.Prepared.handler.Validate(execution.RawOutput)
|
||||
if err != nil {
|
||||
return outcome, nil, &profileExecutionError{operation: "validate generated text", err: err}
|
||||
}
|
||||
metadata, snapshot, reportFacts, err := req.Prepared.renderInputs()
|
||||
if err != nil {
|
||||
return outcome, nil, &profileExecutionError{operation: "copy prepared render inputs", err: err}
|
||||
}
|
||||
renderContext, err := req.Prepared.handler.BuildRenderContext(metadata, snapshot, reportFacts.Collected, reportFacts.Derived, generatedText)
|
||||
if err != nil {
|
||||
return outcome, nil, &profileExecutionError{operation: "build render context", err: err}
|
||||
}
|
||||
rendered, err := req.Prepared.handler.Render(renderContext)
|
||||
if err != nil {
|
||||
return outcome, nil, &profileExecutionError{operation: "render template", err: err}
|
||||
}
|
||||
return outcome, rendered, nil
|
||||
}
|
||||
70
internal/app/profile_execution_test.go
Normal file
70
internal/app/profile_execution_test.go
Normal file
@@ -0,0 +1,70 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/collect"
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptdebug"
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec"
|
||||
)
|
||||
|
||||
func TestExecutePreparedProfileRendersWithoutPublishing(t *testing.T) {
|
||||
prepared, inspection := preparedDailyProfile(t)
|
||||
executor := &generationExecutor{}
|
||||
outputPath := filepath.Join(t.TempDir(), "report.md")
|
||||
outcome, rendered, err := executePreparedProfile(context.Background(), profileExecutionRequest{
|
||||
Prepared: prepared, Prompt: inspection,
|
||||
Profile: promptexec.ProfileInspection{ProfileID: inspection.ProfileID, BackendID: inspection.BackendID, ModelName: inspection.ModelName},
|
||||
Executor: executor,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("executePreparedProfile() error = %v", err)
|
||||
}
|
||||
if len(rendered) == 0 || outcome.ValidationStatus != promptexec.ValidationPassed || outcome.ProfileID != inspection.ProfileID || executor.executeCalls != 1 {
|
||||
t.Fatalf("outcome/rendered/execution calls = %#v/%q/%d", outcome, rendered, executor.executeCalls)
|
||||
}
|
||||
if _, statErr := os.Stat(outputPath); !os.IsNotExist(statErr) {
|
||||
t.Fatalf("execution unexpectedly published %q: %v", outputPath, statErr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecutePreparedProfileKeepsDebugCallbackFailureLocal(t *testing.T) {
|
||||
prepared, inspection := preparedDailyProfile(t)
|
||||
debugWriter, err := promptdebug.NewPromptDebugWriter(t.TempDir())
|
||||
if err != nil {
|
||||
t.Fatalf("NewPromptDebugWriter() error = %v", err)
|
||||
}
|
||||
executor := &generationExecutor{}
|
||||
outcome, rendered, err := executePreparedProfile(context.Background(), profileExecutionRequest{
|
||||
Prepared: prepared, Prompt: inspection,
|
||||
Profile: promptexec.ProfileInspection{ProfileID: inspection.ProfileID, BackendID: inspection.BackendID, ModelName: inspection.ModelName},
|
||||
Executor: executor, DebugWriter: debugWriter,
|
||||
DebugRef: &promptdebug.PromptDebugRef{ReportID: inspectionResolved(t).Definition.ID, ValidDate: "2026-05-29", RunID: "invalid/path"},
|
||||
})
|
||||
var executionErr *profileExecutionError
|
||||
if err == nil || !errors.As(err, &executionErr) || !executionErr.callbackFailure || promptexec.CategoryOf(err) != promptexec.InvalidConfiguration || len(rendered) != 0 || executor.executeCalls != 0 || outcome.LLMDebugPath != "" {
|
||||
t.Fatalf("outcome/rendered/error/execution calls = %#v/%q/%v/%d", outcome, rendered, err, executor.executeCalls)
|
||||
}
|
||||
}
|
||||
|
||||
func preparedDailyProfile(t *testing.T) (preparedReport, PromptInspectionResult) {
|
||||
t.Helper()
|
||||
cfg := generationConfig()
|
||||
resolved, err := ResolveGenerate(GenerateRequest{
|
||||
Config: cfg, Report: ReportDaily,
|
||||
Date: generationTime("2026-05-29T12:00:00-05:00"), Now: generationTime("2026-05-29T08:30:00-05:00"),
|
||||
}, generationTime("2026-05-29T08:30:00-05:00"))
|
||||
if err != nil {
|
||||
t.Fatalf("ResolveGenerate() error = %v", err)
|
||||
}
|
||||
bundle := generationBundle(t)
|
||||
prepared, err := prepareReport(prepareReportRequest{Config: cfg, Resolved: resolved, Collection: collect.Result{Bundle: &bundle}})
|
||||
if err != nil {
|
||||
t.Fatalf("prepareReport() error = %v", err)
|
||||
}
|
||||
return prepared, PromptInspectionResult{PromptID: resolved.Definition.PromptID, PromptVersion: resolved.Definition.PromptVersion, PromptHash: "prompt-hash", ProfileID: "fixture", BackendID: "fixture", ModelName: "fixture-model"}
|
||||
}
|
||||
@@ -5,16 +5,11 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/briefing"
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/collect"
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/fileutil"
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/generatedtext"
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/module"
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptdebug"
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec"
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptinput"
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/report"
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/weatherdata"
|
||||
)
|
||||
|
||||
type promptReportRequest struct {
|
||||
@@ -27,51 +22,7 @@ type promptReportRequest struct {
|
||||
noNotify bool
|
||||
}
|
||||
|
||||
type promptReportWorkflow struct {
|
||||
ctx context.Context
|
||||
req promptReportRequest
|
||||
result *ReportResult
|
||||
briefingMetadata briefing.Metadata
|
||||
reportFacts ReportFacts
|
||||
moduleSnapshot module.Snapshot
|
||||
dataPackage []byte
|
||||
handler generatedtext.Handler
|
||||
debugRef promptdebug.PromptDebugRef
|
||||
callbackFailed bool
|
||||
}
|
||||
|
||||
func generatePromptReport(ctx context.Context, req promptReportRequest) (*ReportResult, error) {
|
||||
workflow, err := newPromptReportWorkflow(ctx, req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := workflow.buildInputs(); err != nil {
|
||||
return workflow.result, err
|
||||
}
|
||||
execution, err := workflow.executePrompt()
|
||||
if err != nil {
|
||||
if workflow.callbackFailed {
|
||||
return workflow.result, err
|
||||
}
|
||||
return workflow.result, workflow.reportError("execute prompt", classifiedPromptError("prompt execution failed", err))
|
||||
}
|
||||
if execution == nil {
|
||||
return workflow.result, workflow.reportError("execute prompt", promptexec.NewError(promptexec.Generation, "prompt executor returned no execution", nil))
|
||||
}
|
||||
workflow.result.ValidationStatus = execution.Validation.Status
|
||||
if err := workflow.writeExecutionDebug(*execution); err != nil {
|
||||
return workflow.result, err
|
||||
}
|
||||
if execution.Validation.Status != promptexec.ValidationPassed && execution.Validation.Status != promptexec.ValidationFailed {
|
||||
return workflow.result, workflow.reportError("validate prompt execution", promptexec.NewError(promptexec.OperationalValidation, "prompt execution did not complete validation", nil))
|
||||
}
|
||||
if execution.Validation.Status == promptexec.ValidationFailed {
|
||||
return workflow.result, workflow.reportError("validate prompt execution", promptexec.NewError(promptexec.ValidationRejected, "prompt output did not satisfy its schema", nil))
|
||||
}
|
||||
return workflow.renderAndPublish(execution.RawOutput)
|
||||
}
|
||||
|
||||
func newPromptReportWorkflow(ctx context.Context, req promptReportRequest) (*promptReportWorkflow, error) {
|
||||
if req.Collection.Bundle == nil {
|
||||
return nil, fmt.Errorf("collected weather bundle is required")
|
||||
}
|
||||
@@ -79,10 +30,36 @@ func newPromptReportWorkflow(ctx context.Context, req promptReportRequest) (*pro
|
||||
if result == nil {
|
||||
result = initialReportResult(req.GenerateRequest, req.Resolved, req.Inspection)
|
||||
}
|
||||
return &promptReportWorkflow{
|
||||
ctx: ctx, req: req,
|
||||
result: result,
|
||||
}, nil
|
||||
prepared, err := prepareReport(prepareReportRequest{Config: req.Config, Resolved: req.Resolved, Collection: req.Collection})
|
||||
if err != nil {
|
||||
return result, generatedPreparationError(req.Resolved, result.RunID, err)
|
||||
}
|
||||
result.SourceWarnings = prepared.sourceWarningsCopy()
|
||||
debugRef := promptdebug.PromptDebugRef{ReportID: result.ReportID, ValidDate: prepared.resolved.ValidPeriod.Start.Format("2006-01-02"), RunID: result.RunID}
|
||||
outcome, rendered, err := executePreparedProfile(ctx, profileExecutionRequest{
|
||||
Prepared: prepared,
|
||||
Prompt: req.Inspection,
|
||||
Profile: promptexec.ProfileInspection{
|
||||
ProfileID: req.Inspection.ProfileID,
|
||||
BackendID: req.Inspection.BackendID,
|
||||
ModelName: req.Inspection.ModelName,
|
||||
},
|
||||
Executor: req.Executor, DebugWriter: req.DebugWriter, DebugRef: &debugRef,
|
||||
})
|
||||
result.ProfileID, result.BackendID, result.ModelName = outcome.ProfileID, outcome.BackendID, outcome.ModelName
|
||||
result.ValidationStatus = outcome.ValidationStatus
|
||||
result.LLMDebugPath = outcome.LLMDebugPath
|
||||
if err != nil {
|
||||
return result, generatedProfileExecutionError(req.Resolved, result.RunID, err)
|
||||
}
|
||||
return publishPromptReport(ctx, promptPublicationRequest{
|
||||
GenerateRequest: req.GenerateRequest,
|
||||
Resolved: req.Resolved,
|
||||
OutputPath: req.OutputPath,
|
||||
Result: result,
|
||||
Markdown: rendered,
|
||||
suppressNotification: req.noNotify,
|
||||
})
|
||||
}
|
||||
|
||||
func initialReportResult(req GenerateRequest, resolved report.Resolved, inspection PromptInspectionResult) *ReportResult {
|
||||
@@ -96,100 +73,51 @@ func initialReportResult(req GenerateRequest, resolved report.Resolved, inspecti
|
||||
}
|
||||
}
|
||||
|
||||
func (w *promptReportWorkflow) buildInputs() error {
|
||||
var err error
|
||||
w.reportFacts, err = BuildReportFacts(ModuleSnapshotRequest{Config: w.req.Config, Resolved: w.req.Resolved}, w.req.Collection.Bundle)
|
||||
if err != nil {
|
||||
return w.reportError("build report facts", err)
|
||||
}
|
||||
w.moduleSnapshot, err = BuildModuleSnapshotFromFacts(ModuleSnapshotRequest{Config: w.req.Config, Resolved: w.req.Resolved}, w.reportFacts)
|
||||
if err != nil {
|
||||
return w.reportError("build module snapshot", err)
|
||||
}
|
||||
w.briefingMetadata = briefing.BuildMetadata(briefingBuildContext(w.req.Config, w.req.Resolved, w.reportFacts.Collected))
|
||||
w.result.SourceWarnings = append([]weatherdata.SourceWarning(nil), w.briefingMetadata.SourceWarnings...)
|
||||
dataPackage, err := promptinput.Build(promptinput.BuildRequest{Metadata: promptMetadata(w.briefingMetadata), Modules: w.moduleSnapshot})
|
||||
if err != nil {
|
||||
return w.reportError("build data package", err)
|
||||
}
|
||||
w.dataPackage, err = promptinput.MarshalYAML(dataPackage)
|
||||
if err != nil {
|
||||
return w.reportError("marshal data package", err)
|
||||
}
|
||||
w.handler, err = generatedtext.LookupDefinition(w.req.Resolved.Definition)
|
||||
if err != nil {
|
||||
return w.reportError("lookup generated text catalog", err)
|
||||
}
|
||||
w.debugRef = promptdebug.PromptDebugRef{ReportID: w.result.ReportID, ValidDate: w.req.Resolved.ValidPeriod.Start.Format("2006-01-02"), RunID: w.result.RunID}
|
||||
return nil
|
||||
type promptPublicationRequest struct {
|
||||
GenerateRequest
|
||||
Resolved report.Resolved
|
||||
OutputPath string
|
||||
Result *ReportResult
|
||||
Markdown []byte
|
||||
suppressNotification bool
|
||||
}
|
||||
|
||||
func (w *promptReportWorkflow) executePrompt() (*promptexec.Execution, error) {
|
||||
captureDebug := w.req.DebugWriter != nil && w.req.DebugWriter.Enabled()
|
||||
return w.req.Executor.Execute(w.ctx, promptexec.ExecuteRequest{PromptID: w.req.Inspection.PromptID, PromptVersion: w.req.Inspection.PromptVersion, ProfileID: w.req.Inspection.ProfileID, DataPackage: w.dataPackage, CaptureDebug: captureDebug}, w.writePreparationDebug)
|
||||
func publishPromptReport(ctx context.Context, req promptPublicationRequest) (*ReportResult, error) {
|
||||
if err := publicationContextError(ctx); err != nil {
|
||||
return req.Result, generatedReportError(req.Resolved, req.Result.RunID, "publish report", err)
|
||||
}
|
||||
if err := fileutil.WriteFileAtomic(req.OutputPath, req.Markdown); err != nil {
|
||||
return req.Result, err
|
||||
}
|
||||
req.Result.OutputPath = req.OutputPath
|
||||
if req.suppressNotification {
|
||||
return req.Result, nil
|
||||
}
|
||||
notification, err := notifyReport(ctx, req.Config, req.Resolved, req.Result.OutputPath, req.Result.RunID, req.Result.GeneratedAt, req.Notifier)
|
||||
req.Result.Notification = notification
|
||||
if err != nil {
|
||||
return req.Result, err
|
||||
}
|
||||
return req.Result, nil
|
||||
}
|
||||
|
||||
func (w *promptReportWorkflow) writePreparationDebug(preparation promptexec.Preparation, debug *promptexec.PreparationDebug) error {
|
||||
w.result.ProfileID, w.result.BackendID, w.result.ModelName = preparation.ProfileID, preparation.BackendID, preparation.ModelName
|
||||
if w.req.DebugWriter == nil {
|
||||
return nil
|
||||
func generatedPreparationError(resolved report.Resolved, runID string, err error) error {
|
||||
var preparation *preparationError
|
||||
if errors.As(err, &preparation) {
|
||||
return generatedReportError(resolved, runID, preparation.operation, preparation.err)
|
||||
}
|
||||
path, err := w.req.DebugWriter.WritePreparation(w.debugRef, preparation, debug)
|
||||
if err != nil {
|
||||
w.callbackFailed = true
|
||||
return promptDebugWriteError(err)
|
||||
}
|
||||
w.result.LLMDebugPath = path
|
||||
return nil
|
||||
return generatedReportError(resolved, runID, "prepare report", err)
|
||||
}
|
||||
|
||||
func (w *promptReportWorkflow) writeExecutionDebug(execution promptexec.Execution) error {
|
||||
if w.req.DebugWriter == nil {
|
||||
return nil
|
||||
func generatedProfileExecutionError(resolved report.Resolved, runID string, err error) error {
|
||||
var execution *profileExecutionError
|
||||
if errors.As(err, &execution) {
|
||||
if execution.callbackFailure {
|
||||
return execution.err
|
||||
}
|
||||
return generatedReportError(resolved, runID, execution.operation, execution.err)
|
||||
}
|
||||
path, err := w.req.DebugWriter.WriteExecution(w.debugRef, execution)
|
||||
if err != nil {
|
||||
return w.reportError("write prompt debug", promptDebugWriteError(err))
|
||||
}
|
||||
if path != "" {
|
||||
w.result.LLMDebugPath = path
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (w *promptReportWorkflow) renderAndPublish(raw []byte) (*ReportResult, error) {
|
||||
generatedText, _, err := w.handler.Validate(raw)
|
||||
if err != nil {
|
||||
return w.result, w.reportError("validate generated text", err)
|
||||
}
|
||||
renderContext, err := w.handler.BuildRenderContext(w.briefingMetadata, w.moduleSnapshot, w.reportFacts.Collected, w.reportFacts.Derived, generatedText)
|
||||
if err != nil {
|
||||
return w.result, w.reportError("build render context", err)
|
||||
}
|
||||
rendered, err := w.handler.Render(renderContext)
|
||||
if err != nil {
|
||||
return w.result, w.reportError("render template", err)
|
||||
}
|
||||
if err := publicationContextError(w.ctx); err != nil {
|
||||
return w.result, w.reportError("publish report", err)
|
||||
}
|
||||
if err := fileutil.WriteFileAtomic(w.req.OutputPath, rendered); err != nil {
|
||||
return w.result, err
|
||||
}
|
||||
w.result.OutputPath = w.req.OutputPath
|
||||
if w.req.noNotify {
|
||||
return w.result, nil
|
||||
}
|
||||
notification, err := notifyReport(w.ctx, w.req.Config, w.req.Resolved, w.result.OutputPath, w.result.RunID, w.result.GeneratedAt, w.req.Notifier)
|
||||
w.result.Notification = notification
|
||||
if err != nil {
|
||||
return w.result, err
|
||||
}
|
||||
return w.result, nil
|
||||
}
|
||||
|
||||
func (w *promptReportWorkflow) reportError(operation string, err error) error {
|
||||
return generatedReportError(w.req.Resolved, w.result.RunID, operation, err)
|
||||
return generatedReportError(resolved, runID, "execute prompt", err)
|
||||
}
|
||||
|
||||
func classifiedPromptError(operation string, err error) error {
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/comparison"
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/config"
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec"
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/report"
|
||||
@@ -39,6 +40,32 @@ type PromptExecutionsInspectionRequest struct {
|
||||
LookupEnv func(string) (string, bool)
|
||||
}
|
||||
|
||||
// ComparisonInspectionRequest contains the explicit profile selection for one
|
||||
// resolved prompt comparison. It intentionally has no configured profile field.
|
||||
type ComparisonInspectionRequest struct {
|
||||
Resolved report.Resolved
|
||||
ProfileIDs []string
|
||||
Executor promptexec.Executor
|
||||
LookupEnv func(string) (string, bool)
|
||||
}
|
||||
|
||||
// ComparisonInspectionResult contains the safe, shared prompt identity and
|
||||
// ordered effective profile identities for a comparison.
|
||||
type ComparisonInspectionResult struct {
|
||||
PromptID string
|
||||
PromptVersion string
|
||||
PromptHash string
|
||||
Profiles []ComparisonProfileInspection
|
||||
}
|
||||
|
||||
// ComparisonProfileInspection contains one requested profile's safe effective
|
||||
// execution identity.
|
||||
type ComparisonProfileInspection struct {
|
||||
ProfileID string
|
||||
BackendID string
|
||||
ModelName string
|
||||
}
|
||||
|
||||
// InspectPromptExecution validates the exact prompt and profile needed for a
|
||||
// report before collection, execution, or durable writes begin.
|
||||
func InspectPromptExecution(ctx context.Context, req PromptInspectionRequest) (PromptInspectionResult, error) {
|
||||
@@ -64,21 +91,9 @@ func InspectPromptExecutions(ctx context.Context, req PromptExecutionsInspection
|
||||
profiles := map[string]promptexec.ProfileInspection{}
|
||||
for _, resolved := range req.Resolved {
|
||||
definition := resolved.Definition
|
||||
if strings.TrimSpace(definition.PromptID) == "" || strings.TrimSpace(definition.PromptVersion) == "" {
|
||||
return nil, promptexec.NewError(promptexec.InvalidConfiguration, "report prompt identity is incomplete", nil)
|
||||
}
|
||||
inspection, err := req.Executor.InspectPrompt(ctx, definition.PromptID, definition.PromptVersion)
|
||||
inspection, err := inspectPromptContract(ctx, req.Executor, definition)
|
||||
if err != nil {
|
||||
return nil, promptInspectionError("prompt inspection failed", err)
|
||||
}
|
||||
if inspection.PromptID != definition.PromptID || inspection.PromptVersion != definition.PromptVersion {
|
||||
return nil, promptexec.NewError(promptexec.InvalidConfiguration, "prompt inspection did not return the requested prompt version", nil)
|
||||
}
|
||||
if !validPromptInput(inspection.Inputs) {
|
||||
return nil, promptexec.NewError(promptexec.InvalidConfiguration, "prompt must declare exactly one required application/yaml data_package input", nil)
|
||||
}
|
||||
if !validPromptOutput(definition, inspection.Output) {
|
||||
return nil, promptexec.NewError(promptexec.InvalidConfiguration, "prompt must declare the report JSON Schema output contract", nil)
|
||||
return nil, err
|
||||
}
|
||||
profileID := req.Promptkit.Profile
|
||||
if profileID == "" {
|
||||
@@ -103,6 +118,62 @@ func InspectPromptExecutions(ctx context.Context, req PromptExecutionsInspection
|
||||
return results, nil
|
||||
}
|
||||
|
||||
// InspectComparisonExecution validates one exact prompt and every explicitly
|
||||
// requested profile before collection or model execution. Profiles are
|
||||
// inspected sequentially in request order. If a profile fails, the returned
|
||||
// partial result retains the prompt identity and successfully inspected prefix.
|
||||
func InspectComparisonExecution(ctx context.Context, req ComparisonInspectionRequest) (ComparisonInspectionResult, error) {
|
||||
if err := comparison.ValidateProfileIDs(req.ProfileIDs); err != nil {
|
||||
return ComparisonInspectionResult{}, promptexec.NewError(promptexec.InvalidRequest, "comparison profile selection is invalid", err)
|
||||
}
|
||||
if req.Executor == nil {
|
||||
return ComparisonInspectionResult{}, promptexec.NewError(promptexec.InvalidConfiguration, "prompt executor is required", nil)
|
||||
}
|
||||
|
||||
inspection, err := inspectPromptContract(ctx, req.Executor, req.Resolved.Definition)
|
||||
if err != nil {
|
||||
return ComparisonInspectionResult{}, comparisonInspectionError("comparison prompt inspection failed", err)
|
||||
}
|
||||
result := ComparisonInspectionResult{
|
||||
PromptID: inspection.PromptID,
|
||||
PromptVersion: inspection.PromptVersion,
|
||||
PromptHash: inspection.PromptHash,
|
||||
Profiles: make([]ComparisonProfileInspection, 0, len(req.ProfileIDs)),
|
||||
}
|
||||
for _, profileID := range req.ProfileIDs {
|
||||
profile, err := inspectPromptProfile(ctx, req.Executor, profileID, req.LookupEnv)
|
||||
if err != nil {
|
||||
return result, comparisonInspectionError("comparison profile inspection failed", err)
|
||||
}
|
||||
result.Profiles = append(result.Profiles, ComparisonProfileInspection{
|
||||
ProfileID: profile.ProfileID,
|
||||
BackendID: profile.BackendID,
|
||||
ModelName: profile.ModelName,
|
||||
})
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func inspectPromptContract(ctx context.Context, executor promptexec.Executor, definition report.Definition) (promptexec.PromptInspection, error) {
|
||||
if strings.TrimSpace(definition.PromptID) == "" || strings.TrimSpace(definition.PromptVersion) == "" {
|
||||
return promptexec.PromptInspection{}, promptexec.NewError(promptexec.InvalidConfiguration, "report prompt identity is incomplete", nil)
|
||||
}
|
||||
inspection, err := executor.InspectPrompt(ctx, definition.PromptID, definition.PromptVersion)
|
||||
if err != nil {
|
||||
return promptexec.PromptInspection{}, promptInspectionError("prompt inspection failed", err)
|
||||
}
|
||||
if inspection.PromptID != definition.PromptID || inspection.PromptVersion != definition.PromptVersion {
|
||||
return promptexec.PromptInspection{}, promptexec.NewError(promptexec.InvalidConfiguration, "prompt inspection did not return the requested prompt version", nil)
|
||||
}
|
||||
if !validPromptInput(inspection.Inputs) {
|
||||
return promptexec.PromptInspection{}, promptexec.NewError(promptexec.InvalidConfiguration, "prompt must declare exactly one required application/yaml data_package input", nil)
|
||||
}
|
||||
if !validPromptOutput(definition, inspection.Output) {
|
||||
return promptexec.PromptInspection{}, promptexec.NewError(promptexec.InvalidConfiguration, "prompt must declare the report JSON Schema output contract", nil)
|
||||
}
|
||||
return inspection, nil
|
||||
}
|
||||
|
||||
func inspectPromptProfile(ctx context.Context, executor promptexec.Executor, profileID string, lookupEnv func(string) (string, bool)) (promptexec.ProfileInspection, error) {
|
||||
profile, err := executor.InspectProfile(ctx, profileID)
|
||||
if err != nil {
|
||||
@@ -140,3 +211,11 @@ func promptInspectionError(operation string, err error) error {
|
||||
}
|
||||
return promptexec.NewError(promptexec.InvalidConfiguration, operation, err)
|
||||
}
|
||||
|
||||
func comparisonInspectionError(operation string, err error) error {
|
||||
category := promptexec.CategoryOf(err)
|
||||
if category == "" {
|
||||
category = promptexec.InvalidConfiguration
|
||||
}
|
||||
return promptexec.NewError(category, operation, err)
|
||||
}
|
||||
|
||||
@@ -3,6 +3,7 @@ package app
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -131,6 +132,98 @@ func TestInspectPromptExecutionsReusesEffectiveProfile(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestInspectComparisonExecutionPreservesOrderedExplicitProfiles(t *testing.T) {
|
||||
resolved := inspectionResolved(t)
|
||||
executor := &inspectionExecutor{
|
||||
prompt: validPromptInspection(resolved.Definition),
|
||||
profiles: map[string]promptexec.ProfileInspection{
|
||||
"weather-light": {ProfileID: "weather-light", BackendID: "local", ModelName: "light-model"},
|
||||
"weather-deep": {ProfileID: "weather-deep", BackendID: "cloud", ModelName: "deep-model"},
|
||||
},
|
||||
}
|
||||
profileIDs := []string{"weather-light", "weather-deep"}
|
||||
result, err := InspectComparisonExecution(context.Background(), ComparisonInspectionRequest{
|
||||
Resolved: resolved, ProfileIDs: profileIDs, Executor: executor,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("InspectComparisonExecution() error = %v", err)
|
||||
}
|
||||
if result.PromptID != resolved.Definition.PromptID || result.PromptVersion != resolved.Definition.PromptVersion || result.PromptHash != "prompt-hash" {
|
||||
t.Fatalf("prompt result = %#v", result)
|
||||
}
|
||||
if !reflect.DeepEqual(executor.profileRequests, profileIDs) || len(executor.promptRequests) != 1 || executor.executeRequests != 0 {
|
||||
t.Fatalf("prompt/profile/execute requests = %#v/%#v/%d", executor.promptRequests, executor.profileRequests, executor.executeRequests)
|
||||
}
|
||||
wantProfiles := []ComparisonProfileInspection{
|
||||
{ProfileID: "weather-light", BackendID: "local", ModelName: "light-model"},
|
||||
{ProfileID: "weather-deep", BackendID: "cloud", ModelName: "deep-model"},
|
||||
}
|
||||
if !reflect.DeepEqual(result.Profiles, wantProfiles) {
|
||||
t.Fatalf("profiles = %#v, want %#v", result.Profiles, wantProfiles)
|
||||
}
|
||||
}
|
||||
|
||||
func TestInspectComparisonExecutionRejectsInvalidProfilesBeforeInspection(t *testing.T) {
|
||||
resolved := inspectionResolved(t)
|
||||
for _, profileIDs := range [][]string{
|
||||
{"weather-light"},
|
||||
{"weather-light", " \t"},
|
||||
{"weather-light", "weather-light"},
|
||||
} {
|
||||
t.Run(strings.Join(profileIDs, ","), func(t *testing.T) {
|
||||
executor := &inspectionExecutor{prompt: validPromptInspection(resolved.Definition)}
|
||||
_, err := InspectComparisonExecution(context.Background(), ComparisonInspectionRequest{
|
||||
Resolved: resolved, ProfileIDs: profileIDs, Executor: executor,
|
||||
})
|
||||
if err == nil || promptexec.CategoryOf(err) != promptexec.InvalidRequest {
|
||||
t.Fatalf("error/category = %v/%q, want invalid request", err, promptexec.CategoryOf(err))
|
||||
}
|
||||
if len(executor.promptRequests) != 0 || len(executor.profileRequests) != 0 || executor.executeRequests != 0 {
|
||||
t.Fatalf("invalid profile selection performed prompt/profile/execution work: %#v/%#v/%d", executor.promptRequests, executor.profileRequests, executor.executeRequests)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestInspectComparisonExecutionStopsAtFirstProfileFailure(t *testing.T) {
|
||||
resolved := inspectionResolved(t)
|
||||
executor := &inspectionExecutor{
|
||||
prompt: validPromptInspection(resolved.Definition),
|
||||
profiles: map[string]promptexec.ProfileInspection{
|
||||
"weather-light": {ProfileID: "weather-light", BackendID: "local", ModelName: "light-model"},
|
||||
"missing-key": {ProfileID: "missing-key", APIKeyEnv: "PROMPT_API_KEY"},
|
||||
"weather-deep": {ProfileID: "weather-deep", BackendID: "cloud", ModelName: "deep-model"},
|
||||
},
|
||||
}
|
||||
result, err := InspectComparisonExecution(context.Background(), ComparisonInspectionRequest{
|
||||
Resolved: resolved, ProfileIDs: []string{"weather-light", "missing-key", "weather-deep"}, Executor: executor,
|
||||
LookupEnv: func(string) (string, bool) { return "", false },
|
||||
})
|
||||
if err == nil || promptexec.CategoryOf(err) != promptexec.MissingCredential {
|
||||
t.Fatalf("error/category = %v/%q, want missing credential", err, promptexec.CategoryOf(err))
|
||||
}
|
||||
if !reflect.DeepEqual(executor.profileRequests, []string{"weather-light", "missing-key"}) || len(executor.promptRequests) != 1 || executor.executeRequests != 0 {
|
||||
t.Fatalf("prompt/profile/execute requests = %#v/%#v/%d", executor.promptRequests, executor.profileRequests, executor.executeRequests)
|
||||
}
|
||||
if result.PromptID != resolved.Definition.PromptID || result.PromptVersion != resolved.Definition.PromptVersion || result.PromptHash == "" || len(result.Profiles) != 1 || result.Profiles[0].ProfileID != "weather-light" {
|
||||
t.Fatalf("partial inspection result = %#v", result)
|
||||
}
|
||||
}
|
||||
|
||||
func TestInspectComparisonExecutionStopsBeforeProfileInspectionWhenPromptFails(t *testing.T) {
|
||||
resolved := inspectionResolved(t)
|
||||
executor := &inspectionExecutor{promptErr: promptexec.NewError(promptexec.PromptNotFound, "prompt is unavailable", nil)}
|
||||
_, err := InspectComparisonExecution(context.Background(), ComparisonInspectionRequest{
|
||||
Resolved: resolved, ProfileIDs: []string{"weather-light", "weather-deep"}, Executor: executor,
|
||||
})
|
||||
if err == nil || promptexec.CategoryOf(err) != promptexec.PromptNotFound || !strings.Contains(err.Error(), "comparison prompt") {
|
||||
t.Fatalf("error/category = %v/%q, want prompt-context prompt not found", err, promptexec.CategoryOf(err))
|
||||
}
|
||||
if len(executor.promptRequests) != 1 || len(executor.profileRequests) != 0 || executor.executeRequests != 0 {
|
||||
t.Fatalf("prompt/profile/execute requests = %#v/%#v/%d", executor.promptRequests, executor.profileRequests, executor.executeRequests)
|
||||
}
|
||||
}
|
||||
|
||||
type inspectionPromptRequest struct {
|
||||
id string
|
||||
version string
|
||||
@@ -143,6 +236,7 @@ type inspectionExecutor struct {
|
||||
promptErr error
|
||||
promptRequests []inspectionPromptRequest
|
||||
profileRequests []string
|
||||
executeRequests int
|
||||
}
|
||||
|
||||
func (e *inspectionExecutor) InspectPrompt(_ context.Context, id string, version string) (promptexec.PromptInspection, error) {
|
||||
@@ -166,6 +260,7 @@ func (e *inspectionExecutor) InspectProfile(_ context.Context, id string) (promp
|
||||
}
|
||||
|
||||
func (e *inspectionExecutor) Execute(context.Context, promptexec.ExecuteRequest, promptexec.PreparationCallback) (*promptexec.Execution, error) {
|
||||
e.executeRequests++
|
||||
return nil, errors.New("unexpected execution")
|
||||
}
|
||||
|
||||
|
||||
368
internal/cli/comparison_test.go
Normal file
368
internal/cli/comparison_test.go
Normal file
@@ -0,0 +1,368 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/app"
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/comparison"
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec"
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/timeutil"
|
||||
)
|
||||
|
||||
func TestParseComparisonFlagsPreservesProfileOrder(t *testing.T) {
|
||||
opts, err := parseComparisonFlags(app.ReportDaily, []string{
|
||||
"--profile", "weather-light", "--profile=weather-balanced", "--profile", "weather-deep",
|
||||
"--date", "2026-05-29", "--out-dir", "reports", "--replace", "--quiet",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("parseComparisonFlags() error = %v", err)
|
||||
}
|
||||
wantProfiles := []string{"weather-light", "weather-balanced", "weather-deep"}
|
||||
if !reflect.DeepEqual([]string(opts.ProfileIDs), wantProfiles) || opts.Date != "2026-05-29" || opts.OutputDir != "reports" || !opts.Replace || !opts.Quiet {
|
||||
t.Fatalf("options = %#v", opts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveComparisonActionBuildsExplicitRequest(t *testing.T) {
|
||||
workingDir := t.TempDir()
|
||||
configPath := comparisonConfigPath(t, "weather_api:\n base_url: https://weather.api.example.com/\n units: metric\n timezone: America/Chicago\noutput:\n directory: configured-reports\npromptkit:\n profile: configured-default\n")
|
||||
clock := timeutil.FixedClock{Time: time.Date(2026, 5, 29, 8, 30, 0, 0, time.UTC)}
|
||||
var factoryConfig PromptExecutorConfig
|
||||
factoryCalls := 0
|
||||
executor := &factoryExecutor{}
|
||||
runner := Runner{
|
||||
Clock: clock, WorkingDir: workingDir,
|
||||
ExecutorFactory: func(value PromptExecutorConfig) (promptexec.Executor, error) {
|
||||
factoryCalls++
|
||||
factoryConfig = value
|
||||
return executor, nil
|
||||
},
|
||||
}
|
||||
req, opts, err := runner.resolveComparisonAction([]string{
|
||||
"daily", "--date", "2026-05-29", "--profile", "weather-light", "--profile", "weather-deep",
|
||||
"--out-dir", "bundles/../comparison", "--replace", "--llm-debug-dir", "/tmp/debug", "--units", "imperial", "--tz", "America/New_York", "--config", configPath,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("resolveComparisonAction() error = %v", err)
|
||||
}
|
||||
if factoryCalls != 1 || req.Executor != executor || req.Clock != clock || req.WorkingDir != workingDir || req.OutputDir != filepath.Join(workingDir, "comparison") || !req.Replace || req.LLMDebugDir != "/tmp/debug" {
|
||||
t.Fatalf("request/factory calls = %#v/%#v/%d", req, factoryConfig, factoryCalls)
|
||||
}
|
||||
if got, want := req.ProfileIDs, []string{"weather-light", "weather-deep"}; !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("profile IDs = %#v, want %#v", got, want)
|
||||
}
|
||||
if req.Config.WeatherAPI.Units != "imperial" || req.Config.WeatherAPI.Timezone != "America/New_York" || req.Config.Output.Directory != "configured-reports" || factoryConfig.Profile != "" || opts.Quiet {
|
||||
t.Fatalf("configuration/options/factory config = %#v/%#v/%#v", req.Config, opts, factoryConfig)
|
||||
}
|
||||
location, loadErr := time.LoadLocation("America/New_York")
|
||||
if loadErr != nil || req.Date.Location().String() != location.String() || req.Date.Format("2006-01-02") != "2026-05-29" {
|
||||
t.Fatalf("date/location = %v/%v", req.Date, loadErr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveComparisonActionMatchesReportDatePolicies(t *testing.T) {
|
||||
configPath := comparisonConfigPath(t, "weather_api:\n base_url: https://weather.api.example.com/\n timezone: America/Chicago\n")
|
||||
runner := comparisonRunner(t, t.TempDir())
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
args []string
|
||||
wantDay string
|
||||
wantErr bool
|
||||
}{
|
||||
{name: "daily requires date", args: []string{"daily", "--profile", "one", "--profile", "two", "--config", configPath}, wantErr: true},
|
||||
{name: "today uses current local date", args: []string{"today", "--profile", "one", "--profile", "two", "--config", configPath}, wantDay: "2026-05-29"},
|
||||
{name: "today accepts date", args: []string{"today", "--date", "2026-05-30", "--profile", "one", "--profile", "two", "--config", configPath}, wantDay: "2026-05-30"},
|
||||
{name: "tomorrow rejects date", args: []string{"tomorrow", "--date", "2026-05-30", "--profile", "one", "--profile", "two", "--config", configPath}, wantErr: true},
|
||||
{name: "hourly rejects date", args: []string{"hourly", "--date", "2026-05-30", "--profile", "one", "--profile", "two", "--config", configPath}, wantErr: true},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
req, _, err := runner.resolveComparisonAction(test.args)
|
||||
if test.wantErr {
|
||||
if err == nil {
|
||||
t.Fatal("resolveComparisonAction() error = nil")
|
||||
}
|
||||
return
|
||||
}
|
||||
if err != nil || req.Date.Format("2006-01-02") != test.wantDay {
|
||||
t.Fatalf("request/error = %#v/%v", req, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveComparisonActionLeavesConfiguredOutputWithoutOverride(t *testing.T) {
|
||||
workingDir := t.TempDir()
|
||||
configPath := comparisonConfigPath(t, "weather_api:\n base_url: https://weather.api.example.com/\noutput:\n directory: configured/../reports\n")
|
||||
req, _, err := comparisonRunner(t, workingDir).resolveComparisonAction([]string{
|
||||
"daily", "--date", "2026-05-29", "--profile", "weather-light", "--profile", "weather-deep", "--config", configPath,
|
||||
})
|
||||
if err != nil || req.OutputDir != "" || req.Config.Output.Directory != "configured/../reports" {
|
||||
t.Fatalf("request/error = %#v/%v", req, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestComparisonInputFailuresDoNotConstructOrExecute(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
args []string
|
||||
}{
|
||||
{name: "missing report", args: nil},
|
||||
{name: "unknown report", args: []string{"unknown", "--profile", "one", "--profile", "two"}},
|
||||
{name: "single profile", args: []string{"today", "--profile", "one"}},
|
||||
{name: "blank profile", args: []string{"today", "--profile", "one", "--profile", " \t"}},
|
||||
{name: "duplicate profile", args: []string{"today", "--profile", "one", "--profile", "one"}},
|
||||
{name: "unexpected argument", args: []string{"today", "extra", "--profile", "one", "--profile", "two"}},
|
||||
{name: "unsupported output flag", args: []string{"today", "--out", "report.md", "--profile", "one", "--profile", "two"}},
|
||||
{name: "configuration failure", args: []string{"today", "--profile", "one", "--profile", "two", "--config", filepath.Join(t.TempDir(), "missing.yml")}},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
factoryCalls, applicationCalls := 0, 0
|
||||
runner := Runner{
|
||||
Clock: timeutil.FixedClock{Time: time.Date(2026, 5, 29, 8, 30, 0, 0, time.UTC)},
|
||||
ExecutorFactory: func(PromptExecutorConfig) (promptexec.Executor, error) {
|
||||
factoryCalls++
|
||||
return &factoryExecutor{}, nil
|
||||
},
|
||||
compareDetailed: func(context.Context, app.ComparisonRequest) (*app.ComparisonResult, error) {
|
||||
applicationCalls++
|
||||
return nil, nil
|
||||
},
|
||||
}
|
||||
var stdout, stderr bytes.Buffer
|
||||
if err := runner.Run(context.Background(), append([]string{"compare"}, test.args...), &stdout, &stderr); err == nil {
|
||||
t.Fatal("Run() error = nil")
|
||||
}
|
||||
if factoryCalls != 0 || applicationCalls != 0 || stdout.Len() != 0 || stderr.Len() != 0 {
|
||||
t.Fatalf("factory/application calls/output = %d/%d/%q/%q", factoryCalls, applicationCalls, stdout.String(), stderr.String())
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompareCommandUsesOneExecutorAndInjectedApplication(t *testing.T) {
|
||||
workingDir := t.TempDir()
|
||||
configPath := comparisonConfigPath(t, "weather_api:\n base_url: https://weather.api.example.com/\n")
|
||||
executor := &factoryExecutor{}
|
||||
factoryCalls, applicationCalls := 0, 0
|
||||
var received app.ComparisonRequest
|
||||
wantResult := &app.ComparisonResult{ComparisonID: "comparison_test"}
|
||||
comparisonErr := errors.New("comparison completed with 1 failed profiles")
|
||||
runner := Runner{
|
||||
Clock: timeutil.FixedClock{Time: time.Date(2026, 5, 29, 8, 30, 0, 0, time.UTC)}, WorkingDir: workingDir,
|
||||
ExecutorFactory: func(PromptExecutorConfig) (promptexec.Executor, error) {
|
||||
factoryCalls++
|
||||
return executor, nil
|
||||
},
|
||||
compareDetailed: func(_ context.Context, req app.ComparisonRequest) (*app.ComparisonResult, error) {
|
||||
applicationCalls++
|
||||
received = req
|
||||
return wantResult, comparisonErr
|
||||
},
|
||||
}
|
||||
var stdout, stderr bytes.Buffer
|
||||
err := runner.Run(context.Background(), []string{
|
||||
"compare", "daily", "--date", "2026-05-29", "--profile", "weather-light", "--profile", "weather-deep", "--out-dir", "comparison", "--replace", "--config", configPath,
|
||||
}, &stdout, &stderr)
|
||||
if !errors.Is(err, comparisonErr) || factoryCalls != 1 || applicationCalls != 1 || received.Executor != executor || received.OutputDir != filepath.Join(workingDir, "comparison") || !received.Replace || stdout.Len() == 0 || stderr.Len() != 0 {
|
||||
t.Fatalf("error/calls/request/output = %v/%d/%d/%#v/%q/%q", err, factoryCalls, applicationCalls, received, stdout.String(), stderr.String())
|
||||
}
|
||||
if !reflect.DeepEqual(received.ProfileIDs, []string{"weather-light", "weather-deep"}) {
|
||||
t.Fatalf("profile IDs = %#v", received.ProfileIDs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompareCommandWritesStructuredPartialFailure(t *testing.T) {
|
||||
workingDir := t.TempDir()
|
||||
configPath := comparisonConfigPath(t, "weather_api:\n base_url: https://weather.api.example.com/\n")
|
||||
profileFailure := comparison.NewSafeError("generation", "execute prompt failed")
|
||||
result := comparisonResult("/reports/comparison-daily", []app.ComparisonProfileResult{
|
||||
{Position: 1, ProfileID: "weather-light", BackendID: "local", ModelName: "light", Status: comparison.StatusSucceeded, ValidationStatus: promptexec.ValidationPassed, ReportPath: "/reports/comparison-daily/01-weather-light.md", LLMDebugPath: "/debug/comparison-light"},
|
||||
{Position: 2, ProfileID: "weather-deep", BackendID: "cloud", ModelName: "deep", Status: comparison.StatusFailed, Error: &profileFailure},
|
||||
})
|
||||
partialErr := errors.New("comparison completed with 1 failed profiles")
|
||||
factoryCalls, applicationCalls := 0, 0
|
||||
runner := Runner{
|
||||
Clock: timeutil.FixedClock{Time: time.Date(2026, 5, 29, 8, 30, 0, 0, time.UTC)}, WorkingDir: workingDir,
|
||||
ExecutorFactory: func(PromptExecutorConfig) (promptexec.Executor, error) {
|
||||
factoryCalls++
|
||||
return &factoryExecutor{}, nil
|
||||
},
|
||||
compareDetailed: func(context.Context, app.ComparisonRequest) (*app.ComparisonResult, error) {
|
||||
applicationCalls++
|
||||
return result, partialErr
|
||||
},
|
||||
}
|
||||
var stdout, stderr bytes.Buffer
|
||||
err := runner.Run(context.Background(), []string{"compare", "daily", "--date", "2026-05-29", "--profile", "weather-light", "--profile", "weather-deep", "--config", configPath}, &stdout, &stderr)
|
||||
if !errors.Is(err, partialErr) || factoryCalls != 1 || applicationCalls != 1 || stderr.Len() != 0 {
|
||||
t.Fatalf("error/calls/stderr = %v/%d/%d/%q", err, factoryCalls, applicationCalls, stderr.String())
|
||||
}
|
||||
var summary comparisonSummary
|
||||
if err := json.Unmarshal(stdout.Bytes(), &summary); err != nil {
|
||||
t.Fatalf("decode summary: %v\n%s", err, stdout.String())
|
||||
}
|
||||
if summary.Command != commandCompare || summary.Status != summaryStatusFailed || summary.Error == nil || summary.Error.Message != partialErr.Error() || summary.OutputDirectory != result.OutputDirectory || len(summary.Results) != 2 || summary.Results[0].ReportPath != result.Results[0].ReportPath || summary.Results[0].LLMDebugPath != result.Results[0].LLMDebugPath || summary.Results[1].Error == nil {
|
||||
t.Fatalf("summary = %#v", summary)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompareCommandReportsCommittedBundleWhenCleanupFails(t *testing.T) {
|
||||
workingDir := t.TempDir()
|
||||
configPath := comparisonConfigPath(t, "weather_api:\n base_url: https://weather.api.example.com/\n")
|
||||
outputDirectory := filepath.Join(workingDir, "comparison-daily")
|
||||
result := comparisonResult(outputDirectory, []app.ComparisonProfileResult{
|
||||
{Position: 1, ProfileID: "weather-light", ModelName: "light", Status: comparison.StatusSucceeded, ValidationStatus: promptexec.ValidationPassed, ReportPath: filepath.Join(outputDirectory, "01-weather-light.md")},
|
||||
{Position: 2, ProfileID: "weather-deep", ModelName: "deep", Status: comparison.StatusSucceeded, ValidationStatus: promptexec.ValidationPassed, ReportPath: filepath.Join(outputDirectory, "02-weather-deep.md")},
|
||||
})
|
||||
backupPath := filepath.Join(workingDir, ".comparison-daily.backup-retained")
|
||||
cleanupCause := errors.New("filesystem cleanup detail")
|
||||
cleanupErr := &comparison.PublicationCleanupError{RetainedBackupPath: backupPath, Err: cleanupCause}
|
||||
runner := comparisonRunner(t, workingDir)
|
||||
runner.compareDetailed = func(context.Context, app.ComparisonRequest) (*app.ComparisonResult, error) {
|
||||
return result, cleanupErr
|
||||
}
|
||||
|
||||
var stdout, stderr bytes.Buffer
|
||||
err := runner.Run(context.Background(), []string{"compare", "daily", "--date", "2026-05-29", "--profile", "weather-light", "--profile", "weather-deep", "--config", configPath}, &stdout, &stderr)
|
||||
if !errors.Is(err, cleanupCause) || stderr.Len() != 0 {
|
||||
t.Fatalf("Run() error/stderr = %v/%q", err, stderr.String())
|
||||
}
|
||||
var summary comparisonSummary
|
||||
if err := json.Unmarshal(stdout.Bytes(), &summary); err != nil {
|
||||
t.Fatalf("decode summary: %v\n%s", err, stdout.String())
|
||||
}
|
||||
if summary.Status != summaryStatusFailed || summary.Error == nil || summary.Error.Category != "publication_cleanup" || summary.Error.Message != "comparison published but cleanup did not complete" || summary.ManifestPath != result.ManifestPath || summary.DataPackagePath != result.DataPackagePath || summary.Results[0].ReportPath == "" {
|
||||
t.Fatalf("summary = %#v", summary)
|
||||
}
|
||||
for _, unsafe := range []string{cleanupCause.Error(), backupPath} {
|
||||
if strings.Contains(stdout.String(), unsafe) {
|
||||
t.Fatalf("summary contains unsafe recovery detail %q: %s", unsafe, stdout.String())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompareCommandWritesSuccessForDefaultAndExplicitDestinations(t *testing.T) {
|
||||
workingDir := t.TempDir()
|
||||
configPath := comparisonConfigPath(t, "weather_api:\n base_url: https://weather.api.example.com/\noutput:\n directory: configured-reports\n")
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
outputArgument []string
|
||||
outputDirectory string
|
||||
wantRequestPath string
|
||||
}{
|
||||
{name: "configured default", outputDirectory: filepath.Join(workingDir, "configured-reports", "comparison-daily-2026-05-29")},
|
||||
{name: "explicit directory", outputArgument: []string{"--out-dir", "published"}, outputDirectory: filepath.Join(workingDir, "published"), wantRequestPath: filepath.Join(workingDir, "published")},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
var received app.ComparisonRequest
|
||||
runner := comparisonRunner(t, workingDir)
|
||||
runner.compareDetailed = func(_ context.Context, req app.ComparisonRequest) (*app.ComparisonResult, error) {
|
||||
received = req
|
||||
return comparisonResult(test.outputDirectory, []app.ComparisonProfileResult{
|
||||
{Position: 1, ProfileID: "weather-light", ModelName: "light", Status: comparison.StatusSucceeded, ValidationStatus: promptexec.ValidationPassed, ReportPath: filepath.Join(test.outputDirectory, "01-weather-light.md")},
|
||||
{Position: 2, ProfileID: "weather-deep", ModelName: "deep", Status: comparison.StatusSucceeded, ValidationStatus: promptexec.ValidationPassed, ReportPath: filepath.Join(test.outputDirectory, "02-weather-deep.md")},
|
||||
}), nil
|
||||
}
|
||||
args := []string{"compare", "daily", "--date", "2026-05-29", "--profile", "weather-light", "--profile", "weather-deep", "--config", configPath}
|
||||
args = append(args, test.outputArgument...)
|
||||
var stdout, stderr bytes.Buffer
|
||||
if err := runner.Run(context.Background(), args, &stdout, &stderr); err != nil {
|
||||
t.Fatalf("Run() error = %v", err)
|
||||
}
|
||||
var summary comparisonSummary
|
||||
if err := json.Unmarshal(stdout.Bytes(), &summary); err != nil || summary.Status != summaryStatusSucceeded || summary.OutputDirectory != test.outputDirectory || stderr.Len() != 0 || received.OutputDir != test.wantRequestPath {
|
||||
t.Fatalf("summary/error/stderr/request = %#v/%v/%q/%#v", summary, err, stderr.String(), received)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompareCommandQuietPreservesFailure(t *testing.T) {
|
||||
configPath := comparisonConfigPath(t, "weather_api:\n base_url: https://weather.api.example.com/\n")
|
||||
failure := errors.New("comparison completed with 2 failed profiles")
|
||||
calls := 0
|
||||
runner := comparisonRunner(t, t.TempDir())
|
||||
runner.compareDetailed = func(context.Context, app.ComparisonRequest) (*app.ComparisonResult, error) {
|
||||
calls++
|
||||
return comparisonResult("/reports/comparison-daily", nil), failure
|
||||
}
|
||||
var stdout, stderr bytes.Buffer
|
||||
err := runner.Run(context.Background(), []string{"compare", "daily", "--date", "2026-05-29", "--profile", "weather-light", "--profile", "weather-deep", "--quiet", "--config", configPath}, &stdout, &stderr)
|
||||
if !errors.Is(err, failure) || calls != 1 || stdout.Len() != 0 || stderr.Len() != 0 {
|
||||
t.Fatalf("error/calls/stdout/stderr = %v/%d/%q/%q", err, calls, stdout.String(), stderr.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompareCommandLeavesPreExecutionFailuresUnstructured(t *testing.T) {
|
||||
runner := comparisonRunner(t, t.TempDir())
|
||||
called := false
|
||||
runner.compareDetailed = func(context.Context, app.ComparisonRequest) (*app.ComparisonResult, error) {
|
||||
called = true
|
||||
return nil, nil
|
||||
}
|
||||
var stdout, stderr bytes.Buffer
|
||||
err := runner.Run(context.Background(), []string{"compare", "today", "--profile", "only-one"}, &stdout, &stderr)
|
||||
if err == nil || called || stdout.Len() != 0 || stderr.Len() != 0 {
|
||||
t.Fatalf("error/called/stdout/stderr = %v/%t/%q/%q", err, called, stdout.String(), stderr.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompareHelpIncludesCommand(t *testing.T) {
|
||||
var stdout, stderr bytes.Buffer
|
||||
if err := (Runner{}).Run(context.Background(), []string{"--help"}, &stdout, &stderr); err != nil || !strings.Contains(stdout.String(), "weatherreporter compare REPORT") || !strings.Contains(stdout.String(), "--profile PROFILE") {
|
||||
t.Fatalf("help/error = %q/%v", stdout.String(), err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompareHelpDoesNotRequireConfiguration(t *testing.T) {
|
||||
var stdout, stderr bytes.Buffer
|
||||
if err := (Runner{}).Run(context.Background(), []string{"compare", "--help"}, &stdout, &stderr); err != nil || !strings.Contains(stdout.String(), "weatherreporter compare REPORT") || stderr.Len() != 0 {
|
||||
t.Fatalf("help/error/stderr = %q/%v/%q", stdout.String(), err, stderr.String())
|
||||
}
|
||||
}
|
||||
|
||||
func comparisonRunner(t *testing.T, workingDir string) Runner {
|
||||
t.Helper()
|
||||
return Runner{
|
||||
Clock: timeutil.FixedClock{Time: time.Date(2026, 5, 29, 8, 30, 0, 0, time.UTC)}, WorkingDir: workingDir,
|
||||
ExecutorFactory: func(PromptExecutorConfig) (promptexec.Executor, error) { return &factoryExecutor{}, nil },
|
||||
}
|
||||
}
|
||||
|
||||
func comparisonConfigPath(t *testing.T, contents string) string {
|
||||
t.Helper()
|
||||
path := filepath.Join(t.TempDir(), "config.yml")
|
||||
if err := os.WriteFile(path, []byte(contents), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return path
|
||||
}
|
||||
|
||||
func comparisonResult(outputDirectory string, results []app.ComparisonProfileResult) *app.ComparisonResult {
|
||||
started := time.Date(2026, 5, 29, 13, 30, 0, 0, time.UTC)
|
||||
result := &app.ComparisonResult{
|
||||
ComparisonID: "comparison_run-123", ReportID: "daily", ReportName: "Daily Report",
|
||||
PromptID: "weather.daily_generated_text", PromptVersion: "2.0.0", PromptHash: strings.Repeat("a", 64),
|
||||
StartedAt: started, FinishedAt: started.Add(time.Minute), Timezone: "America/Chicago",
|
||||
ValidPeriod: timeutil.Period{Start: started, End: started.Add(24 * time.Hour)}, OutputDirectory: outputDirectory,
|
||||
ManifestPath: filepath.Join(outputDirectory, "comparison.json"), DataPackagePath: filepath.Join(outputDirectory, "data-package.yml"),
|
||||
Results: append([]app.ComparisonProfileResult(nil), results...), Total: len(results),
|
||||
}
|
||||
for _, profile := range results {
|
||||
if profile.Status == "succeeded" {
|
||||
result.Succeeded++
|
||||
} else {
|
||||
result.Failed++
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
@@ -1,9 +1,16 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/app"
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/comparison"
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec"
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/report"
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/timeutil"
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/weatherdata"
|
||||
@@ -12,6 +19,7 @@ import (
|
||||
const (
|
||||
commandGenerate = "generate"
|
||||
commandRun = "run"
|
||||
commandCompare = "compare"
|
||||
|
||||
summaryStatusSucceeded = "succeeded"
|
||||
summaryStatusFailed = "failed"
|
||||
@@ -67,6 +75,41 @@ type batchSummary struct {
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
type comparisonSummary struct {
|
||||
Command string `json:"command"`
|
||||
ComparisonID string `json:"comparisonId"`
|
||||
ReportID report.ID `json:"reportId"`
|
||||
ReportName string `json:"reportName"`
|
||||
PromptID string `json:"promptId"`
|
||||
PromptVersion string `json:"promptVersion"`
|
||||
PromptHash string `json:"promptHash"`
|
||||
Status string `json:"status"`
|
||||
StartedAt time.Time `json:"startedAt"`
|
||||
FinishedAt time.Time `json:"finishedAt"`
|
||||
Timezone string `json:"timezone"`
|
||||
ValidPeriod timeutil.Period `json:"validPeriod"`
|
||||
OutputDirectory string `json:"outputDirectory"`
|
||||
ManifestPath string `json:"manifestPath,omitempty"`
|
||||
DataPackagePath string `json:"dataPackagePath,omitempty"`
|
||||
Total int `json:"total"`
|
||||
Succeeded int `json:"succeeded"`
|
||||
Failed int `json:"failed"`
|
||||
Results []comparisonProfileSummary `json:"results"`
|
||||
Error *comparison.SafeError `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
type comparisonProfileSummary struct {
|
||||
Position int `json:"position"`
|
||||
ProfileID string `json:"profileId"`
|
||||
BackendID string `json:"backendId,omitempty"`
|
||||
ModelName string `json:"modelName"`
|
||||
Status string `json:"status"`
|
||||
ValidationStatus string `json:"validationStatus,omitempty"`
|
||||
ReportPath string `json:"reportPath,omitempty"`
|
||||
LLMDebugPath string `json:"llmDebugPath,omitempty"`
|
||||
Error *comparison.SafeError `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
func newGenerateSummary(result *app.ReportResult, err error) generateSummary {
|
||||
summary := generateSummary{Command: commandGenerate}
|
||||
if result == nil {
|
||||
@@ -139,6 +182,87 @@ func newBatchSummary(result *app.BatchResult) batchSummary {
|
||||
return summary
|
||||
}
|
||||
|
||||
func newComparisonSummary(result *app.ComparisonResult, err error) comparisonSummary {
|
||||
summary := comparisonSummary{Command: commandCompare, Results: []comparisonProfileSummary{}}
|
||||
if result == nil {
|
||||
return summary
|
||||
}
|
||||
summary.ComparisonID = result.ComparisonID
|
||||
summary.ReportID, summary.ReportName = result.ReportID, result.ReportName
|
||||
summary.PromptID, summary.PromptVersion, summary.PromptHash = result.PromptID, result.PromptVersion, result.PromptHash
|
||||
summary.StartedAt, summary.FinishedAt = result.StartedAt, result.FinishedAt
|
||||
summary.Timezone, summary.ValidPeriod = result.Timezone, result.ValidPeriod
|
||||
summary.OutputDirectory = result.OutputDirectory
|
||||
summary.ManifestPath, summary.DataPackagePath = result.ManifestPath, result.DataPackagePath
|
||||
summary.Total, summary.Succeeded, summary.Failed = result.Total, result.Succeeded, result.Failed
|
||||
for _, profile := range result.Results {
|
||||
summary.Results = append(summary.Results, comparisonProfileSummary{
|
||||
Position: profile.Position, ProfileID: profile.ProfileID, BackendID: profile.BackendID, ModelName: profile.ModelName,
|
||||
Status: profile.Status, ValidationStatus: string(profile.ValidationStatus), ReportPath: profile.ReportPath,
|
||||
LLMDebugPath: profile.LLMDebugPath, Error: profile.Error,
|
||||
})
|
||||
}
|
||||
summary.Status = comparisonSummaryStatus(result, err)
|
||||
if err != nil {
|
||||
summary.Error = safeComparisonSummaryError(err)
|
||||
}
|
||||
return summary
|
||||
}
|
||||
|
||||
func comparisonSummaryStatus(result *app.ComparisonResult, err error) string {
|
||||
if result == nil || err != nil || result.Total < 2 || result.Succeeded != result.Total || result.Failed != 0 || result.ManifestPath == "" || result.DataPackagePath == "" {
|
||||
return summaryStatusFailed
|
||||
}
|
||||
return summaryStatusSucceeded
|
||||
}
|
||||
|
||||
func safeComparisonSummaryError(err error) *comparison.SafeError {
|
||||
if aggregate, ok := comparisonAggregateErrorMessage(err); ok {
|
||||
safe := comparison.NewSafeError("application", aggregate)
|
||||
return &safe
|
||||
}
|
||||
var cleanupErr *comparison.PublicationCleanupError
|
||||
if errors.As(err, &cleanupErr) {
|
||||
safe := comparison.NewSafeError("publication_cleanup", "comparison published but cleanup did not complete")
|
||||
return &safe
|
||||
}
|
||||
var destinationErr *comparison.DestinationError
|
||||
if errors.As(err, &destinationErr) {
|
||||
safe := comparison.NewSafeError("destination_"+string(destinationErr.Kind), "comparison destination preflight failed")
|
||||
return &safe
|
||||
}
|
||||
if category := promptexec.CategoryOf(err); category != "" {
|
||||
safe := comparison.NewSafeError(string(category), "comparison prompt operation failed")
|
||||
return &safe
|
||||
}
|
||||
if errors.Is(err, context.DeadlineExceeded) {
|
||||
safe := comparison.NewSafeError("deadline_exceeded", "comparison deadline exceeded")
|
||||
return &safe
|
||||
}
|
||||
if errors.Is(err, context.Canceled) {
|
||||
safe := comparison.NewSafeError("canceled", "comparison canceled")
|
||||
return &safe
|
||||
}
|
||||
safe := comparison.NewSafeError("application", "comparison did not complete")
|
||||
return &safe
|
||||
}
|
||||
|
||||
func comparisonAggregateErrorMessage(err error) (string, bool) {
|
||||
const prefix = "comparison completed with "
|
||||
const suffix = " failed profiles"
|
||||
for candidate := err; candidate != nil; candidate = errors.Unwrap(candidate) {
|
||||
value := candidate.Error()
|
||||
if !strings.HasPrefix(value, prefix) || !strings.HasSuffix(value, suffix) {
|
||||
continue
|
||||
}
|
||||
count, parseErr := strconv.Atoi(strings.TrimSuffix(strings.TrimPrefix(value, prefix), suffix))
|
||||
if parseErr == nil && count > 0 {
|
||||
return fmt.Sprintf("comparison completed with %d failed profiles", count), true
|
||||
}
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
|
||||
func batchSummaryStatus(result *app.BatchResult) string {
|
||||
if result == nil {
|
||||
return ""
|
||||
|
||||
@@ -1,12 +1,16 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/app"
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/comparison"
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec"
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/report"
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/timeutil"
|
||||
@@ -29,3 +33,179 @@ func TestGenerateSummaryUsesActiveResultFields(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestComparisonSummaryUsesLockedOrderAndSafeFields(t *testing.T) {
|
||||
started := time.Date(2026, 5, 29, 13, 30, 0, 0, time.UTC)
|
||||
profileFailure := comparison.NewSafeError("generation", "execute prompt failed")
|
||||
result := &app.ComparisonResult{
|
||||
ComparisonID: "comparison_run-123", ReportID: report.Daily, ReportName: "Daily Report",
|
||||
PromptID: "weather.daily_generated_text", PromptVersion: "2.0.0", PromptHash: strings.Repeat("a", 64),
|
||||
StartedAt: started, FinishedAt: started.Add(time.Minute), Timezone: "America/Chicago",
|
||||
ValidPeriod: timeutil.Period{Start: started, End: started.Add(24 * time.Hour)}, OutputDirectory: "/reports/comparison-daily",
|
||||
ManifestPath: "/reports/comparison-daily/comparison.json", DataPackagePath: "/reports/comparison-daily/data-package.yml",
|
||||
Total: 2, Succeeded: 1, Failed: 1,
|
||||
Results: []app.ComparisonProfileResult{
|
||||
{Position: 1, ProfileID: "weather-light", BackendID: "local", ModelName: "light", Status: comparison.StatusSucceeded, ValidationStatus: promptexec.ValidationPassed, ReportPath: "/reports/comparison-daily/01-weather-light.md"},
|
||||
{Position: 2, ProfileID: "weather-deep", BackendID: "cloud", ModelName: "deep", Status: comparison.StatusFailed, Error: &profileFailure},
|
||||
},
|
||||
}
|
||||
summary := newComparisonSummary(result, errors.New("comparison completed with 1 failed profiles"))
|
||||
if summary.Command != commandCompare || summary.Status != summaryStatusFailed || summary.Error == nil || summary.Error.Message != "comparison completed with 1 failed profiles" || len(summary.Results) != 2 || summary.Results[0].ReportPath == "" || summary.Results[1].Error == nil {
|
||||
t.Fatalf("summary = %#v", summary)
|
||||
}
|
||||
data, err := json.Marshal(summary)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
previous := -1
|
||||
for _, field := range []string{"command", "comparisonId", "reportId", "reportName", "promptId", "promptVersion", "promptHash", "status", "startedAt", "finishedAt", "timezone", "validPeriod", "outputDirectory", "manifestPath", "dataPackagePath", "total", "succeeded", "failed", "results", "error"} {
|
||||
position := strings.Index(string(data), `"`+field+`":`)
|
||||
if field == "error" {
|
||||
position = strings.LastIndex(string(data), `"`+field+`":`)
|
||||
}
|
||||
if position <= previous {
|
||||
t.Fatalf("field order for %q in %s", field, data)
|
||||
}
|
||||
previous = position
|
||||
}
|
||||
for _, unsafe := range []string{"provider response", "rawOutput", "renderedPrompt", "endpoint"} {
|
||||
if strings.Contains(string(data), unsafe) {
|
||||
t.Fatalf("summary includes unsafe content %q: %s", unsafe, data)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestComparisonSummaryOmitsUnpublishedArtifactsAndBoundsErrors(t *testing.T) {
|
||||
result := &app.ComparisonResult{ComparisonID: "comparison_run-123", ReportID: report.Daily, Results: []app.ComparisonProfileResult{{Position: 1, ProfileID: "weather-light", ModelName: "light", Status: comparison.StatusFailed}}}
|
||||
summary := newComparisonSummary(result, context.Canceled)
|
||||
if summary.Status != summaryStatusFailed || summary.Error == nil || summary.Error.Message != "comparison canceled" || summary.Results == nil {
|
||||
t.Fatalf("summary = %#v", summary)
|
||||
}
|
||||
data, err := json.Marshal(summary)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var fields map[string]json.RawMessage
|
||||
if err := json.Unmarshal(data, &fields); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, omitted := range []string{"manifestPath", "dataPackagePath"} {
|
||||
if _, exists := fields[omitted]; exists {
|
||||
t.Fatalf("summary includes unpublished %s: %s", omitted, data)
|
||||
}
|
||||
}
|
||||
publicationResult := &app.ComparisonResult{
|
||||
ComparisonID: "comparison_run-123", ReportID: report.Daily, OutputDirectory: "/reports/comparison-daily", Total: 2, Succeeded: 2,
|
||||
Results: []app.ComparisonProfileResult{
|
||||
{Position: 1, ProfileID: "weather-light", ModelName: "light", Status: comparison.StatusSucceeded},
|
||||
{Position: 2, ProfileID: "weather-deep", ModelName: "deep", Status: comparison.StatusSucceeded},
|
||||
},
|
||||
}
|
||||
unsafe := errors.New("comparison completed with 1 failed profiles; provider response contains sensitive material")
|
||||
publicationSummary := newComparisonSummary(publicationResult, unsafe)
|
||||
if publicationSummary.Status != summaryStatusFailed || publicationSummary.Error == nil || publicationSummary.Error.Message != "comparison did not complete" || strings.Contains(publicationSummary.Error.Message, "sensitive") || publicationSummary.ManifestPath != "" || publicationSummary.DataPackagePath != "" {
|
||||
safe := publicationSummary.Error
|
||||
t.Fatalf("safe error = %#v", safe)
|
||||
}
|
||||
}
|
||||
|
||||
func TestComparisonSummaryClassifiesCompleteAndAllFailedResults(t *testing.T) {
|
||||
started := time.Date(2026, 5, 29, 13, 30, 0, 0, time.UTC)
|
||||
failure := comparison.NewSafeError("generation", "execute prompt failed")
|
||||
complete := &app.ComparisonResult{
|
||||
ComparisonID: "comparison_run-123", ReportID: report.Daily, PromptID: "weather.daily_generated_text", PromptVersion: "2.0.0", PromptHash: strings.Repeat("a", 64),
|
||||
StartedAt: started, FinishedAt: started, Timezone: "America/Chicago", ValidPeriod: timeutil.Period{Start: started, End: started.Add(time.Hour)},
|
||||
OutputDirectory: "/reports/comparison-daily", ManifestPath: "/reports/comparison-daily/comparison.json", DataPackagePath: "/reports/comparison-daily/data-package.yml",
|
||||
Total: 2, Succeeded: 2,
|
||||
Results: []app.ComparisonProfileResult{
|
||||
{Position: 1, ProfileID: "weather-light", ModelName: "light", Status: comparison.StatusSucceeded, ValidationStatus: promptexec.ValidationPassed, ReportPath: "/reports/comparison-daily/01-weather-light.md"},
|
||||
{Position: 2, ProfileID: "weather-deep", ModelName: "deep", Status: comparison.StatusSucceeded, ValidationStatus: promptexec.ValidationPassed, ReportPath: "/reports/comparison-daily/02-weather-deep.md"},
|
||||
},
|
||||
}
|
||||
if summary := newComparisonSummary(complete, nil); summary.Status != summaryStatusSucceeded || summary.Error != nil {
|
||||
t.Fatalf("complete summary = %#v", summary)
|
||||
}
|
||||
allFailed := *complete
|
||||
allFailed.Succeeded, allFailed.Failed = 0, 2
|
||||
allFailed.Results = []app.ComparisonProfileResult{
|
||||
{Position: 1, ProfileID: "weather-light", ModelName: "light", Status: comparison.StatusFailed, Error: &failure},
|
||||
{Position: 2, ProfileID: "weather-deep", ModelName: "deep", Status: comparison.StatusFailed, Error: &failure},
|
||||
}
|
||||
summary := newComparisonSummary(&allFailed, errors.New("comparison completed with 2 failed profiles"))
|
||||
if summary.Status != summaryStatusFailed || summary.Error == nil || summary.Error.Message != "comparison completed with 2 failed profiles" || summary.Results[0].ReportPath != "" || summary.Results[1].Error == nil {
|
||||
t.Fatalf("all-failed summary = %#v", summary)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSafeComparisonSummaryErrorClassifiesWrappedFailures(t *testing.T) {
|
||||
unsafeDetail := "unsafe filesystem and provider detail"
|
||||
unsafeCause := errors.New(unsafeDetail)
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
err error
|
||||
category string
|
||||
message string
|
||||
}{
|
||||
{
|
||||
name: "aggregate profile failure",
|
||||
err: fmt.Errorf("outer wrapper: %w", errors.New("comparison completed with 2 failed profiles")),
|
||||
category: "application",
|
||||
message: "comparison completed with 2 failed profiles",
|
||||
},
|
||||
{
|
||||
name: "canceled",
|
||||
err: fmt.Errorf("outer wrapper: %w", context.Canceled),
|
||||
category: "canceled",
|
||||
message: "comparison canceled",
|
||||
},
|
||||
{
|
||||
name: "deadline exceeded",
|
||||
err: fmt.Errorf("outer wrapper: %w", context.DeadlineExceeded),
|
||||
category: "deadline_exceeded",
|
||||
message: "comparison deadline exceeded",
|
||||
},
|
||||
{
|
||||
name: "prompt operation",
|
||||
err: fmt.Errorf("outer wrapper: %w", promptexec.NewError(promptexec.Generation, "unsafe prompt detail", fmt.Errorf("%w: %s", context.Canceled, unsafeDetail))),
|
||||
category: string(promptexec.Generation),
|
||||
message: "comparison prompt operation failed",
|
||||
},
|
||||
{
|
||||
name: "destination preflight",
|
||||
err: fmt.Errorf("outer wrapper: %w", &comparison.DestinationError{
|
||||
Kind: comparison.DestinationNotEmpty, Target: "/tmp/" + unsafeDetail, Err: fmt.Errorf("%w: %s", context.Canceled, unsafeDetail),
|
||||
}),
|
||||
category: "destination_not_empty",
|
||||
message: "comparison destination preflight failed",
|
||||
},
|
||||
{
|
||||
name: "publication cleanup",
|
||||
err: fmt.Errorf("outer wrapper: %w", &comparison.PublicationCleanupError{
|
||||
RetainedBackupPath: "/tmp/" + unsafeDetail, Err: fmt.Errorf("%w: %s", context.Canceled, unsafeDetail),
|
||||
}),
|
||||
category: "publication_cleanup",
|
||||
message: "comparison published but cleanup did not complete",
|
||||
},
|
||||
{
|
||||
name: "unknown",
|
||||
err: fmt.Errorf("outer wrapper: %w", unsafeCause),
|
||||
category: "application",
|
||||
message: "comparison did not complete",
|
||||
},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
result := &app.ComparisonResult{ComparisonID: "comparison_test", ReportID: report.Daily}
|
||||
summary := newComparisonSummary(result, test.err)
|
||||
if summary.Error == nil || summary.Error.Category != test.category || summary.Error.Message != test.message {
|
||||
t.Fatalf("summary error = %#v, want %q/%q", summary.Error, test.category, test.message)
|
||||
}
|
||||
data, err := json.Marshal(summary)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if strings.Contains(string(data), unsafeDetail) {
|
||||
t.Fatalf("summary includes unsafe detail: %s", data)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/app"
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/buildinfo"
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/comparison"
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/config"
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/report"
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/timeutil"
|
||||
@@ -26,6 +27,7 @@ Usage:
|
||||
weatherreporter generate hourly [--config PATH] [--units VALUE] [--tz NAME] [--out PATH] [--llm-debug-dir PATH] [--quiet]
|
||||
weatherreporter run morning [--config PATH] [--units VALUE] [--tz NAME] [--out-dir PATH] [--llm-debug-dir PATH] [--quiet]
|
||||
weatherreporter run evening [--config PATH] [--units VALUE] [--tz NAME] [--out-dir PATH] [--llm-debug-dir PATH] [--quiet]
|
||||
weatherreporter compare REPORT --profile PROFILE --profile PROFILE [--config PATH] [--units VALUE] [--tz NAME] [--date YYYY-MM-DD] [--out-dir PATH] [--replace] [--llm-debug-dir PATH] [--quiet]
|
||||
|
||||
Options:
|
||||
-h, --help Show this help message.
|
||||
@@ -35,8 +37,10 @@ Options:
|
||||
--tz NAME Override weather API timezone.
|
||||
--out PATH Write the generated Markdown report to PATH.
|
||||
--llm-debug-dir PATH Write sensitive prompt debug artifacts under PATH.
|
||||
--out-dir PATH Write generated Markdown reports beneath PATH for run commands.
|
||||
--quiet Suppress successful generate and run output.
|
||||
--profile PROFILE Select a prompt profile for compare; repeat for every profile.
|
||||
--out-dir PATH Write generated Markdown reports beneath PATH for run commands, or select the exact comparison directory.
|
||||
--replace Authorize replacement of a recognized comparison bundle.
|
||||
--quiet Suppress successful action output.
|
||||
`
|
||||
|
||||
type Runner struct {
|
||||
@@ -45,6 +49,7 @@ type Runner struct {
|
||||
Version string
|
||||
WorkingDir string
|
||||
runBatchDetailed func(context.Context, app.BatchRequest) (*app.BatchResult, error)
|
||||
compareDetailed func(context.Context, app.ComparisonRequest) (*app.ComparisonResult, error)
|
||||
}
|
||||
|
||||
func Run(ctx context.Context, args []string, stdout io.Writer, stderr io.Writer) error {
|
||||
@@ -59,6 +64,10 @@ func (r Runner) Run(ctx context.Context, args []string, stdout io.Writer, stderr
|
||||
_, err := fmt.Fprint(stdout, helpText)
|
||||
return err
|
||||
}
|
||||
if len(args) == 2 && args[0] == "compare" && (args[1] == "--help" || args[1] == "-h") {
|
||||
_, err := fmt.Fprint(stdout, helpText)
|
||||
return err
|
||||
}
|
||||
if args[0] == "--version" {
|
||||
if len(args) != 1 {
|
||||
return fmt.Errorf("--version does not accept arguments")
|
||||
@@ -107,6 +116,23 @@ func (r Runner) Run(ctx context.Context, args []string, stdout io.Writer, stderr
|
||||
}
|
||||
}
|
||||
return err
|
||||
case "compare":
|
||||
req, opts, err := r.resolveComparisonAction(args[1:])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
compareDetailed := r.compareDetailed
|
||||
if compareDetailed == nil {
|
||||
compareDetailed = app.CompareDetailed
|
||||
}
|
||||
result, err := compareDetailed(ctx, req)
|
||||
if result != nil {
|
||||
summary := newComparisonSummary(result, err)
|
||||
if encodeErr := writeActionResult(stdout, stderr, summary, outputOptions{Quiet: opts.Quiet}, nil); encodeErr != nil {
|
||||
return encodeErr
|
||||
}
|
||||
}
|
||||
return err
|
||||
default:
|
||||
return fmt.Errorf("unknown command %q", args[0])
|
||||
}
|
||||
@@ -127,6 +153,24 @@ type generateOptions struct {
|
||||
Date string
|
||||
}
|
||||
|
||||
type comparisonOptions struct {
|
||||
commonOptions
|
||||
Date string
|
||||
ProfileIDs profileValues
|
||||
Replace bool
|
||||
}
|
||||
|
||||
type profileValues []string
|
||||
|
||||
func (values *profileValues) String() string {
|
||||
return ""
|
||||
}
|
||||
|
||||
func (values *profileValues) Set(value string) error {
|
||||
*values = append(*values, value)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r Runner) resolveGenerate(args []string) (app.GenerateRequest, error) {
|
||||
req, _, err := r.resolveGenerateAction(args)
|
||||
return req, err
|
||||
@@ -212,6 +256,76 @@ func (r Runner) resolveRun(args []string) (app.BatchRequest, error) {
|
||||
return req, err
|
||||
}
|
||||
|
||||
func (r Runner) resolveComparisonAction(args []string) (app.ComparisonRequest, commonOptions, error) {
|
||||
if r.Clock == nil {
|
||||
r.Clock = timeutil.SystemClock{}
|
||||
}
|
||||
if len(args) == 0 {
|
||||
return app.ComparisonRequest{}, commonOptions{}, fmt.Errorf("compare requires a report name")
|
||||
}
|
||||
if _, err := report.IDForCommandName(args[0]); err != nil {
|
||||
return app.ComparisonRequest{}, commonOptions{}, fmt.Errorf("unknown compare report %q", args[0])
|
||||
}
|
||||
reportKind := app.ReportKind(args[0])
|
||||
opts, err := parseComparisonFlags(reportKind, args[1:])
|
||||
if err != nil {
|
||||
return app.ComparisonRequest{}, commonOptions{}, err
|
||||
}
|
||||
if err := comparison.ValidateProfileIDs(opts.ProfileIDs); err != nil {
|
||||
return app.ComparisonRequest{}, commonOptions{}, err
|
||||
}
|
||||
cfg, err := config.Load(config.LoadOptions{
|
||||
Path: opts.ConfigPath,
|
||||
Units: opts.Units,
|
||||
Timezone: opts.Timezone,
|
||||
})
|
||||
if err != nil {
|
||||
return app.ComparisonRequest{}, commonOptions{}, err
|
||||
}
|
||||
location, err := timeutil.LoadLocation(cfg.WeatherAPI.Timezone)
|
||||
if err != nil {
|
||||
return app.ComparisonRequest{}, commonOptions{}, err
|
||||
}
|
||||
workingDir, err := r.workingDir()
|
||||
if err != nil {
|
||||
return app.ComparisonRequest{}, commonOptions{}, err
|
||||
}
|
||||
outputDir, err := resolveOutputOverride(workingDir, opts.OutputDir)
|
||||
if err != nil {
|
||||
return app.ComparisonRequest{}, commonOptions{}, err
|
||||
}
|
||||
|
||||
req := app.ComparisonRequest{
|
||||
Config: cfg, Report: reportKind, ProfileIDs: append([]string(nil), opts.ProfileIDs...),
|
||||
WorkingDir: workingDir, OutputDir: outputDir, Replace: opts.Replace, LLMDebugDir: opts.LLMDebugDir, Clock: r.Clock,
|
||||
}
|
||||
switch reportKind {
|
||||
case app.ReportDaily:
|
||||
if opts.Date == "" {
|
||||
return app.ComparisonRequest{}, commonOptions{}, fmt.Errorf("compare daily requires --date YYYY-MM-DD")
|
||||
}
|
||||
req.Date, err = timeutil.ParseLocalDate(opts.Date, location)
|
||||
case app.ReportToday:
|
||||
if opts.Date == "" {
|
||||
req.Date = timeutil.LocalDate(r.Clock.Now(), location)
|
||||
} else {
|
||||
req.Date, err = timeutil.ParseLocalDate(opts.Date, location)
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
return app.ComparisonRequest{}, commonOptions{}, err
|
||||
}
|
||||
|
||||
promptkitConfig := cfg.Promptkit
|
||||
promptkitConfig.Profile = ""
|
||||
executor, err := r.promptExecutor(promptkitConfig)
|
||||
if err != nil {
|
||||
return app.ComparisonRequest{}, commonOptions{}, err
|
||||
}
|
||||
req.Executor = executor
|
||||
return req, opts.commonOptions, nil
|
||||
}
|
||||
|
||||
func (r Runner) resolveRunAction(args []string) (app.BatchRequest, commonOptions, error) {
|
||||
if r.Clock == nil {
|
||||
r.Clock = timeutil.SystemClock{}
|
||||
@@ -288,6 +402,27 @@ func parseRunFlags(args []string) (commonOptions, error) {
|
||||
return opts, nil
|
||||
}
|
||||
|
||||
func parseComparisonFlags(report app.ReportKind, args []string) (comparisonOptions, error) {
|
||||
fs := flag.NewFlagSet("compare "+string(report), flag.ContinueOnError)
|
||||
fs.SetOutput(io.Discard)
|
||||
opts := comparisonOptions{}
|
||||
addCommonFlags(fs, &opts.commonOptions, false)
|
||||
fs.StringVar(&opts.OutputDir, "out-dir", "", "comparison bundle directory")
|
||||
fs.BoolVar(&opts.Replace, "replace", false, "replace a recognized comparison bundle")
|
||||
fs.BoolVar(&opts.Quiet, "quiet", false, "suppress successful action output")
|
||||
fs.Var(&opts.ProfileIDs, "profile", "prompt profile ID")
|
||||
if report == app.ReportDaily || report == app.ReportToday {
|
||||
fs.StringVar(&opts.Date, "date", "", "report date in YYYY-MM-DD")
|
||||
}
|
||||
if err := fs.Parse(args); err != nil {
|
||||
return comparisonOptions{}, err
|
||||
}
|
||||
if fs.NArg() > 0 {
|
||||
return comparisonOptions{}, fmt.Errorf("unexpected argument %q", fs.Arg(0))
|
||||
}
|
||||
return opts, nil
|
||||
}
|
||||
|
||||
func addCommonFlags(fs *flag.FlagSet, opts *commonOptions, includeOutput bool) {
|
||||
fs.StringVar(&opts.ConfigPath, "config", "", "configuration file path")
|
||||
fs.StringVar(&opts.Units, "units", "", "weather API units")
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -50,7 +51,8 @@ func TestResolveRunActionConstructsOneExecutor(t *testing.T) {
|
||||
func TestResolveGenerateActionUsesInjectedWorkingDirectoryForOutputOverrides(t *testing.T) {
|
||||
workingDir := t.TempDir()
|
||||
configPath := filepath.Join(t.TempDir(), "config.yml")
|
||||
if err := os.WriteFile(configPath, []byte("weather_api:\n base_url: https://weather.api.example.com/\n"), 0o600); err != nil {
|
||||
const configuredDirectory = "configured/../reports"
|
||||
if err := os.WriteFile(configPath, []byte("weather_api:\n base_url: https://weather.api.example.com/\noutput:\n directory: "+configuredDirectory+"\n"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
absoluteOutput := filepath.Join(t.TempDir(), "daily.md")
|
||||
@@ -76,13 +78,76 @@ func TestResolveGenerateActionUsesInjectedWorkingDirectoryForOutputOverrides(t *
|
||||
args = append(args, "--out", scenario.out)
|
||||
}
|
||||
req, _, err := runner.resolveGenerateAction(args)
|
||||
if err != nil || req.WorkingDir != workingDir || req.OutputPath != scenario.want {
|
||||
if err != nil || req.WorkingDir != workingDir || req.OutputPath != scenario.want || req.Config.Output.Directory != configuredDirectory {
|
||||
t.Fatalf("resolveGenerateAction() request/error = %#v/%v", req, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveActionsPreserveConfiguredOutputDirectoryWithoutAnOverride(t *testing.T) {
|
||||
workingDir := t.TempDir()
|
||||
absoluteDirectory := filepath.Join(t.TempDir(), "reports")
|
||||
for _, configuredDirectory := range []string{"configured/../reports", absoluteDirectory} {
|
||||
t.Run(configuredDirectory, func(t *testing.T) {
|
||||
configPath := filepath.Join(t.TempDir(), "config.yml")
|
||||
contents := "weather_api:\n base_url: https://weather.api.example.com/\noutput:\n directory: " + strconv.Quote(configuredDirectory) + "\n"
|
||||
if err := os.WriteFile(configPath, []byte(contents), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
runner := Runner{
|
||||
Clock: timeutil.FixedClock{Time: time.Date(2026, 5, 29, 8, 0, 0, 0, time.UTC)},
|
||||
WorkingDir: workingDir,
|
||||
ExecutorFactory: func(PromptExecutorConfig) (promptexec.Executor, error) {
|
||||
return &factoryExecutor{}, nil
|
||||
},
|
||||
}
|
||||
|
||||
generateReq, _, generateErr := runner.resolveGenerateAction([]string{"daily", "--date", "2026-05-29", "--config", configPath})
|
||||
if generateErr != nil || generateReq.Config.Output.Directory != configuredDirectory || generateReq.OutputPath != "" {
|
||||
t.Fatalf("resolveGenerateAction() request/error = %#v/%v", generateReq, generateErr)
|
||||
}
|
||||
|
||||
batchReq, _, batchErr := runner.resolveRunAction([]string{"morning", "--config", configPath})
|
||||
if batchErr != nil || batchReq.Config.Output.Directory != configuredDirectory || batchReq.OutputDir != "" {
|
||||
t.Fatalf("resolveRunAction() request/error = %#v/%v", batchReq, batchErr)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveRunActionUsesInjectedWorkingDirectoryForOutputOverrides(t *testing.T) {
|
||||
workingDir := t.TempDir()
|
||||
configPath := filepath.Join(t.TempDir(), "config.yml")
|
||||
const configuredDirectory = "configured/../reports"
|
||||
if err := os.WriteFile(configPath, []byte("weather_api:\n base_url: https://weather.api.example.com/\noutput:\n directory: "+configuredDirectory+"\n"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
absoluteOutput := filepath.Join(t.TempDir(), "reports")
|
||||
for _, scenario := range []struct {
|
||||
name string
|
||||
out string
|
||||
want string
|
||||
}{
|
||||
{name: "relative", out: "reports/../published", want: filepath.Join(workingDir, "published")},
|
||||
{name: "absolute", out: absoluteOutput, want: absoluteOutput},
|
||||
} {
|
||||
t.Run(scenario.name, func(t *testing.T) {
|
||||
runner := Runner{
|
||||
Clock: timeutil.FixedClock{Time: time.Date(2026, 5, 29, 8, 0, 0, 0, time.UTC)},
|
||||
WorkingDir: workingDir,
|
||||
ExecutorFactory: func(PromptExecutorConfig) (promptexec.Executor, error) {
|
||||
return &factoryExecutor{}, nil
|
||||
},
|
||||
}
|
||||
req, _, err := runner.resolveRunAction([]string{"morning", "--config", configPath, "--out-dir", scenario.out})
|
||||
if err != nil || req.WorkingDir != workingDir || req.OutputDir != scenario.want || req.Config.Output.Directory != configuredDirectory {
|
||||
t.Fatalf("resolveRunAction() request/error = %#v/%v", req, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunActionReturnsFailureForBatchNotificationFailure(t *testing.T) {
|
||||
configPath := filepath.Join(t.TempDir(), "config.yml")
|
||||
if err := os.WriteFile(configPath, []byte("weather_api:\n base_url: https://weather.api.example.com/\n"), 0o600); err != nil {
|
||||
|
||||
385
internal/comparison/comparison.go
Normal file
385
internal/comparison/comparison.go
Normal file
@@ -0,0 +1,385 @@
|
||||
// Package comparison owns logical profile-comparison bundle contracts and
|
||||
// guarded filesystem destination planning and publication.
|
||||
package comparison
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"path"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
)
|
||||
|
||||
const (
|
||||
// SchemaVersion identifies the supported comparison manifest schema.
|
||||
SchemaVersion = "weatherreporter.comparison.v1"
|
||||
|
||||
// ManifestFilename is the canonical name of a comparison manifest.
|
||||
ManifestFilename = "comparison.json"
|
||||
// DataPackageFilename is the canonical name of the shared prompt input.
|
||||
DataPackageFilename = "data-package.yml"
|
||||
|
||||
// StatusSucceeded identifies a profile that produced a validated report.
|
||||
StatusSucceeded = "succeeded"
|
||||
// StatusFailed identifies a profile that did not produce a report.
|
||||
StatusFailed = "failed"
|
||||
|
||||
maxProfileSlugBytes = 64
|
||||
maxErrorMessageBytes = 1024
|
||||
)
|
||||
|
||||
// Manifest is the versioned, authoritative index of a comparison bundle.
|
||||
// Field declaration order is the JSON field order.
|
||||
type Manifest struct {
|
||||
SchemaVersion string `json:"schemaVersion"`
|
||||
ComparisonID string `json:"comparisonId"`
|
||||
StartedAt time.Time `json:"startedAt"`
|
||||
FinishedAt time.Time `json:"finishedAt"`
|
||||
ReportID string `json:"reportId"`
|
||||
ValidPeriod ValidPeriod `json:"validPeriod"`
|
||||
Timezone string `json:"timezone"`
|
||||
PromptID string `json:"promptId"`
|
||||
PromptVersion string `json:"promptVersion"`
|
||||
PromptHash string `json:"promptHash"`
|
||||
DataPackage DataPackageReference `json:"dataPackage"`
|
||||
Total int `json:"total"`
|
||||
Succeeded int `json:"succeeded"`
|
||||
Failed int `json:"failed"`
|
||||
Results []Result `json:"results"`
|
||||
}
|
||||
|
||||
// ValidPeriod records the report's resolved half-open period.
|
||||
type ValidPeriod struct {
|
||||
Start time.Time `json:"start"`
|
||||
End time.Time `json:"end"`
|
||||
}
|
||||
|
||||
// DataPackageReference identifies and verifies the shared prompt input.
|
||||
type DataPackageReference struct {
|
||||
Path string `json:"path"`
|
||||
SHA256 string `json:"sha256"`
|
||||
}
|
||||
|
||||
// Result records one explicitly selected profile in selection order.
|
||||
type Result struct {
|
||||
Position int `json:"position"`
|
||||
ProfileID string `json:"profileId"`
|
||||
BackendID string `json:"backendId,omitempty"`
|
||||
ModelName string `json:"modelName"`
|
||||
Status string `json:"status"`
|
||||
ValidationStatus string `json:"validationStatus,omitempty"`
|
||||
ReportPath string `json:"reportPath,omitempty"`
|
||||
Error *SafeError `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
// SafeError contains bounded, operator-safe failure information only.
|
||||
type SafeError struct {
|
||||
Category string `json:"category"`
|
||||
Message string `json:"message"`
|
||||
}
|
||||
|
||||
// LogicalBundle contains every byte required to publish a comparison without
|
||||
// coupling the comparison contract to a filesystem implementation.
|
||||
type LogicalBundle struct {
|
||||
Manifest Manifest
|
||||
DataPackage []byte
|
||||
Reports []BundleReport
|
||||
}
|
||||
|
||||
// BundleReport is one rendered Markdown document in a logical bundle.
|
||||
type BundleReport struct {
|
||||
Position int
|
||||
Path string
|
||||
Markdown []byte
|
||||
}
|
||||
|
||||
// ValidateProfileIDs requires at least two distinct, nonblank profile IDs. It
|
||||
// intentionally preserves accepted IDs unchanged and treats case distinctly.
|
||||
func ValidateProfileIDs(profileIDs []string) error {
|
||||
if len(profileIDs) < 2 {
|
||||
return fmt.Errorf("comparison requires at least two profile IDs")
|
||||
}
|
||||
|
||||
seen := make(map[string]struct{}, len(profileIDs))
|
||||
for _, profileID := range profileIDs {
|
||||
if strings.TrimSpace(profileID) == "" {
|
||||
return fmt.Errorf("profile ID must not be blank")
|
||||
}
|
||||
if _, duplicate := seen[profileID]; duplicate {
|
||||
return fmt.Errorf("duplicate profile ID %q", profileID)
|
||||
}
|
||||
seen[profileID] = struct{}{}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// ProfileSlug returns a deterministic, filesystem-safe representation of a
|
||||
// logical profile ID. The logical ID remains authoritative in the manifest.
|
||||
func ProfileSlug(profileID string) string {
|
||||
var builder strings.Builder
|
||||
lastReplacement := false
|
||||
for _, r := range profileID {
|
||||
if isSlugRune(r) {
|
||||
builder.WriteRune(r)
|
||||
lastReplacement = false
|
||||
continue
|
||||
}
|
||||
if !lastReplacement {
|
||||
builder.WriteByte('-')
|
||||
lastReplacement = true
|
||||
}
|
||||
}
|
||||
|
||||
slug := strings.Trim(builder.String(), "-_")
|
||||
if len(slug) > maxProfileSlugBytes {
|
||||
slug = strings.Trim(slug[:maxProfileSlugBytes], "-_")
|
||||
}
|
||||
if slug == "" {
|
||||
return "profile"
|
||||
}
|
||||
return slug
|
||||
}
|
||||
|
||||
func isSlugRune(r rune) bool {
|
||||
return r >= 'a' && r <= 'z' ||
|
||||
r >= 'A' && r <= 'Z' ||
|
||||
r >= '0' && r <= '9' ||
|
||||
r == '-' || r == '_'
|
||||
}
|
||||
|
||||
// OrdinalWidth returns the zero-padding width for a comparison of count
|
||||
// profiles.
|
||||
func OrdinalWidth(count int) int {
|
||||
width := 2
|
||||
for value := count; value >= 100; value /= 10 {
|
||||
width++
|
||||
}
|
||||
return width
|
||||
}
|
||||
|
||||
// ReportFilename derives a report's deterministic bundle filename.
|
||||
func ReportFilename(position, profileCount int, profileID string) (string, error) {
|
||||
if profileCount < 1 {
|
||||
return "", fmt.Errorf("profile count must be positive")
|
||||
}
|
||||
if position < 1 || position > profileCount {
|
||||
return "", fmt.Errorf("profile position %d is outside 1..%d", position, profileCount)
|
||||
}
|
||||
return fmt.Sprintf("%0*d-%s.md", OrdinalWidth(profileCount), position, ProfileSlug(profileID)), nil
|
||||
}
|
||||
|
||||
// BuildComparisonID derives the stable comparison identity for a resolved
|
||||
// report run.
|
||||
func BuildComparisonID(reportRunID string) (string, error) {
|
||||
if strings.TrimSpace(reportRunID) == "" {
|
||||
return "", fmt.Errorf("report run ID must not be blank")
|
||||
}
|
||||
return "comparison_" + reportRunID, nil
|
||||
}
|
||||
|
||||
// DefaultDirectoryName derives the bundle directory name for a resolved report
|
||||
// output filename.
|
||||
func DefaultDirectoryName(reportOutputName string) (string, error) {
|
||||
if !strings.HasSuffix(reportOutputName, ".md") {
|
||||
return "", fmt.Errorf("report output name %q must end in .md", reportOutputName)
|
||||
}
|
||||
if !isArtifactBasename(reportOutputName) {
|
||||
return "", fmt.Errorf("report output name %q must be a basename", reportOutputName)
|
||||
}
|
||||
stem := strings.TrimSuffix(reportOutputName, ".md")
|
||||
if stem == "" {
|
||||
return "", fmt.Errorf("report output name %q has an empty stem", reportOutputName)
|
||||
}
|
||||
return "comparison-" + stem, nil
|
||||
}
|
||||
|
||||
// SHA256 returns a lowercase hexadecimal SHA-256 digest.
|
||||
func SHA256(content []byte) string {
|
||||
digest := sha256.Sum256(content)
|
||||
return hex.EncodeToString(digest[:])
|
||||
}
|
||||
|
||||
// NewSafeError returns bounded, valid UTF-8 error information suitable for a
|
||||
// comparison manifest.
|
||||
func NewSafeError(category, message string) SafeError {
|
||||
return SafeError{Category: category, Message: TruncateErrorMessage(message)}
|
||||
}
|
||||
|
||||
// TruncateErrorMessage returns a valid UTF-8 message of at most 1,024 bytes.
|
||||
func TruncateErrorMessage(message string) string {
|
||||
message = strings.ToValidUTF8(message, "\uFFFD")
|
||||
if len(message) <= maxErrorMessageBytes {
|
||||
return message
|
||||
}
|
||||
|
||||
end := maxErrorMessageBytes
|
||||
for end > 0 && !utf8.RuneStart(message[end]) {
|
||||
end--
|
||||
}
|
||||
return message[:end]
|
||||
}
|
||||
|
||||
// Validate checks every invariant required for a current comparison manifest.
|
||||
func (manifest Manifest) Validate() error {
|
||||
if manifest.SchemaVersion != SchemaVersion {
|
||||
return fmt.Errorf("unsupported comparison schema version %q", manifest.SchemaVersion)
|
||||
}
|
||||
if strings.TrimSpace(manifest.ComparisonID) == "" {
|
||||
return fmt.Errorf("comparison ID must not be blank")
|
||||
}
|
||||
if manifest.StartedAt.IsZero() || manifest.FinishedAt.IsZero() {
|
||||
return fmt.Errorf("comparison timestamps must be nonzero")
|
||||
}
|
||||
if manifest.StartedAt.Location() != time.UTC || manifest.FinishedAt.Location() != time.UTC {
|
||||
return fmt.Errorf("comparison timestamps must use UTC")
|
||||
}
|
||||
if manifest.FinishedAt.Before(manifest.StartedAt) {
|
||||
return fmt.Errorf("comparison finish time precedes start time")
|
||||
}
|
||||
if manifest.ValidPeriod.Start.IsZero() || manifest.ValidPeriod.End.IsZero() || !manifest.ValidPeriod.End.After(manifest.ValidPeriod.Start) {
|
||||
return fmt.Errorf("valid period must have a nonempty increasing range")
|
||||
}
|
||||
if strings.TrimSpace(manifest.ReportID) == "" || strings.TrimSpace(manifest.Timezone) == "" {
|
||||
return fmt.Errorf("report ID and timezone must not be blank")
|
||||
}
|
||||
if strings.TrimSpace(manifest.PromptID) == "" || strings.TrimSpace(manifest.PromptVersion) == "" || !isSHA256(manifest.PromptHash) {
|
||||
return fmt.Errorf("prompt identity is invalid")
|
||||
}
|
||||
if manifest.DataPackage.Path != DataPackageFilename || !isSHA256(manifest.DataPackage.SHA256) {
|
||||
return fmt.Errorf("data package reference is invalid")
|
||||
}
|
||||
if manifest.Total < 2 || manifest.Total != len(manifest.Results) {
|
||||
return fmt.Errorf("comparison result count is invalid")
|
||||
}
|
||||
if manifest.Succeeded < 0 || manifest.Failed < 0 || manifest.Succeeded+manifest.Failed != manifest.Total {
|
||||
return fmt.Errorf("comparison result totals are inconsistent")
|
||||
}
|
||||
|
||||
profiles := make(map[string]struct{}, len(manifest.Results))
|
||||
reportPaths := make(map[string]struct{}, len(manifest.Results))
|
||||
succeeded, failed := 0, 0
|
||||
for index, result := range manifest.Results {
|
||||
if result.Position != index+1 {
|
||||
return fmt.Errorf("result position %d is not ordered", result.Position)
|
||||
}
|
||||
if strings.TrimSpace(result.ProfileID) == "" {
|
||||
return fmt.Errorf("result %d has a blank profile ID", result.Position)
|
||||
}
|
||||
if _, duplicate := profiles[result.ProfileID]; duplicate {
|
||||
return fmt.Errorf("result %d duplicates profile ID %q", result.Position, result.ProfileID)
|
||||
}
|
||||
profiles[result.ProfileID] = struct{}{}
|
||||
if strings.TrimSpace(result.ModelName) == "" {
|
||||
return fmt.Errorf("result %d has a blank model name", result.Position)
|
||||
}
|
||||
|
||||
switch result.Status {
|
||||
case StatusSucceeded:
|
||||
succeeded++
|
||||
if result.ValidationStatus != "passed" {
|
||||
return fmt.Errorf("successful result %d did not pass validation", result.Position)
|
||||
}
|
||||
if result.Error != nil || !isReportPath(result.ReportPath) {
|
||||
return fmt.Errorf("successful result %d has invalid report details", result.Position)
|
||||
}
|
||||
if _, duplicate := reportPaths[result.ReportPath]; duplicate {
|
||||
return fmt.Errorf("result %d duplicates report path %q", result.Position, result.ReportPath)
|
||||
}
|
||||
reportPaths[result.ReportPath] = struct{}{}
|
||||
case StatusFailed:
|
||||
failed++
|
||||
if result.ReportPath != "" || result.Error == nil || !isValidationStatus(result.ValidationStatus) {
|
||||
return fmt.Errorf("failed result %d has invalid failure details", result.Position)
|
||||
}
|
||||
if err := result.Error.validate(); err != nil {
|
||||
return fmt.Errorf("failed result %d: %w", result.Position, err)
|
||||
}
|
||||
default:
|
||||
return fmt.Errorf("result %d has unsupported status %q", result.Position, result.Status)
|
||||
}
|
||||
}
|
||||
if succeeded != manifest.Succeeded || failed != manifest.Failed {
|
||||
return fmt.Errorf("comparison status counts do not match results")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (safeError SafeError) validate() error {
|
||||
if strings.TrimSpace(safeError.Category) == "" || strings.TrimSpace(safeError.Message) == "" {
|
||||
return fmt.Errorf("safe error category and message must not be blank")
|
||||
}
|
||||
if !utf8.ValidString(safeError.Message) || len(safeError.Message) > maxErrorMessageBytes {
|
||||
return fmt.Errorf("safe error message is not bounded UTF-8")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func isValidationStatus(status string) bool {
|
||||
return status == "" || status == "failed" || status == "skipped"
|
||||
}
|
||||
|
||||
func isSHA256(value string) bool {
|
||||
if len(value) != sha256.Size*2 {
|
||||
return false
|
||||
}
|
||||
for _, r := range value {
|
||||
if (r < '0' || r > '9') && (r < 'a' || r > 'f') {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func isReportPath(value string) bool {
|
||||
return strings.HasSuffix(value, ".md") && isArtifactBasename(value) && value != ManifestFilename && value != DataPackageFilename
|
||||
}
|
||||
|
||||
func isArtifactBasename(value string) bool {
|
||||
return value != "" && value != "." && value != ".." &&
|
||||
!strings.ContainsAny(value, "/\\") && path.Base(value) == value
|
||||
}
|
||||
|
||||
// Validate checks that the manifest and in-memory bundle payloads agree.
|
||||
func (bundle LogicalBundle) Validate() error {
|
||||
if err := bundle.Manifest.Validate(); err != nil {
|
||||
return err
|
||||
}
|
||||
if SHA256(bundle.DataPackage) != bundle.Manifest.DataPackage.SHA256 {
|
||||
return fmt.Errorf("data package digest does not match manifest")
|
||||
}
|
||||
|
||||
if len(bundle.Reports) != bundle.Manifest.Succeeded {
|
||||
return fmt.Errorf("bundle report count does not match manifest")
|
||||
}
|
||||
reportIndex := 0
|
||||
for _, result := range bundle.Manifest.Results {
|
||||
if result.Status != StatusSucceeded {
|
||||
continue
|
||||
}
|
||||
report := bundle.Reports[reportIndex]
|
||||
if report.Position != result.Position || report.Path != result.ReportPath || !isReportPath(report.Path) {
|
||||
return fmt.Errorf("bundle report for result %d does not match manifest", result.Position)
|
||||
}
|
||||
reportIndex++
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// EncodeManifest validates and deterministically encodes a manifest as the
|
||||
// canonical two-space-indented JSON document with one trailing newline.
|
||||
func EncodeManifest(manifest Manifest) ([]byte, error) {
|
||||
if err := manifest.Validate(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
encoded, err := json.MarshalIndent(manifest, "", " ")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("encode comparison manifest: %w", err)
|
||||
}
|
||||
return append(encoded, '\n'), nil
|
||||
}
|
||||
314
internal/comparison/comparison_test.go
Normal file
314
internal/comparison/comparison_test.go
Normal file
@@ -0,0 +1,314 @@
|
||||
package comparison
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
)
|
||||
|
||||
func TestValidateProfileIDs(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
profileIDs []string
|
||||
wantErr bool
|
||||
}{
|
||||
{name: "accepts exact case distinct IDs", profileIDs: []string{"weather-light", "Weather-Light"}},
|
||||
{name: "preserves surrounding whitespace", profileIDs: []string{" weather-light", "weather-deep "}},
|
||||
{name: "rejects one ID", profileIDs: []string{"weather-light"}, wantErr: true},
|
||||
{name: "rejects blank ID", profileIDs: []string{"weather-light", " \t\n"}, wantErr: true},
|
||||
{name: "rejects exact duplicate", profileIDs: []string{"weather-light", "weather-light"}, wantErr: true},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
err := ValidateProfileIDs(test.profileIDs)
|
||||
if (err != nil) != test.wantErr {
|
||||
t.Fatalf("ValidateProfileIDs(%q) error = %v, want error %t", test.profileIDs, err, test.wantErr)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestProfileSlug(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
long := strings.Repeat("a", 62) + "-_more"
|
||||
tests := []struct {
|
||||
profileID string
|
||||
want string
|
||||
}{
|
||||
{profileID: "weather-light", want: "weather-light"},
|
||||
{profileID: "model.v1", want: "model-v1"},
|
||||
{profileID: "nested/path\\name", want: "nested-path-name"},
|
||||
{profileID: " weather\tlight\n", want: "weather-light"},
|
||||
{profileID: "\x00\x01", want: "profile"},
|
||||
{profileID: "météo-東京", want: "m-t-o"},
|
||||
{profileID: "___", want: "profile"},
|
||||
{profileID: long, want: strings.Repeat("a", 62)},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.profileID, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
if got := ProfileSlug(test.profileID); got != test.want {
|
||||
t.Fatalf("ProfileSlug(%q) = %q, want %q", test.profileID, got, test.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestReportNaming(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
if got := OrdinalWidth(99); got != 2 {
|
||||
t.Fatalf("OrdinalWidth(99) = %d, want 2", got)
|
||||
}
|
||||
if got := OrdinalWidth(100); got != 3 {
|
||||
t.Fatalf("OrdinalWidth(100) = %d, want 3", got)
|
||||
}
|
||||
if got, err := ReportFilename(1, 3, "weather.light"); err != nil || got != "01-weather-light.md" {
|
||||
t.Fatalf("ReportFilename() = %q, %v, want %q, nil", got, err, "01-weather-light.md")
|
||||
}
|
||||
if got, err := ReportFilename(100, 100, "weather-light"); err != nil || got != "100-weather-light.md" {
|
||||
t.Fatalf("ReportFilename() = %q, %v, want %q, nil", got, err, "100-weather-light.md")
|
||||
}
|
||||
first, err := ReportFilename(1, 2, "model.v1")
|
||||
if err != nil {
|
||||
t.Fatalf("first ReportFilename() error = %v", err)
|
||||
}
|
||||
second, err := ReportFilename(2, 2, "model/v1")
|
||||
if err != nil {
|
||||
t.Fatalf("second ReportFilename() error = %v", err)
|
||||
}
|
||||
if first == second {
|
||||
t.Fatalf("normalized profile collisions produced the same filename %q", first)
|
||||
}
|
||||
if _, err := ReportFilename(0, 2, "weather-light"); err == nil {
|
||||
t.Fatal("ReportFilename accepted position zero")
|
||||
}
|
||||
if _, err := ReportFilename(1, 0, "weather-light"); err == nil {
|
||||
t.Fatal("ReportFilename accepted zero profile count")
|
||||
}
|
||||
if got, err := BuildComparisonID("daily-2026-08-24"); err != nil || got != "comparison_daily-2026-08-24" {
|
||||
t.Fatalf("BuildComparisonID() = %q, %v", got, err)
|
||||
}
|
||||
if _, err := BuildComparisonID(" \t"); err == nil {
|
||||
t.Fatal("BuildComparisonID accepted blank run ID")
|
||||
}
|
||||
if got, err := DefaultDirectoryName("daily-2026-08-24.md"); err != nil || got != "comparison-daily-2026-08-24" {
|
||||
t.Fatalf("DefaultDirectoryName() = %q, %v", got, err)
|
||||
}
|
||||
for _, name := range []string{"daily.txt", "nested/daily.md", ".md"} {
|
||||
if _, err := DefaultDirectoryName(name); err == nil {
|
||||
t.Fatalf("DefaultDirectoryName(%q) accepted invalid name", name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestManifestEncodingAndRoundTrip(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
manifest := validManifest()
|
||||
encoded, err := EncodeManifest(manifest)
|
||||
if err != nil {
|
||||
t.Fatalf("EncodeManifest() error = %v", err)
|
||||
}
|
||||
want := "{\n" +
|
||||
" \"schemaVersion\": \"weatherreporter.comparison.v1\",\n" +
|
||||
" \"comparisonId\": \"comparison_daily-2026-08-24\",\n" +
|
||||
" \"startedAt\": \"2026-08-24T12:00:00Z\",\n" +
|
||||
" \"finishedAt\": \"2026-08-24T12:01:00Z\",\n" +
|
||||
" \"reportId\": \"daily\",\n" +
|
||||
" \"validPeriod\": {\n" +
|
||||
" \"start\": \"2026-08-24T00:00:00-04:00\",\n" +
|
||||
" \"end\": \"2026-08-25T00:00:00-04:00\"\n" +
|
||||
" },\n" +
|
||||
" \"timezone\": \"America/New_York\",\n" +
|
||||
" \"promptId\": \"daily-report\",\n" +
|
||||
" \"promptVersion\": \"2026-08-01\",\n" +
|
||||
" \"promptHash\": \"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\",\n" +
|
||||
" \"dataPackage\": {\n" +
|
||||
" \"path\": \"data-package.yml\",\n" +
|
||||
" \"sha256\": \"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\"\n" +
|
||||
" },\n" +
|
||||
" \"total\": 2,\n" +
|
||||
" \"succeeded\": 1,\n" +
|
||||
" \"failed\": 1,\n" +
|
||||
" \"results\": [\n" +
|
||||
" {\n" +
|
||||
" \"position\": 1,\n" +
|
||||
" \"profileId\": \"weather-light\",\n" +
|
||||
" \"backendId\": \"openai\",\n" +
|
||||
" \"modelName\": \"gpt-5-mini\",\n" +
|
||||
" \"status\": \"succeeded\",\n" +
|
||||
" \"validationStatus\": \"passed\",\n" +
|
||||
" \"reportPath\": \"01-weather-light.md\"\n" +
|
||||
" },\n" +
|
||||
" {\n" +
|
||||
" \"position\": 2,\n" +
|
||||
" \"profileId\": \"weather-deep\",\n" +
|
||||
" \"modelName\": \"gpt-5\",\n" +
|
||||
" \"status\": \"failed\",\n" +
|
||||
" \"error\": {\n" +
|
||||
" \"category\": \"application\",\n" +
|
||||
" \"message\": \"generated text was rejected\"\n" +
|
||||
" }\n" +
|
||||
" }\n" +
|
||||
" ]\n" +
|
||||
"}\n"
|
||||
if string(encoded) != want {
|
||||
t.Fatalf("EncodeManifest() =\n%s\nwant\n%s", encoded, want)
|
||||
}
|
||||
|
||||
var decoded Manifest
|
||||
if err := json.Unmarshal(encoded, &decoded); err != nil {
|
||||
t.Fatalf("json.Unmarshal() error = %v", err)
|
||||
}
|
||||
if err := decoded.Validate(); err != nil {
|
||||
t.Fatalf("decoded manifest validation error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestManifestValidateRejectsInvariants(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
mutate func(*Manifest)
|
||||
}{
|
||||
{name: "schema version", mutate: func(manifest *Manifest) { manifest.SchemaVersion = "v0" }},
|
||||
{name: "non UTC timestamp", mutate: func(manifest *Manifest) { manifest.StartedAt = manifest.StartedAt.In(time.FixedZone("UTC", 0)) }},
|
||||
{name: "reversed timestamps", mutate: func(manifest *Manifest) { manifest.FinishedAt = manifest.StartedAt.Add(-time.Second) }},
|
||||
{name: "empty valid period", mutate: func(manifest *Manifest) { manifest.ValidPeriod.End = manifest.ValidPeriod.Start }},
|
||||
{name: "bad prompt hash", mutate: func(manifest *Manifest) { manifest.PromptHash = "ABC" }},
|
||||
{name: "bad data package path", mutate: func(manifest *Manifest) { manifest.DataPackage.Path = "nested/data-package.yml" }},
|
||||
{name: "inconsistent totals", mutate: func(manifest *Manifest) { manifest.Succeeded = 2 }},
|
||||
{name: "unordered position", mutate: func(manifest *Manifest) { manifest.Results[1].Position = 3 }},
|
||||
{name: "duplicate profile", mutate: func(manifest *Manifest) { manifest.Results[1].ProfileID = manifest.Results[0].ProfileID }},
|
||||
{name: "unsupported status", mutate: func(manifest *Manifest) { manifest.Results[1].Status = "skipped" }},
|
||||
{name: "successful result without report", mutate: func(manifest *Manifest) { manifest.Results[0].ReportPath = "" }},
|
||||
{name: "successful result with error", mutate: func(manifest *Manifest) {
|
||||
manifest.Results[0].Error = &SafeError{Category: "application", Message: "bad"}
|
||||
}},
|
||||
{name: "successful result without passed validation", mutate: func(manifest *Manifest) { manifest.Results[0].ValidationStatus = "failed" }},
|
||||
{name: "failed result with report", mutate: func(manifest *Manifest) { manifest.Results[1].ReportPath = "02-weather-deep.md" }},
|
||||
{name: "failed result without error", mutate: func(manifest *Manifest) { manifest.Results[1].Error = nil }},
|
||||
{name: "traversal report path", mutate: func(manifest *Manifest) { manifest.Results[0].ReportPath = "../report.md" }},
|
||||
{name: "duplicate report path", mutate: func(manifest *Manifest) {
|
||||
manifest.Results[1] = Result{Position: 2, ProfileID: "weather-deep", ModelName: "gpt-5", Status: StatusSucceeded, ValidationStatus: "passed", ReportPath: manifest.Results[0].ReportPath}
|
||||
manifest.Succeeded = 2
|
||||
manifest.Failed = 0
|
||||
}},
|
||||
{name: "oversized error", mutate: func(manifest *Manifest) { manifest.Results[1].Error.Message = strings.Repeat("x", 1025) }},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
manifest := validManifest()
|
||||
test.mutate(&manifest)
|
||||
if err := manifest.Validate(); err == nil {
|
||||
t.Fatal("Manifest.Validate() succeeded for invalid manifest")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestLogicalBundleValidate(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
dataPackage := []byte("report: daily\n")
|
||||
manifest := validManifest()
|
||||
manifest.DataPackage.SHA256 = SHA256(dataPackage)
|
||||
bundle := LogicalBundle{
|
||||
Manifest: manifest,
|
||||
DataPackage: dataPackage,
|
||||
Reports: []BundleReport{{
|
||||
Position: 1,
|
||||
Path: "01-weather-light.md",
|
||||
Markdown: []byte("# Daily\n"),
|
||||
}},
|
||||
}
|
||||
if err := bundle.Validate(); err != nil {
|
||||
t.Fatalf("LogicalBundle.Validate() error = %v", err)
|
||||
}
|
||||
|
||||
bundle.Reports[0].Path = "other.md"
|
||||
if err := bundle.Validate(); err == nil {
|
||||
t.Fatal("LogicalBundle.Validate() accepted mismatched report path")
|
||||
}
|
||||
bundle.Reports[0].Path = "01-weather-light.md"
|
||||
bundle.Reports[0].Position = 2
|
||||
if err := bundle.Validate(); err == nil {
|
||||
t.Fatal("LogicalBundle.Validate() accepted unordered report position")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSHA256AndTruncateErrorMessage(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
if got, want := SHA256([]byte("weather")), "e5e72beb4e3c6926d3dc9e3e2ef7833ba50cd919c2460a782b244fd071e920de"; got != want {
|
||||
t.Fatalf("SHA256() = %q, want %q", got, want)
|
||||
}
|
||||
message := strings.Repeat("€", 400)
|
||||
got := TruncateErrorMessage(message)
|
||||
if len(got) > 1024 || !utf8.ValidString(got) {
|
||||
t.Fatalf("TruncateErrorMessage() returned %d bytes of valid UTF-8 = %t", len(got), utf8.ValidString(got))
|
||||
}
|
||||
if want := strings.Repeat("€", 341); got != want {
|
||||
t.Fatalf("TruncateErrorMessage() = %q, want %q", got, want)
|
||||
}
|
||||
invalid := string([]byte{'x', 0xff, 'y'})
|
||||
if got := TruncateErrorMessage(invalid); !utf8.ValidString(got) {
|
||||
t.Fatal("TruncateErrorMessage() retained invalid UTF-8")
|
||||
}
|
||||
}
|
||||
|
||||
func validManifest() Manifest {
|
||||
newYork := time.FixedZone("-0400", -4*60*60)
|
||||
return Manifest{
|
||||
SchemaVersion: SchemaVersion,
|
||||
ComparisonID: "comparison_daily-2026-08-24",
|
||||
StartedAt: time.Date(2026, time.August, 24, 12, 0, 0, 0, time.UTC),
|
||||
FinishedAt: time.Date(2026, time.August, 24, 12, 1, 0, 0, time.UTC),
|
||||
ReportID: "daily",
|
||||
ValidPeriod: ValidPeriod{
|
||||
Start: time.Date(2026, time.August, 24, 0, 0, 0, 0, newYork),
|
||||
End: time.Date(2026, time.August, 25, 0, 0, 0, 0, newYork),
|
||||
},
|
||||
Timezone: "America/New_York",
|
||||
PromptID: "daily-report",
|
||||
PromptVersion: "2026-08-01",
|
||||
PromptHash: strings.Repeat("a", 64),
|
||||
DataPackage: DataPackageReference{
|
||||
Path: DataPackageFilename,
|
||||
SHA256: strings.Repeat("a", 64),
|
||||
},
|
||||
Total: 2,
|
||||
Succeeded: 1,
|
||||
Failed: 1,
|
||||
Results: []Result{
|
||||
{
|
||||
Position: 1,
|
||||
ProfileID: "weather-light",
|
||||
BackendID: "openai",
|
||||
ModelName: "gpt-5-mini",
|
||||
Status: StatusSucceeded,
|
||||
ValidationStatus: "passed",
|
||||
ReportPath: "01-weather-light.md",
|
||||
},
|
||||
{
|
||||
Position: 2,
|
||||
ProfileID: "weather-deep",
|
||||
ModelName: "gpt-5",
|
||||
Status: StatusFailed,
|
||||
Error: &SafeError{Category: "application", Message: "generated text was rejected"},
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
460
internal/comparison/publish.go
Normal file
460
internal/comparison/publish.go
Normal file
@@ -0,0 +1,460 @@
|
||||
package comparison
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// DestinationErrorKind identifies a comparison destination safety failure.
|
||||
type DestinationErrorKind string
|
||||
|
||||
const (
|
||||
DestinationInvalidPath DestinationErrorKind = "invalid_path"
|
||||
DestinationFilesystemRoot DestinationErrorKind = "filesystem_root"
|
||||
DestinationWorkingDirectory DestinationErrorKind = "working_directory"
|
||||
DestinationSymlink DestinationErrorKind = "symlink"
|
||||
DestinationNotDirectory DestinationErrorKind = "not_directory"
|
||||
DestinationNotEmpty DestinationErrorKind = "not_empty"
|
||||
DestinationUnrecognized DestinationErrorKind = "unrecognized"
|
||||
DestinationInspection DestinationErrorKind = "inspection"
|
||||
)
|
||||
|
||||
// DestinationError provides inspectable context without making filesystem
|
||||
// implementation details part of a manifest or command-result schema.
|
||||
type DestinationError struct {
|
||||
Kind DestinationErrorKind
|
||||
Target string
|
||||
Err error
|
||||
}
|
||||
|
||||
func (err *DestinationError) Error() string {
|
||||
if err.Err == nil {
|
||||
return fmt.Sprintf("comparison destination %q is %s", err.Target, err.Kind)
|
||||
}
|
||||
return fmt.Sprintf("comparison destination %q is %s: %v", err.Target, err.Kind, err.Err)
|
||||
}
|
||||
|
||||
func (err *DestinationError) Unwrap() error {
|
||||
return err.Err
|
||||
}
|
||||
|
||||
// DestinationPlan records a preflighted, exact bundle directory. Callers must
|
||||
// pass it to Publish rather than recreating destination policy themselves.
|
||||
type DestinationPlan struct {
|
||||
WorkingDirectory string
|
||||
Target string
|
||||
Replace bool
|
||||
state destinationState
|
||||
}
|
||||
|
||||
type destinationState uint8
|
||||
|
||||
const (
|
||||
destinationAbsent destinationState = iota
|
||||
destinationEmpty
|
||||
destinationBundle
|
||||
)
|
||||
|
||||
// ErrUnrecognizedBundle marks a directory that is not a valid current-schema
|
||||
// Weatherreporter comparison bundle.
|
||||
var ErrUnrecognizedBundle = errors.New("unrecognized comparison bundle")
|
||||
|
||||
// PlanDestination performs the read-only comparison destination preflight.
|
||||
func PlanDestination(workingDirectory, target string, replace bool) (DestinationPlan, error) {
|
||||
workingDirectory, err := absoluteCleanPath(workingDirectory)
|
||||
if err != nil {
|
||||
return DestinationPlan{}, newDestinationError(DestinationInvalidPath, workingDirectory, err)
|
||||
}
|
||||
target, err = absoluteCleanPath(target)
|
||||
if err != nil {
|
||||
return DestinationPlan{}, newDestinationError(DestinationInvalidPath, target, err)
|
||||
}
|
||||
if filepath.Dir(target) == target {
|
||||
return DestinationPlan{}, newDestinationError(DestinationFilesystemRoot, target, nil)
|
||||
}
|
||||
if target == workingDirectory {
|
||||
return DestinationPlan{}, newDestinationError(DestinationWorkingDirectory, target, nil)
|
||||
}
|
||||
|
||||
plan := DestinationPlan{WorkingDirectory: workingDirectory, Target: target, Replace: replace}
|
||||
info, err := os.Lstat(target)
|
||||
if err != nil {
|
||||
if !errors.Is(err, os.ErrNotExist) {
|
||||
return DestinationPlan{}, newDestinationError(DestinationInspection, target, err)
|
||||
}
|
||||
if err := inspectMissingDestinationParent(target); err != nil {
|
||||
return DestinationPlan{}, err
|
||||
}
|
||||
return plan, nil
|
||||
}
|
||||
if info.Mode()&os.ModeSymlink != 0 {
|
||||
return DestinationPlan{}, newDestinationError(DestinationSymlink, target, nil)
|
||||
}
|
||||
if !info.IsDir() {
|
||||
return DestinationPlan{}, newDestinationError(DestinationNotDirectory, target, nil)
|
||||
}
|
||||
|
||||
entries, err := os.ReadDir(target)
|
||||
if err != nil {
|
||||
return DestinationPlan{}, newDestinationError(DestinationInspection, target, err)
|
||||
}
|
||||
if len(entries) == 0 {
|
||||
plan.state = destinationEmpty
|
||||
return plan, nil
|
||||
}
|
||||
if !replace {
|
||||
return DestinationPlan{}, newDestinationError(DestinationNotEmpty, target, nil)
|
||||
}
|
||||
if _, err := RecognizeBundle(target); err != nil {
|
||||
return DestinationPlan{}, newDestinationError(DestinationUnrecognized, target, err)
|
||||
}
|
||||
plan.state = destinationBundle
|
||||
return plan, nil
|
||||
}
|
||||
|
||||
func absoluteCleanPath(value string) (string, error) {
|
||||
if strings.TrimSpace(value) == "" {
|
||||
return "", fmt.Errorf("path is required")
|
||||
}
|
||||
if !filepath.IsAbs(value) {
|
||||
return "", fmt.Errorf("path %q must be absolute", value)
|
||||
}
|
||||
return filepath.Clean(value), nil
|
||||
}
|
||||
|
||||
func inspectMissingDestinationParent(target string) error {
|
||||
for parent := filepath.Dir(target); ; parent = filepath.Dir(parent) {
|
||||
info, err := os.Lstat(parent)
|
||||
if err == nil {
|
||||
if info.Mode()&os.ModeSymlink != 0 {
|
||||
resolved, statErr := os.Stat(parent)
|
||||
if statErr != nil {
|
||||
return newDestinationError(DestinationInspection, target, statErr)
|
||||
}
|
||||
if !resolved.IsDir() {
|
||||
return newDestinationError(DestinationNotDirectory, target, nil)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if !info.IsDir() {
|
||||
return newDestinationError(DestinationNotDirectory, target, nil)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if !errors.Is(err, os.ErrNotExist) {
|
||||
return newDestinationError(DestinationInspection, target, err)
|
||||
}
|
||||
if filepath.Dir(parent) == parent {
|
||||
return newDestinationError(DestinationInspection, target, fmt.Errorf("no existing directory ancestor"))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func newDestinationError(kind DestinationErrorKind, target string, err error) error {
|
||||
return &DestinationError{Kind: kind, Target: target, Err: err}
|
||||
}
|
||||
|
||||
// RecognizeBundle verifies that directory contains exactly one valid current
|
||||
// comparison bundle. It never follows bundle entries through symlinks.
|
||||
func RecognizeBundle(directory string) (Manifest, error) {
|
||||
info, err := os.Lstat(directory)
|
||||
if err != nil {
|
||||
return Manifest{}, unrecognizedBundleError("inspect directory", err)
|
||||
}
|
||||
if info.Mode()&os.ModeSymlink != 0 || !info.IsDir() {
|
||||
return Manifest{}, unrecognizedBundleError("directory is not a real directory", nil)
|
||||
}
|
||||
|
||||
entries, err := os.ReadDir(directory)
|
||||
if err != nil {
|
||||
return Manifest{}, unrecognizedBundleError("read directory", err)
|
||||
}
|
||||
manifestData, err := readBundleFile(directory, ManifestFilename)
|
||||
if err != nil {
|
||||
return Manifest{}, err
|
||||
}
|
||||
manifest, err := decodeManifest(manifestData)
|
||||
if err != nil {
|
||||
return Manifest{}, err
|
||||
}
|
||||
if err := manifest.Validate(); err != nil {
|
||||
return Manifest{}, unrecognizedBundleError("validate manifest", err)
|
||||
}
|
||||
|
||||
expected := map[string]struct{}{
|
||||
ManifestFilename: {},
|
||||
DataPackageFilename: {},
|
||||
}
|
||||
for _, result := range manifest.Results {
|
||||
if result.Status == StatusSucceeded {
|
||||
expected[result.ReportPath] = struct{}{}
|
||||
}
|
||||
}
|
||||
if len(entries) != len(expected) {
|
||||
return Manifest{}, unrecognizedBundleError("directory entries do not match manifest", nil)
|
||||
}
|
||||
for _, entry := range entries {
|
||||
if _, ok := expected[entry.Name()]; !ok {
|
||||
return Manifest{}, unrecognizedBundleError("directory has an undeclared entry", nil)
|
||||
}
|
||||
if _, err := readBundleFile(directory, entry.Name()); err != nil {
|
||||
return Manifest{}, err
|
||||
}
|
||||
}
|
||||
|
||||
dataPackage, err := readBundleFile(directory, DataPackageFilename)
|
||||
if err != nil {
|
||||
return Manifest{}, err
|
||||
}
|
||||
if SHA256(dataPackage) != manifest.DataPackage.SHA256 {
|
||||
return Manifest{}, unrecognizedBundleError("data package digest does not match manifest", nil)
|
||||
}
|
||||
return manifest, nil
|
||||
}
|
||||
|
||||
func readBundleFile(directory, name string) ([]byte, error) {
|
||||
if !isArtifactBasename(name) {
|
||||
return nil, unrecognizedBundleError("bundle entry name is unsafe", nil)
|
||||
}
|
||||
filePath := filepath.Join(directory, name)
|
||||
info, err := os.Lstat(filePath)
|
||||
if err != nil {
|
||||
return nil, unrecognizedBundleError("inspect bundle entry", err)
|
||||
}
|
||||
if info.Mode()&os.ModeSymlink != 0 || !info.Mode().IsRegular() {
|
||||
return nil, unrecognizedBundleError("bundle entry is not a regular file", nil)
|
||||
}
|
||||
data, err := os.ReadFile(filePath)
|
||||
if err != nil {
|
||||
return nil, unrecognizedBundleError("read bundle entry", err)
|
||||
}
|
||||
return data, nil
|
||||
}
|
||||
|
||||
func decodeManifest(data []byte) (Manifest, error) {
|
||||
decoder := json.NewDecoder(bytes.NewReader(data))
|
||||
decoder.DisallowUnknownFields()
|
||||
var manifest Manifest
|
||||
if err := decoder.Decode(&manifest); err != nil {
|
||||
return Manifest{}, unrecognizedBundleError("decode manifest", err)
|
||||
}
|
||||
var trailing any
|
||||
if err := decoder.Decode(&trailing); err != io.EOF {
|
||||
if err == nil {
|
||||
return Manifest{}, unrecognizedBundleError("manifest has multiple JSON values", nil)
|
||||
}
|
||||
return Manifest{}, unrecognizedBundleError("decode manifest", err)
|
||||
}
|
||||
return manifest, nil
|
||||
}
|
||||
|
||||
func unrecognizedBundleError(action string, err error) error {
|
||||
if err == nil {
|
||||
return fmt.Errorf("%w: %s", ErrUnrecognizedBundle, action)
|
||||
}
|
||||
return fmt.Errorf("%w: %s: %v", ErrUnrecognizedBundle, action, err)
|
||||
}
|
||||
|
||||
// PublicationResult describes the durable state of a publication attempt.
|
||||
type PublicationResult struct {
|
||||
Committed bool
|
||||
RetainedBackupPath string
|
||||
}
|
||||
|
||||
// PublicationCleanupError reports that a committed bundle could not remove its
|
||||
// prior sibling backup. The new bundle remains installed and the backup path
|
||||
// is retained for operator recovery.
|
||||
type PublicationCleanupError struct {
|
||||
RetainedBackupPath string
|
||||
Err error
|
||||
}
|
||||
|
||||
func (err *PublicationCleanupError) Error() string {
|
||||
return fmt.Sprintf("remove comparison backup %q: %v", err.RetainedBackupPath, err.Err)
|
||||
}
|
||||
|
||||
func (err *PublicationCleanupError) Unwrap() error {
|
||||
return err.Err
|
||||
}
|
||||
|
||||
// Publish writes a complete logical bundle through a private sibling directory
|
||||
// and atomically installs it at a preflighted destination.
|
||||
func Publish(ctx context.Context, plan DestinationPlan, bundle LogicalBundle) (PublicationResult, error) {
|
||||
return publish(ctx, plan, bundle, publishOperations{rename: os.Rename, removeAll: os.RemoveAll})
|
||||
}
|
||||
|
||||
type publishOperations struct {
|
||||
rename func(string, string) error
|
||||
removeAll func(string) error
|
||||
beforeCommit func()
|
||||
}
|
||||
|
||||
func publish(ctx context.Context, plan DestinationPlan, bundle LogicalBundle, operations publishOperations) (PublicationResult, error) {
|
||||
if operations.rename == nil {
|
||||
operations.rename = os.Rename
|
||||
}
|
||||
if operations.removeAll == nil {
|
||||
operations.removeAll = os.RemoveAll
|
||||
}
|
||||
if err := bundle.Validate(); err != nil {
|
||||
return PublicationResult{}, fmt.Errorf("validate comparison bundle: %w", err)
|
||||
}
|
||||
manifestData, err := EncodeManifest(bundle.Manifest)
|
||||
if err != nil {
|
||||
return PublicationResult{}, err
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return PublicationResult{}, err
|
||||
}
|
||||
plan, err = PlanDestination(plan.WorkingDirectory, plan.Target, plan.Replace)
|
||||
if err != nil {
|
||||
return PublicationResult{}, err
|
||||
}
|
||||
|
||||
if err := os.MkdirAll(filepath.Dir(plan.Target), 0o755); err != nil {
|
||||
return PublicationResult{}, fmt.Errorf("create comparison destination parent %q: %w", filepath.Dir(plan.Target), err)
|
||||
}
|
||||
temporaryDirectory, err := os.MkdirTemp(filepath.Dir(plan.Target), "."+filepath.Base(plan.Target)+".staging-")
|
||||
if err != nil {
|
||||
return PublicationResult{}, fmt.Errorf("create comparison staging directory: %w", err)
|
||||
}
|
||||
if err := os.Chmod(temporaryDirectory, 0o700); err != nil {
|
||||
os.RemoveAll(temporaryDirectory)
|
||||
return PublicationResult{}, fmt.Errorf("secure comparison staging directory: %w", err)
|
||||
}
|
||||
defer func() {
|
||||
if temporaryDirectory != "" {
|
||||
_ = os.RemoveAll(temporaryDirectory)
|
||||
}
|
||||
}()
|
||||
|
||||
if err := writeLogicalBundle(ctx, temporaryDirectory, bundle, manifestData); err != nil {
|
||||
return PublicationResult{}, err
|
||||
}
|
||||
|
||||
currentPlan, err := PlanDestination(plan.WorkingDirectory, plan.Target, plan.Replace)
|
||||
if err != nil {
|
||||
return PublicationResult{}, err
|
||||
}
|
||||
if operations.beforeCommit != nil {
|
||||
operations.beforeCommit()
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return PublicationResult{}, err
|
||||
}
|
||||
if currentPlan.state == destinationAbsent {
|
||||
if err := operations.rename(temporaryDirectory, currentPlan.Target); err != nil {
|
||||
return PublicationResult{}, fmt.Errorf("publish comparison bundle to %q: %w", currentPlan.Target, err)
|
||||
}
|
||||
temporaryDirectory = ""
|
||||
return PublicationResult{Committed: true}, nil
|
||||
}
|
||||
|
||||
backupDirectory, err := uniqueSiblingPath(filepath.Dir(currentPlan.Target), "."+filepath.Base(currentPlan.Target)+".backup-")
|
||||
if err != nil {
|
||||
return PublicationResult{}, err
|
||||
}
|
||||
if err := operations.rename(currentPlan.Target, backupDirectory); err != nil {
|
||||
return PublicationResult{}, fmt.Errorf("back up comparison destination %q: %w", currentPlan.Target, err)
|
||||
}
|
||||
if err := authorizeMovedDestination(currentPlan, backupDirectory); err != nil {
|
||||
return PublicationResult{}, restoreMovedDestination(operations, backupDirectory, currentPlan.Target, err)
|
||||
}
|
||||
if err := operations.rename(temporaryDirectory, currentPlan.Target); err != nil {
|
||||
return PublicationResult{}, restoreMovedDestination(operations, backupDirectory, currentPlan.Target, fmt.Errorf("replace comparison destination %q: %w", currentPlan.Target, err))
|
||||
}
|
||||
temporaryDirectory = ""
|
||||
if err := operations.removeAll(backupDirectory); err != nil {
|
||||
cleanupErr := &PublicationCleanupError{RetainedBackupPath: backupDirectory, Err: err}
|
||||
return PublicationResult{Committed: true, RetainedBackupPath: backupDirectory}, cleanupErr
|
||||
}
|
||||
return PublicationResult{Committed: true}, nil
|
||||
}
|
||||
|
||||
func authorizeMovedDestination(plan DestinationPlan, backupDirectory string) error {
|
||||
backupPlan, err := PlanDestination(plan.WorkingDirectory, backupDirectory, plan.Replace)
|
||||
if err != nil {
|
||||
return fmt.Errorf("authorize moved comparison destination %q: %w", backupDirectory, err)
|
||||
}
|
||||
if backupPlan.state != destinationEmpty && backupPlan.state != destinationBundle {
|
||||
return fmt.Errorf("authorize moved comparison destination %q: destination disappeared", backupDirectory)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func restoreMovedDestination(operations publishOperations, backupDirectory, target string, cause error) error {
|
||||
if _, err := os.Lstat(target); err == nil {
|
||||
return errors.Join(
|
||||
cause,
|
||||
fmt.Errorf("restore prior comparison destination from %q: destination %q reappeared", backupDirectory, target),
|
||||
)
|
||||
} else if !errors.Is(err, os.ErrNotExist) {
|
||||
return errors.Join(
|
||||
cause,
|
||||
fmt.Errorf("inspect comparison destination %q before restoring from %q: %w", target, backupDirectory, err),
|
||||
)
|
||||
}
|
||||
if restoreErr := operations.rename(backupDirectory, target); restoreErr != nil {
|
||||
return errors.Join(
|
||||
cause,
|
||||
fmt.Errorf("restore prior comparison destination from %q: %w", backupDirectory, restoreErr),
|
||||
)
|
||||
}
|
||||
return cause
|
||||
}
|
||||
|
||||
func writeLogicalBundle(ctx context.Context, directory string, bundle LogicalBundle, manifestData []byte) error {
|
||||
if err := writeBundleFile(ctx, filepath.Join(directory, DataPackageFilename), bundle.DataPackage); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, report := range bundle.Reports {
|
||||
if err := writeBundleFile(ctx, filepath.Join(directory, report.Path), report.Markdown); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if err := writeBundleFile(ctx, filepath.Join(directory, ManifestFilename), manifestData); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func writeBundleFile(ctx context.Context, filePath string, content []byte) error {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
file, err := os.OpenFile(filePath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600)
|
||||
if err != nil {
|
||||
return fmt.Errorf("create comparison bundle file %q: %w", filePath, err)
|
||||
}
|
||||
if _, err := file.Write(content); err != nil {
|
||||
file.Close()
|
||||
return fmt.Errorf("write comparison bundle file %q: %w", filePath, err)
|
||||
}
|
||||
if err := file.Close(); err != nil {
|
||||
return fmt.Errorf("close comparison bundle file %q: %w", filePath, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func uniqueSiblingPath(parent, prefix string) (string, error) {
|
||||
file, err := os.CreateTemp(parent, prefix)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("reserve comparison backup path: %w", err)
|
||||
}
|
||||
path := file.Name()
|
||||
if err := file.Close(); err != nil {
|
||||
os.Remove(path)
|
||||
return "", fmt.Errorf("close comparison backup reservation: %w", err)
|
||||
}
|
||||
if err := os.Remove(path); err != nil {
|
||||
return "", fmt.Errorf("release comparison backup reservation: %w", err)
|
||||
}
|
||||
return path, nil
|
||||
}
|
||||
754
internal/comparison/publish_test.go
Normal file
754
internal/comparison/publish_test.go
Normal file
@@ -0,0 +1,754 @@
|
||||
package comparison
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestPlanDestination(t *testing.T) {
|
||||
workingDirectory := t.TempDir()
|
||||
absent := filepath.Join(workingDirectory, "absent")
|
||||
plan, err := PlanDestination(workingDirectory, absent, false)
|
||||
if err != nil {
|
||||
t.Fatalf("PlanDestination() error = %v", err)
|
||||
}
|
||||
if plan.Target != absent || plan.state != destinationAbsent {
|
||||
t.Fatalf("PlanDestination() = %#v, want absent target", plan)
|
||||
}
|
||||
nested := filepath.Join(workingDirectory, "missing-parent", "comparison")
|
||||
if _, err := PlanDestination(workingDirectory, nested, false); err != nil {
|
||||
t.Fatalf("PlanDestination(nested) error = %v", err)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Dir(nested)); !errors.Is(err, os.ErrNotExist) {
|
||||
t.Fatalf("read-only plan created parent: stat error = %v", err)
|
||||
}
|
||||
|
||||
empty := filepath.Join(workingDirectory, "empty")
|
||||
if err := os.Mkdir(empty, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
plan, err = PlanDestination(workingDirectory, empty, false)
|
||||
if err != nil || plan.state != destinationEmpty {
|
||||
t.Fatalf("PlanDestination(empty) = %#v, %v", plan, err)
|
||||
}
|
||||
|
||||
file := filepath.Join(workingDirectory, "file")
|
||||
if err := os.WriteFile(file, []byte("not a directory"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
assertDestinationErrorKind(t, workingDirectory, file, false, DestinationNotDirectory)
|
||||
assertDestinationErrorKind(t, workingDirectory, workingDirectory, false, DestinationWorkingDirectory)
|
||||
assertDestinationErrorKind(t, workingDirectory, string(os.PathSeparator), false, DestinationFilesystemRoot)
|
||||
assertDestinationErrorKind(t, workingDirectory, "relative", false, DestinationInvalidPath)
|
||||
|
||||
link := filepath.Join(workingDirectory, "link")
|
||||
if err := os.Symlink(empty, link); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
assertDestinationErrorKind(t, workingDirectory, link, false, DestinationSymlink)
|
||||
|
||||
dangling := filepath.Join(workingDirectory, "dangling")
|
||||
if err := os.Symlink(filepath.Join(workingDirectory, "missing"), dangling); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
assertDestinationErrorKind(t, workingDirectory, dangling, false, DestinationSymlink)
|
||||
assertDestinationErrorKind(t, workingDirectory, filepath.Join(dangling, "child"), false, DestinationInspection)
|
||||
|
||||
nonempty := filepath.Join(workingDirectory, "nonempty")
|
||||
if err := os.Mkdir(nonempty, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(nonempty, "extra"), []byte("x"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
assertDestinationErrorKind(t, workingDirectory, nonempty, false, DestinationNotEmpty)
|
||||
assertDestinationErrorKind(t, workingDirectory, nonempty, true, DestinationUnrecognized)
|
||||
}
|
||||
|
||||
func TestPlanDestinationRejectsUnreadableDirectory(t *testing.T) {
|
||||
if os.Geteuid() == 0 {
|
||||
t.Skip("root can inspect directories regardless of mode")
|
||||
}
|
||||
workingDirectory := t.TempDir()
|
||||
target := filepath.Join(workingDirectory, "unreadable")
|
||||
if err := os.Mkdir(target, 0o700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.Chmod(target, 0); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = os.Chmod(target, 0o700) })
|
||||
assertDestinationErrorKind(t, workingDirectory, target, false, DestinationInspection)
|
||||
}
|
||||
|
||||
func TestRecognizeBundleRejectsUnsafeAndMalformedContents(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
mutate func(t *testing.T, directory string)
|
||||
}{
|
||||
{name: "extra file", mutate: func(t *testing.T, directory string) {
|
||||
t.Helper()
|
||||
if err := os.WriteFile(filepath.Join(directory, "extra.txt"), []byte("extra"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}},
|
||||
{name: "subdirectory", mutate: func(t *testing.T, directory string) {
|
||||
t.Helper()
|
||||
if err := os.Mkdir(filepath.Join(directory, "nested"), 0o700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}},
|
||||
{name: "missing report", mutate: func(t *testing.T, directory string) {
|
||||
t.Helper()
|
||||
if err := os.Remove(filepath.Join(directory, "01-weather-light.md")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}},
|
||||
{name: "symlink report", mutate: func(t *testing.T, directory string) {
|
||||
t.Helper()
|
||||
path := filepath.Join(directory, "01-weather-light.md")
|
||||
if err := os.Remove(path); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.Symlink(filepath.Join(directory, DataPackageFilename), path); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}},
|
||||
{name: "digest mismatch", mutate: func(t *testing.T, directory string) {
|
||||
t.Helper()
|
||||
if err := os.WriteFile(filepath.Join(directory, DataPackageFilename), []byte("altered"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}},
|
||||
{name: "unknown manifest field", mutate: func(t *testing.T, directory string) {
|
||||
t.Helper()
|
||||
path := filepath.Join(directory, ManifestFilename)
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
data = append(data[:len(data)-2], []byte(",\n \"unknown\": true\n}\n")...)
|
||||
if err := os.WriteFile(path, data, 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}},
|
||||
{name: "traversal report path", mutate: func(t *testing.T, directory string) {
|
||||
t.Helper()
|
||||
path := filepath.Join(directory, ManifestFilename)
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var manifest Manifest
|
||||
if err := json.Unmarshal(data, &manifest); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
manifest.Results[0].ReportPath = "../outside.md"
|
||||
data, err = json.Marshal(manifest)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(path, data, 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
directory := publishTestBundle(t, testBundle())
|
||||
test.mutate(t, directory)
|
||||
if _, err := RecognizeBundle(directory); !errors.Is(err, ErrUnrecognizedBundle) {
|
||||
t.Fatalf("RecognizeBundle() error = %v, want ErrUnrecognizedBundle", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestPlanDestinationAcceptsRecognizedReplacement(t *testing.T) {
|
||||
directory := publishTestBundle(t, testBundle())
|
||||
plan, err := PlanDestination(filepath.Dir(directory), directory, true)
|
||||
if err != nil {
|
||||
t.Fatalf("PlanDestination() error = %v", err)
|
||||
}
|
||||
if plan.state != destinationBundle {
|
||||
t.Fatal("PlanDestination() did not record recognized existing destination")
|
||||
}
|
||||
}
|
||||
|
||||
func TestPublishReauthorizesMovedDestination(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
replace bool
|
||||
requiresSymlink bool
|
||||
prepare func(t *testing.T, workingDirectory, target string)
|
||||
mutate func(t *testing.T, target string)
|
||||
verify func(t *testing.T, target string)
|
||||
}{
|
||||
{
|
||||
name: "regular file after absent preflight",
|
||||
mutate: func(t *testing.T, target string) {
|
||||
t.Helper()
|
||||
if err := os.WriteFile(target, []byte("unrelated file"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
},
|
||||
verify: func(t *testing.T, target string) {
|
||||
t.Helper()
|
||||
if data := readFile(t, target); string(data) != "unrelated file" {
|
||||
t.Fatalf("unrelated file data = %q", data)
|
||||
}
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "nonempty directory after empty preflight",
|
||||
prepare: func(t *testing.T, _, target string) {
|
||||
t.Helper()
|
||||
if err := os.Mkdir(target, 0o700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
},
|
||||
mutate: writeUnrecognizedDirectory,
|
||||
verify: func(t *testing.T, target string) {
|
||||
t.Helper()
|
||||
if data := readFile(t, filepath.Join(target, "unrelated")); string(data) != "unrelated" {
|
||||
t.Fatalf("unrelated data = %q", data)
|
||||
}
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "regular file after empty preflight",
|
||||
prepare: func(t *testing.T, _, target string) {
|
||||
t.Helper()
|
||||
if err := os.Mkdir(target, 0o700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
},
|
||||
mutate: func(t *testing.T, target string) {
|
||||
t.Helper()
|
||||
if err := os.WriteFile(target, []byte("unrelated file"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
},
|
||||
verify: func(t *testing.T, target string) {
|
||||
t.Helper()
|
||||
if data := readFile(t, target); string(data) != "unrelated file" {
|
||||
t.Fatalf("unrelated file data = %q", data)
|
||||
}
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "symlink after empty preflight",
|
||||
requiresSymlink: true,
|
||||
prepare: func(t *testing.T, _, target string) {
|
||||
t.Helper()
|
||||
if err := os.Mkdir(target, 0o700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
},
|
||||
mutate: func(t *testing.T, target string) {
|
||||
t.Helper()
|
||||
if err := os.Symlink("unrelated-target", target); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
},
|
||||
verify: func(t *testing.T, target string) {
|
||||
t.Helper()
|
||||
info, err := os.Lstat(target)
|
||||
if err != nil || info.Mode()&os.ModeSymlink == 0 {
|
||||
t.Fatalf("symlink stat = %v, %v", info, err)
|
||||
}
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "unrecognized directory after bundle preflight",
|
||||
replace: true,
|
||||
prepare: func(t *testing.T, workingDirectory, target string) {
|
||||
t.Helper()
|
||||
plan, err := PlanDestination(workingDirectory, target, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := Publish(context.Background(), plan, testBundle()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
},
|
||||
mutate: writeUnrecognizedDirectory,
|
||||
verify: func(t *testing.T, target string) {
|
||||
t.Helper()
|
||||
if data := readFile(t, filepath.Join(target, "unrelated")); string(data) != "unrelated" {
|
||||
t.Fatalf("unrelated data = %q", data)
|
||||
}
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
if test.requiresSymlink && runtime.GOOS == "windows" {
|
||||
t.Skip("symlink replacement coverage requires Unix symlink semantics")
|
||||
}
|
||||
workingDirectory := t.TempDir()
|
||||
target := filepath.Join(workingDirectory, "comparison-daily")
|
||||
if test.prepare != nil {
|
||||
test.prepare(t, workingDirectory, target)
|
||||
}
|
||||
plan, err := PlanDestination(workingDirectory, target, test.replace)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, err = publish(context.Background(), plan, testBundle(), publishOperations{
|
||||
rename: os.Rename,
|
||||
beforeCommit: func() {
|
||||
if err := os.RemoveAll(target); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
test.mutate(t, target)
|
||||
},
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("publish() succeeded despite unauthorized replacement")
|
||||
}
|
||||
test.verify(t, target)
|
||||
assertOnlyDestinationEntry(t, workingDirectory, filepath.Base(target))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestPublishRetainsUnauthorizedMovedDestinationWhenRestoreFails(t *testing.T) {
|
||||
workingDirectory := t.TempDir()
|
||||
target := filepath.Join(workingDirectory, "comparison-daily")
|
||||
if err := os.Mkdir(target, 0o700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
plan, err := PlanDestination(workingDirectory, target, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var backupPath string
|
||||
calls := 0
|
||||
_, err = publish(context.Background(), plan, testBundle(), publishOperations{
|
||||
rename: func(oldPath, newPath string) error {
|
||||
calls++
|
||||
if calls == 1 {
|
||||
backupPath = newPath
|
||||
}
|
||||
if calls == 2 {
|
||||
return errors.New("restore failed")
|
||||
}
|
||||
return os.Rename(oldPath, newPath)
|
||||
},
|
||||
beforeCommit: func() {
|
||||
if err := os.RemoveAll(target); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(target, []byte("unrelated file"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
},
|
||||
})
|
||||
if err == nil || !strings.Contains(err.Error(), backupPath) {
|
||||
t.Fatalf("publish() error = %v, want retained backup path %q", err, backupPath)
|
||||
}
|
||||
if data := readFile(t, backupPath); string(data) != "unrelated file" {
|
||||
t.Fatalf("retained backup data = %q", data)
|
||||
}
|
||||
if _, err := os.Lstat(target); !errors.Is(err, os.ErrNotExist) {
|
||||
t.Fatalf("target stat error = %v, want not exist", err)
|
||||
}
|
||||
assertOnlyDestinationEntry(t, workingDirectory, filepath.Base(backupPath))
|
||||
}
|
||||
|
||||
func TestPublishRetainsMovedDestinationWhenTargetReappears(t *testing.T) {
|
||||
workingDirectory := t.TempDir()
|
||||
target := filepath.Join(workingDirectory, "comparison-daily")
|
||||
if err := os.Mkdir(target, 0o700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
plan, err := PlanDestination(workingDirectory, target, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var backupPath string
|
||||
_, err = publish(context.Background(), plan, testBundle(), publishOperations{
|
||||
rename: func(oldPath, newPath string) error {
|
||||
if backupPath != "" {
|
||||
return os.Rename(oldPath, newPath)
|
||||
}
|
||||
backupPath = newPath
|
||||
if err := os.Rename(oldPath, newPath); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(backupPath, "unrelated"), []byte("unrelated"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(target, []byte("reappeared"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return nil
|
||||
},
|
||||
})
|
||||
if err == nil || !strings.Contains(err.Error(), "reappeared") || !strings.Contains(err.Error(), backupPath) {
|
||||
t.Fatalf("publish() error = %v, want reappeared target and retained backup path %q", err, backupPath)
|
||||
}
|
||||
if data := readFile(t, target); string(data) != "reappeared" {
|
||||
t.Fatalf("reappeared target data = %q", data)
|
||||
}
|
||||
if data := readFile(t, filepath.Join(backupPath, "unrelated")); string(data) != "unrelated" {
|
||||
t.Fatalf("retained backup data = %q", data)
|
||||
}
|
||||
assertDestinationEntries(t, workingDirectory, filepath.Base(target), filepath.Base(backupPath))
|
||||
}
|
||||
|
||||
func TestPublishWritesAndReplacesBundle(t *testing.T) {
|
||||
workingDirectory := t.TempDir()
|
||||
target := filepath.Join(workingDirectory, "comparison-daily")
|
||||
plan, err := PlanDestination(workingDirectory, target, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := Publish(context.Background(), plan, testBundle()); err != nil {
|
||||
t.Fatalf("Publish() error = %v", err)
|
||||
}
|
||||
if _, err := RecognizeBundle(target); err != nil {
|
||||
t.Fatalf("RecognizeBundle() error = %v", err)
|
||||
}
|
||||
assertMode(t, target, 0o700)
|
||||
for _, name := range []string{ManifestFilename, DataPackageFilename, "01-weather-light.md"} {
|
||||
assertMode(t, filepath.Join(target, name), 0o600)
|
||||
}
|
||||
|
||||
next := testBundle()
|
||||
next.Manifest.Results[0].ProfileID = "weather-balanced"
|
||||
next.Manifest.Results[0].ReportPath = "01-weather-balanced.md"
|
||||
next.Reports[0].Path = "01-weather-balanced.md"
|
||||
next.Reports[0].Markdown = []byte("# Replacement\n")
|
||||
plan, err = PlanDestination(workingDirectory, target, true)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := Publish(context.Background(), plan, next); err != nil {
|
||||
t.Fatalf("Publish(replace) error = %v", err)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(target, "01-weather-light.md")); !errors.Is(err, os.ErrNotExist) {
|
||||
t.Fatalf("stale report stat error = %v, want not exist", err)
|
||||
}
|
||||
if data, err := os.ReadFile(filepath.Join(target, "01-weather-balanced.md")); err != nil || string(data) != "# Replacement\n" {
|
||||
t.Fatalf("replacement report = %q, %v", data, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPublishReplacesEmptyDirectory(t *testing.T) {
|
||||
workingDirectory := t.TempDir()
|
||||
target := filepath.Join(workingDirectory, "comparison-daily")
|
||||
if err := os.Mkdir(target, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
plan, err := PlanDestination(workingDirectory, target, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := Publish(context.Background(), plan, testBundle()); err != nil {
|
||||
t.Fatalf("Publish() error = %v", err)
|
||||
}
|
||||
if _, err := RecognizeBundle(target); err != nil {
|
||||
t.Fatalf("RecognizeBundle() error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPublishReportsCommittedBundleWhenBackupCleanupFails(t *testing.T) {
|
||||
workingDirectory := t.TempDir()
|
||||
target := filepath.Join(workingDirectory, "comparison-daily")
|
||||
initialPlan, err := PlanDestination(workingDirectory, target, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := Publish(context.Background(), initialPlan, testBundle()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
previous := directorySnapshot(t, target)
|
||||
|
||||
next := testBundle()
|
||||
next.Reports[0].Markdown = []byte("# Next\n")
|
||||
plan, err := PlanDestination(workingDirectory, target, true)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cleanupCause := errors.New("backup removal failed")
|
||||
var backupPath string
|
||||
result, err := publish(context.Background(), plan, next, publishOperations{
|
||||
rename: os.Rename,
|
||||
removeAll: func(path string) error {
|
||||
backupPath = path
|
||||
return cleanupCause
|
||||
},
|
||||
})
|
||||
var cleanupErr *PublicationCleanupError
|
||||
if !result.Committed || result.RetainedBackupPath != backupPath || !filepath.IsAbs(backupPath) || !errors.As(err, &cleanupErr) || cleanupErr.RetainedBackupPath != backupPath || !errors.Is(err, cleanupCause) {
|
||||
t.Fatalf("publish() result/error = %#v/%v", result, err)
|
||||
}
|
||||
if _, err := RecognizeBundle(target); err != nil {
|
||||
t.Fatalf("new bundle recognition error = %v", err)
|
||||
}
|
||||
if retained := directorySnapshot(t, backupPath); !equalSnapshots(previous, retained) {
|
||||
t.Fatalf("retained backup = %#v, want %#v", retained, previous)
|
||||
}
|
||||
manifestData, err := os.ReadFile(filepath.Join(target, ManifestFilename))
|
||||
if err != nil || strings.Contains(string(manifestData), backupPath) {
|
||||
t.Fatalf("manifest/backup path = %q/%q, error = %v", manifestData, backupPath, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPublishRestoresExistingBundleAfterReplacementFailure(t *testing.T) {
|
||||
workingDirectory := t.TempDir()
|
||||
target := filepath.Join(workingDirectory, "comparison-daily")
|
||||
initial := testBundle()
|
||||
initialPlan, err := PlanDestination(workingDirectory, target, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := Publish(context.Background(), initialPlan, initial); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
before := directorySnapshot(t, target)
|
||||
|
||||
next := testBundle()
|
||||
next.Reports[0].Markdown = []byte("# New\n")
|
||||
plan, err := PlanDestination(workingDirectory, target, true)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
calls := 0
|
||||
publication, err := publish(context.Background(), plan, next, publishOperations{rename: func(oldPath, newPath string) error {
|
||||
calls++
|
||||
if calls == 2 {
|
||||
return errors.New("replace failed")
|
||||
}
|
||||
return os.Rename(oldPath, newPath)
|
||||
}})
|
||||
if publication.Committed || err == nil {
|
||||
t.Fatal("publish() succeeded despite replacement failure")
|
||||
}
|
||||
after := directorySnapshot(t, target)
|
||||
if !equalSnapshots(before, after) {
|
||||
t.Fatal("failed publication changed the prior bundle")
|
||||
}
|
||||
entries, err := os.ReadDir(workingDirectory)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(entries) != 1 || entries[0].Name() != filepath.Base(target) {
|
||||
t.Fatalf("failed publication left sibling artifacts: %#v", entries)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPublishRetainsBackupWhenRestorationFails(t *testing.T) {
|
||||
workingDirectory := t.TempDir()
|
||||
target := filepath.Join(workingDirectory, "comparison-daily")
|
||||
initialPlan, err := PlanDestination(workingDirectory, target, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := Publish(context.Background(), initialPlan, testBundle()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
plan, err := PlanDestination(workingDirectory, target, true)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var backupPath string
|
||||
calls := 0
|
||||
_, err = publish(context.Background(), plan, testBundle(), publishOperations{rename: func(oldPath, newPath string) error {
|
||||
calls++
|
||||
if calls == 1 {
|
||||
backupPath = newPath
|
||||
}
|
||||
if calls == 2 || calls == 3 {
|
||||
return errors.New("rename failed")
|
||||
}
|
||||
return os.Rename(oldPath, newPath)
|
||||
}})
|
||||
if err == nil || !strings.Contains(err.Error(), backupPath) {
|
||||
t.Fatalf("publish() error = %v, want retained backup path %q", err, backupPath)
|
||||
}
|
||||
if info, statErr := os.Stat(backupPath); statErr != nil || !info.IsDir() {
|
||||
t.Fatalf("backup stat = %v, %v, want retained directory", info, statErr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPublishCancellationBeforeCommitLeavesNoDestination(t *testing.T) {
|
||||
workingDirectory := t.TempDir()
|
||||
target := filepath.Join(workingDirectory, "comparison-daily")
|
||||
plan, err := PlanDestination(workingDirectory, target, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
if publication, err := publish(ctx, plan, testBundle(), publishOperations{rename: os.Rename, beforeCommit: cancel}); publication.Committed || !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("Publish() error = %v, want context cancellation", err)
|
||||
}
|
||||
if _, err := os.Lstat(target); !errors.Is(err, os.ErrNotExist) {
|
||||
t.Fatalf("destination stat error = %v, want not exist", err)
|
||||
}
|
||||
}
|
||||
|
||||
func assertDestinationErrorKind(t *testing.T, workingDirectory, target string, replace bool, want DestinationErrorKind) {
|
||||
t.Helper()
|
||||
_, err := PlanDestination(workingDirectory, target, replace)
|
||||
var destinationError *DestinationError
|
||||
if !errors.As(err, &destinationError) || destinationError.Kind != want {
|
||||
t.Fatalf("PlanDestination(%q) error = %v, want destination error %q", target, err, want)
|
||||
}
|
||||
}
|
||||
|
||||
func publishTestBundle(t *testing.T, bundle LogicalBundle) string {
|
||||
t.Helper()
|
||||
workingDirectory := t.TempDir()
|
||||
target := filepath.Join(workingDirectory, "comparison-daily")
|
||||
plan, err := PlanDestination(workingDirectory, target, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := Publish(context.Background(), plan, bundle); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return target
|
||||
}
|
||||
|
||||
func writeUnrecognizedDirectory(t *testing.T, target string) {
|
||||
t.Helper()
|
||||
if err := os.Mkdir(target, 0o700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(target, "unrelated"), []byte("unrelated"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func readFile(t *testing.T, path string) []byte {
|
||||
t.Helper()
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return data
|
||||
}
|
||||
|
||||
func assertOnlyDestinationEntry(t *testing.T, directory, want string) {
|
||||
t.Helper()
|
||||
assertDestinationEntries(t, directory, want)
|
||||
}
|
||||
|
||||
func assertDestinationEntries(t *testing.T, directory string, wants ...string) {
|
||||
t.Helper()
|
||||
entries, err := os.ReadDir(directory)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(entries) != len(wants) {
|
||||
t.Fatalf("directory entries = %#v, want %#v", entries, wants)
|
||||
}
|
||||
for _, want := range wants {
|
||||
found := false
|
||||
for _, entry := range entries {
|
||||
if entry.Name() == want {
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Fatalf("directory entries = %#v, missing %q", entries, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func testBundle() LogicalBundle {
|
||||
dataPackage := []byte("report: daily\n")
|
||||
manifest := validManifest()
|
||||
manifest.DataPackage.SHA256 = SHA256(dataPackage)
|
||||
return LogicalBundle{
|
||||
Manifest: manifest,
|
||||
DataPackage: dataPackage,
|
||||
Reports: []BundleReport{{
|
||||
Position: 1,
|
||||
Path: "01-weather-light.md",
|
||||
Markdown: []byte("# Daily\n"),
|
||||
}},
|
||||
}
|
||||
}
|
||||
|
||||
func assertMode(t *testing.T, path string, want os.FileMode) {
|
||||
t.Helper()
|
||||
info, err := os.Stat(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := info.Mode().Perm(); got != want {
|
||||
t.Fatalf("mode for %q = %#o, want %#o", path, got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func directorySnapshot(t *testing.T, directory string) map[string]string {
|
||||
t.Helper()
|
||||
entries, err := os.ReadDir(directory)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
snapshot := make(map[string]string, len(entries))
|
||||
for _, entry := range entries {
|
||||
data, err := os.ReadFile(filepath.Join(directory, entry.Name()))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
snapshot[entry.Name()] = string(data)
|
||||
}
|
||||
return snapshot
|
||||
}
|
||||
|
||||
func equalSnapshots(left, right map[string]string) bool {
|
||||
if len(left) != len(right) {
|
||||
return false
|
||||
}
|
||||
for name, value := range left {
|
||||
if right[name] != value {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func TestRecognizeBundleRejectsTrailingJSONValue(t *testing.T) {
|
||||
directory := publishTestBundle(t, testBundle())
|
||||
path := filepath.Join(directory, ManifestFilename)
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(path, append(data, []byte("{}")...), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := RecognizeBundle(directory); !errors.Is(err, ErrUnrecognizedBundle) {
|
||||
t.Fatalf("RecognizeBundle() error = %v, want ErrUnrecognizedBundle", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPlanDestinationRejectsWhitespaceTarget(t *testing.T) {
|
||||
workingDirectory := t.TempDir()
|
||||
assertDestinationErrorKind(t, workingDirectory, " \t", false, DestinationInvalidPath)
|
||||
}
|
||||
|
||||
func TestRecognizeBundleDoesNotAcceptNonRegularManifest(t *testing.T) {
|
||||
directory := t.TempDir()
|
||||
if err := os.Mkdir(filepath.Join(directory, ManifestFilename), 0o700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := RecognizeBundle(directory); !strings.Contains(err.Error(), "regular file") {
|
||||
t.Fatalf("RecognizeBundle() error = %v, want regular file rejection", err)
|
||||
}
|
||||
}
|
||||
@@ -25,6 +25,7 @@ type Config struct {
|
||||
WeatherAPI WeatherAPIConfig `yaml:"weather_api"`
|
||||
Location LocationConfig `yaml:"location"`
|
||||
Secrets SecretsConfig `yaml:"secrets"`
|
||||
Output OutputConfig `yaml:"output"`
|
||||
Notify NotifyConfig `yaml:"notify"`
|
||||
MissingSource MissingSourceConfig `yaml:"missing_source"`
|
||||
Promptkit PromptkitConfig `yaml:"promptkit"`
|
||||
@@ -51,6 +52,10 @@ type SecretsConfig struct {
|
||||
Directory string `yaml:"directory"`
|
||||
}
|
||||
|
||||
type OutputConfig struct {
|
||||
Directory string `yaml:"directory"`
|
||||
}
|
||||
|
||||
type NotifyConfig struct {
|
||||
Distributor DistributorNotifyConfig `yaml:"distributor"`
|
||||
}
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -37,6 +38,9 @@ func TestDefaults(t *testing.T) {
|
||||
if cfg.Secrets.Directory != "" {
|
||||
t.Fatalf("Secrets.Directory = %q, want empty", cfg.Secrets.Directory)
|
||||
}
|
||||
if cfg.Output.Directory != "" {
|
||||
t.Fatalf("Output.Directory = %q, want empty", cfg.Output.Directory)
|
||||
}
|
||||
if cfg.Notify.Distributor.Enabled {
|
||||
t.Fatalf("Notify.Distributor.Enabled = true, want false")
|
||||
}
|
||||
@@ -78,6 +82,120 @@ func TestDefaults(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestOutputDirectoryLoading(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
yaml string
|
||||
wantValue string
|
||||
wantErr string
|
||||
}{
|
||||
{
|
||||
name: "omitted",
|
||||
yaml: "{}\n",
|
||||
wantValue: "",
|
||||
},
|
||||
{
|
||||
name: "explicit empty",
|
||||
yaml: `
|
||||
output:
|
||||
directory: ""
|
||||
`,
|
||||
wantValue: "",
|
||||
},
|
||||
{
|
||||
name: "absolute path",
|
||||
yaml: `
|
||||
output:
|
||||
directory: /var/lib/weatherreporter/reports
|
||||
`,
|
||||
wantValue: "/var/lib/weatherreporter/reports",
|
||||
},
|
||||
{
|
||||
name: "relative path",
|
||||
yaml: `
|
||||
output:
|
||||
directory: reports/../published
|
||||
`,
|
||||
wantValue: "reports/../published",
|
||||
},
|
||||
{
|
||||
name: "whitespace only",
|
||||
yaml: `
|
||||
output:
|
||||
directory: " \t "
|
||||
`,
|
||||
wantErr: "output.directory",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
cfg, err := LoadFile(writeConfig(t, tt.yaml))
|
||||
if tt.wantErr != "" {
|
||||
if err == nil || !strings.Contains(err.Error(), tt.wantErr) {
|
||||
t.Fatalf("LoadFile() error = %v, want %q", err, tt.wantErr)
|
||||
}
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("LoadFile() error = %v", err)
|
||||
}
|
||||
if cfg.Output.Directory != tt.wantValue {
|
||||
t.Fatalf("Output.Directory = %q, want %q", cfg.Output.Directory, tt.wantValue)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestOutputDirectoryValidationIsConsistentForLoadedAndConstructedConfigs(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
directory string
|
||||
wantErr string
|
||||
}{
|
||||
{name: "empty"},
|
||||
{name: "absolute path", directory: "/var/lib/weatherreporter/reports"},
|
||||
{name: "relative path", directory: "reports/../published"},
|
||||
{name: "whitespace only", directory: " \t ", wantErr: "output.directory"},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
yaml := "output:\n directory: " + strconv.Quote(tt.directory) + "\n"
|
||||
_, loadErr := LoadFile(writeConfig(t, yaml))
|
||||
|
||||
cfg := Defaults()
|
||||
cfg.Output.Directory = tt.directory
|
||||
validateErr := Validate(cfg)
|
||||
|
||||
if (loadErr == nil) != (validateErr == nil) {
|
||||
t.Fatalf("LoadFile() error = %v, Validate() error = %v", loadErr, validateErr)
|
||||
}
|
||||
if tt.wantErr != "" {
|
||||
if loadErr == nil || !strings.Contains(loadErr.Error(), tt.wantErr) {
|
||||
t.Fatalf("LoadFile() error = %v, want %q", loadErr, tt.wantErr)
|
||||
}
|
||||
if validateErr == nil || !strings.Contains(validateErr.Error(), tt.wantErr) {
|
||||
t.Fatalf("Validate() error = %v, want %q", validateErr, tt.wantErr)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestOutputDirectoryRejectsUnknownFields(t *testing.T) {
|
||||
_, err := LoadFile(writeConfig(t, `
|
||||
output:
|
||||
location: reports
|
||||
`))
|
||||
if err == nil {
|
||||
t.Fatal("LoadFile() error = nil, want strict decoding error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "field location not found") {
|
||||
t.Fatalf("LoadFile() error = %q, want output field rejection", err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadExampleConfig(t *testing.T) {
|
||||
cfg, err := LoadFile(filepath.Join("..", "..", "examples", "config.yml"))
|
||||
if err != nil {
|
||||
@@ -96,6 +214,9 @@ func TestLoadExampleConfig(t *testing.T) {
|
||||
if cfg.Location.ID != "home" || cfg.Location.Name != "Brentwood" || cfg.Location.Region != "St. Louis Metro" {
|
||||
t.Fatalf("Location = %#v, want example location", cfg.Location)
|
||||
}
|
||||
if cfg.Output.Directory != "/var/lib/weatherreporter/reports" {
|
||||
t.Fatalf("Output.Directory = %q, want maintained example value", cfg.Output.Directory)
|
||||
}
|
||||
if cfg.Notify.Distributor.PipelineIDTemplate != "weatherreporter.{report_id}" {
|
||||
t.Fatalf("PipelineIDTemplate = %q, want example pipeline template", cfg.Notify.Distributor.PipelineIDTemplate)
|
||||
}
|
||||
@@ -1674,6 +1795,9 @@ func TestLoadSecretsRejectsInvalidDirectoryEntries(t *testing.T) {
|
||||
t.Cleanup(func() {
|
||||
_ = os.Chmod(path, 0o600)
|
||||
})
|
||||
if _, err := os.ReadFile(path); err == nil {
|
||||
t.Skip("test process can read files without permission bits")
|
||||
}
|
||||
},
|
||||
wantErr: "read secret file",
|
||||
},
|
||||
|
||||
@@ -21,6 +21,9 @@ func Defaults() Config {
|
||||
Secrets: SecretsConfig{
|
||||
Directory: "",
|
||||
},
|
||||
Output: OutputConfig{
|
||||
Directory: "",
|
||||
},
|
||||
Notify: NotifyConfig{
|
||||
Distributor: DistributorNotifyConfig{
|
||||
Enabled: false,
|
||||
|
||||
@@ -15,6 +15,9 @@ func Validate(cfg Config) error {
|
||||
if err := validateReportDistributorPathOverrides(cfg); err != nil {
|
||||
return err
|
||||
}
|
||||
if cfg.Output.Directory != "" && strings.TrimSpace(cfg.Output.Directory) == "" {
|
||||
return fmt.Errorf("output.directory must not be blank when configured")
|
||||
}
|
||||
if cfg.WeatherAPI.BaseURL != "" {
|
||||
parsed, err := url.Parse(cfg.WeatherAPI.BaseURL)
|
||||
if err != nil || parsed.Scheme == "" || parsed.Host == "" {
|
||||
|
||||
@@ -20,14 +20,46 @@ Distinguish the main weather outcome from its caveat. If showers and thunderstor
|
||||
|
||||
# Forecast discussion
|
||||
|
||||
Use narrative products to explain the “why” behind the local forecast when useful. Useful context may include synoptic pattern, fronts or boundaries, shortwaves, troughs or ridges, instability, moisture, shear, forcing, capping, regional placement of precipitation or severe-weather chances, hazards, timing windows, confidence, uncertainty, conditional outcomes, and relevant notes about following days.
|
||||
The forecast discussion should explain the local forecast, not merely restate it or summarize every narrative product. Write for an informed reader who wants to understand what is driving the weather, what the important limitations are, and what the pattern may imply next.
|
||||
|
||||
In most cases, include three paragraphs: a two-to-four sentence relevant local or regional setup; a two-to-four sentence main uncertainty or conditional factor when present; and a two-to-four sentence next-day or broader-pattern note when supported.
|
||||
Select the one to three source-supported details that most improve understanding of the forecast. When useful, connect them in this order:
|
||||
|
||||
1. the relevant weather feature, mechanism, or change;
|
||||
2. the timing, parameter, or forecast evidence that matters;
|
||||
3. the practical implication for precipitation, temperatures, hazards, confidence, or planning.
|
||||
|
||||
For example, explain not only that a front is approaching, but—when supported—whether its timing, instability, moisture, or movement changes the expected storm coverage, hazard ceiling, rainfall potential, or temperature trend. The reader should not need prior meteorological knowledge to understand why a technical detail matters.
|
||||
|
||||
Prefer explanatory synthesis over enumeration. Do not include a technical term or numerical parameter merely because it is available. Include it only when it materially helps explain the forecast, and translate its significance into plain language. Prefer the interpretation stated or clearly supported by the narrative products rather than independently diagnosing the atmosphere from isolated raw values.
|
||||
|
||||
Usually provide two or three paragraphs:
|
||||
|
||||
- Explain the main setup and its practical consequence for the valid period.
|
||||
- Explain the most important uncertainty, conditional factor, or limiting factor when one exists.
|
||||
- When supported, explain a meaningful consequence for the following day or broader pattern, including the causal connection. Do not add a broader-pattern paragraph merely to fill space.
|
||||
|
||||
If the narrative products do not provide a useful mechanism or broader explanation, keep the discussion concise and focus on supported timing, confidence, and alternate outcomes. Never invent a mechanism to explain the local forecast.
|
||||
|
||||
# Precipitation timing
|
||||
|
||||
When precipitation windows are present, use one to four sentences to give practical context about a supported frontal, convective, or stratiform setup; expected type, intensity, and duration; and uncertainty in onset or duration. When no precipitation windows are present, return an empty string (`""`) for `precipitation_timing`.
|
||||
When precipitation windows are present, use one to four sentences to explain the supported timing, precipitation type, likely coverage or continuity, and the most important uncertainty. Mention intensity, duration, or the responsible weather feature only when those details are explicitly supported by the supplied data.
|
||||
|
||||
# Narrative source selection
|
||||
Distinguish a regional setup favorable for precipitation from the local probability of receiving it. When no precipitation windows are present, return an empty string (`""`) for `precipitation_timing`.
|
||||
|
||||
Use `briefing.derived_daily_summary`, `briefing.derived_daypart_summaries`, `briefing.narrative_products.narrative_forecast.periods`, and `briefing.raw_data.hourly_forecast.periods` as primary sources. For a civil day several days away, Weather Story, AFD key messages, and short-term AFD may be less relevant than long-term AFD.
|
||||
# Source selection and grounding
|
||||
|
||||
Use the local derived summaries, narrative point forecast, and hourly forecast as the primary sources for what is expected at the configured location: timing, probability, temperatures, winds, and local conditions.
|
||||
|
||||
Use the AFD, Weather Story, applicable risk products, and alerts to explain why the weather is expected, the broader regional context, forecast confidence, conditional outcomes, and the ceiling for hazards. For a civil day several days away, the long-term AFD will often be more relevant than the short-term AFD.
|
||||
|
||||
Do not allow regional AFD language to override or strengthen the local point forecast. A regional expectation for thunderstorms does not make precipitation likely at the configured location when the local probability is lower. Clearly distinguish regional support for an event from the probability that the location itself will be affected.
|
||||
|
||||
Preserve relationships stated in the sources. Do not infer a front, trough, boundary, dry-air intrusion, clearing trend, instability level, severe threat, or rainfall hazard solely from changes in hourly probabilities or conditions. The absence of an alert or SPC outlook does not by itself prove that severe weather is impossible.
|
||||
|
||||
When a source describes a regional or conditional hazard, preserve its stated geographic scope, timing, and uncertainty. Do not infer that the hazard is either expected or excluded at the configured location unless the source or an applicable location-specific product supports that conclusion. When a risk product overlaps only part of the report period, state or respect that limited validity.
|
||||
|
||||
Before returning the JSON, verify silently that:
|
||||
|
||||
- every weather mechanism, parameter, hazard, and causal relationship is supported by the supplied data;
|
||||
- every technical detail included has a clear practical significance for the reader;
|
||||
- regional forecast language has not been converted into a stronger local forecast.
|
||||
@@ -269,7 +269,23 @@ func ensureSecureDirectory(path string) error {
|
||||
info, err := os.Lstat(current)
|
||||
if os.IsNotExist(err) {
|
||||
if err := os.Mkdir(current, debugDirectoryMode); err != nil {
|
||||
return err
|
||||
if !os.IsExist(err) {
|
||||
return err
|
||||
}
|
||||
info, err = os.Lstat(current)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if info.Mode()&os.ModeSymlink != 0 {
|
||||
return fmt.Errorf("directory component %q must not be a symlink", current)
|
||||
}
|
||||
if !info.IsDir() {
|
||||
return fmt.Errorf("directory component %q is not a directory", current)
|
||||
}
|
||||
if err := os.Chmod(current, debugDirectoryMode); err != nil {
|
||||
return err
|
||||
}
|
||||
continue
|
||||
}
|
||||
if err := os.Chmod(current, debugDirectoryMode); err != nil {
|
||||
return err
|
||||
|
||||
@@ -1,10 +1,12 @@
|
||||
package promptdebug
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -91,6 +93,73 @@ func TestPromptDebugWriterAtomicallyReplacesArtifacts(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestPromptDebugWriterCreatesSharedMissingAncestorsConcurrently(t *testing.T) {
|
||||
root := filepath.Join(t.TempDir(), "debug")
|
||||
writer, err := NewPromptDebugWriter(root)
|
||||
if err != nil {
|
||||
t.Fatalf("NewPromptDebugWriter() error = %v", err)
|
||||
}
|
||||
|
||||
const writerCount = 8
|
||||
start := make(chan struct{})
|
||||
type writeResult struct {
|
||||
directory string
|
||||
err error
|
||||
}
|
||||
results := make(chan writeResult, writerCount)
|
||||
var writers sync.WaitGroup
|
||||
for index := 0; index < writerCount; index++ {
|
||||
writers.Add(1)
|
||||
go func(index int) {
|
||||
defer writers.Done()
|
||||
<-start
|
||||
directory, err := writer.WritePreparation(PromptDebugRef{
|
||||
ReportID: report.Daily, ValidDate: "2026-05-29", RunID: fmt.Sprintf("run-%02d", index),
|
||||
}, promptDebugPreparationFixture(), nil)
|
||||
results <- writeResult{directory: directory, err: err}
|
||||
}(index)
|
||||
}
|
||||
close(start)
|
||||
|
||||
finished := make(chan struct{})
|
||||
go func() {
|
||||
writers.Wait()
|
||||
close(finished)
|
||||
}()
|
||||
select {
|
||||
case <-finished:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("concurrent prompt debug writes did not finish")
|
||||
}
|
||||
close(results)
|
||||
|
||||
directories := map[string]struct{}{}
|
||||
for result := range results {
|
||||
if result.err != nil {
|
||||
t.Fatalf("WritePreparation() error = %v", result.err)
|
||||
}
|
||||
if _, duplicate := directories[result.directory]; duplicate {
|
||||
t.Fatalf("duplicate debug directory %q", result.directory)
|
||||
}
|
||||
directories[result.directory] = struct{}{}
|
||||
if _, err := os.Stat(filepath.Join(result.directory, "preparation.json")); err != nil {
|
||||
t.Fatalf("preparation artifact %q: %v", result.directory, err)
|
||||
}
|
||||
}
|
||||
if len(directories) != writerCount {
|
||||
t.Fatalf("debug directories = %#v", directories)
|
||||
}
|
||||
if runtime.GOOS != "windows" {
|
||||
assertPromptDebugMode(t, root, debugDirectoryMode)
|
||||
assertPromptDebugMode(t, filepath.Join(root, "daily"), debugDirectoryMode)
|
||||
assertPromptDebugMode(t, filepath.Join(root, "daily", "2026-05-29"), debugDirectoryMode)
|
||||
for directory := range directories {
|
||||
assertPromptDebugMode(t, directory, debugDirectoryMode)
|
||||
assertPromptDebugMode(t, filepath.Join(directory, "preparation.json"), debugFileMode)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestPromptDebugWriterDisabledDoesNotAccessFilesystem(t *testing.T) {
|
||||
writer, err := NewPromptDebugWriter("")
|
||||
if err != nil {
|
||||
|
||||
Reference in New Issue
Block a user