Compare commits
16 Commits
af9cb0c0dc
...
eed47b4f68
| Author | SHA1 | Date | |
|---|---|---|---|
| eed47b4f68 | |||
| 6c185b8d0e | |||
| faf547e4a8 | |||
| acb476a142 | |||
| 606b4423f1 | |||
| 1716702c99 | |||
| e0229d9c90 | |||
| ccf6b66880 | |||
| b489c56a48 | |||
| d39e42de30 | |||
| d642791c10 | |||
| 236e3d16c4 | |||
| 4fa873983d | |||
| de1ae896b3 | |||
| 6173e50d25 | |||
| 3bca2f41f7 |
@@ -4,7 +4,9 @@ Weatherreporter is a Go CLI that turns normalized weather data into
|
|||||||
human-facing Markdown reports.
|
human-facing Markdown reports.
|
||||||
|
|
||||||
It produces a Markdown report at an operator-owned destination and can upload
|
It produces a Markdown report at an operator-owned destination and can upload
|
||||||
the completed output through Distributor.
|
the completed output through Distributor. It can also compare explicitly
|
||||||
|
selected Promptkit profiles against one shared prepared report and publish a
|
||||||
|
local comparison bundle.
|
||||||
|
|
||||||
## Quickstart
|
## Quickstart
|
||||||
|
|
||||||
@@ -24,5 +26,6 @@ guide](docs/operations.md) for command and operating details.
|
|||||||
- [CLI reference](docs/cli.md)
|
- [CLI reference](docs/cli.md)
|
||||||
- [Configuration reference](docs/config.md)
|
- [Configuration reference](docs/config.md)
|
||||||
- [Operations guide](docs/operations.md)
|
- [Operations guide](docs/operations.md)
|
||||||
|
- [Comparison bundle contract](docs/integrations/comparison-bundle.md)
|
||||||
- [Development guide](docs/development.md)
|
- [Development guide](docs/development.md)
|
||||||
- [Architecture policy](docs/policy/architecture.md)
|
- [Architecture policy](docs/policy/architecture.md)
|
||||||
|
|||||||
84
docs/cli.md
84
docs/cli.md
@@ -1,7 +1,8 @@
|
|||||||
# Weatherreporter CLI
|
# Weatherreporter CLI
|
||||||
|
|
||||||
`weatherreporter` generates Markdown weather reports and runs report batches.
|
`weatherreporter` generates Markdown weather reports, runs report batches, and
|
||||||
It has no command for inspecting prior runs or application-owned state.
|
compares explicitly selected Promptkit profiles against one prepared report. It
|
||||||
|
has no command for inspecting prior runs or application-owned state.
|
||||||
|
|
||||||
## Shortest Useful Command
|
## Shortest Useful Command
|
||||||
|
|
||||||
@@ -25,6 +26,7 @@ weatherreporter generate tomorrow [--config PATH] [--units VALUE] [--tz NAME] [-
|
|||||||
weatherreporter generate hourly [--config PATH] [--units VALUE] [--tz NAME] [--out PATH] [--llm-debug-dir PATH] [--quiet]
|
weatherreporter generate hourly [--config PATH] [--units VALUE] [--tz NAME] [--out PATH] [--llm-debug-dir PATH] [--quiet]
|
||||||
weatherreporter run morning [--config PATH] [--units VALUE] [--tz NAME] [--out-dir PATH] [--llm-debug-dir PATH] [--quiet]
|
weatherreporter run morning [--config PATH] [--units VALUE] [--tz NAME] [--out-dir PATH] [--llm-debug-dir PATH] [--quiet]
|
||||||
weatherreporter run evening [--config PATH] [--units VALUE] [--tz NAME] [--out-dir PATH] [--llm-debug-dir PATH] [--quiet]
|
weatherreporter run evening [--config PATH] [--units VALUE] [--tz NAME] [--out-dir PATH] [--llm-debug-dir PATH] [--quiet]
|
||||||
|
weatherreporter compare REPORT --profile PROFILE --profile PROFILE [--config PATH] [--units VALUE] [--tz NAME] [--date YYYY-MM-DD] [--out-dir PATH] [--replace] [--llm-debug-dir PATH] [--quiet]
|
||||||
```
|
```
|
||||||
|
|
||||||
`weatherreporter --version` prints the version embedded in the executable.
|
`weatherreporter --version` prints the version embedded in the executable.
|
||||||
@@ -38,10 +40,13 @@ builds report `development`.
|
|||||||
| `generate tomorrow` | Uses the next local civil day and writes `tomorrow.md` by default. |
|
| `generate tomorrow` | Uses the next local civil day and writes `tomorrow.md` by default. |
|
||||||
| `generate hourly` | Covers the next six hours in the effective report timezone and writes `hourly.md` by default. It does not accept `--date`, `--hours`, or `--duration`. |
|
| `generate hourly` | Covers the next six hours in the effective report timezone and writes `hourly.md` by default. It does not accept `--date`, `--hours`, or `--duration`. |
|
||||||
| `run morning` and `run evening` | Run their defined report batches beneath the configured output directory, or the current directory when none is configured. `--out-dir` selects another directory. `--out` is not accepted. |
|
| `run morning` and `run evening` | Run their defined report batches beneath the configured output directory, or the current directory when none is configured. `--out-dir` selects another directory. `--out` is not accepted. |
|
||||||
|
| `compare REPORT` | Accepts `daily`, `today`, `tomorrow`, or `hourly`. It requires at least two distinct, nonblank `--profile` values in their supplied order. Daily requires `--date`; Today accepts it optionally; Tomorrow and Hourly do not accept it. |
|
||||||
|
|
||||||
`generate` accepts the four report command names shown above. `run` accepts
|
`generate` accepts the four report command names shown above. `run` accepts
|
||||||
only `morning` and `evening`. Batch membership and notification ordering are
|
only `morning` and `evening`. `compare` always requires explicit profile
|
||||||
described in the [operations guide](operations.md).
|
selection: `promptkit.profile` is not used as a comparison default. Batch
|
||||||
|
membership and notification ordering are described in the
|
||||||
|
[operations guide](operations.md).
|
||||||
|
|
||||||
## Output, Errors, And Quiet Mode
|
## Output, Errors, And Quiet Mode
|
||||||
|
|
||||||
@@ -50,21 +55,25 @@ For `generate`, the report's default filename is placed beneath
|
|||||||
PATH` selects one complete output file instead. A relative path is resolved
|
PATH` selects one complete output file instead. A relative path is resolved
|
||||||
from the current directory; an absolute path is used as given. For a batch,
|
from the current directory; an absolute path is used as given. For a batch,
|
||||||
the configured directory has the same role and `--out-dir PATH` selects its
|
the configured directory has the same role and `--out-dir PATH` selects its
|
||||||
output directory instead. Successful summaries always report the resulting
|
output directory instead. For `compare`, `--out-dir PATH` selects one exact
|
||||||
absolute `outputPath` values. See the [configuration reference](config.md) for
|
bundle directory; otherwise the report-derived comparison directory is placed
|
||||||
the field's validation and path rules.
|
beneath the configured directory or current directory. `--replace` is required
|
||||||
|
to replace an existing nonempty recognized comparison bundle. See the
|
||||||
|
[configuration reference](config.md) for the field's validation and path rules
|
||||||
|
and the [comparison bundle contract](integrations/comparison-bundle.md) for the
|
||||||
|
bundle format.
|
||||||
|
|
||||||
Outputs are written atomically. A generation, rendering, write, or cancellation
|
Outputs are written atomically. A generation, rendering, write, or cancellation
|
||||||
failure before publication leaves an existing destination unchanged. A
|
failure before publication leaves an existing destination unchanged. A
|
||||||
notification failure occurs after publication, so the newly written output
|
notification failure occurs after publication, so the newly written output
|
||||||
remains available.
|
remains available.
|
||||||
|
|
||||||
Action commands (`generate` and `run`) write a JSON summary to stdout unless
|
Action commands (`generate`, `run`, and `compare`) write a JSON summary to
|
||||||
`--quiet` is set. `run` also writes compact per-report and batch status lines
|
stdout unless `--quiet` is set. `run` also writes compact per-report and batch
|
||||||
to stderr. A pre-run error, such as an invalid flag, missing required argument,
|
status lines to stderr. A pre-run error, such as an invalid flag, missing
|
||||||
or configuration-load failure, produces no partial JSON summary. When an
|
required argument, or configuration-load failure, produces no partial JSON
|
||||||
action fails after it has produced a result, its summary has `"status": "failed"`
|
summary. When an action fails after it has produced a result, its summary has
|
||||||
and an `error` field.
|
`"status": "failed"` and an `error` field.
|
||||||
|
|
||||||
`--quiet` is supported by action commands only. It suppresses action summaries
|
`--quiet` is supported by action commands only. It suppresses action summaries
|
||||||
and routine batch status output; it does not suppress command errors.
|
and routine batch status output; it does not suppress command errors.
|
||||||
@@ -113,19 +122,53 @@ report=today status=succeeded output="/srv/weather/reports/today.md"
|
|||||||
batch=morning total=2 succeeded=2 failed=0
|
batch=morning total=2 succeeded=2 failed=0
|
||||||
```
|
```
|
||||||
|
|
||||||
|
### Compare Summary
|
||||||
|
|
||||||
|
A comparison summary contains these fields in this order: `command`,
|
||||||
|
`comparisonId`, `reportId`, `reportName`, `promptId`, `promptVersion`,
|
||||||
|
`promptHash`, `status`, `startedAt`, `finishedAt`, `timezone`, `validPeriod`,
|
||||||
|
`outputDirectory`, `manifestPath`, `dataPackagePath`, `total`, `succeeded`,
|
||||||
|
`failed`, `results`, and optional `error`. Published artifact paths and each
|
||||||
|
successful `results[].reportPath` are absolute. `results` preserves the
|
||||||
|
supplied profile order and each item contains `position`, `profileId`, optional
|
||||||
|
`backendId`, `modelName`, `status`, optional `validationStatus`, optional
|
||||||
|
`reportPath`, optional `llmDebugPath`, and optional safe `error`.
|
||||||
|
|
||||||
|
The comparison status is `succeeded` only when every selected profile succeeds
|
||||||
|
and the bundle is published. Individual profile failures still publish a
|
||||||
|
complete partial bundle and return a failed command result. Cancellation or a
|
||||||
|
failure before publication omits the artifact paths and returns a safe
|
||||||
|
top-level error; the resolved `outputDirectory` and finalized timestamp remain
|
||||||
|
when available. The safe error includes only a category and message: aggregate
|
||||||
|
and unclassified application failures use `application`; cancellation uses
|
||||||
|
`canceled`; deadlines use `deadline_exceeded`; prompt execution uses its
|
||||||
|
published Promptkit category; destination failures use `destination_<kind>`;
|
||||||
|
and committed cleanup failures use `publication_cleanup`. It does not expose
|
||||||
|
provider diagnostics, filesystem causes, or recovery paths. See the
|
||||||
|
[comparison bundle contract](integrations/comparison-bundle.md) for durable
|
||||||
|
artifact fields and failure invariants.
|
||||||
|
|
||||||
|
If the bundle is published but cleanup of its replaced prior bundle fails, the
|
||||||
|
summary still includes the published artifact paths and has status `failed`.
|
||||||
|
Its JSON error is `publication_cleanup` with the message `comparison published
|
||||||
|
but cleanup did not complete`; the returned command error identifies the
|
||||||
|
retained backup path for operator recovery.
|
||||||
|
|
||||||
## Flag Reference
|
## Flag Reference
|
||||||
|
|
||||||
| Flag | Accepted by | Meaning |
|
| Flag | Accepted by | Meaning |
|
||||||
| --- | --- | --- |
|
| --- | --- | --- |
|
||||||
| `-h`, `--help` | top level | Show help. |
|
| `-h`, `--help` | top level, `compare` | Show help without loading configuration or contacting a provider. |
|
||||||
| `--config PATH` | all commands | Load `PATH` instead of `/usr/local/etc/weatherreporter/config.yml`. |
|
| `--config PATH` | all commands | Load `PATH` instead of `/usr/local/etc/weatherreporter/config.yml`. |
|
||||||
| `--units VALUE` | `generate`, `run` | Override `weather_api.units` for this command. |
|
| `--units VALUE` | `generate`, `run`, `compare` | Override `weather_api.units` for this command. |
|
||||||
| `--tz NAME` | `generate`, `run` | Override `weather_api.timezone` for this command. |
|
| `--tz NAME` | `generate`, `run`, `compare` | Override `weather_api.timezone` for this command. |
|
||||||
| `--out PATH` | every `generate` command | Write the report to this complete file destination instead of the configured or current-directory default. |
|
| `--out PATH` | every `generate` command | Write the report to this complete file destination instead of the configured or current-directory default. |
|
||||||
| `--llm-debug-dir PATH` | every `generate` and `run` command | Write requested sensitive prompt diagnostics under this absolute path. |
|
| `--llm-debug-dir PATH` | every `generate`, `run`, and `compare` command | Write requested sensitive prompt diagnostics under this absolute path. |
|
||||||
| `--out-dir PATH` | `run morning`, `run evening` | Write batch reports beneath this directory instead of the configured or current-directory default. |
|
| `--profile PROFILE` | `compare` | Select one explicit profile. Repeat at least twice with distinct, nonblank IDs. |
|
||||||
| `--quiet` | `generate`, `run` | Suppress action summaries and routine batch status output. |
|
| `--out-dir PATH` | `run morning`, `run evening`, `compare` | Write batch reports beneath this directory, or select the exact comparison directory. |
|
||||||
| `--date YYYY-MM-DD` | `generate daily`, `generate today` | Required for Daily; optional for Today. |
|
| `--replace` | `compare` | Authorize replacement of a recognized nonempty comparison bundle. |
|
||||||
|
| `--quiet` | `generate`, `run`, `compare` | Suppress successful action output and routine batch status output. |
|
||||||
|
| `--date YYYY-MM-DD` | `generate daily`, `generate today`, `compare daily`, `compare today` | Required for Daily; optional for Today. |
|
||||||
|
|
||||||
Distributor notification is configured through `notify.distributor`; there are
|
Distributor notification is configured through `notify.distributor`; there are
|
||||||
no Distributor-specific CLI flags. See the [configuration reference](config.md).
|
no Distributor-specific CLI flags. See the [configuration reference](config.md).
|
||||||
@@ -138,4 +181,5 @@ weatherreporter generate today --out ./reports/today.md
|
|||||||
weatherreporter generate hourly --out /srv/weather/hourly.md
|
weatherreporter generate hourly --out /srv/weather/hourly.md
|
||||||
weatherreporter generate today --llm-debug-dir /var/tmp/weatherreporter-debug
|
weatherreporter generate today --llm-debug-dir /var/tmp/weatherreporter-debug
|
||||||
weatherreporter run morning --out-dir ./reports --llm-debug-dir /var/tmp/weatherreporter-debug
|
weatherreporter run morning --out-dir ./reports --llm-debug-dir /var/tmp/weatherreporter-debug
|
||||||
|
weatherreporter compare daily --date 2026-05-29 --profile weather-light --profile weather-balanced --out-dir ./comparison-daily-2026-05-29
|
||||||
```
|
```
|
||||||
|
|||||||
@@ -84,7 +84,7 @@ this directory, never in the YAML file.
|
|||||||
### `output`
|
### `output`
|
||||||
|
|
||||||
`output.directory` selects the ordinary operator-owned publication directory
|
`output.directory` selects the ordinary operator-owned publication directory
|
||||||
for both individual reports and batches.
|
for individual reports, batches, and the default parent of comparison bundles.
|
||||||
|
|
||||||
| Field | Default | Rules |
|
| Field | Default | Rules |
|
||||||
| --- | --- | --- |
|
| --- | --- | --- |
|
||||||
@@ -101,6 +101,8 @@ publication.
|
|||||||
|
|
||||||
For one `generate` command, `--out` is a complete file destination and takes
|
For one `generate` command, `--out` is a complete file destination and takes
|
||||||
precedence over `output.directory`. For `run`, `--out-dir` takes precedence.
|
precedence over `output.directory`. For `run`, `--out-dir` takes precedence.
|
||||||
|
For `compare`, `--out-dir` selects its exact bundle directory; without it, the
|
||||||
|
comparison's report-derived directory is placed beneath `output.directory`.
|
||||||
Those explicit flags do not inspect or rebase beneath the configured directory.
|
Those explicit flags do not inspect or rebase beneath the configured directory.
|
||||||
See the [CLI reference](cli.md) for command selection and the [operations
|
See the [CLI reference](cli.md) for command selection and the [operations
|
||||||
guide](operations.md) for publication and failure handling.
|
guide](operations.md) for publication and failure handling.
|
||||||
@@ -166,11 +168,11 @@ source keys are `observations`, `current`, `narrative`, `alerts`, `discussion`,
|
|||||||
### `promptkit`
|
### `promptkit`
|
||||||
|
|
||||||
Promptkit configuration selects the executor and prompt/profile checks for
|
Promptkit configuration selects the executor and prompt/profile checks for
|
||||||
every `generate` and `run` command. A top-level `scriptorium:` configuration
|
every `generate`, `run`, and `compare` command. A top-level `scriptorium:` configuration
|
||||||
key is rejected with a migration error; it is not translated or ignored.
|
key is rejected with a migration error; it is not translated or ignored.
|
||||||
|
|
||||||
Prompt debug capture has no YAML setting. Use `--llm-debug-dir PATH` on an
|
Prompt debug capture has no YAML setting. Use `--llm-debug-dir PATH` on an
|
||||||
individual `generate` or `run` command when explicitly needed.
|
individual `generate`, `run`, or `compare` command when explicitly needed.
|
||||||
|
|
||||||
| Field | Default | Rules |
|
| Field | Default | Rules |
|
||||||
| --- | --- | --- |
|
| --- | --- | --- |
|
||||||
|
|||||||
@@ -21,14 +21,15 @@ boundaries and invariants.
|
|||||||
| Adding, changing, reviewing, or deleting tests | [Testing policy](policy/testing.md) and focused package tests | The policy defines risk-based sufficiency, durable test boundaries, doubles, and test-maintenance criteria. |
|
| Adding, changing, reviewing, or deleting tests | [Testing policy](policy/testing.md) and focused package tests | The policy defines risk-based sufficiency, durable test boundaries, doubles, and test-maintenance criteria. |
|
||||||
| CLI commands, flags, output, quiet mode, or command wiring | [CLI reference](cli.md) and [CLI internals](internal/cli.md) | The reference owns the user contract; the internal guide owns command composition and output flow. |
|
| CLI commands, flags, output, quiet mode, or command wiring | [CLI reference](cli.md) and [CLI internals](internal/cli.md) | The reference owns the user contract; the internal guide owns command composition and output flow. |
|
||||||
| Configuration fields, defaults, loading, overrides, validation, or secrets | [Configuration reference](config.md), [architecture policy](policy/architecture.md), and tests under `internal/config` | These separate the user-visible contract, architectural rules, and executable behavior. |
|
| Configuration fields, defaults, loading, overrides, validation, or secrets | [Configuration reference](config.md), [architecture policy](policy/architecture.md), and tests under `internal/config` | These separate the user-visible contract, architectural rules, and executable behavior. |
|
||||||
| Top-level generation, batch, collection, output publication, or notification workflow | [App orchestration internals](internal/app-orchestration.md) | It owns workflow ordering, output publication, failure propagation, and orchestration invariants. |
|
| Top-level generation, batch, comparison, collection, output publication, or notification workflow | [App orchestration internals](internal/app-orchestration.md), [comparison execution internals](internal/comparison-execution.md), and [comparison publication internals](internal/comparison-publication.md) | They own workflow ordering, concurrent profile execution, output publication, failure propagation, and orchestration invariants. |
|
||||||
| Weather API transport, source envelopes, source warnings, or collection | [Weather API integration](integrations/weatherapi.md), [weather-data internals](internal/weather-data.md), and [collection internals](internal/collect.md) | These separate the external contract, normalized source facts, and app-facing collection behavior. |
|
| Weather API transport, source envelopes, source warnings, or collection | [Weather API integration](integrations/weatherapi.md), [weather-data internals](internal/weather-data.md), and [collection internals](internal/collect.md) | These separate the external contract, normalized source facts, and app-facing collection behavior. |
|
||||||
| Forecast periods, weather derivation, collected facts, or derived facts | [Forecast derivation internals](internal/forecast-derivation.md) and [fact contracts](internal/facts.md) | They own deterministic derivation and the fact boundaries used by reports. |
|
| Forecast periods, weather derivation, collected facts, or derived facts | [Forecast derivation internals](internal/forecast-derivation.md) and [fact contracts](internal/facts.md) | They own deterministic derivation and the fact boundaries used by reports. |
|
||||||
| Report definitions, valid periods, report IDs, output naming, or batch composition | [Report registry internals](internal/report-registry.md) and [app orchestration internals](internal/app-orchestration.md) | Report definitions own selection and period rules; orchestration owns execution. |
|
| Report definitions, valid periods, report IDs, output naming, or batch composition | [Report registry internals](internal/report-registry.md) and [app orchestration internals](internal/app-orchestration.md) | Report definitions own selection and period rules; orchestration owns execution. |
|
||||||
| Module IDs, module composition, briefing values, or prompt-facing exports | [Module contract internals](internal/module.md), [module builder internals](internal/briefing.md), and [prompt-input internals](internal/prompt-input.md) | These own module contracts, value construction, and the curated prompt-package boundary. |
|
| Module IDs, module composition, briefing values, or prompt-facing exports | [Module contract internals](internal/module.md), [module builder internals](internal/briefing.md), and [prompt-input internals](internal/prompt-input.md) | These own module contracts, value construction, and the curated prompt-package boundary. |
|
||||||
| Prompt execution, profiles, prompt inputs, or result handling | `internal/promptexec`, the Promptkit adapter, and [prompt-input internals](internal/prompt-input.md) | These separate the executor contract and input construction. |
|
| Prompt execution, profiles, prepared report inputs, or result handling | `internal/promptexec`, the Promptkit adapter, [prepared report internals](internal/prepared-report.md), and [prompt-input internals](internal/prompt-input.md) | These separate the executor contract, immutable preparation, and input construction. |
|
||||||
|
| Durable comparison bundles or their compatibility | [Comparison bundle contract](integrations/comparison-bundle.md) and [comparison publication internals](internal/comparison-publication.md) | The integration document owns the external schema; internals own how it is published. |
|
||||||
| Generated-text schemas, validation, render contexts, templates, or Markdown rendering | [Generated-text internals](internal/generatedtext.md), [report-template internals](internal/reporttemplate.md), and [report template guide](templates.md) | These own structured text, renderer implementation, and the maintainer-facing template surface. |
|
| Generated-text schemas, validation, render contexts, templates, or Markdown rendering | [Generated-text internals](internal/generatedtext.md), [report-template internals](internal/reporttemplate.md), and [report template guide](templates.md) | These own structured text, renderer implementation, and the maintainer-facing template surface. |
|
||||||
| Output destinations, atomic publication, prompt diagnosis, or legacy cleanup | [Operations guide](operations.md) and [App orchestration internals](internal/app-orchestration.md) | Operations owns operator workflows; app internals owns the implementation boundary. |
|
| Output destinations, atomic publication, prompt diagnosis, or legacy cleanup | [Operations guide](operations.md), [App orchestration internals](internal/app-orchestration.md), and [comparison publication internals](internal/comparison-publication.md) | Operations owns operator workflows; internals own implementation boundaries. |
|
||||||
| Distributor bundles, uploads, notification results, or failures | [Distributor adapter internals](internal/distributor-adapter.md), [Distributor integration contracts](integrations/distributor/), and [operations guide](operations.md) | These separate adapter behavior, external contracts, and operational lifecycle. |
|
| Distributor bundles, uploads, notification results, or failures | [Distributor adapter internals](internal/distributor-adapter.md), [Distributor integration contracts](integrations/distributor/), and [operations guide](operations.md) | These separate adapter behavior, external contracts, and operational lifecycle. |
|
||||||
| Maintained example configuration | [Configuration reference](config.md) and files under `examples/` | The reference owns field meaning; examples own complete copyable files. |
|
| Maintained example configuration | [Configuration reference](config.md) and files under `examples/` | The reference owns field meaning; examples own complete copyable files. |
|
||||||
| Release preparation, tagging, publication, or verification | [Release procedure](release.md) | It owns version selection, release-note preparation, candidate validation, tag publication, CI behavior, and post-publication checks. |
|
| Release preparation, tagging, publication, or verification | [Release procedure](release.md) | It owns version selection, release-note preparation, candidate validation, tag publication, CI behavior, and post-publication checks. |
|
||||||
@@ -44,7 +45,8 @@ present before introducing a new package or abstraction.
|
|||||||
| --- | --- |
|
| --- | --- |
|
||||||
| `cmd/weatherreporter` | Binary entry point. |
|
| `cmd/weatherreporter` | Binary entry point. |
|
||||||
| `internal/cli` | Command parsing, flags, help, output, and command wiring. |
|
| `internal/cli` | Command parsing, flags, help, output, and command wiring. |
|
||||||
| `internal/app` | Stateless generation, batches, collection coordination, output publication, and notification. |
|
| `internal/app` | Stateless generation, batches, comparisons, collection coordination, output publication, and notification. |
|
||||||
|
| `internal/comparison` | Comparison identities, logical bundles, guarded destinations, and atomic bundle publication. |
|
||||||
| `internal/config` | Configuration defaults, loading, precedence, secrets, and validation. |
|
| `internal/config` | Configuration defaults, loading, precedence, secrets, and validation. |
|
||||||
| `internal/adapters` | Weather API, Promptkit, and Distributor boundaries. |
|
| `internal/adapters` | Weather API, Promptkit, and Distributor boundaries. |
|
||||||
| `internal/weatherdata`, `internal/forecast`, `internal/facts` | Normalized source facts and deterministic derivation. |
|
| `internal/weatherdata`, `internal/forecast`, `internal/facts` | Normalized source facts and deterministic derivation. |
|
||||||
|
|||||||
92
docs/integrations/comparison-bundle.md
Normal file
92
docs/integrations/comparison-bundle.md
Normal file
@@ -0,0 +1,92 @@
|
|||||||
|
# Comparison Bundle Contract
|
||||||
|
|
||||||
|
A comparison bundle is the durable, flat artifact produced when one report is
|
||||||
|
executed with multiple explicit Promptkit profiles. This document is the
|
||||||
|
canonical contract for consumers of those bundles. Command invocation and JSON
|
||||||
|
action summaries belong to the [CLI reference](../cli.md); destination handling
|
||||||
|
and retention belong to the [operations guide](../operations.md).
|
||||||
|
|
||||||
|
## Version And Layout
|
||||||
|
|
||||||
|
The current and only supported manifest schema version is
|
||||||
|
`weatherreporter.comparison.v1`. A bundle directory contains exactly these
|
||||||
|
regular, non-symlinked files:
|
||||||
|
|
||||||
|
```text
|
||||||
|
comparison.json
|
||||||
|
data-package.yml
|
||||||
|
NN-profile-slug.md
|
||||||
|
```
|
||||||
|
|
||||||
|
`comparison.json` is the manifest and `data-package.yml` is the exact YAML
|
||||||
|
input supplied to every selected profile. There is one Markdown file for each
|
||||||
|
successful result and none for failed results. `NN` is the one-based selected
|
||||||
|
profile position, zero padded to at least two digits (and widened only when
|
||||||
|
needed for 100 or more profiles). The profile slug preserves ASCII letters,
|
||||||
|
digits, `-`, and `_`; each run of other characters becomes one `-`; edge `-`
|
||||||
|
and `_` characters are removed; the value is capped at 64 bytes; and an empty
|
||||||
|
slug becomes `profile`. Logical profile IDs remain authoritative in the
|
||||||
|
manifest.
|
||||||
|
|
||||||
|
All manifest paths are basenames relative to the bundle root. They never use
|
||||||
|
path separators, `.` or `..`. The CLI reports absolute paths only after a
|
||||||
|
bundle has been published.
|
||||||
|
|
||||||
|
## Manifest Schema
|
||||||
|
|
||||||
|
The manifest is UTF-8 JSON, encoded as two-space-indented JSON with one
|
||||||
|
trailing newline. Its fields appear in this order:
|
||||||
|
|
||||||
|
```text
|
||||||
|
schemaVersion, comparisonId, startedAt, finishedAt, reportId, validPeriod,
|
||||||
|
timezone, promptId, promptVersion, promptHash, dataPackage, total, succeeded,
|
||||||
|
failed, results
|
||||||
|
```
|
||||||
|
|
||||||
|
`validPeriod` contains `start` and `end`; it is a nonempty half-open period.
|
||||||
|
`dataPackage` contains `path` (always `data-package.yml`) and `sha256` (the
|
||||||
|
lowercase, 64-character SHA-256 digest of that file's exact bytes). `results`
|
||||||
|
is in the explicit profile-selection order. Its result-object fields appear in
|
||||||
|
this order:
|
||||||
|
|
||||||
|
```text
|
||||||
|
position, profileId, backendId, modelName, status, validationStatus,
|
||||||
|
reportPath, error
|
||||||
|
```
|
||||||
|
|
||||||
|
`startedAt` and `finishedAt` are nonzero UTC timestamps, and the latter is not
|
||||||
|
earlier than the former. `validPeriod` retains its resolved time offset.
|
||||||
|
`reportId`, `timezone`, prompt identity, model name, and comparison ID are
|
||||||
|
nonblank. `promptHash` and `dataPackage.sha256` are lowercase SHA-256 digests.
|
||||||
|
|
||||||
|
## Result Invariants
|
||||||
|
|
||||||
|
`total` is at least two and equals the number of results. Positions are
|
||||||
|
contiguous from one, profile IDs are distinct and nonblank, and
|
||||||
|
`succeeded + failed == total`.
|
||||||
|
|
||||||
|
A successful result has `status: "succeeded"`, `validationStatus: "passed"`,
|
||||||
|
a unique Markdown `reportPath`, and no `error`. A failed result has
|
||||||
|
`status: "failed"`, no `reportPath`, and an `error` object with nonblank
|
||||||
|
`category` and `message`. Its validation status is absent, `failed`, or
|
||||||
|
`skipped`. Error messages are valid UTF-8 and no longer than 1,024 bytes.
|
||||||
|
`backendId` and `validationStatus` are omitted when unavailable.
|
||||||
|
|
||||||
|
Every successful Markdown file is declared by exactly one successful result.
|
||||||
|
The directory contains no extra entries. Consumers can therefore verify the
|
||||||
|
data-package digest and the full manifest-to-file mapping without scanning a
|
||||||
|
larger workspace.
|
||||||
|
|
||||||
|
## Compatibility And Sensitivity
|
||||||
|
|
||||||
|
Weatherreporter recognizes a replaceable bundle only when it exactly satisfies
|
||||||
|
the current version, schema, file set, file types, relative-path rules, and
|
||||||
|
data-package digest. It rejects unknown manifest fields, multiple JSON values,
|
||||||
|
extra entries, symlinks, and future or otherwise unsupported versions. Treat a
|
||||||
|
bundle that fails recognition as an ordinary directory, not as a compatible
|
||||||
|
bundle.
|
||||||
|
|
||||||
|
The manifest contains safe operational provenance, but `data-package.yml` and
|
||||||
|
the generated Markdown can contain sensitive weather or location context. Do
|
||||||
|
not assume these artifacts are safe for public distribution. Handle retention,
|
||||||
|
access, and deletion according to the [operations guide](../operations.md).
|
||||||
@@ -29,6 +29,22 @@ Profiles that require a direct API key are unsupported; a profile that reports `
|
|||||||
|
|
||||||
Promptkit receives the YAML data package as an inline input and returns structured JSON that Weatherreporter validates before rendering its own Markdown template. Safe active provenance remains in memory. Content-rich diagnostics are opt-in through `--llm-debug-dir`; see [operations](../operations.md) for retention and permissions.
|
Promptkit receives the YAML data package as an inline input and returns structured JSON that Weatherreporter validates before rendering its own Markdown template. Safe active provenance remains in memory. Content-rich diagnostics are opt-in through `--llm-debug-dir`; see [operations](../operations.md) for retention and permissions.
|
||||||
|
|
||||||
|
## Comparison Execution
|
||||||
|
|
||||||
|
For `compare`, Weatherreporter validates one exact prompt and every explicitly
|
||||||
|
selected profile before weather collection. It prepares one deterministic YAML
|
||||||
|
data package, retains immutable copies of the report inputs, and executes every
|
||||||
|
profile against the same exact data-package bytes. Each profile remains an
|
||||||
|
independent Promptkit execution: one provider or validation failure does not
|
||||||
|
stop its peers, while caller cancellation applies to every in-flight execution.
|
||||||
|
|
||||||
|
Weatherreporter starts selected profile executions concurrently and does not
|
||||||
|
add an application-level concurrency limit. Promptkit owns backend capacity and
|
||||||
|
any profile or backend concurrency policy. The durable comparison output and
|
||||||
|
its compatibility rules are defined by the
|
||||||
|
[comparison bundle contract](comparison-bundle.md); the user-facing command
|
||||||
|
contract is in the [CLI reference](../cli.md).
|
||||||
|
|
||||||
The generated-text schemas require `summary`, `forecast_discussion`, and `precipitation_timing`, and reject additional properties. Prompts return an empty string for `precipitation_timing` when the deterministic package contains no precipitation windows.
|
The generated-text schemas require `summary`, `forecast_discussion`, and `precipitation_timing`, and reject additional properties. Prompts return an empty string for `precipitation_timing` when the deterministic package contains no precipitation windows.
|
||||||
|
|
||||||
Prompt/profile configuration and the maintained local override example are owned by the [configuration reference](../config.md). Adapter construction and mapping are documented in the [Promptkit adapter internals](../internal/promptkit-adapter.md).
|
Prompt/profile configuration and the maintained local override example are owned by the [configuration reference](../config.md). Adapter construction and mapping are documented in the [Promptkit adapter internals](../internal/promptkit-adapter.md).
|
||||||
|
|||||||
@@ -1,6 +1,9 @@
|
|||||||
# Application Orchestration Internals
|
# Application Orchestration Internals
|
||||||
|
|
||||||
`internal/app` owns stateless report generation, batch execution, atomic output publication, and notification coordination after `internal/cli` has parsed arguments and loaded configuration. The user contract is owned by the [CLI reference](../cli.md) and [operations guide](../operations.md).
|
`internal/app` owns stateless report generation, batch execution, comparison
|
||||||
|
orchestration, atomic output publication, and notification coordination after
|
||||||
|
`internal/cli` has parsed arguments and loaded configuration. The user contract
|
||||||
|
is owned by the [CLI reference](../cli.md) and [operations guide](../operations.md).
|
||||||
|
|
||||||
## Single-Report Flow
|
## Single-Report Flow
|
||||||
|
|
||||||
@@ -16,6 +19,36 @@ Failures return an active partial result with safe identity, profile, warning, v
|
|||||||
|
|
||||||
Each item has an independent result. A failed item does not stop later items; successful items retain their published output paths. Per-report notification is suppressed during a batch. Batch notification runs only after every planned report has published successfully. It is skipped when any item failed. Batch result counters count report items only; a batch notification failure is represented by the top-level notification result and still produces a failed batch outcome.
|
Each item has an independent result. A failed item does not stop later items; successful items retain their published output paths. Per-report notification is suppressed during a batch. Batch notification runs only after every planned report has published successfully. It is skipped when any item failed. Batch result counters count report items only; a batch notification failure is represented by the top-level notification result and still produces a failed batch outcome.
|
||||||
|
|
||||||
|
## Comparisons
|
||||||
|
|
||||||
|
`CompareDetailed` validates ordered explicit profile IDs, resolves the report,
|
||||||
|
and preflights the exact bundle destination before initializing optional prompt
|
||||||
|
debugging, prompt inspection, or collection. It then inspects the one prompt
|
||||||
|
and every selected profile, collects once, and delegates shared report
|
||||||
|
construction to the prepared-report flow. It does not accept a notifier.
|
||||||
|
|
||||||
|
Once the destination is resolved, the partial result retains its absolute
|
||||||
|
output directory even when later preflight, debug initialization, inspection,
|
||||||
|
collection, or preparation fails. Every initialized result is finalized with a
|
||||||
|
finished timestamp. If prompt inspection succeeds before a later profile
|
||||||
|
inspection fails, the partial result retains the resolved prompt ID, version,
|
||||||
|
and hash. Artifact paths are added only after publication commits.
|
||||||
|
|
||||||
|
The comparison execution core starts each inspected profile independently,
|
||||||
|
keeps results in selection order, and waits for all started work. Independent
|
||||||
|
profile failures are recorded and do not stop peers. Context cancellation marks
|
||||||
|
unfinished work and prevents publication. Details of prepared values, execution
|
||||||
|
and debugging, and publication are documented in [prepared report
|
||||||
|
internals](prepared-report.md), [comparison execution
|
||||||
|
internals](comparison-execution.md), and [comparison publication
|
||||||
|
internals](comparison-publication.md).
|
||||||
|
|
||||||
|
When publication has committed its new bundle, application results contain the
|
||||||
|
absolute manifest, data-package, and successful report paths even if removal of
|
||||||
|
the previous sibling backup then fails. That cleanup failure is still returned
|
||||||
|
as an operational error rather than treating the new bundle as unpublished;
|
||||||
|
the returned error retains the recovery path and underlying filesystem cause.
|
||||||
|
|
||||||
## Boundaries And Verification
|
## Boundaries And Verification
|
||||||
|
|
||||||
The package does not parse flags, load YAML, implement transport, construct provider SDKs, or define report-period policy. Prompt, profile, weather, and Distributor implementations remain behind project-owned contracts.
|
The package does not parse flags, load YAML, implement transport, construct provider SDKs, or define report-period policy. Prompt, profile, weather, and Distributor implementations remain behind project-owned contracts.
|
||||||
|
|||||||
27
docs/internal/comparison-execution.md
Normal file
27
docs/internal/comparison-execution.md
Normal file
@@ -0,0 +1,27 @@
|
|||||||
|
# Comparison Execution Internals
|
||||||
|
|
||||||
|
The comparison execution core receives an already prepared report and an
|
||||||
|
already inspected, ordered profile list. It initializes an outcome for every
|
||||||
|
selected profile, launches each started profile in its own goroutine, and
|
||||||
|
waits for every started goroutine before returning. Results retain the supplied
|
||||||
|
selection order even though execution completes in an arbitrary order.
|
||||||
|
|
||||||
|
Every profile uses the exact inspected prompt identity and a private copy of
|
||||||
|
the same prepared data package. Provider, generated-text validation, rendering,
|
||||||
|
or debug-write failure becomes that profile's safe failed outcome and does not
|
||||||
|
cancel its peers. The application deliberately imposes no additional semaphore:
|
||||||
|
Promptkit owns backend capacity. Cancellation or a deadline marks unfinished
|
||||||
|
outcomes as skipped or failed, joins work, and prevents bundle publication.
|
||||||
|
|
||||||
|
When debugging is enabled, each execution receives a deterministic reference
|
||||||
|
derived from the comparison identity, ordered profile position, and safe
|
||||||
|
profile slug. This keeps concurrent captures separate. The debug writer itself
|
||||||
|
owns secure-root validation and file permissions. It safely creates shared
|
||||||
|
missing ancestors during concurrent writes, then rejects symlink and non-
|
||||||
|
directory components. Operational retention and sensitivity are documented in
|
||||||
|
the [operations guide](../operations.md).
|
||||||
|
|
||||||
|
The output result and its safe errors are converted into the durable contract
|
||||||
|
only by comparison publication. See [comparison publication
|
||||||
|
internals](comparison-publication.md) and the external [comparison bundle
|
||||||
|
contract](../integrations/comparison-bundle.md).
|
||||||
30
docs/internal/comparison-publication.md
Normal file
30
docs/internal/comparison-publication.md
Normal file
@@ -0,0 +1,30 @@
|
|||||||
|
# Comparison Publication Internals
|
||||||
|
|
||||||
|
`internal/comparison` separates the logical bundle from filesystem mechanics.
|
||||||
|
The application builds a validated manifest, exact shared data-package bytes,
|
||||||
|
and only the Markdown files for successful profiles. The durable layout,
|
||||||
|
schema, and compatibility rules are owned by the [comparison bundle
|
||||||
|
contract](../integrations/comparison-bundle.md).
|
||||||
|
|
||||||
|
Destination planning is read-only. It requires an exact absolute target that
|
||||||
|
is neither the filesystem root nor the working directory, rejects unsafe
|
||||||
|
symlinks and non-directories, accepts a missing or empty directory, and permits
|
||||||
|
replacement only for a recognized current bundle. Publication rechecks that
|
||||||
|
authorization immediately before it writes a private sibling staging directory.
|
||||||
|
For replacement, it moves the prior bundle to a private sibling backup,
|
||||||
|
reauthorizes that moved entry, and restores it if installing the new bundle
|
||||||
|
fails.
|
||||||
|
|
||||||
|
The new bundle is committed only after the staged directory has been installed
|
||||||
|
at the target. From that point its artifact paths are authoritative: a failure
|
||||||
|
to remove the retained sibling backup does not roll back the new bundle.
|
||||||
|
Publication returns an inspectable cleanup error with the absolute backup path
|
||||||
|
and underlying filesystem cause so an operator can recover or remove that
|
||||||
|
backup manually.
|
||||||
|
|
||||||
|
The application preflights before prompt inspection and collection, then
|
||||||
|
preflights again before publication. A cancellation or any failure before the
|
||||||
|
commit leaves the prior destination untouched. Completed bundles include
|
||||||
|
partial profile results; comparison publication never coordinates Distributor
|
||||||
|
notification. Operator-facing lifecycle and cleanup are in the
|
||||||
|
[operations guide](../operations.md).
|
||||||
20
docs/internal/prepared-report.md
Normal file
20
docs/internal/prepared-report.md
Normal file
@@ -0,0 +1,20 @@
|
|||||||
|
# Prepared Report Internals
|
||||||
|
|
||||||
|
`internal/app` builds a `preparedReport` after collection and before profile
|
||||||
|
execution. This is the immutable boundary shared by ordinary report generation
|
||||||
|
and profile comparison; it is not a durable artifact.
|
||||||
|
|
||||||
|
Preparation builds report facts, the configured module snapshot, briefing
|
||||||
|
metadata, the curated prompt-input package, serialized YAML, and the
|
||||||
|
generated-text definition. It deep-copies mutable facts, snapshots, metadata,
|
||||||
|
and data-package bytes before returning them. Consumers receive independent
|
||||||
|
copies so one execution cannot change another's input or rendering context.
|
||||||
|
|
||||||
|
Single-report generation executes one prepared profile and publishes its
|
||||||
|
Markdown. Comparison prepares once, gives every selected profile the same YAML
|
||||||
|
bytes, and only then assembles the resulting logical bundle. The prompt-input
|
||||||
|
shape is owned by [prompt-input internals](prompt-input.md); profile execution
|
||||||
|
semantics are owned by [Promptkit integration](../integrations/promptkit.md).
|
||||||
|
|
||||||
|
Preparation failure has no publication side effects. Tests for this boundary
|
||||||
|
cover mutation isolation, byte equality, and reuse by both execution paths.
|
||||||
@@ -72,6 +72,38 @@ For a single report, Distributor notification follows the atomic output write.
|
|||||||
See the [configuration reference](config.md) for pipeline, bundle,
|
See the [configuration reference](config.md) for pipeline, bundle,
|
||||||
idempotency-key, and per-report path templates.
|
idempotency-key, and per-report path templates.
|
||||||
|
|
||||||
|
## Comparison Bundles
|
||||||
|
|
||||||
|
Use `compare` when an operator needs to evaluate explicit Promptkit profiles
|
||||||
|
against the same report input. The command writes one flat, operator-owned
|
||||||
|
bundle directory and never sends a Distributor notification. Command syntax,
|
||||||
|
profile validation, JSON output, and exit behavior belong to the
|
||||||
|
[CLI reference](cli.md); the durable file contract belongs to the
|
||||||
|
[comparison bundle contract](integrations/comparison-bundle.md).
|
||||||
|
|
||||||
|
The output destination follows the normal `output.directory` fallback. An
|
||||||
|
explicit `--out-dir` takes precedence and names the exact bundle directory,
|
||||||
|
not a parent to be combined with another name. The standard names are derived
|
||||||
|
from the report output name, such as `comparison-today` and
|
||||||
|
`comparison-daily-2026-05-29`; see the [configuration reference](config.md)
|
||||||
|
for output-directory resolution.
|
||||||
|
|
||||||
|
A comparison bundle contains the shared data package, a manifest, and one
|
||||||
|
Markdown file for every successful profile. Treat all of these files as
|
||||||
|
potentially sensitive: the data package and generated reports can contain
|
||||||
|
location or forecast context. Weatherreporter creates no application-owned
|
||||||
|
history, retention store, or cleanup job. Retain, archive, or remove only the
|
||||||
|
specific bundle directories your operating policy permits.
|
||||||
|
|
||||||
|
The destination is preflighted before prompt inspection and collection, then
|
||||||
|
rechecked immediately before an atomic publish. A missing or empty directory
|
||||||
|
is usable. A nonempty directory can be replaced only when `--replace` is given
|
||||||
|
and it is recognized as a current Weatherreporter comparison bundle; ordinary
|
||||||
|
directories, symlinks, and unsafe destinations are rejected. Cancellation and
|
||||||
|
all failures before publication preserve an existing bundle. Profile failures
|
||||||
|
are different: the command publishes a complete partial bundle, with failed
|
||||||
|
profiles represented in the manifest and no Markdown file for those profiles.
|
||||||
|
|
||||||
## Local Prompt Profile Override
|
## Local Prompt Profile Override
|
||||||
|
|
||||||
Hourly normally selects the embedded `weather-light` profile. To use a local
|
Hourly normally selects the embedded `weather-light` profile. To use a local
|
||||||
@@ -99,9 +131,11 @@ weatherreporter generate today --llm-debug-dir /var/tmp/weatherreporter-debug
|
|||||||
The directory must be absolute. Requested captures are written with restrictive
|
The directory must be absolute. Requested captures are written with restrictive
|
||||||
permissions beneath the supplied directory, organized by report and run. They
|
permissions beneath the supplied directory, organized by report and run. They
|
||||||
can contain rendered prompts and generated output, so limit access to trusted
|
can contain rendered prompts and generated output, so limit access to trusted
|
||||||
operators and remove the captures when they are no longer needed. Normal output,
|
operators and remove the captures when they are no longer needed. Comparison
|
||||||
summaries, and routine logs omit that sensitive content. Debug capture is never
|
captures additionally identify each selected profile so concurrent executions
|
||||||
created for an ordinary command without `--llm-debug-dir`.
|
remain distinct. Normal output, summaries, and routine logs omit that sensitive
|
||||||
|
content. Debug capture is never created for an ordinary command without
|
||||||
|
`--llm-debug-dir`.
|
||||||
|
|
||||||
If capture creation or writing fails, the affected run fails rather than
|
If capture creation or writing fails, the affected run fails rather than
|
||||||
silently continuing without the requested diagnostics.
|
silently continuing without the requested diagnostics.
|
||||||
@@ -111,8 +145,20 @@ silently continuing without the requested diagnostics.
|
|||||||
Start with the command error and JSON summary. For a report generation failure,
|
Start with the command error and JSON summary. For a report generation failure,
|
||||||
the selected destination was not replaced; for a notification failure, inspect
|
the selected destination was not replaced; for a notification failure, inspect
|
||||||
the completed destination and the notification result. For a batch failure,
|
the completed destination and the notification result. For a batch failure,
|
||||||
use the per-report statuses and retain successful output files. Enable explicit
|
use the per-report statuses and retain successful output files. For a comparison
|
||||||
debug capture only when content-rich Promptkit diagnostics are necessary.
|
failure, inspect the published manifest when its path is present: individual
|
||||||
|
profile failures retain their safe result and successful Markdown files, while
|
||||||
|
cancellation and pre-publication errors leave the prior destination unchanged.
|
||||||
|
|
||||||
|
If a replacement commits but cleanup of its prior sibling backup fails, the new
|
||||||
|
bundle remains valid and its artifact paths appear in the failed command
|
||||||
|
summary. The summary records a safe `publication_cleanup` error, while the
|
||||||
|
returned command error reports the retained backup path. Preserve that backup
|
||||||
|
until it has been inspected and cleaned up manually; do not remove the new
|
||||||
|
bundle to retry that cleanup.
|
||||||
|
|
||||||
|
Enable explicit debug capture only when content-rich Promptkit diagnostics are
|
||||||
|
necessary.
|
||||||
|
|
||||||
Weatherreporter does not retain runs for later inspection, resume failed work,
|
Weatherreporter does not retain runs for later inspection, resume failed work,
|
||||||
or provide automatic cleanup, archival, remote state, daemon operation, or
|
or provide automatic cleanup, archival, remote state, daemon operation, or
|
||||||
|
|||||||
@@ -15,6 +15,11 @@ and renders repository-owned Markdown in memory. Completed Markdown is
|
|||||||
atomically published to an operator-owned output destination and may then be
|
atomically published to an operator-owned output destination and may then be
|
||||||
uploaded through Distributor.
|
uploaded through Distributor.
|
||||||
|
|
||||||
|
An explicit profile comparison prepares one report input once, executes the
|
||||||
|
same exact prompt and data package across selected profiles concurrently, and
|
||||||
|
atomically publishes one operator-owned comparison bundle. It remains local:
|
||||||
|
it does not create application state or send a Distributor notification.
|
||||||
|
|
||||||
The supported report products are Daily, Today, Tomorrow, and Hourly. A batch
|
The supported report products are Daily, Today, Tomorrow, and Hourly. A batch
|
||||||
collects once, validates its complete candidate prompt/profile set before
|
collects once, validates its complete candidate prompt/profile set before
|
||||||
collection, then determines and validates every planned output destination
|
collection, then determines and validates every planned output destination
|
||||||
@@ -29,6 +34,8 @@ planned report succeeds.
|
|||||||
- `internal/config` owns defaults, loading, validation, and secret loading.
|
- `internal/config` owns defaults, loading, validation, and secret loading.
|
||||||
- `internal/app` owns in-memory workflow order, partial results, atomic output
|
- `internal/app` owns in-memory workflow order, partial results, atomic output
|
||||||
publication, and notification coordination through project-owned contracts.
|
publication, and notification coordination through project-owned contracts.
|
||||||
|
- `internal/comparison` owns comparison identity, durable logical bundle
|
||||||
|
validation, safe destination recognition, and atomic bundle publication.
|
||||||
- Deterministic domain packages own weather derivation, report periods, modules,
|
- Deterministic domain packages own weather derivation, report periods, modules,
|
||||||
generated-text validation, and template contexts.
|
generated-text validation, and template contexts.
|
||||||
- `internal/adapters/weatherapi`, `internal/adapters/promptkit`, and
|
- `internal/adapters/weatherapi`, `internal/adapters/promptkit`, and
|
||||||
@@ -47,6 +54,9 @@ directly.
|
|||||||
before collection.
|
before collection.
|
||||||
- Prompt and profile validation completes before weather collection. Raw output
|
- Prompt and profile validation completes before weather collection. Raw output
|
||||||
is validated before template rendering.
|
is validated before template rendering.
|
||||||
|
- Comparison validates every explicit profile before collection, prepares one
|
||||||
|
immutable report input, and delegates backend capacity to Promptkit rather
|
||||||
|
than adding an application-wide execution limit.
|
||||||
- Generated text fills defined prose slots only. Deterministic facts remain
|
- Generated text fills defined prose slots only. Deterministic facts remain
|
||||||
authoritative and repository-owned templates produce all Markdown output.
|
authoritative and repository-owned templates produce all Markdown output.
|
||||||
- Sensitive rendered prompts, schemas, input bodies, provider endpoints, and
|
- Sensitive rendered prompts, schemas, input bodies, provider endpoints, and
|
||||||
@@ -64,11 +74,16 @@ directly.
|
|||||||
does not remove a newly published output.
|
does not remove a newly published output.
|
||||||
- Configuration or explicit CLI input selects that operator-owned destination;
|
- Configuration or explicit CLI input selects that operator-owned destination;
|
||||||
it does not create an application-owned state boundary.
|
it does not create an application-owned state boundary.
|
||||||
|
- Comparison bundles are flat, versioned operator outputs. Their guarded
|
||||||
|
replacement accepts only a recognized current bundle; cancellation and every
|
||||||
|
pre-publication failure preserve a prior bundle, while individual profile
|
||||||
|
failures can publish a complete partial bundle.
|
||||||
- Distributor uploads use only the published Markdown output, never a scan of
|
- Distributor uploads use only the published Markdown output, never a scan of
|
||||||
local files. Single notification follows publication; batch notification
|
local files. Single notification follows publication; batch notification
|
||||||
follows publication of every selected report. Batch counters describe report
|
follows publication of every selected report. Batch counters describe report
|
||||||
outcomes only; a failed batch notification is represented separately at the
|
outcomes only; a failed batch notification is represented separately at the
|
||||||
batch level.
|
batch level.
|
||||||
|
- Comparison never invokes Distributor notification.
|
||||||
- Default tests are deterministic, offline, and use Promptkit/provider fakes
|
- Default tests are deterministic, offline, and use Promptkit/provider fakes
|
||||||
rather than live provider calls. See the [testing policy](testing.md).
|
rather than live provider calls. See the [testing policy](testing.md).
|
||||||
|
|
||||||
|
|||||||
398
docs/roadmap/implementation.md
Normal file
398
docs/roadmap/implementation.md
Normal file
@@ -0,0 +1,398 @@
|
|||||||
|
# LLM Profile Comparison Implementation Plan
|
||||||
|
|
||||||
|
Status: Complete.
|
||||||
|
|
||||||
|
## Purpose And Authority
|
||||||
|
|
||||||
|
This document is the ordered implementation plan for the accepted [LLM
|
||||||
|
Profile Comparison Roadmap](profile-comparison.md). The roadmap owns the
|
||||||
|
feature purpose, policy, scope, and desired end state. This plan records the
|
||||||
|
completed implementation and records the corrective work completed during
|
||||||
|
post-implementation review.
|
||||||
|
|
||||||
|
All implementation work in this plan is complete. The recorded work leaves the
|
||||||
|
repository compiling, tested, documented to its implemented boundary, and
|
||||||
|
internally coherent.
|
||||||
|
|
||||||
|
## Implementation Rules
|
||||||
|
|
||||||
|
The following rules governed every implementation stage:
|
||||||
|
|
||||||
|
- Read `docs/development.md`, the task-specific documents it identifies, all
|
||||||
|
files under `docs/policy/`, and the feature roadmap before changing code.
|
||||||
|
- Preserve the existing `generate`, `run`, and `compare` command contracts
|
||||||
|
except for the explicit comparison corrections defined below.
|
||||||
|
- Keep Promptkit types and calls behind `internal/adapters/promptkit` and the
|
||||||
|
dependency-neutral `internal/promptexec` interface.
|
||||||
|
- Keep comparison artifacts operator-owned and explicit. They are not durable
|
||||||
|
application state and must never be discovered or consumed implicitly by a
|
||||||
|
later invocation.
|
||||||
|
- Preserve comparison's prepare-once, execute-concurrently, order-results-by-
|
||||||
|
selection, publish-on-profile-failure, and never-notify invariants.
|
||||||
|
- Do not add a Weatherreporter concurrency limit. Promptkit owns backend
|
||||||
|
capacity.
|
||||||
|
- Never expose provider bodies, prompts, schemas, model output, endpoints,
|
||||||
|
credentials, or arbitrary wrapped error text in normal JSON summaries or
|
||||||
|
manifests.
|
||||||
|
- Use deterministic, offline, credential-free tests. Test filesystem safety,
|
||||||
|
concurrency, and recovery through the narrowest stable behavioral boundary;
|
||||||
|
do not rely on timing-only sleeps or host permission behavior.
|
||||||
|
- Run `gofmt` on changed Go files and `git diff --check` in every stage. Run
|
||||||
|
focused tests while developing and `GOWORK=off go test -count=1 ./...` before
|
||||||
|
completing each stage. Stages involving concurrency or filesystem mutation
|
||||||
|
must also run affected packages with `-race`.
|
||||||
|
- Do not commit, tag, push, or prepare a release unless the implementing prompt
|
||||||
|
separately requests it.
|
||||||
|
|
||||||
|
## Completed Stages
|
||||||
|
|
||||||
|
### Stage 1: Comparison Artifact And Naming Contracts
|
||||||
|
|
||||||
|
Added the dependency-neutral comparison model, schema version, manifest
|
||||||
|
validation and encoding, safe errors, deterministic profile filenames,
|
||||||
|
comparison identities, default directory names, and content hashing.
|
||||||
|
|
||||||
|
### Stage 2: Destination Recognition And Transactional Publication
|
||||||
|
|
||||||
|
Added read-only destination planning, strict recognition of current comparison
|
||||||
|
bundles, private sibling staging, guarded replacement, rollback, and atomic
|
||||||
|
directory publication.
|
||||||
|
|
||||||
|
### Stage 3: Ordered Multi-Profile Preflight
|
||||||
|
|
||||||
|
Added exact prompt inspection followed by sequential profile inspection before
|
||||||
|
weather collection, including effective backend, model, and credential checks.
|
||||||
|
|
||||||
|
### Stage 4: Immutable Shared Report Preparation
|
||||||
|
|
||||||
|
Extracted one immutable prepared-report value so comparison collection,
|
||||||
|
derivation, module construction, and data-package serialization happen once.
|
||||||
|
|
||||||
|
### Stage 5: Profile Execution And In-Memory Rendering
|
||||||
|
|
||||||
|
Separated profile-specific Promptkit execution, generated-text validation, and
|
||||||
|
Markdown rendering from output publication while preserving ordinary report
|
||||||
|
generation behavior.
|
||||||
|
|
||||||
|
### Stage 6: Concurrent Ordered Profile Execution
|
||||||
|
|
||||||
|
Added one goroutine per selected profile using one shared executor and one
|
||||||
|
prepared input, deterministic debug identities, isolated profile failures,
|
||||||
|
joined cancellation, and selection-ordered results.
|
||||||
|
|
||||||
|
### Stage 7: Application-Level Comparison
|
||||||
|
|
||||||
|
Added `app.CompareDetailed`, coherent complete and partial bundle construction,
|
||||||
|
aggregate profile-failure behavior, absolute published paths, and the
|
||||||
|
application-level guarantee that comparison never notifies Distributor.
|
||||||
|
|
||||||
|
### Stage 8: Compare Command Parsing
|
||||||
|
|
||||||
|
Added the `compare` command request path, repeatable ordered `--profile`, exact
|
||||||
|
`--out-dir`, guarded `--replace`, applicable common flags, validation, and one
|
||||||
|
executor construction per invocation.
|
||||||
|
|
||||||
|
### Stage 9: CLI Results And Exit Behavior
|
||||||
|
|
||||||
|
Added structured success and failure summaries, quiet-mode suppression,
|
||||||
|
ordered per-profile results, safe bounded errors, and nonzero exit behavior for
|
||||||
|
partial or command-level failure.
|
||||||
|
|
||||||
|
### Stage 10: Canonical Documentation And Initial Validation
|
||||||
|
|
||||||
|
Documented the implemented CLI, operations, Promptkit integration, comparison
|
||||||
|
bundle, application orchestration, execution, publication, architecture, and
|
||||||
|
development contracts, then passed the original repository-wide validation
|
||||||
|
gate.
|
||||||
|
|
||||||
|
## Stage 11: Make Concurrent Prompt Debug Creation Race-Safe
|
||||||
|
|
||||||
|
### Goal
|
||||||
|
|
||||||
|
Ensure concurrent comparison profiles can create their distinct debug runs
|
||||||
|
under one new report/date directory without spuriously failing or leaving a
|
||||||
|
test goroutine blocked.
|
||||||
|
|
||||||
|
### Work
|
||||||
|
|
||||||
|
1. Update `internal/promptdebug.ensureSecureDirectory` so concurrent creation
|
||||||
|
of the same missing directory is idempotent. If `os.Mkdir` reports that the
|
||||||
|
path already exists, inspect the path with `Lstat` and accept it only when it
|
||||||
|
is the expected real directory. Continue to reject symlinks, non-directories,
|
||||||
|
unsafe modes, and every unrelated filesystem error.
|
||||||
|
2. Preserve the existing absolute-path, containment, `0700` directory, `0600`
|
||||||
|
file, and no-symlink guarantees. Do not weaken debug-root validation or make
|
||||||
|
all `EEXIST` errors successful.
|
||||||
|
3. Add a focused prompt-debug concurrency regression that starts multiple
|
||||||
|
writers beneath a shared missing ancestor, joins every goroutine, and
|
||||||
|
verifies every expected artifact and permission invariant.
|
||||||
|
4. Make comparison execution test barriers time-bounded and failure-aware. A
|
||||||
|
callback failure before executor entry must fail the test promptly rather
|
||||||
|
than leave `waitForProfileStarts` waiting forever.
|
||||||
|
5. Retain distinct deterministic debug references and profile-local debug
|
||||||
|
failure behavior.
|
||||||
|
|
||||||
|
### Tests And Exit Criteria
|
||||||
|
|
||||||
|
- The focused prompt-debug concurrency test passes repeatedly and with the race
|
||||||
|
detector.
|
||||||
|
- `TestExecuteComparisonProfilesUsesDistinctDeterministicDebugReferences`
|
||||||
|
cannot hang when a profile fails before reaching the fake executor.
|
||||||
|
- Run, at minimum:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
GOWORK=off go test -count=100 ./internal/promptdebug
|
||||||
|
GOWORK=off go test -count=100 -run TestExecuteComparisonProfilesUsesDistinctDeterministicDebugReferences ./internal/app
|
||||||
|
GOWORK=off go test -race -count=1 ./internal/promptdebug ./internal/app
|
||||||
|
GOWORK=off go test -count=1 ./...
|
||||||
|
```
|
||||||
|
|
||||||
|
## Stage 12: Make Replacement Authorization Commit-Safe
|
||||||
|
|
||||||
|
### Goal
|
||||||
|
|
||||||
|
Prevent a destination changed after the final read-only preflight from being
|
||||||
|
treated as the previously authorized empty directory or recognized bundle and
|
||||||
|
then deleted during replacement.
|
||||||
|
|
||||||
|
### Work
|
||||||
|
|
||||||
|
1. Replace `DestinationPlan.Exists` as the publication decision with an
|
||||||
|
explicit destination-state classification: absent, empty real directory, or
|
||||||
|
recognized current-schema bundle. Keep `Replace` in the plan so publication
|
||||||
|
can apply the same authorization policy at commit time.
|
||||||
|
2. Continue to call `PlanDestination` immediately before publication. For an
|
||||||
|
absent target, install staging with one rename; a concurrently created
|
||||||
|
target must cause that rename to fail without modifying the new target.
|
||||||
|
3. For an existing target, rename that exact filesystem entry to the unique
|
||||||
|
sibling backup before deleting or installing anything. Classify the moved
|
||||||
|
backup while it is at its stable backup path and authorize it under the
|
||||||
|
original replacement policy:
|
||||||
|
|
||||||
|
- an empty real directory is allowed with or without `Replace`;
|
||||||
|
- a recognized current-schema comparison bundle is allowed only with
|
||||||
|
`Replace`; and
|
||||||
|
- a file, symlink, unrecognized/nonempty directory, unreadable entry, or
|
||||||
|
other classification failure is not allowed.
|
||||||
|
|
||||||
|
4. Treat this post-move classification as the destructive-action
|
||||||
|
authorization point. If it fails, restore the moved entry to the target and
|
||||||
|
return an error without installing staging. If the target has concurrently
|
||||||
|
reappeared or restoration otherwise fails, retain the backup and return an
|
||||||
|
actionable joined or typed error that identifies its recovery path; never
|
||||||
|
delete either entry to force restoration.
|
||||||
|
5. Install staging only after the moved target has passed authorization. Never
|
||||||
|
remove a backup that did not pass post-move authorization.
|
||||||
|
6. Preserve the existing final cancellation linearization rule: cancellation
|
||||||
|
observed before the rename transaction prevents replacement; after the
|
||||||
|
transaction starts, finish commit or rollback rather than abandoning it.
|
||||||
|
7. Add a package-private filesystem-operation seam only if needed for
|
||||||
|
deterministic tests. Keep the public destination and publication APIs free
|
||||||
|
of test-only hooks.
|
||||||
|
|
||||||
|
### Tests And Exit Criteria
|
||||||
|
|
||||||
|
- Deterministically replace an initially accepted destination after final
|
||||||
|
preflight but before its move with each consequential unauthorized type:
|
||||||
|
unrelated nonempty directory, regular file, and symlink. Publication must
|
||||||
|
fail, staging must not become the target, and the moved entry must be restored
|
||||||
|
or retained at a reported recovery path.
|
||||||
|
- Cover an initially empty directory whose contents change before its move and
|
||||||
|
a recognized bundle swapped for an unrecognized directory.
|
||||||
|
- Retain coverage for absent targets, empty directories, recognized bundle
|
||||||
|
replacement, cancellation before commit, install failure, successful
|
||||||
|
rollback, failed rollback, and cleanup of ordinary staging failures.
|
||||||
|
- Run `GOWORK=off go test -race -count=1 ./internal/comparison` and the
|
||||||
|
repository-wide standard test command.
|
||||||
|
|
||||||
|
## Stage 13: Represent Committed Publication Cleanup Failures Accurately
|
||||||
|
|
||||||
|
### Goal
|
||||||
|
|
||||||
|
Keep application and CLI results truthful when the new comparison bundle has
|
||||||
|
been committed but removal of the old sibling backup fails.
|
||||||
|
|
||||||
|
### Work
|
||||||
|
|
||||||
|
1. Change comparison publication to return a dependency-neutral result as well
|
||||||
|
as an error:
|
||||||
|
|
||||||
|
```go
|
||||||
|
type PublicationResult struct {
|
||||||
|
Committed bool
|
||||||
|
RetainedBackupPath string
|
||||||
|
}
|
||||||
|
|
||||||
|
func Publish(
|
||||||
|
ctx context.Context,
|
||||||
|
plan DestinationPlan,
|
||||||
|
bundle LogicalBundle,
|
||||||
|
) (PublicationResult, error)
|
||||||
|
```
|
||||||
|
|
||||||
|
2. Define `Committed` as meaning the complete staged bundle is now installed
|
||||||
|
at the target. Pre-commit, staging, authorization, install, and successful-
|
||||||
|
rollback failures return `Committed == false`. A successful install returns
|
||||||
|
`Committed == true` even if later backup cleanup fails.
|
||||||
|
3. Add a typed post-commit cleanup error that unwraps its filesystem cause and
|
||||||
|
records the retained backup path for operator recovery. On this error,
|
||||||
|
return `Committed == true` and the absolute retained backup path. Do not
|
||||||
|
roll back or remove the newly committed valid bundle merely because old
|
||||||
|
backup cleanup failed.
|
||||||
|
4. In `app.CompareDetailed`, populate `ManifestPath`, `DataPackagePath`, and
|
||||||
|
successful profile `ReportPath` values whenever publication reports
|
||||||
|
`Committed == true`, before returning any cleanup error.
|
||||||
|
5. Treat post-commit cleanup failure as a command-level operational failure:
|
||||||
|
return the non-nil structured result plus an error, produce status `failed`,
|
||||||
|
and exit nonzero even though the published artifact paths are present. The
|
||||||
|
ordinary safe JSON error must not contain the raw filesystem cause or backup
|
||||||
|
path; the wrapped diagnostic returned on stderr may identify the retained
|
||||||
|
backup for recovery.
|
||||||
|
6. Keep `RetainedBackupPath` out of the versioned comparison manifest. It
|
||||||
|
describes an incomplete local transaction cleanup, not the logical bundle.
|
||||||
|
|
||||||
|
### Tests And Exit Criteria
|
||||||
|
|
||||||
|
- Inject a deterministic backup-removal failure after successful installation
|
||||||
|
and assert the target is the new recognized bundle, the old bundle remains
|
||||||
|
at the reported backup, `Committed` is true, and the error is inspectable by
|
||||||
|
type.
|
||||||
|
- At the application boundary, assert all committed artifact paths are
|
||||||
|
absolute and populated while the method still returns an error.
|
||||||
|
- At the CLI boundary, assert status `failed`, nonzero return, present artifact
|
||||||
|
paths, and a bounded generic safe error with no raw filesystem detail.
|
||||||
|
- Retain tests showing every pre-commit or rolled-back failure omits published
|
||||||
|
artifact paths.
|
||||||
|
- Run comparison, application, and CLI tests with `-race`, then the
|
||||||
|
repository-wide standard test command.
|
||||||
|
|
||||||
|
## Stage 14: Complete Structured Failure Metadata And Classification
|
||||||
|
|
||||||
|
### Goal
|
||||||
|
|
||||||
|
Make every non-nil comparison result a reliable description of the attempted
|
||||||
|
run and preserve useful safe error categories in the top-level CLI summary.
|
||||||
|
|
||||||
|
### Work
|
||||||
|
|
||||||
|
1. In `app.CompareDetailed`, assign the absolute resolved
|
||||||
|
`OutputDirectory` immediately after output-directory resolution and before
|
||||||
|
destination preflight. Do not wait for `PlanDestination` to succeed.
|
||||||
|
2. Once the initial `ComparisonResult` exists, guarantee that every return path
|
||||||
|
sets a nonzero UTC `FinishedAt` that is not before `StartedAt`. Use one
|
||||||
|
centralized finalization path or a defer; do not scatter timestamp writes
|
||||||
|
across individual failures.
|
||||||
|
3. Continue to omit manifest, data-package, and report paths until publication
|
||||||
|
commits. Preserve whatever prompt identity fields have actually been
|
||||||
|
resolved; never invent a hash or profile result for a phase that did not
|
||||||
|
run.
|
||||||
|
4. Update `safeComparisonSummaryError` to use this stable mapping, always
|
||||||
|
passing messages through `comparison.NewSafeError`:
|
||||||
|
|
||||||
|
| Error | Category | Safe message |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| aggregate profile failure | `application` | existing bounded aggregate message |
|
||||||
|
| `context.Canceled` | `canceled` | `comparison canceled` |
|
||||||
|
| `context.DeadlineExceeded` | `deadline_exceeded` | `comparison deadline exceeded` |
|
||||||
|
| categorized `promptexec` error | exact `promptexec.CategoryOf` value | `comparison prompt operation failed` |
|
||||||
|
| `comparison.DestinationError` | `destination_<kind>` | `comparison destination preflight failed` |
|
||||||
|
| post-commit cleanup error | `publication_cleanup` | `comparison published but cleanup did not complete` |
|
||||||
|
| any unknown error | `application` | `comparison did not complete` |
|
||||||
|
|
||||||
|
5. Apply the most specific mapping before a more general wrapped match. In
|
||||||
|
particular, detect the post-commit cleanup and destination types before
|
||||||
|
falling back to a nested filesystem or context cause.
|
||||||
|
6. Do not copy `DestinationError.Target`, wrapped causes, or arbitrary
|
||||||
|
`error.Error()` text into normal JSON. Detailed returned errors remain
|
||||||
|
available on stderr and through Go error inspection.
|
||||||
|
|
||||||
|
### Tests And Exit Criteria
|
||||||
|
|
||||||
|
- Add application tests for destination-preflight, debug initialization,
|
||||||
|
prompt-preflight, collection, and preparation failures. Whenever a non-nil
|
||||||
|
result is returned, assert an absolute output directory, nonzero ordered UTC
|
||||||
|
timestamps, and omission of unpublished artifact paths.
|
||||||
|
- Add table-driven CLI tests for every mapping row, including wrapped errors,
|
||||||
|
and assert that unsafe sentinel text cannot enter serialized output.
|
||||||
|
- Preserve existing ordered profile-level categories and safe messages.
|
||||||
|
- Run application and CLI tests with `-race`, then the repository-wide standard
|
||||||
|
test command.
|
||||||
|
|
||||||
|
## Stage 15: Remove Temporary Seams, Reconcile Documentation, And Validate
|
||||||
|
|
||||||
|
### Goal
|
||||||
|
|
||||||
|
Remove review-discovered maintenance debt, document the corrected implemented
|
||||||
|
behavior in its canonical owners, and complete the release-equivalent gate.
|
||||||
|
|
||||||
|
### Work
|
||||||
|
|
||||||
|
1. Remove the unused `Runner.resolveComparison` wrapper. Remove the
|
||||||
|
test-oriented `Runner.executeComparison` seam if it has no production
|
||||||
|
caller, and rewrite its remaining coverage through `Runner.Run`,
|
||||||
|
`resolveComparisonAction`, or another stable behavioral boundary.
|
||||||
|
2. Remove `comparisonProfileOutcome.err` if production code still does not use
|
||||||
|
it. Keep raw failures in returned/wrapped errors or explicit internal error
|
||||||
|
types; do not retain an otherwise dead field solely for private test
|
||||||
|
assertions.
|
||||||
|
3. Correct the `internal/comparison` package comment so it describes the
|
||||||
|
package's actual ownership of both logical comparison contracts and
|
||||||
|
filesystem destination/publication behavior.
|
||||||
|
4. Update only the canonical current-state documents affected by Stages 11
|
||||||
|
through 14:
|
||||||
|
|
||||||
|
- `docs/internal/comparison-publication.md` owns post-move authorization,
|
||||||
|
commit state, rollback, retained backups, and cleanup mechanics;
|
||||||
|
- `docs/internal/comparison-execution.md` owns concurrency and debug-write
|
||||||
|
behavior;
|
||||||
|
- `docs/internal/app-orchestration.md` owns partial results and committed
|
||||||
|
publication error handling;
|
||||||
|
- `docs/cli.md` owns structured status, safe category, path, and exit
|
||||||
|
behavior; and
|
||||||
|
- `docs/operations.md` owns operator recovery for a retained sibling backup.
|
||||||
|
|
||||||
|
Link rather than duplicating complete contracts, and update architecture or
|
||||||
|
integration documentation only if its existing invariant is inaccurate.
|
||||||
|
5. Mark this plan `Complete` and restore the feature roadmap's implemented
|
||||||
|
status after every exit criterion below passes. Retain or remove the two
|
||||||
|
roadmap documents only according to a later maintainer-directed roadmap
|
||||||
|
cleanup; do not archive them as a second current-state reference in this
|
||||||
|
stage.
|
||||||
|
|
||||||
|
### Tests And Exit Criteria
|
||||||
|
|
||||||
|
- Confirm no production-only helper or field remains solely to support tests,
|
||||||
|
and no test loses meaningful behavioral coverage during cleanup.
|
||||||
|
- Verify changed relative links and fenced examples. Search current-state docs
|
||||||
|
for stale claims about comparison publication, debug behavior, results, or
|
||||||
|
recovery.
|
||||||
|
- Run the release-equivalent local gate:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
set -eu
|
||||||
|
test -z "$(git ls-files go.work go.work.sum)"
|
||||||
|
test ! -e vendor
|
||||||
|
if grep -Eq '^[[:space:]]*replace([[:space:]]|\()' go.mod; then
|
||||||
|
printf '%s\n' 'go.mod contains a replacement' >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
GOWORK=off go test -count=1 ./...
|
||||||
|
GOWORK=off go test -race -count=1 ./...
|
||||||
|
GOWORK=off go vet ./...
|
||||||
|
GOWORK=off go build ./...
|
||||||
|
GOWORK=off go mod tidy -diff
|
||||||
|
unformatted="$(git ls-files '*.go' | while IFS= read -r file; do gofmt -l "$file"; done)"
|
||||||
|
test -z "$unformatted"
|
||||||
|
git diff --check
|
||||||
|
```
|
||||||
|
|
||||||
|
- Run `GOWORK=off go run ./cmd/weatherreporter --help` and
|
||||||
|
`GOWORK=off go run ./cmd/weatherreporter compare --help` without credentials
|
||||||
|
or network access, and confirm that help agrees with `docs/cli.md`.
|
||||||
|
- Inspect the final diff for accidental generated artifacts, secrets,
|
||||||
|
workspaces, vendored dependencies, release notes, or unrelated changes.
|
||||||
|
|
||||||
|
## Open Questions
|
||||||
|
|
||||||
|
None. The roadmap and the contracts in Stages 11 through 15 define the
|
||||||
|
remaining decisions needed to complete the corrective work.
|
||||||
@@ -1,6 +1,6 @@
|
|||||||
# LLM Profile Comparison Roadmap
|
# LLM Profile Comparison Roadmap
|
||||||
|
|
||||||
Status: Accepted; unimplemented.
|
Status: Implemented; retained as the feature roadmap.
|
||||||
|
|
||||||
## Purpose
|
## Purpose
|
||||||
|
|
||||||
@@ -280,8 +280,8 @@ The completed feature includes:
|
|||||||
|
|
||||||
- the `compare` CLI command for every implemented generated-text report;
|
- the `compare` CLI command for every implemented generated-text report;
|
||||||
- repeatable explicit profile selection and validation;
|
- repeatable explicit profile selection and validation;
|
||||||
- configured and CLI output-directory integration after the prerequisite
|
- configured and CLI output-directory integration through the implemented
|
||||||
feature lands;
|
destination policy;
|
||||||
- one-time report resolution, collection, deterministic preparation, and YAML
|
- one-time report resolution, collection, deterministic preparation, and YAML
|
||||||
serialization;
|
serialization;
|
||||||
- concurrent execution through one Promptkit executor with backend capacity
|
- concurrent execution through one Promptkit executor with backend capacity
|
||||||
@@ -353,9 +353,10 @@ owns the statelessness, concurrency, notification, and publication invariants.
|
|||||||
|
|
||||||
The [Promptkit integration guide](../integrations/promptkit.md) should describe
|
The [Promptkit integration guide](../integrations/promptkit.md) should describe
|
||||||
the consumer-visible multi-profile execution boundary without duplicating
|
the consumer-visible multi-profile execution boundary without duplicating
|
||||||
Promptkit's backend-capacity reference. App orchestration, prompt input,
|
Promptkit's backend-capacity reference. The versioned manifest and flat bundle
|
||||||
generated text, prompt debugging, and any new bundle implementation details
|
format belong in a focused contract under `docs/integrations/`. App
|
||||||
belong in focused documents under `docs/internal/`.
|
orchestration, prompt input, generated text, prompt debugging, and bundle
|
||||||
|
publication mechanics belong in focused documents under `docs/internal/`.
|
||||||
|
|
||||||
Current-state documentation must not describe profile comparison as available
|
Current-state documentation must not describe profile comparison as available
|
||||||
until the implementation lands.
|
until the implementation lands.
|
||||||
@@ -393,5 +394,5 @@ affected canonical documentation must describe the implemented behavior.
|
|||||||
|
|
||||||
## Open Questions
|
## Open Questions
|
||||||
|
|
||||||
None. The scope, prerequisites, user intent, and target behavior required for a
|
None. The scope, prerequisites, user intent, and target behavior are defined
|
||||||
future staged implementation plan are defined above.
|
above.
|
||||||
|
|||||||
240
internal/app/comparison.go
Normal file
240
internal/app/comparison.go
Normal file
@@ -0,0 +1,240 @@
|
|||||||
|
package app
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/comparison"
|
||||||
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/config"
|
||||||
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptdebug"
|
||||||
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec"
|
||||||
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/report"
|
||||||
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/timeutil"
|
||||||
|
)
|
||||||
|
|
||||||
|
// ComparisonRequest describes one explicit, multi-profile report comparison.
|
||||||
|
// It deliberately does not accept a notifier: comparison publication is local.
|
||||||
|
type ComparisonRequest struct {
|
||||||
|
Config config.Config
|
||||||
|
Report ReportKind
|
||||||
|
ProfileIDs []string
|
||||||
|
WorkingDir string
|
||||||
|
OutputDir string
|
||||||
|
Replace bool
|
||||||
|
LLMDebugDir string
|
||||||
|
Date time.Time
|
||||||
|
Clock timeutil.Clock
|
||||||
|
Collector Collector
|
||||||
|
Executor promptexec.Executor
|
||||||
|
}
|
||||||
|
|
||||||
|
// ComparisonResult records the resolved comparison and profile outcomes.
|
||||||
|
type ComparisonResult struct {
|
||||||
|
ComparisonID string
|
||||||
|
ReportID report.ID
|
||||||
|
ReportName string
|
||||||
|
PromptID string
|
||||||
|
PromptVersion string
|
||||||
|
PromptHash string
|
||||||
|
StartedAt time.Time
|
||||||
|
FinishedAt time.Time
|
||||||
|
Timezone string
|
||||||
|
ValidPeriod timeutil.Period
|
||||||
|
OutputDirectory string
|
||||||
|
ManifestPath string
|
||||||
|
DataPackagePath string
|
||||||
|
Total int
|
||||||
|
Succeeded int
|
||||||
|
Failed int
|
||||||
|
Results []ComparisonProfileResult
|
||||||
|
}
|
||||||
|
|
||||||
|
// ComparisonProfileResult records one explicitly selected profile.
|
||||||
|
type ComparisonProfileResult struct {
|
||||||
|
Position int
|
||||||
|
ProfileID string
|
||||||
|
BackendID string
|
||||||
|
ModelName string
|
||||||
|
Status string
|
||||||
|
ValidationStatus promptexec.ValidationStatus
|
||||||
|
ReportPath string
|
||||||
|
LLMDebugPath string
|
||||||
|
Error *comparison.SafeError
|
||||||
|
}
|
||||||
|
|
||||||
|
type comparisonPublisher func(context.Context, comparison.DestinationPlan, comparison.LogicalBundle) (comparison.PublicationResult, error)
|
||||||
|
|
||||||
|
// CompareDetailed assembles, executes, and atomically publishes a comparison
|
||||||
|
// bundle. Profile failures publish a complete partial bundle. Failures before
|
||||||
|
// commit leave the destination untouched; a post-commit cleanup failure leaves
|
||||||
|
// the new bundle installed and returns its artifact paths with an error.
|
||||||
|
func CompareDetailed(ctx context.Context, req ComparisonRequest) (*ComparisonResult, error) {
|
||||||
|
return compareDetailed(ctx, req, comparison.Publish)
|
||||||
|
}
|
||||||
|
|
||||||
|
func compareDetailed(ctx context.Context, req ComparisonRequest, publish comparisonPublisher) (*ComparisonResult, error) {
|
||||||
|
if err := comparison.ValidateProfileIDs(req.ProfileIDs); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
clock := req.Clock
|
||||||
|
if clock == nil {
|
||||||
|
clock = timeutil.SystemClock{}
|
||||||
|
}
|
||||||
|
now := clock.Now()
|
||||||
|
resolved, err := ResolveGenerate(GenerateRequest{Config: req.Config, Report: req.Report, Date: req.Date}, now)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
metadata := resolved.Metadata()
|
||||||
|
comparisonID, err := comparison.BuildComparisonID(metadata.RunID)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("build comparison identity: %w", err)
|
||||||
|
}
|
||||||
|
result := initialComparisonResult(req, resolved, comparisonID, now.UTC())
|
||||||
|
defer func() {
|
||||||
|
if result.FinishedAt.IsZero() {
|
||||||
|
finalizeComparisonResult(result, clock)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
outputName, err := resolved.OutputName()
|
||||||
|
if err != nil {
|
||||||
|
return result, fmt.Errorf("resolve comparison output name: %w", err)
|
||||||
|
}
|
||||||
|
outputDirectory, err := resolveComparisonOutputDirectory(req.WorkingDir, req.OutputDir, req.Config.Output.Directory, outputName)
|
||||||
|
if err != nil {
|
||||||
|
return result, err
|
||||||
|
}
|
||||||
|
result.OutputDirectory = outputDirectory
|
||||||
|
_, err = comparison.PlanDestination(req.WorkingDir, outputDirectory, req.Replace)
|
||||||
|
if err != nil {
|
||||||
|
return result, fmt.Errorf("preflight comparison destination: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
debugWriter, err := promptdebug.NewPromptDebugWriter(req.LLMDebugDir)
|
||||||
|
if err != nil {
|
||||||
|
return result, promptexec.NewError(promptexec.InvalidConfiguration, "initialize prompt debug", err)
|
||||||
|
}
|
||||||
|
inspection, err := InspectComparisonExecution(ctx, ComparisonInspectionRequest{
|
||||||
|
Resolved: resolved, ProfileIDs: req.ProfileIDs, Executor: req.Executor, LookupEnv: os.LookupEnv,
|
||||||
|
})
|
||||||
|
result.PromptID, result.PromptVersion, result.PromptHash = inspection.PromptID, inspection.PromptVersion, inspection.PromptHash
|
||||||
|
if err != nil {
|
||||||
|
return result, err
|
||||||
|
}
|
||||||
|
|
||||||
|
collection, err := collectWeather(ctx, req.Config, req.Collector)
|
||||||
|
if err != nil {
|
||||||
|
return result, err
|
||||||
|
}
|
||||||
|
prepared, err := prepareReport(prepareReportRequest{Config: req.Config, Resolved: resolved, Collection: *collection})
|
||||||
|
if err != nil {
|
||||||
|
return result, fmt.Errorf("prepare comparison report: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
executed := executeComparisonProfiles(ctx, comparisonExecutionRequest{
|
||||||
|
Prepared: prepared, Inspection: inspection, ComparisonID: comparisonID, DebugWriter: debugWriter, Executor: req.Executor,
|
||||||
|
})
|
||||||
|
finalizeComparisonResult(result, clock)
|
||||||
|
copyComparisonOutcomes(result, executed.Outcomes, false)
|
||||||
|
if executed.Canceled {
|
||||||
|
return result, fmt.Errorf("comparison execution: %w", ctx.Err())
|
||||||
|
}
|
||||||
|
|
||||||
|
bundle := comparisonBundle(result, prepared.dataPackageCopy(), executed.Outcomes)
|
||||||
|
if err := bundle.Validate(); err != nil {
|
||||||
|
return result, fmt.Errorf("build comparison bundle: %w", err)
|
||||||
|
}
|
||||||
|
publicationPlan, err := comparison.PlanDestination(req.WorkingDir, outputDirectory, req.Replace)
|
||||||
|
if err != nil {
|
||||||
|
return result, fmt.Errorf("re-preflight comparison destination: %w", err)
|
||||||
|
}
|
||||||
|
publication, err := publish(ctx, publicationPlan, bundle)
|
||||||
|
if publication.Committed {
|
||||||
|
result.OutputDirectory = publicationPlan.Target
|
||||||
|
result.ManifestPath = filepath.Join(publicationPlan.Target, comparison.ManifestFilename)
|
||||||
|
result.DataPackagePath = filepath.Join(publicationPlan.Target, comparison.DataPackageFilename)
|
||||||
|
copyComparisonOutcomes(result, executed.Outcomes, true)
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return result, fmt.Errorf("publish comparison bundle: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if result.Failed > 0 {
|
||||||
|
return result, fmt.Errorf("comparison completed with %d failed profiles", result.Failed)
|
||||||
|
}
|
||||||
|
return result, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func finalizeComparisonResult(result *ComparisonResult, clock timeutil.Clock) {
|
||||||
|
finishedAt := clock.Now().UTC()
|
||||||
|
if finishedAt.IsZero() {
|
||||||
|
finishedAt = time.Unix(0, 1).UTC()
|
||||||
|
}
|
||||||
|
if finishedAt.Before(result.StartedAt) {
|
||||||
|
finishedAt = result.StartedAt
|
||||||
|
}
|
||||||
|
result.FinishedAt = finishedAt
|
||||||
|
}
|
||||||
|
|
||||||
|
func initialComparisonResult(req ComparisonRequest, resolved report.Resolved, comparisonID string, startedAt time.Time) *ComparisonResult {
|
||||||
|
metadata := resolved.Metadata()
|
||||||
|
return &ComparisonResult{
|
||||||
|
ComparisonID: comparisonID,
|
||||||
|
ReportID: resolved.Definition.ID,
|
||||||
|
ReportName: resolved.Definition.Name,
|
||||||
|
StartedAt: startedAt,
|
||||||
|
Timezone: req.Config.WeatherAPI.Timezone,
|
||||||
|
ValidPeriod: metadata.ValidPeriod,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func copyComparisonOutcomes(result *ComparisonResult, outcomes []comparisonProfileOutcome, published bool) {
|
||||||
|
result.Results = make([]ComparisonProfileResult, len(outcomes))
|
||||||
|
result.Total, result.Succeeded, result.Failed = len(outcomes), 0, 0
|
||||||
|
for index, outcome := range outcomes {
|
||||||
|
profile := ComparisonProfileResult{
|
||||||
|
Position: outcome.Position, ProfileID: outcome.ProfileID, BackendID: outcome.BackendID, ModelName: outcome.ModelName,
|
||||||
|
Status: outcome.Status, ValidationStatus: outcome.ValidationStatus, LLMDebugPath: outcome.LLMDebugPath, Error: outcome.Error,
|
||||||
|
}
|
||||||
|
if published && outcome.Status == comparison.StatusSucceeded {
|
||||||
|
profile.ReportPath = filepath.Join(result.OutputDirectory, outcome.ReportPath)
|
||||||
|
}
|
||||||
|
result.Results[index] = profile
|
||||||
|
if outcome.Status == comparison.StatusSucceeded {
|
||||||
|
result.Succeeded++
|
||||||
|
} else {
|
||||||
|
result.Failed++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func comparisonBundle(result *ComparisonResult, dataPackage []byte, outcomes []comparisonProfileOutcome) comparison.LogicalBundle {
|
||||||
|
manifest := comparison.Manifest{
|
||||||
|
SchemaVersion: comparison.SchemaVersion, ComparisonID: result.ComparisonID,
|
||||||
|
StartedAt: result.StartedAt.UTC(), FinishedAt: result.FinishedAt.UTC(),
|
||||||
|
ReportID: string(result.ReportID), Timezone: result.Timezone,
|
||||||
|
ValidPeriod: comparison.ValidPeriod{Start: result.ValidPeriod.Start, End: result.ValidPeriod.End},
|
||||||
|
PromptID: result.PromptID, PromptVersion: result.PromptVersion, PromptHash: result.PromptHash,
|
||||||
|
DataPackage: comparison.DataPackageReference{Path: comparison.DataPackageFilename, SHA256: comparison.SHA256(dataPackage)},
|
||||||
|
Total: result.Total, Succeeded: result.Succeeded, Failed: result.Failed,
|
||||||
|
Results: make([]comparison.Result, len(outcomes)),
|
||||||
|
}
|
||||||
|
bundle := comparison.LogicalBundle{Manifest: manifest, DataPackage: dataPackage}
|
||||||
|
for index, outcome := range outcomes {
|
||||||
|
manifestResult := comparison.Result{
|
||||||
|
Position: outcome.Position, ProfileID: outcome.ProfileID, BackendID: outcome.BackendID, ModelName: outcome.ModelName,
|
||||||
|
Status: outcome.Status, ValidationStatus: string(outcome.ValidationStatus), Error: outcome.Error,
|
||||||
|
}
|
||||||
|
if outcome.Status == comparison.StatusSucceeded {
|
||||||
|
manifestResult.ReportPath = outcome.ReportPath
|
||||||
|
bundle.Reports = append(bundle.Reports, comparison.BundleReport{Position: outcome.Position, Path: outcome.ReportPath, Markdown: outcome.Markdown})
|
||||||
|
}
|
||||||
|
bundle.Manifest.Results[index] = manifestResult
|
||||||
|
}
|
||||||
|
return bundle
|
||||||
|
}
|
||||||
165
internal/app/comparison_execution.go
Normal file
165
internal/app/comparison_execution.go
Normal file
@@ -0,0 +1,165 @@
|
|||||||
|
package app
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"sync"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/comparison"
|
||||||
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptdebug"
|
||||||
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec"
|
||||||
|
)
|
||||||
|
|
||||||
|
type comparisonExecutionRequest struct {
|
||||||
|
Prepared preparedReport
|
||||||
|
Inspection ComparisonInspectionResult
|
||||||
|
ComparisonID string
|
||||||
|
DebugWriter *promptdebug.PromptDebugWriter
|
||||||
|
Executor promptexec.Executor
|
||||||
|
}
|
||||||
|
|
||||||
|
type comparisonExecutionResult struct {
|
||||||
|
Outcomes []comparisonProfileOutcome
|
||||||
|
Canceled bool
|
||||||
|
}
|
||||||
|
|
||||||
|
type comparisonProfileOutcome struct {
|
||||||
|
Position int
|
||||||
|
ProfileID string
|
||||||
|
BackendID string
|
||||||
|
ModelName string
|
||||||
|
Status string
|
||||||
|
ValidationStatus promptexec.ValidationStatus
|
||||||
|
ReportPath string
|
||||||
|
Markdown []byte
|
||||||
|
LLMDebugPath string
|
||||||
|
Error *comparison.SafeError
|
||||||
|
}
|
||||||
|
|
||||||
|
func executeComparisonProfiles(ctx context.Context, req comparisonExecutionRequest) comparisonExecutionResult {
|
||||||
|
profiles := req.Inspection.Profiles
|
||||||
|
result := comparisonExecutionResult{Outcomes: make([]comparisonProfileOutcome, len(profiles))}
|
||||||
|
for index, profile := range profiles {
|
||||||
|
result.Outcomes[index] = comparisonProfileOutcome{
|
||||||
|
Position: index + 1,
|
||||||
|
ProfileID: profile.ProfileID,
|
||||||
|
BackendID: profile.BackendID,
|
||||||
|
ModelName: profile.ModelName,
|
||||||
|
Status: comparison.StatusFailed,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
var waitGroup sync.WaitGroup
|
||||||
|
for index, profile := range profiles {
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
result.Canceled = true
|
||||||
|
markUnstartedComparisonOutcomes(result.Outcomes[index:], err)
|
||||||
|
break
|
||||||
|
}
|
||||||
|
index, profile := index, profile
|
||||||
|
waitGroup.Add(1)
|
||||||
|
go func() {
|
||||||
|
defer waitGroup.Done()
|
||||||
|
result.Outcomes[index] = executeComparisonProfile(ctx, req, index, profile)
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
waitGroup.Wait()
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
result.Canceled = true
|
||||||
|
for index := range result.Outcomes {
|
||||||
|
if result.Outcomes[index].Status != comparison.StatusSucceeded {
|
||||||
|
markCanceledComparisonOutcome(&result.Outcomes[index], err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
|
||||||
|
func executeComparisonProfile(ctx context.Context, req comparisonExecutionRequest, index int, profile ComparisonProfileInspection) comparisonProfileOutcome {
|
||||||
|
position := index + 1
|
||||||
|
outcome := comparisonProfileOutcome{
|
||||||
|
Position: position, ProfileID: profile.ProfileID, BackendID: profile.BackendID, ModelName: profile.ModelName,
|
||||||
|
Status: comparison.StatusFailed,
|
||||||
|
}
|
||||||
|
debugRef := promptdebug.PromptDebugRef{
|
||||||
|
ReportID: req.Prepared.resolved.Definition.ID,
|
||||||
|
ValidDate: req.Prepared.resolved.ValidPeriod.Start.Format("2006-01-02"),
|
||||||
|
RunID: comparisonDebugRunID(req.ComparisonID, position, len(req.Inspection.Profiles), profile.ProfileID),
|
||||||
|
}
|
||||||
|
execution, markdown, err := executePreparedProfile(ctx, profileExecutionRequest{
|
||||||
|
Prepared: req.Prepared,
|
||||||
|
Prompt: PromptInspectionResult{
|
||||||
|
PromptID: req.Inspection.PromptID, PromptVersion: req.Inspection.PromptVersion, PromptHash: req.Inspection.PromptHash,
|
||||||
|
},
|
||||||
|
Profile: promptexec.ProfileInspection{ProfileID: profile.ProfileID, BackendID: profile.BackendID, ModelName: profile.ModelName},
|
||||||
|
Executor: req.Executor, DebugWriter: req.DebugWriter, DebugRef: &debugRef,
|
||||||
|
})
|
||||||
|
outcome.ProfileID, outcome.BackendID, outcome.ModelName = execution.ProfileID, execution.BackendID, execution.ModelName
|
||||||
|
outcome.ValidationStatus = execution.ValidationStatus
|
||||||
|
outcome.LLMDebugPath = execution.LLMDebugPath
|
||||||
|
if err != nil {
|
||||||
|
safe := comparisonSafeExecutionError(err)
|
||||||
|
outcome.Error = &safe
|
||||||
|
return outcome
|
||||||
|
}
|
||||||
|
reportPath, err := comparison.ReportFilename(position, len(req.Inspection.Profiles), profile.ProfileID)
|
||||||
|
if err != nil {
|
||||||
|
safe := comparison.NewSafeError("application", "derive comparison report filename failed")
|
||||||
|
outcome.Error = &safe
|
||||||
|
return outcome
|
||||||
|
}
|
||||||
|
outcome.Status = comparison.StatusSucceeded
|
||||||
|
outcome.ReportPath = reportPath
|
||||||
|
outcome.Markdown = append([]byte(nil), markdown...)
|
||||||
|
return outcome
|
||||||
|
}
|
||||||
|
|
||||||
|
func comparisonDebugRunID(comparisonID string, position, profileCount int, profileID string) string {
|
||||||
|
return fmt.Sprintf("%s_%0*d-%s", comparisonID, comparison.OrdinalWidth(profileCount), position, comparison.ProfileSlug(profileID))
|
||||||
|
}
|
||||||
|
|
||||||
|
func markUnstartedComparisonOutcomes(outcomes []comparisonProfileOutcome, err error) {
|
||||||
|
for index := range outcomes {
|
||||||
|
markCanceledComparisonOutcome(&outcomes[index], err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func markCanceledComparisonOutcome(outcome *comparisonProfileOutcome, err error) {
|
||||||
|
outcome.Status = comparison.StatusFailed
|
||||||
|
outcome.ValidationStatus = promptexec.ValidationSkipped
|
||||||
|
outcome.ReportPath = ""
|
||||||
|
outcome.Markdown = nil
|
||||||
|
safe := comparisonSafeExecutionError(err)
|
||||||
|
outcome.Error = &safe
|
||||||
|
}
|
||||||
|
|
||||||
|
func comparisonSafeExecutionError(err error) comparison.SafeError {
|
||||||
|
category := promptexec.CategoryOf(err)
|
||||||
|
if category == "" {
|
||||||
|
switch {
|
||||||
|
case errors.Is(err, context.Canceled):
|
||||||
|
category = promptexec.Canceled
|
||||||
|
case errors.Is(err, context.DeadlineExceeded):
|
||||||
|
category = promptexec.DeadlineExceeded
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if category == "" {
|
||||||
|
return comparison.NewSafeError("application", comparisonExecutionMessage(err))
|
||||||
|
}
|
||||||
|
return comparison.NewSafeError(string(category), comparisonExecutionMessage(err))
|
||||||
|
}
|
||||||
|
|
||||||
|
func comparisonExecutionMessage(err error) string {
|
||||||
|
if errors.Is(err, context.Canceled) {
|
||||||
|
return "profile execution canceled"
|
||||||
|
}
|
||||||
|
if errors.Is(err, context.DeadlineExceeded) {
|
||||||
|
return "profile execution deadline exceeded"
|
||||||
|
}
|
||||||
|
var execution *profileExecutionError
|
||||||
|
if errors.As(err, &execution) {
|
||||||
|
return comparison.TruncateErrorMessage(execution.operation + " failed")
|
||||||
|
}
|
||||||
|
return "profile execution failed"
|
||||||
|
}
|
||||||
300
internal/app/comparison_execution_test.go
Normal file
300
internal/app/comparison_execution_test.go
Normal file
@@ -0,0 +1,300 @@
|
|||||||
|
package app
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"reflect"
|
||||||
|
"sync"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/comparison"
|
||||||
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptdebug"
|
||||||
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestExecuteComparisonProfilesRunsOrderedProfilesConcurrently(t *testing.T) {
|
||||||
|
prepared, prompt := preparedDailyProfile(t)
|
||||||
|
profiles := comparisonProfiles(10)
|
||||||
|
executor := newBarrierExecutor(profiles)
|
||||||
|
results := make(chan comparisonExecutionResult, 1)
|
||||||
|
go func() {
|
||||||
|
results <- executeComparisonProfiles(context.Background(), comparisonExecutionRequest{
|
||||||
|
Prepared: prepared, Inspection: comparisonInspection(prompt, profiles), ComparisonID: "comparison_daily", Executor: executor,
|
||||||
|
})
|
||||||
|
}()
|
||||||
|
waitForProfileStarts(t, executor, profiles, results)
|
||||||
|
if executor.maximumInFlight() < 2 {
|
||||||
|
t.Fatalf("maximum in-flight executions = %d, want overlap", executor.maximumInFlight())
|
||||||
|
}
|
||||||
|
for index := len(profiles) - 1; index >= 0; index-- {
|
||||||
|
executor.release(profiles[index].ProfileID)
|
||||||
|
}
|
||||||
|
result := <-results
|
||||||
|
if result.Canceled || len(result.Outcomes) != len(profiles) {
|
||||||
|
t.Fatalf("result = %#v", result)
|
||||||
|
}
|
||||||
|
for index, profile := range profiles {
|
||||||
|
outcome := result.Outcomes[index]
|
||||||
|
wantPath, err := comparison.ReportFilename(index+1, len(profiles), profile.ProfileID)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if outcome.Position != index+1 || outcome.ProfileID != profile.ProfileID || outcome.Status != comparison.StatusSucceeded || outcome.ValidationStatus != promptexec.ValidationPassed || outcome.ReportPath != wantPath || len(outcome.Markdown) == 0 || outcome.Error != nil {
|
||||||
|
t.Fatalf("outcome[%d] = %#v", index, outcome)
|
||||||
|
}
|
||||||
|
request, ok := executor.request(profile.ProfileID)
|
||||||
|
if !ok || request.PromptVersion != prompt.PromptVersion || !bytesEqual(request.DataPackage, prepared.dataPackage) {
|
||||||
|
t.Fatalf("request for %q = %#v, want prompt version %q and shared data package", profile.ProfileID, request, prompt.PromptVersion)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExecuteComparisonProfilesContinuesAfterProfileFailure(t *testing.T) {
|
||||||
|
prepared, prompt := preparedDailyProfile(t)
|
||||||
|
profiles := comparisonProfiles(3)
|
||||||
|
executor := newBarrierExecutor(profiles)
|
||||||
|
executor.setError(profiles[1].ProfileID, errors.New("provider response body must not escape"))
|
||||||
|
results := make(chan comparisonExecutionResult, 1)
|
||||||
|
go func() {
|
||||||
|
results <- executeComparisonProfiles(context.Background(), comparisonExecutionRequest{
|
||||||
|
Prepared: prepared, Inspection: comparisonInspection(prompt, profiles), ComparisonID: "comparison_daily", Executor: executor,
|
||||||
|
})
|
||||||
|
}()
|
||||||
|
waitForProfileStarts(t, executor, profiles, results)
|
||||||
|
for _, profile := range profiles {
|
||||||
|
executor.release(profile.ProfileID)
|
||||||
|
}
|
||||||
|
result := <-results
|
||||||
|
if result.Canceled || result.Outcomes[0].Status != comparison.StatusSucceeded || result.Outcomes[1].Status != comparison.StatusFailed || result.Outcomes[2].Status != comparison.StatusSucceeded {
|
||||||
|
t.Fatalf("outcomes = %#v", result.Outcomes)
|
||||||
|
}
|
||||||
|
failure := result.Outcomes[1]
|
||||||
|
if failure.Error == nil || failure.Error.Category != string(promptexec.Generation) || failure.Error.Message != "execute prompt failed" || failure.ReportPath != "" || len(failure.Markdown) != 0 {
|
||||||
|
t.Fatalf("failure outcome = %#v", failure)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExecuteComparisonProfilesPropagatesCancellationAndJoins(t *testing.T) {
|
||||||
|
prepared, prompt := preparedDailyProfile(t)
|
||||||
|
profiles := comparisonProfiles(4)
|
||||||
|
executor := newBarrierExecutor(profiles)
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
defer cancel()
|
||||||
|
results := make(chan comparisonExecutionResult, 1)
|
||||||
|
go func() {
|
||||||
|
results <- executeComparisonProfiles(ctx, comparisonExecutionRequest{
|
||||||
|
Prepared: prepared, Inspection: comparisonInspection(prompt, profiles), ComparisonID: "comparison_daily", Executor: executor,
|
||||||
|
})
|
||||||
|
}()
|
||||||
|
waitForProfileStarts(t, executor, profiles, results)
|
||||||
|
cancel()
|
||||||
|
result := <-results
|
||||||
|
if !result.Canceled || executor.inFlightCount() != 0 {
|
||||||
|
t.Fatalf("result/in-flight = %#v/%d", result, executor.inFlightCount())
|
||||||
|
}
|
||||||
|
for _, outcome := range result.Outcomes {
|
||||||
|
if outcome.Status != comparison.StatusFailed || outcome.Error == nil || outcome.Error.Category != string(promptexec.Canceled) || outcome.ValidationStatus != promptexec.ValidationSkipped || outcome.ReportPath != "" || len(outcome.Markdown) != 0 {
|
||||||
|
t.Fatalf("canceled outcome = %#v", outcome)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExecuteComparisonProfilesUsesDistinctDeterministicDebugReferences(t *testing.T) {
|
||||||
|
prepared, prompt := preparedDailyProfile(t)
|
||||||
|
profiles := []ComparisonProfileInspection{
|
||||||
|
{ProfileID: "light.one", BackendID: "local", ModelName: "light"},
|
||||||
|
{ProfileID: "deep/two", BackendID: "cloud", ModelName: "deep"},
|
||||||
|
}
|
||||||
|
debugWriter, err := promptdebug.NewPromptDebugWriter(t.TempDir())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewPromptDebugWriter() error = %v", err)
|
||||||
|
}
|
||||||
|
executor := newBarrierExecutor(profiles)
|
||||||
|
results := make(chan comparisonExecutionResult, 1)
|
||||||
|
go func() {
|
||||||
|
results <- executeComparisonProfiles(context.Background(), comparisonExecutionRequest{
|
||||||
|
Prepared: prepared, Inspection: comparisonInspection(prompt, profiles), ComparisonID: "comparison_daily", DebugWriter: debugWriter, Executor: executor,
|
||||||
|
})
|
||||||
|
}()
|
||||||
|
waitForProfileStarts(t, executor, profiles, results)
|
||||||
|
for _, profile := range profiles {
|
||||||
|
executor.release(profile.ProfileID)
|
||||||
|
}
|
||||||
|
result := <-results
|
||||||
|
paths := map[string]struct{}{}
|
||||||
|
for index, outcome := range result.Outcomes {
|
||||||
|
wantName := fmt.Sprintf("comparison_daily_%0*d-%s", comparison.OrdinalWidth(len(profiles)), index+1, comparison.ProfileSlug(outcome.ProfileID))
|
||||||
|
if filepath.Base(outcome.LLMDebugPath) != wantName {
|
||||||
|
t.Fatalf("debug path = %q, want base %q", outcome.LLMDebugPath, wantName)
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(filepath.Join(outcome.LLMDebugPath, "preparation.json")); err != nil {
|
||||||
|
t.Fatalf("preparation artifact %q: %v", outcome.LLMDebugPath, err)
|
||||||
|
}
|
||||||
|
paths[outcome.LLMDebugPath] = struct{}{}
|
||||||
|
}
|
||||||
|
if len(paths) != len(profiles) {
|
||||||
|
t.Fatalf("debug paths = %#v", paths)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type barrierExecutor struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
started chan string
|
||||||
|
callbackFailures chan error
|
||||||
|
releases map[string]chan struct{}
|
||||||
|
requests map[string]promptexec.ExecuteRequest
|
||||||
|
errors map[string]error
|
||||||
|
inFlight int
|
||||||
|
maximum int
|
||||||
|
}
|
||||||
|
|
||||||
|
func newBarrierExecutor(profiles []ComparisonProfileInspection) *barrierExecutor {
|
||||||
|
releases := make(map[string]chan struct{}, len(profiles))
|
||||||
|
for _, profile := range profiles {
|
||||||
|
releases[profile.ProfileID] = make(chan struct{})
|
||||||
|
}
|
||||||
|
return &barrierExecutor{
|
||||||
|
started: make(chan string, len(profiles)), callbackFailures: make(chan error, len(profiles)), releases: releases,
|
||||||
|
requests: make(map[string]promptexec.ExecuteRequest, len(profiles)), errors: map[string]error{},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *barrierExecutor) InspectPrompt(context.Context, string, string) (promptexec.PromptInspection, error) {
|
||||||
|
return promptexec.PromptInspection{}, errors.New("unexpected prompt inspection")
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *barrierExecutor) InspectProfile(context.Context, string) (promptexec.ProfileInspection, error) {
|
||||||
|
return promptexec.ProfileInspection{}, errors.New("unexpected profile inspection")
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *barrierExecutor) Execute(ctx context.Context, req promptexec.ExecuteRequest, callback promptexec.PreparationCallback) (*promptexec.Execution, error) {
|
||||||
|
stamp := time.Date(2026, 5, 29, 15, 0, 0, 0, time.UTC)
|
||||||
|
if err := callback(promptexec.Preparation{PromptID: req.PromptID, PromptVersion: req.PromptVersion, PromptHash: "prompt-hash", ProfileID: req.ProfileID, BackendID: "backend-" + req.ProfileID, ModelName: "model-" + req.ProfileID, StartedAt: stamp, EndedAt: stamp}, nil); err != nil {
|
||||||
|
e.callbackFailures <- err
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
e.mu.Lock()
|
||||||
|
e.requests[req.ProfileID] = promptexec.ExecuteRequest{PromptID: req.PromptID, PromptVersion: req.PromptVersion, ProfileID: req.ProfileID, DataPackage: append([]byte(nil), req.DataPackage...), CaptureDebug: req.CaptureDebug}
|
||||||
|
e.inFlight++
|
||||||
|
if e.inFlight > e.maximum {
|
||||||
|
e.maximum = e.inFlight
|
||||||
|
}
|
||||||
|
release := e.releases[req.ProfileID]
|
||||||
|
e.mu.Unlock()
|
||||||
|
e.started <- req.ProfileID
|
||||||
|
select {
|
||||||
|
case <-release:
|
||||||
|
case <-ctx.Done():
|
||||||
|
e.mu.Lock()
|
||||||
|
e.inFlight--
|
||||||
|
e.mu.Unlock()
|
||||||
|
return nil, ctx.Err()
|
||||||
|
}
|
||||||
|
e.mu.Lock()
|
||||||
|
e.inFlight--
|
||||||
|
err := e.errors[req.ProfileID]
|
||||||
|
e.mu.Unlock()
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return &promptexec.Execution{
|
||||||
|
PromptID: req.PromptID, PromptVersion: req.PromptVersion, PromptHash: "prompt-hash",
|
||||||
|
ProfileID: req.ProfileID, BackendID: "backend-" + req.ProfileID, ModelName: "model-" + req.ProfileID,
|
||||||
|
StartedAt: stamp, EndedAt: stamp, RawOutput: comparisonRawOutput(),
|
||||||
|
Validation: promptexec.NewValidation(promptexec.ValidationPassed, "json_schema", generationDefinitionForPrompt(req.PromptID).GeneratedTextSchemaID+".generated_text.schema.json", nil),
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *barrierExecutor) request(profileID string) (promptexec.ExecuteRequest, bool) {
|
||||||
|
e.mu.Lock()
|
||||||
|
defer e.mu.Unlock()
|
||||||
|
request, ok := e.requests[profileID]
|
||||||
|
return request, ok
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *barrierExecutor) setError(profileID string, err error) {
|
||||||
|
e.mu.Lock()
|
||||||
|
defer e.mu.Unlock()
|
||||||
|
e.errors[profileID] = err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *barrierExecutor) release(profileID string) {
|
||||||
|
close(e.releases[profileID])
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *barrierExecutor) releaseAll() {
|
||||||
|
for _, release := range e.releases {
|
||||||
|
select {
|
||||||
|
case <-release:
|
||||||
|
default:
|
||||||
|
close(release)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *barrierExecutor) maximumInFlight() int {
|
||||||
|
e.mu.Lock()
|
||||||
|
defer e.mu.Unlock()
|
||||||
|
return e.maximum
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *barrierExecutor) inFlightCount() int {
|
||||||
|
e.mu.Lock()
|
||||||
|
defer e.mu.Unlock()
|
||||||
|
return e.inFlight
|
||||||
|
}
|
||||||
|
|
||||||
|
func waitForProfileStarts(t *testing.T, executor *barrierExecutor, profiles []ComparisonProfileInspection, results <-chan comparisonExecutionResult) {
|
||||||
|
t.Helper()
|
||||||
|
timeout := time.NewTimer(5 * time.Second)
|
||||||
|
defer timeout.Stop()
|
||||||
|
seen := map[string]struct{}{}
|
||||||
|
for range profiles {
|
||||||
|
var profileID string
|
||||||
|
select {
|
||||||
|
case profileID = <-executor.started:
|
||||||
|
case err := <-executor.callbackFailures:
|
||||||
|
executor.releaseAll()
|
||||||
|
select {
|
||||||
|
case result := <-results:
|
||||||
|
t.Fatalf("comparison profile preparation failed before executor entry: %v; result: %#v", err, result)
|
||||||
|
case <-timeout.C:
|
||||||
|
t.Fatalf("comparison profile preparation failed before executor entry: %v; comparison did not finish", err)
|
||||||
|
}
|
||||||
|
case result := <-results:
|
||||||
|
t.Fatalf("comparison completed before all profiles started: %#v", result)
|
||||||
|
case <-timeout.C:
|
||||||
|
t.Fatal("timed out waiting for comparison profile starts")
|
||||||
|
}
|
||||||
|
if _, duplicate := seen[profileID]; duplicate {
|
||||||
|
t.Fatalf("duplicate execution start for %q", profileID)
|
||||||
|
}
|
||||||
|
seen[profileID] = struct{}{}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func comparisonProfiles(count int) []ComparisonProfileInspection {
|
||||||
|
profiles := make([]ComparisonProfileInspection, 0, count)
|
||||||
|
for index := 1; index <= count; index++ {
|
||||||
|
profiles = append(profiles, ComparisonProfileInspection{ProfileID: fmt.Sprintf("profile.%02d", index), BackendID: "backend", ModelName: "model"})
|
||||||
|
}
|
||||||
|
return profiles
|
||||||
|
}
|
||||||
|
|
||||||
|
func comparisonInspection(prompt PromptInspectionResult, profiles []ComparisonProfileInspection) ComparisonInspectionResult {
|
||||||
|
return ComparisonInspectionResult{PromptID: prompt.PromptID, PromptVersion: prompt.PromptVersion, PromptHash: prompt.PromptHash, Profiles: profiles}
|
||||||
|
}
|
||||||
|
|
||||||
|
func comparisonRawOutput() []byte {
|
||||||
|
return []byte(`{"summary":"Showers are possible during the selected day.","forecast_discussion":["A front will keep rain chances in the forecast."],"precipitation_timing":"Rain is most likely during the afternoon."}`)
|
||||||
|
}
|
||||||
|
|
||||||
|
func bytesEqual(left, right []byte) bool {
|
||||||
|
return reflect.DeepEqual(left, right)
|
||||||
|
}
|
||||||
|
|
||||||
|
var _ promptexec.Executor = (*barrierExecutor)(nil)
|
||||||
332
internal/app/comparison_test.go
Normal file
332
internal/app/comparison_test.go
Normal file
@@ -0,0 +1,332 @@
|
|||||||
|
package app
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/comparison"
|
||||||
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/config"
|
||||||
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec"
|
||||||
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/timeutil"
|
||||||
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/weatherdata"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestCompareDetailedPublishesOneCoherentBundle(t *testing.T) {
|
||||||
|
cfg := comparisonConfig()
|
||||||
|
bundle := generationBundle(t)
|
||||||
|
workingDir := t.TempDir()
|
||||||
|
executor := &generationExecutor{}
|
||||||
|
inspectedBeforeCollection := false
|
||||||
|
result, err := CompareDetailed(context.Background(), ComparisonRequest{
|
||||||
|
Config: cfg, Report: ReportDaily, ProfileIDs: []string{"weather-light", "weather-deep"},
|
||||||
|
WorkingDir: workingDir, Date: generationTime("2026-05-29T12:00:00-05:00"),
|
||||||
|
Clock: timeutil.FixedClock{Time: generationTime("2026-05-29T08:30:00-05:00")},
|
||||||
|
Collector: &generationCollector{bundle: &bundle, beforeRun: func() {
|
||||||
|
inspectedBeforeCollection = executor.promptInspections == 1 && executor.profileInspections == 2
|
||||||
|
}}, Executor: executor,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("CompareDetailed() error = %v", err)
|
||||||
|
}
|
||||||
|
if result == nil || result.Total != 2 || result.Succeeded != 2 || result.Failed != 0 || executor.promptInspections != 1 || executor.profileInspections != 2 || executor.executeCalls != 2 || !inspectedBeforeCollection || result.ManifestPath == "" || result.DataPackagePath == "" {
|
||||||
|
t.Fatalf("result/executor = %#v/%#v", result, executor)
|
||||||
|
}
|
||||||
|
if result.OutputDirectory != filepath.Dir(result.ManifestPath) || !filepath.IsAbs(result.ManifestPath) || !filepath.IsAbs(result.DataPackagePath) {
|
||||||
|
t.Fatalf("published paths = %#v", result)
|
||||||
|
}
|
||||||
|
for index, profile := range result.Results {
|
||||||
|
if profile.Position != index+1 || profile.Status != comparison.StatusSucceeded || !filepath.IsAbs(profile.ReportPath) || profile.Error != nil {
|
||||||
|
t.Fatalf("profile result = %#v", profile)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
data, readErr := os.ReadFile(result.ManifestPath)
|
||||||
|
if readErr != nil {
|
||||||
|
t.Fatal(readErr)
|
||||||
|
}
|
||||||
|
var manifest comparison.Manifest
|
||||||
|
if err := json.Unmarshal(data, &manifest); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if manifest.ComparisonID != result.ComparisonID || manifest.Total != result.Total || manifest.Succeeded != result.Succeeded || manifest.DataPackage.SHA256 == "" || len(manifest.Results) != 2 {
|
||||||
|
t.Fatalf("manifest = %#v", manifest)
|
||||||
|
}
|
||||||
|
if manifest.Results[0].ReportPath != filepath.Base(result.Results[0].ReportPath) || manifest.Results[1].ReportPath != filepath.Base(result.Results[1].ReportPath) {
|
||||||
|
t.Fatalf("manifest report paths = %#v", manifest.Results)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCompareDetailedPublishesPartialBundleAndReturnsAggregateError(t *testing.T) {
|
||||||
|
cfg := comparisonConfig()
|
||||||
|
bundle := generationBundle(t)
|
||||||
|
executor := &generationExecutor{executeErrors: map[string]error{"weather-deep": errors.New("provider detail must not escape")}}
|
||||||
|
result, err := CompareDetailed(context.Background(), ComparisonRequest{
|
||||||
|
Config: cfg, Report: ReportDaily, ProfileIDs: []string{"weather-light", "weather-deep", "weather-fallback"},
|
||||||
|
WorkingDir: t.TempDir(), Date: generationTime("2026-05-29T12:00:00-05:00"),
|
||||||
|
Clock: timeutil.FixedClock{Time: generationTime("2026-05-29T08:30:00-05:00")},
|
||||||
|
Collector: &generationCollector{bundle: &bundle}, Executor: executor,
|
||||||
|
})
|
||||||
|
if err == nil || err.Error() != "comparison completed with 1 failed profiles" || result == nil || result.Total != 3 || result.Succeeded != 2 || result.Failed != 1 {
|
||||||
|
t.Fatalf("CompareDetailed() result/error = %#v/%v", result, err)
|
||||||
|
}
|
||||||
|
failure := result.Results[1]
|
||||||
|
if failure.Status != comparison.StatusFailed || failure.ReportPath != "" || failure.Error == nil || strings.Contains(failure.Error.Message, "provider detail") {
|
||||||
|
t.Fatalf("failure = %#v", failure)
|
||||||
|
}
|
||||||
|
if _, statErr := os.Stat(result.ManifestPath); statErr != nil {
|
||||||
|
t.Fatalf("partial manifest: %v", statErr)
|
||||||
|
}
|
||||||
|
if _, statErr := os.Stat(filepath.Join(result.OutputDirectory, filepath.Base(result.Results[0].ReportPath))); statErr != nil {
|
||||||
|
t.Fatalf("successful partial report: %v", statErr)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCompareDetailedRetainsCommittedPathsWhenBackupCleanupFails(t *testing.T) {
|
||||||
|
bundle := generationBundle(t)
|
||||||
|
backupPath := filepath.Join(t.TempDir(), ".comparison-daily.backup-retained")
|
||||||
|
cleanupCause := errors.New("backup cleanup failed")
|
||||||
|
publish := func(context.Context, comparison.DestinationPlan, comparison.LogicalBundle) (comparison.PublicationResult, error) {
|
||||||
|
return comparison.PublicationResult{Committed: true, RetainedBackupPath: backupPath}, &comparison.PublicationCleanupError{RetainedBackupPath: backupPath, Err: cleanupCause}
|
||||||
|
}
|
||||||
|
|
||||||
|
result, err := compareDetailed(context.Background(), ComparisonRequest{
|
||||||
|
Config: comparisonConfig(), Report: ReportDaily, ProfileIDs: []string{"weather-light", "weather-deep"},
|
||||||
|
WorkingDir: t.TempDir(), Date: generationTime("2026-05-29T12:00:00-05:00"),
|
||||||
|
Clock: timeutil.FixedClock{Time: generationTime("2026-05-29T08:30:00-05:00")},
|
||||||
|
Collector: &generationCollector{bundle: &bundle}, Executor: &generationExecutor{},
|
||||||
|
}, publish)
|
||||||
|
var cleanupErr *comparison.PublicationCleanupError
|
||||||
|
if result == nil || !errors.As(err, &cleanupErr) || !errors.Is(err, cleanupCause) || cleanupErr.RetainedBackupPath != backupPath || !filepath.IsAbs(result.ManifestPath) || !filepath.IsAbs(result.DataPackagePath) {
|
||||||
|
t.Fatalf("CompareDetailed() result/error = %#v/%v", result, err)
|
||||||
|
}
|
||||||
|
for _, profile := range result.Results {
|
||||||
|
if profile.Status == comparison.StatusSucceeded && !filepath.IsAbs(profile.ReportPath) {
|
||||||
|
t.Fatalf("published profile result = %#v", profile)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCompareDetailedPreflightsBeforePromptOrCollection(t *testing.T) {
|
||||||
|
invalidDestination := filepath.Join(t.TempDir(), "not-a-directory")
|
||||||
|
if err := os.WriteFile(invalidDestination, []byte("x"), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
bundle := generationBundle(t)
|
||||||
|
collector := &generationCollector{bundle: &bundle}
|
||||||
|
executor := &generationExecutor{}
|
||||||
|
result, err := CompareDetailed(context.Background(), ComparisonRequest{
|
||||||
|
Config: comparisonConfig(), Report: ReportDaily, ProfileIDs: []string{"weather-light", "weather-deep"},
|
||||||
|
WorkingDir: t.TempDir(), OutputDir: invalidDestination, Date: generationTime("2026-05-29T12:00:00-05:00"),
|
||||||
|
Clock: timeutil.FixedClock{Time: generationTime("2026-05-29T08:30:00-05:00")}, Collector: collector, Executor: executor,
|
||||||
|
})
|
||||||
|
if err == nil || result == nil || collector.called || executor.promptInspections != 0 || executor.executeCalls != 0 || result.ManifestPath != "" {
|
||||||
|
t.Fatalf("result/error/collector/executor = %#v/%v/%#v/%#v", result, err, collector, executor)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCompareDetailedFinalizesUnpublishedFailures(t *testing.T) {
|
||||||
|
for _, test := range []struct {
|
||||||
|
name string
|
||||||
|
prepare func(t *testing.T, outputDirectory string)
|
||||||
|
debugDir string
|
||||||
|
executor *generationExecutor
|
||||||
|
collector *generationCollector
|
||||||
|
wantPrompt bool
|
||||||
|
wantCollection bool
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "destination preflight",
|
||||||
|
prepare: func(t *testing.T, outputDirectory string) {
|
||||||
|
t.Helper()
|
||||||
|
if err := os.WriteFile(outputDirectory, []byte("not a directory"), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
},
|
||||||
|
executor: &generationExecutor{},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "debug initialization",
|
||||||
|
debugDir: "relative-debug-directory",
|
||||||
|
executor: &generationExecutor{},
|
||||||
|
collector: &generationCollector{},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "prompt preflight",
|
||||||
|
executor: &generationExecutor{inspectErr: promptexec.NewError(promptexec.PromptLoad, "unsafe prompt detail", errors.New("unsafe cause"))},
|
||||||
|
collector: &generationCollector{},
|
||||||
|
wantPrompt: false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "profile preflight",
|
||||||
|
executor: &generationExecutor{profileInspectErrors: map[string]error{
|
||||||
|
"weather-deep": promptexec.NewError(promptexec.MissingCredential, "profile credential is unavailable", errors.New("unsafe cause")),
|
||||||
|
}},
|
||||||
|
collector: &generationCollector{},
|
||||||
|
wantPrompt: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "collection",
|
||||||
|
executor: &generationExecutor{},
|
||||||
|
collector: &generationCollector{err: errors.New("collection failed")},
|
||||||
|
wantPrompt: true,
|
||||||
|
wantCollection: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "preparation",
|
||||||
|
executor: &generationExecutor{},
|
||||||
|
collector: &generationCollector{bundle: &weatherdata.Bundle{}},
|
||||||
|
wantPrompt: true,
|
||||||
|
wantCollection: true,
|
||||||
|
},
|
||||||
|
} {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
workingDirectory := t.TempDir()
|
||||||
|
outputDirectory := filepath.Join(workingDirectory, "comparison-output")
|
||||||
|
if test.prepare != nil {
|
||||||
|
test.prepare(t, outputDirectory)
|
||||||
|
}
|
||||||
|
collector := test.collector
|
||||||
|
if collector == nil {
|
||||||
|
bundle := generationBundle(t)
|
||||||
|
collector = &generationCollector{bundle: &bundle}
|
||||||
|
}
|
||||||
|
result, err := CompareDetailed(context.Background(), ComparisonRequest{
|
||||||
|
Config: comparisonConfig(), Report: ReportDaily, ProfileIDs: []string{"weather-light", "weather-deep"},
|
||||||
|
WorkingDir: workingDirectory, OutputDir: outputDirectory, LLMDebugDir: test.debugDir,
|
||||||
|
Date: generationTime("2026-05-29T12:00:00-05:00"), Clock: timeutil.FixedClock{Time: generationTime("2026-05-29T08:30:00-05:00")},
|
||||||
|
Collector: collector, Executor: test.executor,
|
||||||
|
})
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("CompareDetailed() error = nil")
|
||||||
|
}
|
||||||
|
assertUnpublishedComparisonResult(t, result, outputDirectory)
|
||||||
|
if (result.PromptID != "") != test.wantPrompt || (result.PromptHash != "") != test.wantPrompt {
|
||||||
|
t.Fatalf("prompt identity = %q/%q, want resolved=%t", result.PromptID, result.PromptHash, test.wantPrompt)
|
||||||
|
}
|
||||||
|
if collector.called != test.wantCollection || test.executor.executeCalls != 0 {
|
||||||
|
t.Fatalf("collection/execution = %t/%d, want collection=%t and no execution", collector.called, test.executor.executeCalls, test.wantCollection)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCompareDetailedLeavesDestinationWhenCollectionOrPreparationFails(t *testing.T) {
|
||||||
|
collectionErr := errors.New("weather collection failed")
|
||||||
|
for _, test := range []struct {
|
||||||
|
name string
|
||||||
|
collector *generationCollector
|
||||||
|
}{
|
||||||
|
{name: "collection", collector: &generationCollector{err: collectionErr}},
|
||||||
|
{name: "preparation", collector: &generationCollector{bundle: &weatherdata.Bundle{}}},
|
||||||
|
} {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
workingDir := t.TempDir()
|
||||||
|
executor := &generationExecutor{}
|
||||||
|
result, err := CompareDetailed(context.Background(), ComparisonRequest{
|
||||||
|
Config: comparisonConfig(), Report: ReportDaily, ProfileIDs: []string{"weather-light", "weather-deep"},
|
||||||
|
WorkingDir: workingDir, Date: generationTime("2026-05-29T12:00:00-05:00"),
|
||||||
|
Clock: timeutil.FixedClock{Time: generationTime("2026-05-29T08:30:00-05:00")}, Collector: test.collector, Executor: executor,
|
||||||
|
})
|
||||||
|
if err == nil || result == nil || executor.executeCalls != 0 || result.ManifestPath != "" || result.DataPackagePath != "" {
|
||||||
|
t.Fatalf("CompareDetailed() result/error/executor = %#v/%v/%#v", result, err, executor)
|
||||||
|
}
|
||||||
|
if _, statErr := os.Stat(filepath.Join(workingDir, "comparison-daily-2026-05-29")); !os.IsNotExist(statErr) {
|
||||||
|
t.Fatalf("comparison destination stat error = %v", statErr)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCompareDetailedPublishesManifestWhenEveryProfileFails(t *testing.T) {
|
||||||
|
bundle := generationBundle(t)
|
||||||
|
executor := &generationExecutor{executeErrors: map[string]error{
|
||||||
|
"weather-light": errors.New("first provider failure"), "weather-deep": errors.New("second provider failure"),
|
||||||
|
}}
|
||||||
|
result, err := CompareDetailed(context.Background(), ComparisonRequest{
|
||||||
|
Config: comparisonConfig(), Report: ReportDaily, ProfileIDs: []string{"weather-light", "weather-deep"},
|
||||||
|
WorkingDir: t.TempDir(), Date: generationTime("2026-05-29T12:00:00-05:00"),
|
||||||
|
Clock: timeutil.FixedClock{Time: generationTime("2026-05-29T08:30:00-05:00")},
|
||||||
|
Collector: &generationCollector{bundle: &bundle}, Executor: executor,
|
||||||
|
})
|
||||||
|
if err == nil || err.Error() != "comparison completed with 2 failed profiles" || result == nil || result.Succeeded != 0 || result.Failed != 2 || result.ManifestPath == "" {
|
||||||
|
t.Fatalf("CompareDetailed() result/error = %#v/%v", result, err)
|
||||||
|
}
|
||||||
|
for _, profile := range result.Results {
|
||||||
|
if profile.ReportPath != "" || profile.Error == nil {
|
||||||
|
t.Fatalf("failed profile = %#v", profile)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCompareDetailedCancellationPreservesPublishedBundle(t *testing.T) {
|
||||||
|
workingDir := t.TempDir()
|
||||||
|
bundle := generationBundle(t)
|
||||||
|
request := ComparisonRequest{
|
||||||
|
Config: comparisonConfig(), Report: ReportDaily, ProfileIDs: []string{"weather-light", "weather-deep"},
|
||||||
|
WorkingDir: workingDir, Date: generationTime("2026-05-29T12:00:00-05:00"),
|
||||||
|
Clock: timeutil.FixedClock{Time: generationTime("2026-05-29T08:30:00-05:00")}, Collector: &generationCollector{bundle: &bundle}, Executor: &generationExecutor{},
|
||||||
|
}
|
||||||
|
previous, err := CompareDetailed(context.Background(), request)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("initial CompareDetailed() error = %v", err)
|
||||||
|
}
|
||||||
|
before, err := os.ReadFile(previous.ManifestPath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
request.Replace = true
|
||||||
|
request.Executor = &generationExecutor{cancelBeforeReturn: cancel}
|
||||||
|
result, err := CompareDetailed(ctx, request)
|
||||||
|
if !errors.Is(err, context.Canceled) || result == nil || result.ManifestPath != "" || result.DataPackagePath != "" {
|
||||||
|
t.Fatalf("canceled CompareDetailed() result/error = %#v/%v", result, err)
|
||||||
|
}
|
||||||
|
after, readErr := os.ReadFile(previous.ManifestPath)
|
||||||
|
if readErr != nil || string(after) != string(before) {
|
||||||
|
t.Fatalf("published manifest changed = %q, error = %v", after, readErr)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCompareDetailedLeavesExistingBundleWhenPublicationPreflightChanges(t *testing.T) {
|
||||||
|
workingDir := t.TempDir()
|
||||||
|
target := filepath.Join(workingDir, "comparison-output")
|
||||||
|
bundle := generationBundle(t)
|
||||||
|
executor := &generationExecutor{beforeExecute: func(promptexec.ExecuteRequest) {
|
||||||
|
_ = os.WriteFile(target, []byte("changed"), 0o600)
|
||||||
|
}}
|
||||||
|
result, err := CompareDetailed(context.Background(), ComparisonRequest{
|
||||||
|
Config: comparisonConfig(), Report: ReportDaily, ProfileIDs: []string{"weather-light", "weather-deep"},
|
||||||
|
WorkingDir: workingDir, OutputDir: target, Date: generationTime("2026-05-29T12:00:00-05:00"),
|
||||||
|
Clock: timeutil.FixedClock{Time: generationTime("2026-05-29T08:30:00-05:00")}, Collector: &generationCollector{bundle: &bundle}, Executor: executor,
|
||||||
|
})
|
||||||
|
if err == nil || result == nil || result.ManifestPath != "" || result.DataPackagePath != "" || result.Results[0].ReportPath != "" {
|
||||||
|
t.Fatalf("CompareDetailed() result/error = %#v/%v", result, err)
|
||||||
|
}
|
||||||
|
data, readErr := os.ReadFile(target)
|
||||||
|
if readErr != nil || string(data) != "changed" {
|
||||||
|
t.Fatalf("destination = %q, error = %v", data, readErr)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func comparisonConfig() config.Config {
|
||||||
|
cfg := config.Defaults()
|
||||||
|
cfg.WeatherAPI.Timezone, cfg.Location.ID = "America/Chicago", "home"
|
||||||
|
return cfg
|
||||||
|
}
|
||||||
|
|
||||||
|
func assertUnpublishedComparisonResult(t *testing.T, result *ComparisonResult, outputDirectory string) {
|
||||||
|
t.Helper()
|
||||||
|
if result == nil || result.OutputDirectory != outputDirectory || !filepath.IsAbs(result.OutputDirectory) || result.FinishedAt.IsZero() || result.FinishedAt.Location() != time.UTC || result.FinishedAt.Before(result.StartedAt) || result.ManifestPath != "" || result.DataPackagePath != "" {
|
||||||
|
t.Fatalf("unpublished comparison result = %#v", result)
|
||||||
|
}
|
||||||
|
for _, profile := range result.Results {
|
||||||
|
if profile.ReportPath != "" {
|
||||||
|
t.Fatalf("unpublished profile result = %#v", profile)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -6,6 +6,7 @@ import (
|
|||||||
"errors"
|
"errors"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
"sync"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
@@ -20,6 +21,7 @@ type generationCollector struct {
|
|||||||
bundle *weatherdata.Bundle
|
bundle *weatherdata.Bundle
|
||||||
err error
|
err error
|
||||||
called bool
|
called bool
|
||||||
|
calls int
|
||||||
beforeRun func()
|
beforeRun func()
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -28,57 +30,89 @@ func (c *generationCollector) Run(context.Context, collect.Request) (*collect.Re
|
|||||||
c.beforeRun()
|
c.beforeRun()
|
||||||
}
|
}
|
||||||
c.called = true
|
c.called = true
|
||||||
|
c.calls++
|
||||||
return &collect.Result{Bundle: c.bundle}, c.err
|
return &collect.Result{Bundle: c.bundle}, c.err
|
||||||
}
|
}
|
||||||
|
|
||||||
type generationExecutor struct {
|
type generationExecutor struct {
|
||||||
called bool
|
called bool
|
||||||
promptInspections int
|
executeCalls int
|
||||||
inspectErr error
|
promptInspections int
|
||||||
executeErr error
|
profileInspections int
|
||||||
cancelBeforeReturn context.CancelFunc
|
inspectErr error
|
||||||
validation promptexec.ValidationStatus
|
profileInspectErrors map[string]error
|
||||||
rawOutput []byte
|
executeErr error
|
||||||
failedPrompt string
|
executeErrors map[string]error
|
||||||
|
beforeExecute func(promptexec.ExecuteRequest)
|
||||||
|
cancelBeforeReturn context.CancelFunc
|
||||||
|
validation promptexec.ValidationStatus
|
||||||
|
rawOutput []byte
|
||||||
|
failedPrompt string
|
||||||
}
|
}
|
||||||
|
|
||||||
|
var generationExecutorMu sync.Mutex
|
||||||
|
|
||||||
func (e *generationExecutor) InspectPrompt(_ context.Context, id, version string) (promptexec.PromptInspection, error) {
|
func (e *generationExecutor) InspectPrompt(_ context.Context, id, version string) (promptexec.PromptInspection, error) {
|
||||||
|
generationExecutorMu.Lock()
|
||||||
|
defer generationExecutorMu.Unlock()
|
||||||
e.promptInspections++
|
e.promptInspections++
|
||||||
if e.inspectErr != nil {
|
if e.inspectErr != nil {
|
||||||
return promptexec.PromptInspection{}, e.inspectErr
|
return promptexec.PromptInspection{}, e.inspectErr
|
||||||
}
|
}
|
||||||
definition := generationDefinitionForPrompt(id)
|
definition := generationDefinitionForPrompt(id)
|
||||||
return promptexec.PromptInspection{PromptID: id, PromptVersion: version, PromptHash: "prompt-hash", DefaultProfileID: "fixture", Inputs: []promptexec.InputDefinition{{Name: "data_package", Required: true, ContentType: "application/yaml"}}, Output: promptexec.OutputContract{Format: "json", ValidationMode: "json_schema", SchemaPath: definition.GeneratedTextSchemaID + ".generated_text.schema.json"}}, nil
|
return promptexec.PromptInspection{PromptID: id, PromptVersion: version, PromptHash: generationPromptHash, DefaultProfileID: "fixture", Inputs: []promptexec.InputDefinition{{Name: "data_package", Required: true, ContentType: "application/yaml"}}, Output: promptexec.OutputContract{Format: "json", ValidationMode: "json_schema", SchemaPath: definition.GeneratedTextSchemaID + ".generated_text.schema.json"}}, nil
|
||||||
}
|
}
|
||||||
func (*generationExecutor) InspectProfile(_ context.Context, id string) (promptexec.ProfileInspection, error) {
|
func (e *generationExecutor) InspectProfile(_ context.Context, id string) (promptexec.ProfileInspection, error) {
|
||||||
|
generationExecutorMu.Lock()
|
||||||
|
defer generationExecutorMu.Unlock()
|
||||||
|
e.profileInspections++
|
||||||
|
if err := e.profileInspectErrors[id]; err != nil {
|
||||||
|
return promptexec.ProfileInspection{}, err
|
||||||
|
}
|
||||||
return promptexec.ProfileInspection{ProfileID: id, BackendID: "fixture", ModelName: "fixture-model"}, nil
|
return promptexec.ProfileInspection{ProfileID: id, BackendID: "fixture", ModelName: "fixture-model"}, nil
|
||||||
}
|
}
|
||||||
func (e *generationExecutor) Execute(_ context.Context, req promptexec.ExecuteRequest, callback promptexec.PreparationCallback) (*promptexec.Execution, error) {
|
func (e *generationExecutor) Execute(_ context.Context, req promptexec.ExecuteRequest, callback promptexec.PreparationCallback) (*promptexec.Execution, error) {
|
||||||
stamp := time.Date(2026, 5, 29, 15, 0, 0, 0, time.UTC)
|
stamp := time.Date(2026, 5, 29, 15, 0, 0, 0, time.UTC)
|
||||||
if err := callback(promptexec.Preparation{PromptID: req.PromptID, PromptVersion: req.PromptVersion, PromptHash: "prompt-hash", RenderedPromptHash: "rendered-hash", ProfileID: req.ProfileID, BackendID: "fixture", ModelName: "fixture-model", StartedAt: stamp, EndedAt: stamp}, nil); err != nil {
|
if err := callback(promptexec.Preparation{PromptID: req.PromptID, PromptVersion: req.PromptVersion, PromptHash: generationPromptHash, RenderedPromptHash: "rendered-hash", ProfileID: req.ProfileID, BackendID: "fixture", ModelName: "fixture-model", StartedAt: stamp, EndedAt: stamp}, nil); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
generationExecutorMu.Lock()
|
||||||
e.called = true
|
e.called = true
|
||||||
if e.executeErr != nil {
|
e.executeCalls++
|
||||||
return nil, e.executeErr
|
beforeExecute := e.beforeExecute
|
||||||
}
|
profileErr := e.executeErrors[req.ProfileID]
|
||||||
|
executeErr := e.executeErr
|
||||||
status := e.validation
|
status := e.validation
|
||||||
|
rawOutput := append([]byte(nil), e.rawOutput...)
|
||||||
|
failedPrompt := e.failedPrompt
|
||||||
|
cancelBeforeReturn := e.cancelBeforeReturn
|
||||||
|
generationExecutorMu.Unlock()
|
||||||
|
if beforeExecute != nil {
|
||||||
|
beforeExecute(req)
|
||||||
|
}
|
||||||
|
if profileErr != nil {
|
||||||
|
return nil, profileErr
|
||||||
|
}
|
||||||
|
if executeErr != nil {
|
||||||
|
return nil, executeErr
|
||||||
|
}
|
||||||
if status == "" {
|
if status == "" {
|
||||||
status = promptexec.ValidationPassed
|
status = promptexec.ValidationPassed
|
||||||
}
|
}
|
||||||
if e.failedPrompt == req.PromptID {
|
if failedPrompt == req.PromptID {
|
||||||
status = promptexec.ValidationFailed
|
status = promptexec.ValidationFailed
|
||||||
}
|
}
|
||||||
rawOutput := e.rawOutput
|
|
||||||
if rawOutput == nil {
|
if rawOutput == nil {
|
||||||
rawOutput = []byte(`{"summary":"Showers are possible during the selected day.","forecast_discussion":["A front will keep rain chances in the forecast."],"precipitation_timing":"Rain is most likely during the afternoon."}`)
|
rawOutput = []byte(`{"summary":"Showers are possible during the selected day.","forecast_discussion":["A front will keep rain chances in the forecast."],"precipitation_timing":"Rain is most likely during the afternoon."}`)
|
||||||
}
|
}
|
||||||
if e.cancelBeforeReturn != nil {
|
if cancelBeforeReturn != nil {
|
||||||
e.cancelBeforeReturn()
|
cancelBeforeReturn()
|
||||||
}
|
}
|
||||||
return &promptexec.Execution{RunID: "provider-run", PromptID: req.PromptID, PromptVersion: req.PromptVersion, PromptHash: "prompt-hash", RenderedPromptHash: "rendered-hash", ProfileID: req.ProfileID, BackendID: "fixture", ModelName: "fixture-model", StartedAt: stamp, EndedAt: stamp, RawOutput: rawOutput, Validation: promptexec.NewValidation(status, "json_schema", generationDefinitionForPrompt(req.PromptID).GeneratedTextSchemaID+".generated_text.schema.json", nil)}, nil
|
return &promptexec.Execution{RunID: "provider-run", PromptID: req.PromptID, PromptVersion: req.PromptVersion, PromptHash: generationPromptHash, RenderedPromptHash: "rendered-hash", ProfileID: req.ProfileID, BackendID: "fixture", ModelName: "fixture-model", StartedAt: stamp, EndedAt: stamp, RawOutput: rawOutput, Validation: promptexec.NewValidation(status, "json_schema", generationDefinitionForPrompt(req.PromptID).GeneratedTextSchemaID+".generated_text.schema.json", nil)}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const generationPromptHash = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"
|
||||||
|
|
||||||
func generationDefinitionForPrompt(promptID string) report.Definition {
|
func generationDefinitionForPrompt(promptID string) report.Definition {
|
||||||
for _, definition := range report.DefaultRegistry().All() {
|
for _, definition := range report.DefaultRegistry().All() {
|
||||||
if definition.PromptID == promptID {
|
if definition.PromptID == promptID {
|
||||||
@@ -93,12 +127,13 @@ func TestGenerateDetailedPublishesOnlySelectedOutput(t *testing.T) {
|
|||||||
cfg.WeatherAPI.Timezone, cfg.Location.ID = "America/Chicago", "home"
|
cfg.WeatherAPI.Timezone, cfg.Location.ID = "America/Chicago", "home"
|
||||||
bundle := generationBundle(t)
|
bundle := generationBundle(t)
|
||||||
executor := &generationExecutor{}
|
executor := &generationExecutor{}
|
||||||
|
collector := &generationCollector{bundle: &bundle}
|
||||||
workingDir := t.TempDir()
|
workingDir := t.TempDir()
|
||||||
result, err := GenerateDetailed(context.Background(), GenerateRequest{Config: cfg, Report: ReportDaily, Date: generationTime("2026-05-29T12:00:00-05:00"), Now: generationTime("2026-05-29T08:30:00-05:00"), WorkingDir: workingDir, Collector: &generationCollector{bundle: &bundle}, Executor: executor})
|
result, err := GenerateDetailed(context.Background(), GenerateRequest{Config: cfg, Report: ReportDaily, Date: generationTime("2026-05-29T12:00:00-05:00"), Now: generationTime("2026-05-29T08:30:00-05:00"), WorkingDir: workingDir, Collector: collector, Executor: executor})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("GenerateDetailed() error = %v", err)
|
t.Fatalf("GenerateDetailed() error = %v", err)
|
||||||
}
|
}
|
||||||
if !executor.called || result.OutputPath != filepath.Join(workingDir, "daily-2026-05-29.md") || result.ValidationStatus != promptexec.ValidationPassed || result.ProfileID == "" || result.BackendID == "" || result.ModelName == "" {
|
if !executor.called || executor.executeCalls != 1 || collector.calls != 1 || result.OutputPath != filepath.Join(workingDir, "daily-2026-05-29.md") || result.ValidationStatus != promptexec.ValidationPassed || result.ProfileID == "" || result.BackendID == "" || result.ModelName == "" {
|
||||||
t.Fatalf("result = %#v", result)
|
t.Fatalf("result = %#v", result)
|
||||||
}
|
}
|
||||||
if result.LLMDebugPath != "" {
|
if result.LLMDebugPath != "" {
|
||||||
@@ -332,6 +367,24 @@ func TestGenerateDetailedDoesNotReplaceDirectoryOutput(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestGenerateDetailedWritesRequestedPromptDebugArtifacts(t *testing.T) {
|
||||||
|
bundle := generationBundle(t)
|
||||||
|
debugRoot := t.TempDir()
|
||||||
|
result, err := GenerateDetailed(context.Background(), GenerateRequest{
|
||||||
|
Config: generationConfig(), Report: ReportDaily,
|
||||||
|
Date: generationTime("2026-05-29T12:00:00-05:00"), Now: generationTime("2026-05-29T08:30:00-05:00"),
|
||||||
|
WorkingDir: t.TempDir(), LLMDebugDir: debugRoot, Collector: &generationCollector{bundle: &bundle}, Executor: &generationExecutor{},
|
||||||
|
})
|
||||||
|
if err != nil || result == nil || result.LLMDebugPath == "" {
|
||||||
|
t.Fatalf("GenerateDetailed() result/error = %#v/%v", result, err)
|
||||||
|
}
|
||||||
|
for _, name := range []string{"preparation.json", "execution.json"} {
|
||||||
|
if _, statErr := os.Stat(filepath.Join(result.LLMDebugPath, name)); statErr != nil {
|
||||||
|
t.Fatalf("debug artifact %q: %v", name, statErr)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func generationConfig() config.Config {
|
func generationConfig() config.Config {
|
||||||
cfg := config.Defaults()
|
cfg := config.Defaults()
|
||||||
cfg.WeatherAPI.Timezone, cfg.Location.ID = "America/Chicago", "home"
|
cfg.WeatherAPI.Timezone, cfg.Location.ID = "America/Chicago", "home"
|
||||||
|
|||||||
@@ -6,6 +6,7 @@ import (
|
|||||||
"path/filepath"
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/comparison"
|
||||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/report"
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/report"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -51,6 +52,35 @@ func resolveOutputDirWithConfigured(workingDir, override, configuredDir string)
|
|||||||
return resolveOutputDir(workingDir, directory)
|
return resolveOutputDir(workingDir, directory)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func resolveComparisonOutputDirectory(workingDir, override, configuredDir, reportOutputName string) (string, error) {
|
||||||
|
workingDir, err := validateWorkingDir(workingDir)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
if override != "" {
|
||||||
|
return resolveComparisonDirectoryPath(workingDir, override)
|
||||||
|
}
|
||||||
|
outputDir, err := resolveOutputDir(workingDir, configuredDir)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
name, err := comparison.DefaultDirectoryName(reportOutputName)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
return filepath.Join(outputDir, name), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func resolveComparisonDirectoryPath(workingDir, directory string) (string, error) {
|
||||||
|
if strings.TrimSpace(directory) == "" {
|
||||||
|
return "", fmt.Errorf("comparison output directory is required")
|
||||||
|
}
|
||||||
|
if !filepath.IsAbs(directory) {
|
||||||
|
directory = filepath.Join(workingDir, directory)
|
||||||
|
}
|
||||||
|
return filepath.Clean(directory), nil
|
||||||
|
}
|
||||||
|
|
||||||
func resolveOutputDir(workingDir, override string) (string, error) {
|
func resolveOutputDir(workingDir, override string) (string, error) {
|
||||||
workingDir, err := validateWorkingDir(workingDir)
|
workingDir, err := validateWorkingDir(workingDir)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
@@ -6,6 +6,43 @@ import (
|
|||||||
"testing"
|
"testing"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
func TestResolveComparisonOutputDirectory(t *testing.T) {
|
||||||
|
workingDir := t.TempDir()
|
||||||
|
configured := filepath.Join(workingDir, "configured")
|
||||||
|
blocked := filepath.Join(workingDir, "not-a-directory")
|
||||||
|
if err := os.WriteFile(blocked, []byte("blocked"), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
override string
|
||||||
|
configuredDir string
|
||||||
|
reportOutputName string
|
||||||
|
want string
|
||||||
|
wantErr bool
|
||||||
|
}{
|
||||||
|
{name: "working directory default", reportOutputName: "today.md", want: filepath.Join(workingDir, "comparison-today")},
|
||||||
|
{name: "configured relative directory", configuredDir: "configured", reportOutputName: "tomorrow.md", want: filepath.Join(configured, "comparison-tomorrow")},
|
||||||
|
{name: "configured absolute directory", configuredDir: configured, reportOutputName: "hourly.md", want: filepath.Join(configured, "comparison-hourly")},
|
||||||
|
{name: "relative explicit directory", override: "exact", configuredDir: blocked, reportOutputName: "daily-2026-08-24.md", want: filepath.Join(workingDir, "exact")},
|
||||||
|
{name: "absolute explicit directory", override: filepath.Join(workingDir, "absolute"), reportOutputName: "today.md", want: filepath.Join(workingDir, "absolute")},
|
||||||
|
{name: "invalid report suffix", reportOutputName: "today.txt", wantErr: true},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, test := range tests {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
got, err := resolveComparisonOutputDirectory(workingDir, test.override, test.configuredDir, test.reportOutputName)
|
||||||
|
if (err != nil) != test.wantErr {
|
||||||
|
t.Fatalf("resolveComparisonOutputDirectory() error = %v, want error %t", err, test.wantErr)
|
||||||
|
}
|
||||||
|
if !test.wantErr && got != test.want {
|
||||||
|
t.Fatalf("resolveComparisonOutputDirectory() = %q, want %q", got, test.want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestResolveOutputDirRejectsDanglingSymlinkComponents(t *testing.T) {
|
func TestResolveOutputDirRejectsDanglingSymlinkComponents(t *testing.T) {
|
||||||
workingDir := t.TempDir()
|
workingDir := t.TempDir()
|
||||||
dangling := filepath.Join(workingDir, "dangling")
|
dangling := filepath.Join(workingDir, "dangling")
|
||||||
|
|||||||
150
internal/app/prepared_report.go
Normal file
150
internal/app/prepared_report.go
Normal file
@@ -0,0 +1,150 @@
|
|||||||
|
package app
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/briefing"
|
||||||
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/collect"
|
||||||
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/config"
|
||||||
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/generatedtext"
|
||||||
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/module"
|
||||||
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptinput"
|
||||||
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/report"
|
||||||
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/weatherdata"
|
||||||
|
)
|
||||||
|
|
||||||
|
// preparedReport contains the immutable deterministic inputs shared by prompt
|
||||||
|
// executions for one resolved report.
|
||||||
|
type preparedReport struct {
|
||||||
|
resolved report.Resolved
|
||||||
|
reportFacts ReportFacts
|
||||||
|
moduleSnapshot module.Snapshot
|
||||||
|
briefingMetadata briefing.Metadata
|
||||||
|
sourceWarnings []weatherdata.SourceWarning
|
||||||
|
dataPackage []byte
|
||||||
|
handler generatedtext.Handler
|
||||||
|
}
|
||||||
|
|
||||||
|
type prepareReportRequest struct {
|
||||||
|
Config config.Config
|
||||||
|
Resolved report.Resolved
|
||||||
|
Collection collect.Result
|
||||||
|
}
|
||||||
|
|
||||||
|
type preparationError struct {
|
||||||
|
operation string
|
||||||
|
err error
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *preparationError) Error() string {
|
||||||
|
return e.operation + ": " + e.err.Error()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *preparationError) Unwrap() error {
|
||||||
|
return e.err
|
||||||
|
}
|
||||||
|
|
||||||
|
func prepareReport(req prepareReportRequest) (preparedReport, error) {
|
||||||
|
if req.Collection.Bundle == nil {
|
||||||
|
return preparedReport{}, &preparationError{operation: "prepare report", err: fmt.Errorf("collected weather bundle is required")}
|
||||||
|
}
|
||||||
|
|
||||||
|
reportFacts, err := BuildReportFacts(ModuleSnapshotRequest{Config: req.Config, Resolved: req.Resolved}, req.Collection.Bundle)
|
||||||
|
if err != nil {
|
||||||
|
return preparedReport{}, &preparationError{operation: "build report facts", err: err}
|
||||||
|
}
|
||||||
|
moduleSnapshot, err := BuildModuleSnapshotFromFacts(ModuleSnapshotRequest{Config: req.Config, Resolved: req.Resolved}, reportFacts)
|
||||||
|
if err != nil {
|
||||||
|
return preparedReport{}, &preparationError{operation: "build module snapshot", err: err}
|
||||||
|
}
|
||||||
|
metadata := briefing.BuildMetadata(briefingBuildContext(req.Config, req.Resolved, reportFacts.Collected))
|
||||||
|
dataPackage, err := promptinput.Build(promptinput.BuildRequest{Metadata: promptMetadata(metadata), Modules: moduleSnapshot})
|
||||||
|
if err != nil {
|
||||||
|
return preparedReport{}, &preparationError{operation: "build data package", err: err}
|
||||||
|
}
|
||||||
|
serializedDataPackage, err := promptinput.MarshalYAML(dataPackage)
|
||||||
|
if err != nil {
|
||||||
|
return preparedReport{}, &preparationError{operation: "marshal data package", err: err}
|
||||||
|
}
|
||||||
|
handler, err := generatedtext.LookupDefinition(req.Resolved.Definition)
|
||||||
|
if err != nil {
|
||||||
|
return preparedReport{}, &preparationError{operation: "lookup generated text catalog", err: err}
|
||||||
|
}
|
||||||
|
|
||||||
|
clonedFacts, err := clonePreparedValue(reportFacts)
|
||||||
|
if err != nil {
|
||||||
|
return preparedReport{}, &preparationError{operation: "copy prepared report facts", err: err}
|
||||||
|
}
|
||||||
|
clonedSnapshot, err := clonePreparedValue(moduleSnapshot)
|
||||||
|
if err != nil {
|
||||||
|
return preparedReport{}, &preparationError{operation: "copy prepared module snapshot", err: err}
|
||||||
|
}
|
||||||
|
clonedMetadata, err := clonePreparedValue(metadata)
|
||||||
|
if err != nil {
|
||||||
|
return preparedReport{}, &preparationError{operation: "copy prepared briefing metadata", err: err}
|
||||||
|
}
|
||||||
|
prepared := preparedReport{
|
||||||
|
resolved: cloneResolved(req.Resolved),
|
||||||
|
reportFacts: clonedFacts,
|
||||||
|
moduleSnapshot: clonedSnapshot,
|
||||||
|
briefingMetadata: clonedMetadata,
|
||||||
|
sourceWarnings: append([]weatherdata.SourceWarning(nil), clonedMetadata.SourceWarnings...),
|
||||||
|
dataPackage: append([]byte(nil), serializedDataPackage...),
|
||||||
|
handler: handler,
|
||||||
|
}
|
||||||
|
return prepared, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func cloneResolved(value report.Resolved) report.Resolved {
|
||||||
|
cloned := value
|
||||||
|
cloned.Definition.DistributorPathTemplates = append([]string(nil), value.Definition.DistributorPathTemplates...)
|
||||||
|
cloned.Definition.Modules = make([]module.ConfigItem, len(value.Definition.Modules))
|
||||||
|
for i, item := range value.Definition.Modules {
|
||||||
|
cloned.Definition.Modules[i] = item
|
||||||
|
switch options := item.Options.(type) {
|
||||||
|
case module.AreaForecastDiscussionOptions:
|
||||||
|
options.Sections = append([]string(nil), options.Sections...)
|
||||||
|
cloned.Definition.Modules[i].Options = options
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return cloned
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p preparedReport) dataPackageCopy() []byte {
|
||||||
|
return append([]byte(nil), p.dataPackage...)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p preparedReport) sourceWarningsCopy() []weatherdata.SourceWarning {
|
||||||
|
return append([]weatherdata.SourceWarning(nil), p.sourceWarnings...)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p preparedReport) renderInputs() (briefing.Metadata, module.Snapshot, ReportFacts, error) {
|
||||||
|
metadata, err := clonePreparedValue(p.briefingMetadata)
|
||||||
|
if err != nil {
|
||||||
|
return briefing.Metadata{}, module.Snapshot{}, ReportFacts{}, err
|
||||||
|
}
|
||||||
|
snapshot, err := clonePreparedValue(p.moduleSnapshot)
|
||||||
|
if err != nil {
|
||||||
|
return briefing.Metadata{}, module.Snapshot{}, ReportFacts{}, err
|
||||||
|
}
|
||||||
|
reportFacts, err := clonePreparedValue(p.reportFacts)
|
||||||
|
if err != nil {
|
||||||
|
return briefing.Metadata{}, module.Snapshot{}, ReportFacts{}, err
|
||||||
|
}
|
||||||
|
return metadata, snapshot, reportFacts, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func clonePreparedValue[T any](value T) (T, error) {
|
||||||
|
encoded, err := json.Marshal(value)
|
||||||
|
if err != nil {
|
||||||
|
var zero T
|
||||||
|
return zero, fmt.Errorf("marshal immutable prepared value: %w", err)
|
||||||
|
}
|
||||||
|
var cloned T
|
||||||
|
if err := json.Unmarshal(encoded, &cloned); err != nil {
|
||||||
|
var zero T
|
||||||
|
return zero, fmt.Errorf("unmarshal immutable prepared value: %w", err)
|
||||||
|
}
|
||||||
|
return cloned, nil
|
||||||
|
}
|
||||||
64
internal/app/prepared_report_test.go
Normal file
64
internal/app/prepared_report_test.go
Normal file
@@ -0,0 +1,64 @@
|
|||||||
|
package app
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"reflect"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/collect"
|
||||||
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/weatherdata"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestPrepareReportBuildsImmutableDeterministicInputs(t *testing.T) {
|
||||||
|
cfg := generationConfig()
|
||||||
|
bundle := generationBundle(t)
|
||||||
|
resolved, err := ResolveGenerate(GenerateRequest{
|
||||||
|
Config: cfg, Report: ReportDaily,
|
||||||
|
Date: generationTime("2026-05-29T12:00:00-05:00"), Now: generationTime("2026-05-29T08:30:00-05:00"),
|
||||||
|
}, generationTime("2026-05-29T08:30:00-05:00"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ResolveGenerate() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
request := prepareReportRequest{Config: cfg, Resolved: resolved, Collection: collect.Result{Bundle: &bundle}}
|
||||||
|
prepared, err := prepareReport(request)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("prepareReport() error = %v", err)
|
||||||
|
}
|
||||||
|
repeated, err := prepareReport(request)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("second prepareReport() error = %v", err)
|
||||||
|
}
|
||||||
|
if len(prepared.dataPackage) == 0 || !bytes.Equal(prepared.dataPackage, repeated.dataPackage) || !reflect.DeepEqual(prepared.briefingMetadata, repeated.briefingMetadata) {
|
||||||
|
t.Fatalf("prepared package/metadata are not deterministic: %q/%#v", prepared.dataPackage, prepared.briefingMetadata)
|
||||||
|
}
|
||||||
|
|
||||||
|
originalDataPackage := append([]byte(nil), prepared.dataPackage...)
|
||||||
|
originalMetadata := prepared.briefingMetadata
|
||||||
|
originalWarnings := append([]weatherdata.SourceWarning(nil), prepared.sourceWarnings...)
|
||||||
|
metadata, snapshot, reportFacts, err := prepared.renderInputs()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("renderInputs() error = %v", err)
|
||||||
|
}
|
||||||
|
metadata.SourceWarnings = append(metadata.SourceWarnings, weatherdata.SourceWarning{Source: "test", Message: "consumer mutation"})
|
||||||
|
snapshot.Outputs = nil
|
||||||
|
reportFacts.Collected.Hourly.Periods[0].TextDescription = "consumer mutation"
|
||||||
|
bundle.Hourly.Periods[0].TextDescription = "mutated after preparation"
|
||||||
|
bundle.Warnings = append(bundle.Warnings, weatherdata.SourceWarning{Source: "test", Message: "mutated warning"})
|
||||||
|
if len(bundle.Sources) > 0 {
|
||||||
|
if bundle.Sources[0].Query == nil {
|
||||||
|
bundle.Sources[0].Query = map[string]string{}
|
||||||
|
}
|
||||||
|
bundle.Sources[0].Query["mutated"] = "true"
|
||||||
|
}
|
||||||
|
|
||||||
|
if !bytes.Equal(prepared.dataPackage, originalDataPackage) || !reflect.DeepEqual(prepared.briefingMetadata, originalMetadata) || !reflect.DeepEqual(prepared.sourceWarnings, originalWarnings) {
|
||||||
|
t.Fatalf("prepared values changed after caller mutation: %#v", prepared)
|
||||||
|
}
|
||||||
|
if prepared.reportFacts.Collected.Hourly.Periods[0].TextDescription == "mutated after preparation" {
|
||||||
|
t.Fatal("prepared report facts retain caller-owned weather data")
|
||||||
|
}
|
||||||
|
if prepared.reportFacts.Collected.Hourly.Periods[0].TextDescription == "consumer mutation" || len(prepared.moduleSnapshot.Outputs) == 0 {
|
||||||
|
t.Fatal("prepared report values retain consumer mutation")
|
||||||
|
}
|
||||||
|
}
|
||||||
127
internal/app/profile_execution.go
Normal file
127
internal/app/profile_execution.go
Normal file
@@ -0,0 +1,127 @@
|
|||||||
|
package app
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptdebug"
|
||||||
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec"
|
||||||
|
)
|
||||||
|
|
||||||
|
type profileExecutionRequest struct {
|
||||||
|
Prepared preparedReport
|
||||||
|
Prompt PromptInspectionResult
|
||||||
|
Profile promptexec.ProfileInspection
|
||||||
|
Executor promptexec.Executor
|
||||||
|
DebugWriter *promptdebug.PromptDebugWriter
|
||||||
|
DebugRef *promptdebug.PromptDebugRef
|
||||||
|
}
|
||||||
|
|
||||||
|
type profileExecutionOutcome struct {
|
||||||
|
ProfileID string
|
||||||
|
BackendID string
|
||||||
|
ModelName string
|
||||||
|
ValidationStatus promptexec.ValidationStatus
|
||||||
|
LLMDebugPath string
|
||||||
|
}
|
||||||
|
|
||||||
|
type profileExecutionError struct {
|
||||||
|
operation string
|
||||||
|
err error
|
||||||
|
callbackFailure bool
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *profileExecutionError) Error() string {
|
||||||
|
return e.operation + ": " + e.err.Error()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *profileExecutionError) Unwrap() error {
|
||||||
|
return e.err
|
||||||
|
}
|
||||||
|
|
||||||
|
func executePreparedProfile(ctx context.Context, req profileExecutionRequest) (profileExecutionOutcome, []byte, error) {
|
||||||
|
outcome := profileExecutionOutcome{
|
||||||
|
ProfileID: req.Profile.ProfileID,
|
||||||
|
BackendID: req.Profile.BackendID,
|
||||||
|
ModelName: req.Profile.ModelName,
|
||||||
|
}
|
||||||
|
if req.Executor == nil {
|
||||||
|
return outcome, nil, &profileExecutionError{operation: "execute prompt", err: promptexec.NewError(promptexec.InvalidConfiguration, "prompt executor is required", nil)}
|
||||||
|
}
|
||||||
|
|
||||||
|
callbackFailed := false
|
||||||
|
preparationCallback := func(preparation promptexec.Preparation, debug *promptexec.PreparationDebug) error {
|
||||||
|
outcome.ProfileID, outcome.BackendID, outcome.ModelName = preparation.ProfileID, preparation.BackendID, preparation.ModelName
|
||||||
|
if req.DebugWriter == nil || !req.DebugWriter.Enabled() {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if req.DebugRef == nil {
|
||||||
|
callbackFailed = true
|
||||||
|
return promptDebugWriteError(fmt.Errorf("prompt debug reference is required"))
|
||||||
|
}
|
||||||
|
path, err := req.DebugWriter.WritePreparation(*req.DebugRef, preparation, debug)
|
||||||
|
if err != nil {
|
||||||
|
callbackFailed = true
|
||||||
|
return promptDebugWriteError(err)
|
||||||
|
}
|
||||||
|
outcome.LLMDebugPath = path
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
captureDebug := req.DebugWriter != nil && req.DebugWriter.Enabled()
|
||||||
|
execution, err := req.Executor.Execute(ctx, promptexec.ExecuteRequest{
|
||||||
|
PromptID: req.Prompt.PromptID,
|
||||||
|
PromptVersion: req.Prompt.PromptVersion,
|
||||||
|
ProfileID: req.Profile.ProfileID,
|
||||||
|
DataPackage: req.Prepared.dataPackageCopy(),
|
||||||
|
CaptureDebug: captureDebug,
|
||||||
|
}, preparationCallback)
|
||||||
|
if err != nil {
|
||||||
|
if callbackFailed {
|
||||||
|
return outcome, nil, &profileExecutionError{operation: "execute prompt", err: err, callbackFailure: true}
|
||||||
|
}
|
||||||
|
return outcome, nil, &profileExecutionError{operation: "execute prompt", err: classifiedPromptError("prompt execution failed", err)}
|
||||||
|
}
|
||||||
|
if execution == nil {
|
||||||
|
return outcome, nil, &profileExecutionError{operation: "execute prompt", err: promptexec.NewError(promptexec.Generation, "prompt executor returned no execution", nil)}
|
||||||
|
}
|
||||||
|
|
||||||
|
outcome.ValidationStatus = execution.Validation.Status
|
||||||
|
if req.DebugWriter != nil && req.DebugWriter.Enabled() {
|
||||||
|
if req.DebugRef == nil {
|
||||||
|
return outcome, nil, &profileExecutionError{operation: "write prompt debug", err: promptDebugWriteError(fmt.Errorf("prompt debug reference is required"))}
|
||||||
|
}
|
||||||
|
path, err := req.DebugWriter.WriteExecution(*req.DebugRef, *execution)
|
||||||
|
if err != nil {
|
||||||
|
return outcome, nil, &profileExecutionError{operation: "write prompt debug", err: promptDebugWriteError(err)}
|
||||||
|
}
|
||||||
|
if path != "" {
|
||||||
|
outcome.LLMDebugPath = path
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if execution.Validation.Status != promptexec.ValidationPassed && execution.Validation.Status != promptexec.ValidationFailed {
|
||||||
|
return outcome, nil, &profileExecutionError{operation: "validate prompt execution", err: promptexec.NewError(promptexec.OperationalValidation, "prompt execution did not complete validation", nil)}
|
||||||
|
}
|
||||||
|
if execution.Validation.Status == promptexec.ValidationFailed {
|
||||||
|
return outcome, nil, &profileExecutionError{operation: "validate prompt execution", err: promptexec.NewError(promptexec.ValidationRejected, "prompt output did not satisfy its schema", nil)}
|
||||||
|
}
|
||||||
|
|
||||||
|
generatedText, _, err := req.Prepared.handler.Validate(execution.RawOutput)
|
||||||
|
if err != nil {
|
||||||
|
return outcome, nil, &profileExecutionError{operation: "validate generated text", err: err}
|
||||||
|
}
|
||||||
|
metadata, snapshot, reportFacts, err := req.Prepared.renderInputs()
|
||||||
|
if err != nil {
|
||||||
|
return outcome, nil, &profileExecutionError{operation: "copy prepared render inputs", err: err}
|
||||||
|
}
|
||||||
|
renderContext, err := req.Prepared.handler.BuildRenderContext(metadata, snapshot, reportFacts.Collected, reportFacts.Derived, generatedText)
|
||||||
|
if err != nil {
|
||||||
|
return outcome, nil, &profileExecutionError{operation: "build render context", err: err}
|
||||||
|
}
|
||||||
|
rendered, err := req.Prepared.handler.Render(renderContext)
|
||||||
|
if err != nil {
|
||||||
|
return outcome, nil, &profileExecutionError{operation: "render template", err: err}
|
||||||
|
}
|
||||||
|
return outcome, rendered, nil
|
||||||
|
}
|
||||||
70
internal/app/profile_execution_test.go
Normal file
70
internal/app/profile_execution_test.go
Normal file
@@ -0,0 +1,70 @@
|
|||||||
|
package app
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/collect"
|
||||||
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptdebug"
|
||||||
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestExecutePreparedProfileRendersWithoutPublishing(t *testing.T) {
|
||||||
|
prepared, inspection := preparedDailyProfile(t)
|
||||||
|
executor := &generationExecutor{}
|
||||||
|
outputPath := filepath.Join(t.TempDir(), "report.md")
|
||||||
|
outcome, rendered, err := executePreparedProfile(context.Background(), profileExecutionRequest{
|
||||||
|
Prepared: prepared, Prompt: inspection,
|
||||||
|
Profile: promptexec.ProfileInspection{ProfileID: inspection.ProfileID, BackendID: inspection.BackendID, ModelName: inspection.ModelName},
|
||||||
|
Executor: executor,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("executePreparedProfile() error = %v", err)
|
||||||
|
}
|
||||||
|
if len(rendered) == 0 || outcome.ValidationStatus != promptexec.ValidationPassed || outcome.ProfileID != inspection.ProfileID || executor.executeCalls != 1 {
|
||||||
|
t.Fatalf("outcome/rendered/execution calls = %#v/%q/%d", outcome, rendered, executor.executeCalls)
|
||||||
|
}
|
||||||
|
if _, statErr := os.Stat(outputPath); !os.IsNotExist(statErr) {
|
||||||
|
t.Fatalf("execution unexpectedly published %q: %v", outputPath, statErr)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExecutePreparedProfileKeepsDebugCallbackFailureLocal(t *testing.T) {
|
||||||
|
prepared, inspection := preparedDailyProfile(t)
|
||||||
|
debugWriter, err := promptdebug.NewPromptDebugWriter(t.TempDir())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewPromptDebugWriter() error = %v", err)
|
||||||
|
}
|
||||||
|
executor := &generationExecutor{}
|
||||||
|
outcome, rendered, err := executePreparedProfile(context.Background(), profileExecutionRequest{
|
||||||
|
Prepared: prepared, Prompt: inspection,
|
||||||
|
Profile: promptexec.ProfileInspection{ProfileID: inspection.ProfileID, BackendID: inspection.BackendID, ModelName: inspection.ModelName},
|
||||||
|
Executor: executor, DebugWriter: debugWriter,
|
||||||
|
DebugRef: &promptdebug.PromptDebugRef{ReportID: inspectionResolved(t).Definition.ID, ValidDate: "2026-05-29", RunID: "invalid/path"},
|
||||||
|
})
|
||||||
|
var executionErr *profileExecutionError
|
||||||
|
if err == nil || !errors.As(err, &executionErr) || !executionErr.callbackFailure || promptexec.CategoryOf(err) != promptexec.InvalidConfiguration || len(rendered) != 0 || executor.executeCalls != 0 || outcome.LLMDebugPath != "" {
|
||||||
|
t.Fatalf("outcome/rendered/error/execution calls = %#v/%q/%v/%d", outcome, rendered, err, executor.executeCalls)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func preparedDailyProfile(t *testing.T) (preparedReport, PromptInspectionResult) {
|
||||||
|
t.Helper()
|
||||||
|
cfg := generationConfig()
|
||||||
|
resolved, err := ResolveGenerate(GenerateRequest{
|
||||||
|
Config: cfg, Report: ReportDaily,
|
||||||
|
Date: generationTime("2026-05-29T12:00:00-05:00"), Now: generationTime("2026-05-29T08:30:00-05:00"),
|
||||||
|
}, generationTime("2026-05-29T08:30:00-05:00"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ResolveGenerate() error = %v", err)
|
||||||
|
}
|
||||||
|
bundle := generationBundle(t)
|
||||||
|
prepared, err := prepareReport(prepareReportRequest{Config: cfg, Resolved: resolved, Collection: collect.Result{Bundle: &bundle}})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("prepareReport() error = %v", err)
|
||||||
|
}
|
||||||
|
return prepared, PromptInspectionResult{PromptID: resolved.Definition.PromptID, PromptVersion: resolved.Definition.PromptVersion, PromptHash: "prompt-hash", ProfileID: "fixture", BackendID: "fixture", ModelName: "fixture-model"}
|
||||||
|
}
|
||||||
@@ -5,16 +5,11 @@ import (
|
|||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/briefing"
|
|
||||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/collect"
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/collect"
|
||||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/fileutil"
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/fileutil"
|
||||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/generatedtext"
|
|
||||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/module"
|
|
||||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptdebug"
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptdebug"
|
||||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec"
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec"
|
||||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptinput"
|
|
||||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/report"
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/report"
|
||||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/weatherdata"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
type promptReportRequest struct {
|
type promptReportRequest struct {
|
||||||
@@ -27,51 +22,7 @@ type promptReportRequest struct {
|
|||||||
noNotify bool
|
noNotify bool
|
||||||
}
|
}
|
||||||
|
|
||||||
type promptReportWorkflow struct {
|
|
||||||
ctx context.Context
|
|
||||||
req promptReportRequest
|
|
||||||
result *ReportResult
|
|
||||||
briefingMetadata briefing.Metadata
|
|
||||||
reportFacts ReportFacts
|
|
||||||
moduleSnapshot module.Snapshot
|
|
||||||
dataPackage []byte
|
|
||||||
handler generatedtext.Handler
|
|
||||||
debugRef promptdebug.PromptDebugRef
|
|
||||||
callbackFailed bool
|
|
||||||
}
|
|
||||||
|
|
||||||
func generatePromptReport(ctx context.Context, req promptReportRequest) (*ReportResult, error) {
|
func generatePromptReport(ctx context.Context, req promptReportRequest) (*ReportResult, error) {
|
||||||
workflow, err := newPromptReportWorkflow(ctx, req)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
if err := workflow.buildInputs(); err != nil {
|
|
||||||
return workflow.result, err
|
|
||||||
}
|
|
||||||
execution, err := workflow.executePrompt()
|
|
||||||
if err != nil {
|
|
||||||
if workflow.callbackFailed {
|
|
||||||
return workflow.result, err
|
|
||||||
}
|
|
||||||
return workflow.result, workflow.reportError("execute prompt", classifiedPromptError("prompt execution failed", err))
|
|
||||||
}
|
|
||||||
if execution == nil {
|
|
||||||
return workflow.result, workflow.reportError("execute prompt", promptexec.NewError(promptexec.Generation, "prompt executor returned no execution", nil))
|
|
||||||
}
|
|
||||||
workflow.result.ValidationStatus = execution.Validation.Status
|
|
||||||
if err := workflow.writeExecutionDebug(*execution); err != nil {
|
|
||||||
return workflow.result, err
|
|
||||||
}
|
|
||||||
if execution.Validation.Status != promptexec.ValidationPassed && execution.Validation.Status != promptexec.ValidationFailed {
|
|
||||||
return workflow.result, workflow.reportError("validate prompt execution", promptexec.NewError(promptexec.OperationalValidation, "prompt execution did not complete validation", nil))
|
|
||||||
}
|
|
||||||
if execution.Validation.Status == promptexec.ValidationFailed {
|
|
||||||
return workflow.result, workflow.reportError("validate prompt execution", promptexec.NewError(promptexec.ValidationRejected, "prompt output did not satisfy its schema", nil))
|
|
||||||
}
|
|
||||||
return workflow.renderAndPublish(execution.RawOutput)
|
|
||||||
}
|
|
||||||
|
|
||||||
func newPromptReportWorkflow(ctx context.Context, req promptReportRequest) (*promptReportWorkflow, error) {
|
|
||||||
if req.Collection.Bundle == nil {
|
if req.Collection.Bundle == nil {
|
||||||
return nil, fmt.Errorf("collected weather bundle is required")
|
return nil, fmt.Errorf("collected weather bundle is required")
|
||||||
}
|
}
|
||||||
@@ -79,10 +30,36 @@ func newPromptReportWorkflow(ctx context.Context, req promptReportRequest) (*pro
|
|||||||
if result == nil {
|
if result == nil {
|
||||||
result = initialReportResult(req.GenerateRequest, req.Resolved, req.Inspection)
|
result = initialReportResult(req.GenerateRequest, req.Resolved, req.Inspection)
|
||||||
}
|
}
|
||||||
return &promptReportWorkflow{
|
prepared, err := prepareReport(prepareReportRequest{Config: req.Config, Resolved: req.Resolved, Collection: req.Collection})
|
||||||
ctx: ctx, req: req,
|
if err != nil {
|
||||||
result: result,
|
return result, generatedPreparationError(req.Resolved, result.RunID, err)
|
||||||
}, nil
|
}
|
||||||
|
result.SourceWarnings = prepared.sourceWarningsCopy()
|
||||||
|
debugRef := promptdebug.PromptDebugRef{ReportID: result.ReportID, ValidDate: prepared.resolved.ValidPeriod.Start.Format("2006-01-02"), RunID: result.RunID}
|
||||||
|
outcome, rendered, err := executePreparedProfile(ctx, profileExecutionRequest{
|
||||||
|
Prepared: prepared,
|
||||||
|
Prompt: req.Inspection,
|
||||||
|
Profile: promptexec.ProfileInspection{
|
||||||
|
ProfileID: req.Inspection.ProfileID,
|
||||||
|
BackendID: req.Inspection.BackendID,
|
||||||
|
ModelName: req.Inspection.ModelName,
|
||||||
|
},
|
||||||
|
Executor: req.Executor, DebugWriter: req.DebugWriter, DebugRef: &debugRef,
|
||||||
|
})
|
||||||
|
result.ProfileID, result.BackendID, result.ModelName = outcome.ProfileID, outcome.BackendID, outcome.ModelName
|
||||||
|
result.ValidationStatus = outcome.ValidationStatus
|
||||||
|
result.LLMDebugPath = outcome.LLMDebugPath
|
||||||
|
if err != nil {
|
||||||
|
return result, generatedProfileExecutionError(req.Resolved, result.RunID, err)
|
||||||
|
}
|
||||||
|
return publishPromptReport(ctx, promptPublicationRequest{
|
||||||
|
GenerateRequest: req.GenerateRequest,
|
||||||
|
Resolved: req.Resolved,
|
||||||
|
OutputPath: req.OutputPath,
|
||||||
|
Result: result,
|
||||||
|
Markdown: rendered,
|
||||||
|
suppressNotification: req.noNotify,
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
func initialReportResult(req GenerateRequest, resolved report.Resolved, inspection PromptInspectionResult) *ReportResult {
|
func initialReportResult(req GenerateRequest, resolved report.Resolved, inspection PromptInspectionResult) *ReportResult {
|
||||||
@@ -96,100 +73,51 @@ func initialReportResult(req GenerateRequest, resolved report.Resolved, inspecti
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (w *promptReportWorkflow) buildInputs() error {
|
type promptPublicationRequest struct {
|
||||||
var err error
|
GenerateRequest
|
||||||
w.reportFacts, err = BuildReportFacts(ModuleSnapshotRequest{Config: w.req.Config, Resolved: w.req.Resolved}, w.req.Collection.Bundle)
|
Resolved report.Resolved
|
||||||
if err != nil {
|
OutputPath string
|
||||||
return w.reportError("build report facts", err)
|
Result *ReportResult
|
||||||
}
|
Markdown []byte
|
||||||
w.moduleSnapshot, err = BuildModuleSnapshotFromFacts(ModuleSnapshotRequest{Config: w.req.Config, Resolved: w.req.Resolved}, w.reportFacts)
|
suppressNotification bool
|
||||||
if err != nil {
|
|
||||||
return w.reportError("build module snapshot", err)
|
|
||||||
}
|
|
||||||
w.briefingMetadata = briefing.BuildMetadata(briefingBuildContext(w.req.Config, w.req.Resolved, w.reportFacts.Collected))
|
|
||||||
w.result.SourceWarnings = append([]weatherdata.SourceWarning(nil), w.briefingMetadata.SourceWarnings...)
|
|
||||||
dataPackage, err := promptinput.Build(promptinput.BuildRequest{Metadata: promptMetadata(w.briefingMetadata), Modules: w.moduleSnapshot})
|
|
||||||
if err != nil {
|
|
||||||
return w.reportError("build data package", err)
|
|
||||||
}
|
|
||||||
w.dataPackage, err = promptinput.MarshalYAML(dataPackage)
|
|
||||||
if err != nil {
|
|
||||||
return w.reportError("marshal data package", err)
|
|
||||||
}
|
|
||||||
w.handler, err = generatedtext.LookupDefinition(w.req.Resolved.Definition)
|
|
||||||
if err != nil {
|
|
||||||
return w.reportError("lookup generated text catalog", err)
|
|
||||||
}
|
|
||||||
w.debugRef = promptdebug.PromptDebugRef{ReportID: w.result.ReportID, ValidDate: w.req.Resolved.ValidPeriod.Start.Format("2006-01-02"), RunID: w.result.RunID}
|
|
||||||
return nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (w *promptReportWorkflow) executePrompt() (*promptexec.Execution, error) {
|
func publishPromptReport(ctx context.Context, req promptPublicationRequest) (*ReportResult, error) {
|
||||||
captureDebug := w.req.DebugWriter != nil && w.req.DebugWriter.Enabled()
|
if err := publicationContextError(ctx); err != nil {
|
||||||
return w.req.Executor.Execute(w.ctx, promptexec.ExecuteRequest{PromptID: w.req.Inspection.PromptID, PromptVersion: w.req.Inspection.PromptVersion, ProfileID: w.req.Inspection.ProfileID, DataPackage: w.dataPackage, CaptureDebug: captureDebug}, w.writePreparationDebug)
|
return req.Result, generatedReportError(req.Resolved, req.Result.RunID, "publish report", err)
|
||||||
|
}
|
||||||
|
if err := fileutil.WriteFileAtomic(req.OutputPath, req.Markdown); err != nil {
|
||||||
|
return req.Result, err
|
||||||
|
}
|
||||||
|
req.Result.OutputPath = req.OutputPath
|
||||||
|
if req.suppressNotification {
|
||||||
|
return req.Result, nil
|
||||||
|
}
|
||||||
|
notification, err := notifyReport(ctx, req.Config, req.Resolved, req.Result.OutputPath, req.Result.RunID, req.Result.GeneratedAt, req.Notifier)
|
||||||
|
req.Result.Notification = notification
|
||||||
|
if err != nil {
|
||||||
|
return req.Result, err
|
||||||
|
}
|
||||||
|
return req.Result, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (w *promptReportWorkflow) writePreparationDebug(preparation promptexec.Preparation, debug *promptexec.PreparationDebug) error {
|
func generatedPreparationError(resolved report.Resolved, runID string, err error) error {
|
||||||
w.result.ProfileID, w.result.BackendID, w.result.ModelName = preparation.ProfileID, preparation.BackendID, preparation.ModelName
|
var preparation *preparationError
|
||||||
if w.req.DebugWriter == nil {
|
if errors.As(err, &preparation) {
|
||||||
return nil
|
return generatedReportError(resolved, runID, preparation.operation, preparation.err)
|
||||||
}
|
}
|
||||||
path, err := w.req.DebugWriter.WritePreparation(w.debugRef, preparation, debug)
|
return generatedReportError(resolved, runID, "prepare report", err)
|
||||||
if err != nil {
|
|
||||||
w.callbackFailed = true
|
|
||||||
return promptDebugWriteError(err)
|
|
||||||
}
|
|
||||||
w.result.LLMDebugPath = path
|
|
||||||
return nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (w *promptReportWorkflow) writeExecutionDebug(execution promptexec.Execution) error {
|
func generatedProfileExecutionError(resolved report.Resolved, runID string, err error) error {
|
||||||
if w.req.DebugWriter == nil {
|
var execution *profileExecutionError
|
||||||
return nil
|
if errors.As(err, &execution) {
|
||||||
|
if execution.callbackFailure {
|
||||||
|
return execution.err
|
||||||
|
}
|
||||||
|
return generatedReportError(resolved, runID, execution.operation, execution.err)
|
||||||
}
|
}
|
||||||
path, err := w.req.DebugWriter.WriteExecution(w.debugRef, execution)
|
return generatedReportError(resolved, runID, "execute prompt", err)
|
||||||
if err != nil {
|
|
||||||
return w.reportError("write prompt debug", promptDebugWriteError(err))
|
|
||||||
}
|
|
||||||
if path != "" {
|
|
||||||
w.result.LLMDebugPath = path
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (w *promptReportWorkflow) renderAndPublish(raw []byte) (*ReportResult, error) {
|
|
||||||
generatedText, _, err := w.handler.Validate(raw)
|
|
||||||
if err != nil {
|
|
||||||
return w.result, w.reportError("validate generated text", err)
|
|
||||||
}
|
|
||||||
renderContext, err := w.handler.BuildRenderContext(w.briefingMetadata, w.moduleSnapshot, w.reportFacts.Collected, w.reportFacts.Derived, generatedText)
|
|
||||||
if err != nil {
|
|
||||||
return w.result, w.reportError("build render context", err)
|
|
||||||
}
|
|
||||||
rendered, err := w.handler.Render(renderContext)
|
|
||||||
if err != nil {
|
|
||||||
return w.result, w.reportError("render template", err)
|
|
||||||
}
|
|
||||||
if err := publicationContextError(w.ctx); err != nil {
|
|
||||||
return w.result, w.reportError("publish report", err)
|
|
||||||
}
|
|
||||||
if err := fileutil.WriteFileAtomic(w.req.OutputPath, rendered); err != nil {
|
|
||||||
return w.result, err
|
|
||||||
}
|
|
||||||
w.result.OutputPath = w.req.OutputPath
|
|
||||||
if w.req.noNotify {
|
|
||||||
return w.result, nil
|
|
||||||
}
|
|
||||||
notification, err := notifyReport(w.ctx, w.req.Config, w.req.Resolved, w.result.OutputPath, w.result.RunID, w.result.GeneratedAt, w.req.Notifier)
|
|
||||||
w.result.Notification = notification
|
|
||||||
if err != nil {
|
|
||||||
return w.result, err
|
|
||||||
}
|
|
||||||
return w.result, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (w *promptReportWorkflow) reportError(operation string, err error) error {
|
|
||||||
return generatedReportError(w.req.Resolved, w.result.RunID, operation, err)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func classifiedPromptError(operation string, err error) error {
|
func classifiedPromptError(operation string, err error) error {
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ import (
|
|||||||
"os"
|
"os"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/comparison"
|
||||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/config"
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/config"
|
||||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec"
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec"
|
||||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/report"
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/report"
|
||||||
@@ -39,6 +40,32 @@ type PromptExecutionsInspectionRequest struct {
|
|||||||
LookupEnv func(string) (string, bool)
|
LookupEnv func(string) (string, bool)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ComparisonInspectionRequest contains the explicit profile selection for one
|
||||||
|
// resolved prompt comparison. It intentionally has no configured profile field.
|
||||||
|
type ComparisonInspectionRequest struct {
|
||||||
|
Resolved report.Resolved
|
||||||
|
ProfileIDs []string
|
||||||
|
Executor promptexec.Executor
|
||||||
|
LookupEnv func(string) (string, bool)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ComparisonInspectionResult contains the safe, shared prompt identity and
|
||||||
|
// ordered effective profile identities for a comparison.
|
||||||
|
type ComparisonInspectionResult struct {
|
||||||
|
PromptID string
|
||||||
|
PromptVersion string
|
||||||
|
PromptHash string
|
||||||
|
Profiles []ComparisonProfileInspection
|
||||||
|
}
|
||||||
|
|
||||||
|
// ComparisonProfileInspection contains one requested profile's safe effective
|
||||||
|
// execution identity.
|
||||||
|
type ComparisonProfileInspection struct {
|
||||||
|
ProfileID string
|
||||||
|
BackendID string
|
||||||
|
ModelName string
|
||||||
|
}
|
||||||
|
|
||||||
// InspectPromptExecution validates the exact prompt and profile needed for a
|
// InspectPromptExecution validates the exact prompt and profile needed for a
|
||||||
// report before collection, execution, or durable writes begin.
|
// report before collection, execution, or durable writes begin.
|
||||||
func InspectPromptExecution(ctx context.Context, req PromptInspectionRequest) (PromptInspectionResult, error) {
|
func InspectPromptExecution(ctx context.Context, req PromptInspectionRequest) (PromptInspectionResult, error) {
|
||||||
@@ -64,21 +91,9 @@ func InspectPromptExecutions(ctx context.Context, req PromptExecutionsInspection
|
|||||||
profiles := map[string]promptexec.ProfileInspection{}
|
profiles := map[string]promptexec.ProfileInspection{}
|
||||||
for _, resolved := range req.Resolved {
|
for _, resolved := range req.Resolved {
|
||||||
definition := resolved.Definition
|
definition := resolved.Definition
|
||||||
if strings.TrimSpace(definition.PromptID) == "" || strings.TrimSpace(definition.PromptVersion) == "" {
|
inspection, err := inspectPromptContract(ctx, req.Executor, definition)
|
||||||
return nil, promptexec.NewError(promptexec.InvalidConfiguration, "report prompt identity is incomplete", nil)
|
|
||||||
}
|
|
||||||
inspection, err := req.Executor.InspectPrompt(ctx, definition.PromptID, definition.PromptVersion)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, promptInspectionError("prompt inspection failed", err)
|
return nil, err
|
||||||
}
|
|
||||||
if inspection.PromptID != definition.PromptID || inspection.PromptVersion != definition.PromptVersion {
|
|
||||||
return nil, promptexec.NewError(promptexec.InvalidConfiguration, "prompt inspection did not return the requested prompt version", nil)
|
|
||||||
}
|
|
||||||
if !validPromptInput(inspection.Inputs) {
|
|
||||||
return nil, promptexec.NewError(promptexec.InvalidConfiguration, "prompt must declare exactly one required application/yaml data_package input", nil)
|
|
||||||
}
|
|
||||||
if !validPromptOutput(definition, inspection.Output) {
|
|
||||||
return nil, promptexec.NewError(promptexec.InvalidConfiguration, "prompt must declare the report JSON Schema output contract", nil)
|
|
||||||
}
|
}
|
||||||
profileID := req.Promptkit.Profile
|
profileID := req.Promptkit.Profile
|
||||||
if profileID == "" {
|
if profileID == "" {
|
||||||
@@ -103,6 +118,62 @@ func InspectPromptExecutions(ctx context.Context, req PromptExecutionsInspection
|
|||||||
return results, nil
|
return results, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// InspectComparisonExecution validates one exact prompt and every explicitly
|
||||||
|
// requested profile before collection or model execution. Profiles are
|
||||||
|
// inspected sequentially in request order. If a profile fails, the returned
|
||||||
|
// partial result retains the prompt identity and successfully inspected prefix.
|
||||||
|
func InspectComparisonExecution(ctx context.Context, req ComparisonInspectionRequest) (ComparisonInspectionResult, error) {
|
||||||
|
if err := comparison.ValidateProfileIDs(req.ProfileIDs); err != nil {
|
||||||
|
return ComparisonInspectionResult{}, promptexec.NewError(promptexec.InvalidRequest, "comparison profile selection is invalid", err)
|
||||||
|
}
|
||||||
|
if req.Executor == nil {
|
||||||
|
return ComparisonInspectionResult{}, promptexec.NewError(promptexec.InvalidConfiguration, "prompt executor is required", nil)
|
||||||
|
}
|
||||||
|
|
||||||
|
inspection, err := inspectPromptContract(ctx, req.Executor, req.Resolved.Definition)
|
||||||
|
if err != nil {
|
||||||
|
return ComparisonInspectionResult{}, comparisonInspectionError("comparison prompt inspection failed", err)
|
||||||
|
}
|
||||||
|
result := ComparisonInspectionResult{
|
||||||
|
PromptID: inspection.PromptID,
|
||||||
|
PromptVersion: inspection.PromptVersion,
|
||||||
|
PromptHash: inspection.PromptHash,
|
||||||
|
Profiles: make([]ComparisonProfileInspection, 0, len(req.ProfileIDs)),
|
||||||
|
}
|
||||||
|
for _, profileID := range req.ProfileIDs {
|
||||||
|
profile, err := inspectPromptProfile(ctx, req.Executor, profileID, req.LookupEnv)
|
||||||
|
if err != nil {
|
||||||
|
return result, comparisonInspectionError("comparison profile inspection failed", err)
|
||||||
|
}
|
||||||
|
result.Profiles = append(result.Profiles, ComparisonProfileInspection{
|
||||||
|
ProfileID: profile.ProfileID,
|
||||||
|
BackendID: profile.BackendID,
|
||||||
|
ModelName: profile.ModelName,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
return result, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func inspectPromptContract(ctx context.Context, executor promptexec.Executor, definition report.Definition) (promptexec.PromptInspection, error) {
|
||||||
|
if strings.TrimSpace(definition.PromptID) == "" || strings.TrimSpace(definition.PromptVersion) == "" {
|
||||||
|
return promptexec.PromptInspection{}, promptexec.NewError(promptexec.InvalidConfiguration, "report prompt identity is incomplete", nil)
|
||||||
|
}
|
||||||
|
inspection, err := executor.InspectPrompt(ctx, definition.PromptID, definition.PromptVersion)
|
||||||
|
if err != nil {
|
||||||
|
return promptexec.PromptInspection{}, promptInspectionError("prompt inspection failed", err)
|
||||||
|
}
|
||||||
|
if inspection.PromptID != definition.PromptID || inspection.PromptVersion != definition.PromptVersion {
|
||||||
|
return promptexec.PromptInspection{}, promptexec.NewError(promptexec.InvalidConfiguration, "prompt inspection did not return the requested prompt version", nil)
|
||||||
|
}
|
||||||
|
if !validPromptInput(inspection.Inputs) {
|
||||||
|
return promptexec.PromptInspection{}, promptexec.NewError(promptexec.InvalidConfiguration, "prompt must declare exactly one required application/yaml data_package input", nil)
|
||||||
|
}
|
||||||
|
if !validPromptOutput(definition, inspection.Output) {
|
||||||
|
return promptexec.PromptInspection{}, promptexec.NewError(promptexec.InvalidConfiguration, "prompt must declare the report JSON Schema output contract", nil)
|
||||||
|
}
|
||||||
|
return inspection, nil
|
||||||
|
}
|
||||||
|
|
||||||
func inspectPromptProfile(ctx context.Context, executor promptexec.Executor, profileID string, lookupEnv func(string) (string, bool)) (promptexec.ProfileInspection, error) {
|
func inspectPromptProfile(ctx context.Context, executor promptexec.Executor, profileID string, lookupEnv func(string) (string, bool)) (promptexec.ProfileInspection, error) {
|
||||||
profile, err := executor.InspectProfile(ctx, profileID)
|
profile, err := executor.InspectProfile(ctx, profileID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -140,3 +211,11 @@ func promptInspectionError(operation string, err error) error {
|
|||||||
}
|
}
|
||||||
return promptexec.NewError(promptexec.InvalidConfiguration, operation, err)
|
return promptexec.NewError(promptexec.InvalidConfiguration, operation, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func comparisonInspectionError(operation string, err error) error {
|
||||||
|
category := promptexec.CategoryOf(err)
|
||||||
|
if category == "" {
|
||||||
|
category = promptexec.InvalidConfiguration
|
||||||
|
}
|
||||||
|
return promptexec.NewError(category, operation, err)
|
||||||
|
}
|
||||||
|
|||||||
@@ -3,6 +3,7 @@ package app
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"errors"
|
"errors"
|
||||||
|
"reflect"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
@@ -131,6 +132,98 @@ func TestInspectPromptExecutionsReusesEffectiveProfile(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestInspectComparisonExecutionPreservesOrderedExplicitProfiles(t *testing.T) {
|
||||||
|
resolved := inspectionResolved(t)
|
||||||
|
executor := &inspectionExecutor{
|
||||||
|
prompt: validPromptInspection(resolved.Definition),
|
||||||
|
profiles: map[string]promptexec.ProfileInspection{
|
||||||
|
"weather-light": {ProfileID: "weather-light", BackendID: "local", ModelName: "light-model"},
|
||||||
|
"weather-deep": {ProfileID: "weather-deep", BackendID: "cloud", ModelName: "deep-model"},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
profileIDs := []string{"weather-light", "weather-deep"}
|
||||||
|
result, err := InspectComparisonExecution(context.Background(), ComparisonInspectionRequest{
|
||||||
|
Resolved: resolved, ProfileIDs: profileIDs, Executor: executor,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("InspectComparisonExecution() error = %v", err)
|
||||||
|
}
|
||||||
|
if result.PromptID != resolved.Definition.PromptID || result.PromptVersion != resolved.Definition.PromptVersion || result.PromptHash != "prompt-hash" {
|
||||||
|
t.Fatalf("prompt result = %#v", result)
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(executor.profileRequests, profileIDs) || len(executor.promptRequests) != 1 || executor.executeRequests != 0 {
|
||||||
|
t.Fatalf("prompt/profile/execute requests = %#v/%#v/%d", executor.promptRequests, executor.profileRequests, executor.executeRequests)
|
||||||
|
}
|
||||||
|
wantProfiles := []ComparisonProfileInspection{
|
||||||
|
{ProfileID: "weather-light", BackendID: "local", ModelName: "light-model"},
|
||||||
|
{ProfileID: "weather-deep", BackendID: "cloud", ModelName: "deep-model"},
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(result.Profiles, wantProfiles) {
|
||||||
|
t.Fatalf("profiles = %#v, want %#v", result.Profiles, wantProfiles)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestInspectComparisonExecutionRejectsInvalidProfilesBeforeInspection(t *testing.T) {
|
||||||
|
resolved := inspectionResolved(t)
|
||||||
|
for _, profileIDs := range [][]string{
|
||||||
|
{"weather-light"},
|
||||||
|
{"weather-light", " \t"},
|
||||||
|
{"weather-light", "weather-light"},
|
||||||
|
} {
|
||||||
|
t.Run(strings.Join(profileIDs, ","), func(t *testing.T) {
|
||||||
|
executor := &inspectionExecutor{prompt: validPromptInspection(resolved.Definition)}
|
||||||
|
_, err := InspectComparisonExecution(context.Background(), ComparisonInspectionRequest{
|
||||||
|
Resolved: resolved, ProfileIDs: profileIDs, Executor: executor,
|
||||||
|
})
|
||||||
|
if err == nil || promptexec.CategoryOf(err) != promptexec.InvalidRequest {
|
||||||
|
t.Fatalf("error/category = %v/%q, want invalid request", err, promptexec.CategoryOf(err))
|
||||||
|
}
|
||||||
|
if len(executor.promptRequests) != 0 || len(executor.profileRequests) != 0 || executor.executeRequests != 0 {
|
||||||
|
t.Fatalf("invalid profile selection performed prompt/profile/execution work: %#v/%#v/%d", executor.promptRequests, executor.profileRequests, executor.executeRequests)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestInspectComparisonExecutionStopsAtFirstProfileFailure(t *testing.T) {
|
||||||
|
resolved := inspectionResolved(t)
|
||||||
|
executor := &inspectionExecutor{
|
||||||
|
prompt: validPromptInspection(resolved.Definition),
|
||||||
|
profiles: map[string]promptexec.ProfileInspection{
|
||||||
|
"weather-light": {ProfileID: "weather-light", BackendID: "local", ModelName: "light-model"},
|
||||||
|
"missing-key": {ProfileID: "missing-key", APIKeyEnv: "PROMPT_API_KEY"},
|
||||||
|
"weather-deep": {ProfileID: "weather-deep", BackendID: "cloud", ModelName: "deep-model"},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
result, err := InspectComparisonExecution(context.Background(), ComparisonInspectionRequest{
|
||||||
|
Resolved: resolved, ProfileIDs: []string{"weather-light", "missing-key", "weather-deep"}, Executor: executor,
|
||||||
|
LookupEnv: func(string) (string, bool) { return "", false },
|
||||||
|
})
|
||||||
|
if err == nil || promptexec.CategoryOf(err) != promptexec.MissingCredential {
|
||||||
|
t.Fatalf("error/category = %v/%q, want missing credential", err, promptexec.CategoryOf(err))
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(executor.profileRequests, []string{"weather-light", "missing-key"}) || len(executor.promptRequests) != 1 || executor.executeRequests != 0 {
|
||||||
|
t.Fatalf("prompt/profile/execute requests = %#v/%#v/%d", executor.promptRequests, executor.profileRequests, executor.executeRequests)
|
||||||
|
}
|
||||||
|
if result.PromptID != resolved.Definition.PromptID || result.PromptVersion != resolved.Definition.PromptVersion || result.PromptHash == "" || len(result.Profiles) != 1 || result.Profiles[0].ProfileID != "weather-light" {
|
||||||
|
t.Fatalf("partial inspection result = %#v", result)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestInspectComparisonExecutionStopsBeforeProfileInspectionWhenPromptFails(t *testing.T) {
|
||||||
|
resolved := inspectionResolved(t)
|
||||||
|
executor := &inspectionExecutor{promptErr: promptexec.NewError(promptexec.PromptNotFound, "prompt is unavailable", nil)}
|
||||||
|
_, err := InspectComparisonExecution(context.Background(), ComparisonInspectionRequest{
|
||||||
|
Resolved: resolved, ProfileIDs: []string{"weather-light", "weather-deep"}, Executor: executor,
|
||||||
|
})
|
||||||
|
if err == nil || promptexec.CategoryOf(err) != promptexec.PromptNotFound || !strings.Contains(err.Error(), "comparison prompt") {
|
||||||
|
t.Fatalf("error/category = %v/%q, want prompt-context prompt not found", err, promptexec.CategoryOf(err))
|
||||||
|
}
|
||||||
|
if len(executor.promptRequests) != 1 || len(executor.profileRequests) != 0 || executor.executeRequests != 0 {
|
||||||
|
t.Fatalf("prompt/profile/execute requests = %#v/%#v/%d", executor.promptRequests, executor.profileRequests, executor.executeRequests)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
type inspectionPromptRequest struct {
|
type inspectionPromptRequest struct {
|
||||||
id string
|
id string
|
||||||
version string
|
version string
|
||||||
@@ -143,6 +236,7 @@ type inspectionExecutor struct {
|
|||||||
promptErr error
|
promptErr error
|
||||||
promptRequests []inspectionPromptRequest
|
promptRequests []inspectionPromptRequest
|
||||||
profileRequests []string
|
profileRequests []string
|
||||||
|
executeRequests int
|
||||||
}
|
}
|
||||||
|
|
||||||
func (e *inspectionExecutor) InspectPrompt(_ context.Context, id string, version string) (promptexec.PromptInspection, error) {
|
func (e *inspectionExecutor) InspectPrompt(_ context.Context, id string, version string) (promptexec.PromptInspection, error) {
|
||||||
@@ -166,6 +260,7 @@ func (e *inspectionExecutor) InspectProfile(_ context.Context, id string) (promp
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (e *inspectionExecutor) Execute(context.Context, promptexec.ExecuteRequest, promptexec.PreparationCallback) (*promptexec.Execution, error) {
|
func (e *inspectionExecutor) Execute(context.Context, promptexec.ExecuteRequest, promptexec.PreparationCallback) (*promptexec.Execution, error) {
|
||||||
|
e.executeRequests++
|
||||||
return nil, errors.New("unexpected execution")
|
return nil, errors.New("unexpected execution")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
368
internal/cli/comparison_test.go
Normal file
368
internal/cli/comparison_test.go
Normal file
@@ -0,0 +1,368 @@
|
|||||||
|
package cli
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"reflect"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/app"
|
||||||
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/comparison"
|
||||||
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec"
|
||||||
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/timeutil"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestParseComparisonFlagsPreservesProfileOrder(t *testing.T) {
|
||||||
|
opts, err := parseComparisonFlags(app.ReportDaily, []string{
|
||||||
|
"--profile", "weather-light", "--profile=weather-balanced", "--profile", "weather-deep",
|
||||||
|
"--date", "2026-05-29", "--out-dir", "reports", "--replace", "--quiet",
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("parseComparisonFlags() error = %v", err)
|
||||||
|
}
|
||||||
|
wantProfiles := []string{"weather-light", "weather-balanced", "weather-deep"}
|
||||||
|
if !reflect.DeepEqual([]string(opts.ProfileIDs), wantProfiles) || opts.Date != "2026-05-29" || opts.OutputDir != "reports" || !opts.Replace || !opts.Quiet {
|
||||||
|
t.Fatalf("options = %#v", opts)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestResolveComparisonActionBuildsExplicitRequest(t *testing.T) {
|
||||||
|
workingDir := t.TempDir()
|
||||||
|
configPath := comparisonConfigPath(t, "weather_api:\n base_url: https://weather.api.example.com/\n units: metric\n timezone: America/Chicago\noutput:\n directory: configured-reports\npromptkit:\n profile: configured-default\n")
|
||||||
|
clock := timeutil.FixedClock{Time: time.Date(2026, 5, 29, 8, 30, 0, 0, time.UTC)}
|
||||||
|
var factoryConfig PromptExecutorConfig
|
||||||
|
factoryCalls := 0
|
||||||
|
executor := &factoryExecutor{}
|
||||||
|
runner := Runner{
|
||||||
|
Clock: clock, WorkingDir: workingDir,
|
||||||
|
ExecutorFactory: func(value PromptExecutorConfig) (promptexec.Executor, error) {
|
||||||
|
factoryCalls++
|
||||||
|
factoryConfig = value
|
||||||
|
return executor, nil
|
||||||
|
},
|
||||||
|
}
|
||||||
|
req, opts, err := runner.resolveComparisonAction([]string{
|
||||||
|
"daily", "--date", "2026-05-29", "--profile", "weather-light", "--profile", "weather-deep",
|
||||||
|
"--out-dir", "bundles/../comparison", "--replace", "--llm-debug-dir", "/tmp/debug", "--units", "imperial", "--tz", "America/New_York", "--config", configPath,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("resolveComparisonAction() error = %v", err)
|
||||||
|
}
|
||||||
|
if factoryCalls != 1 || req.Executor != executor || req.Clock != clock || req.WorkingDir != workingDir || req.OutputDir != filepath.Join(workingDir, "comparison") || !req.Replace || req.LLMDebugDir != "/tmp/debug" {
|
||||||
|
t.Fatalf("request/factory calls = %#v/%#v/%d", req, factoryConfig, factoryCalls)
|
||||||
|
}
|
||||||
|
if got, want := req.ProfileIDs, []string{"weather-light", "weather-deep"}; !reflect.DeepEqual(got, want) {
|
||||||
|
t.Fatalf("profile IDs = %#v, want %#v", got, want)
|
||||||
|
}
|
||||||
|
if req.Config.WeatherAPI.Units != "imperial" || req.Config.WeatherAPI.Timezone != "America/New_York" || req.Config.Output.Directory != "configured-reports" || factoryConfig.Profile != "" || opts.Quiet {
|
||||||
|
t.Fatalf("configuration/options/factory config = %#v/%#v/%#v", req.Config, opts, factoryConfig)
|
||||||
|
}
|
||||||
|
location, loadErr := time.LoadLocation("America/New_York")
|
||||||
|
if loadErr != nil || req.Date.Location().String() != location.String() || req.Date.Format("2006-01-02") != "2026-05-29" {
|
||||||
|
t.Fatalf("date/location = %v/%v", req.Date, loadErr)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestResolveComparisonActionMatchesReportDatePolicies(t *testing.T) {
|
||||||
|
configPath := comparisonConfigPath(t, "weather_api:\n base_url: https://weather.api.example.com/\n timezone: America/Chicago\n")
|
||||||
|
runner := comparisonRunner(t, t.TempDir())
|
||||||
|
for _, test := range []struct {
|
||||||
|
name string
|
||||||
|
args []string
|
||||||
|
wantDay string
|
||||||
|
wantErr bool
|
||||||
|
}{
|
||||||
|
{name: "daily requires date", args: []string{"daily", "--profile", "one", "--profile", "two", "--config", configPath}, wantErr: true},
|
||||||
|
{name: "today uses current local date", args: []string{"today", "--profile", "one", "--profile", "two", "--config", configPath}, wantDay: "2026-05-29"},
|
||||||
|
{name: "today accepts date", args: []string{"today", "--date", "2026-05-30", "--profile", "one", "--profile", "two", "--config", configPath}, wantDay: "2026-05-30"},
|
||||||
|
{name: "tomorrow rejects date", args: []string{"tomorrow", "--date", "2026-05-30", "--profile", "one", "--profile", "two", "--config", configPath}, wantErr: true},
|
||||||
|
{name: "hourly rejects date", args: []string{"hourly", "--date", "2026-05-30", "--profile", "one", "--profile", "two", "--config", configPath}, wantErr: true},
|
||||||
|
} {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
req, _, err := runner.resolveComparisonAction(test.args)
|
||||||
|
if test.wantErr {
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("resolveComparisonAction() error = nil")
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err != nil || req.Date.Format("2006-01-02") != test.wantDay {
|
||||||
|
t.Fatalf("request/error = %#v/%v", req, err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestResolveComparisonActionLeavesConfiguredOutputWithoutOverride(t *testing.T) {
|
||||||
|
workingDir := t.TempDir()
|
||||||
|
configPath := comparisonConfigPath(t, "weather_api:\n base_url: https://weather.api.example.com/\noutput:\n directory: configured/../reports\n")
|
||||||
|
req, _, err := comparisonRunner(t, workingDir).resolveComparisonAction([]string{
|
||||||
|
"daily", "--date", "2026-05-29", "--profile", "weather-light", "--profile", "weather-deep", "--config", configPath,
|
||||||
|
})
|
||||||
|
if err != nil || req.OutputDir != "" || req.Config.Output.Directory != "configured/../reports" {
|
||||||
|
t.Fatalf("request/error = %#v/%v", req, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestComparisonInputFailuresDoNotConstructOrExecute(t *testing.T) {
|
||||||
|
for _, test := range []struct {
|
||||||
|
name string
|
||||||
|
args []string
|
||||||
|
}{
|
||||||
|
{name: "missing report", args: nil},
|
||||||
|
{name: "unknown report", args: []string{"unknown", "--profile", "one", "--profile", "two"}},
|
||||||
|
{name: "single profile", args: []string{"today", "--profile", "one"}},
|
||||||
|
{name: "blank profile", args: []string{"today", "--profile", "one", "--profile", " \t"}},
|
||||||
|
{name: "duplicate profile", args: []string{"today", "--profile", "one", "--profile", "one"}},
|
||||||
|
{name: "unexpected argument", args: []string{"today", "extra", "--profile", "one", "--profile", "two"}},
|
||||||
|
{name: "unsupported output flag", args: []string{"today", "--out", "report.md", "--profile", "one", "--profile", "two"}},
|
||||||
|
{name: "configuration failure", args: []string{"today", "--profile", "one", "--profile", "two", "--config", filepath.Join(t.TempDir(), "missing.yml")}},
|
||||||
|
} {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
factoryCalls, applicationCalls := 0, 0
|
||||||
|
runner := Runner{
|
||||||
|
Clock: timeutil.FixedClock{Time: time.Date(2026, 5, 29, 8, 30, 0, 0, time.UTC)},
|
||||||
|
ExecutorFactory: func(PromptExecutorConfig) (promptexec.Executor, error) {
|
||||||
|
factoryCalls++
|
||||||
|
return &factoryExecutor{}, nil
|
||||||
|
},
|
||||||
|
compareDetailed: func(context.Context, app.ComparisonRequest) (*app.ComparisonResult, error) {
|
||||||
|
applicationCalls++
|
||||||
|
return nil, nil
|
||||||
|
},
|
||||||
|
}
|
||||||
|
var stdout, stderr bytes.Buffer
|
||||||
|
if err := runner.Run(context.Background(), append([]string{"compare"}, test.args...), &stdout, &stderr); err == nil {
|
||||||
|
t.Fatal("Run() error = nil")
|
||||||
|
}
|
||||||
|
if factoryCalls != 0 || applicationCalls != 0 || stdout.Len() != 0 || stderr.Len() != 0 {
|
||||||
|
t.Fatalf("factory/application calls/output = %d/%d/%q/%q", factoryCalls, applicationCalls, stdout.String(), stderr.String())
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCompareCommandUsesOneExecutorAndInjectedApplication(t *testing.T) {
|
||||||
|
workingDir := t.TempDir()
|
||||||
|
configPath := comparisonConfigPath(t, "weather_api:\n base_url: https://weather.api.example.com/\n")
|
||||||
|
executor := &factoryExecutor{}
|
||||||
|
factoryCalls, applicationCalls := 0, 0
|
||||||
|
var received app.ComparisonRequest
|
||||||
|
wantResult := &app.ComparisonResult{ComparisonID: "comparison_test"}
|
||||||
|
comparisonErr := errors.New("comparison completed with 1 failed profiles")
|
||||||
|
runner := Runner{
|
||||||
|
Clock: timeutil.FixedClock{Time: time.Date(2026, 5, 29, 8, 30, 0, 0, time.UTC)}, WorkingDir: workingDir,
|
||||||
|
ExecutorFactory: func(PromptExecutorConfig) (promptexec.Executor, error) {
|
||||||
|
factoryCalls++
|
||||||
|
return executor, nil
|
||||||
|
},
|
||||||
|
compareDetailed: func(_ context.Context, req app.ComparisonRequest) (*app.ComparisonResult, error) {
|
||||||
|
applicationCalls++
|
||||||
|
received = req
|
||||||
|
return wantResult, comparisonErr
|
||||||
|
},
|
||||||
|
}
|
||||||
|
var stdout, stderr bytes.Buffer
|
||||||
|
err := runner.Run(context.Background(), []string{
|
||||||
|
"compare", "daily", "--date", "2026-05-29", "--profile", "weather-light", "--profile", "weather-deep", "--out-dir", "comparison", "--replace", "--config", configPath,
|
||||||
|
}, &stdout, &stderr)
|
||||||
|
if !errors.Is(err, comparisonErr) || factoryCalls != 1 || applicationCalls != 1 || received.Executor != executor || received.OutputDir != filepath.Join(workingDir, "comparison") || !received.Replace || stdout.Len() == 0 || stderr.Len() != 0 {
|
||||||
|
t.Fatalf("error/calls/request/output = %v/%d/%d/%#v/%q/%q", err, factoryCalls, applicationCalls, received, stdout.String(), stderr.String())
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(received.ProfileIDs, []string{"weather-light", "weather-deep"}) {
|
||||||
|
t.Fatalf("profile IDs = %#v", received.ProfileIDs)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCompareCommandWritesStructuredPartialFailure(t *testing.T) {
|
||||||
|
workingDir := t.TempDir()
|
||||||
|
configPath := comparisonConfigPath(t, "weather_api:\n base_url: https://weather.api.example.com/\n")
|
||||||
|
profileFailure := comparison.NewSafeError("generation", "execute prompt failed")
|
||||||
|
result := comparisonResult("/reports/comparison-daily", []app.ComparisonProfileResult{
|
||||||
|
{Position: 1, ProfileID: "weather-light", BackendID: "local", ModelName: "light", Status: comparison.StatusSucceeded, ValidationStatus: promptexec.ValidationPassed, ReportPath: "/reports/comparison-daily/01-weather-light.md", LLMDebugPath: "/debug/comparison-light"},
|
||||||
|
{Position: 2, ProfileID: "weather-deep", BackendID: "cloud", ModelName: "deep", Status: comparison.StatusFailed, Error: &profileFailure},
|
||||||
|
})
|
||||||
|
partialErr := errors.New("comparison completed with 1 failed profiles")
|
||||||
|
factoryCalls, applicationCalls := 0, 0
|
||||||
|
runner := Runner{
|
||||||
|
Clock: timeutil.FixedClock{Time: time.Date(2026, 5, 29, 8, 30, 0, 0, time.UTC)}, WorkingDir: workingDir,
|
||||||
|
ExecutorFactory: func(PromptExecutorConfig) (promptexec.Executor, error) {
|
||||||
|
factoryCalls++
|
||||||
|
return &factoryExecutor{}, nil
|
||||||
|
},
|
||||||
|
compareDetailed: func(context.Context, app.ComparisonRequest) (*app.ComparisonResult, error) {
|
||||||
|
applicationCalls++
|
||||||
|
return result, partialErr
|
||||||
|
},
|
||||||
|
}
|
||||||
|
var stdout, stderr bytes.Buffer
|
||||||
|
err := runner.Run(context.Background(), []string{"compare", "daily", "--date", "2026-05-29", "--profile", "weather-light", "--profile", "weather-deep", "--config", configPath}, &stdout, &stderr)
|
||||||
|
if !errors.Is(err, partialErr) || factoryCalls != 1 || applicationCalls != 1 || stderr.Len() != 0 {
|
||||||
|
t.Fatalf("error/calls/stderr = %v/%d/%d/%q", err, factoryCalls, applicationCalls, stderr.String())
|
||||||
|
}
|
||||||
|
var summary comparisonSummary
|
||||||
|
if err := json.Unmarshal(stdout.Bytes(), &summary); err != nil {
|
||||||
|
t.Fatalf("decode summary: %v\n%s", err, stdout.String())
|
||||||
|
}
|
||||||
|
if summary.Command != commandCompare || summary.Status != summaryStatusFailed || summary.Error == nil || summary.Error.Message != partialErr.Error() || summary.OutputDirectory != result.OutputDirectory || len(summary.Results) != 2 || summary.Results[0].ReportPath != result.Results[0].ReportPath || summary.Results[0].LLMDebugPath != result.Results[0].LLMDebugPath || summary.Results[1].Error == nil {
|
||||||
|
t.Fatalf("summary = %#v", summary)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCompareCommandReportsCommittedBundleWhenCleanupFails(t *testing.T) {
|
||||||
|
workingDir := t.TempDir()
|
||||||
|
configPath := comparisonConfigPath(t, "weather_api:\n base_url: https://weather.api.example.com/\n")
|
||||||
|
outputDirectory := filepath.Join(workingDir, "comparison-daily")
|
||||||
|
result := comparisonResult(outputDirectory, []app.ComparisonProfileResult{
|
||||||
|
{Position: 1, ProfileID: "weather-light", ModelName: "light", Status: comparison.StatusSucceeded, ValidationStatus: promptexec.ValidationPassed, ReportPath: filepath.Join(outputDirectory, "01-weather-light.md")},
|
||||||
|
{Position: 2, ProfileID: "weather-deep", ModelName: "deep", Status: comparison.StatusSucceeded, ValidationStatus: promptexec.ValidationPassed, ReportPath: filepath.Join(outputDirectory, "02-weather-deep.md")},
|
||||||
|
})
|
||||||
|
backupPath := filepath.Join(workingDir, ".comparison-daily.backup-retained")
|
||||||
|
cleanupCause := errors.New("filesystem cleanup detail")
|
||||||
|
cleanupErr := &comparison.PublicationCleanupError{RetainedBackupPath: backupPath, Err: cleanupCause}
|
||||||
|
runner := comparisonRunner(t, workingDir)
|
||||||
|
runner.compareDetailed = func(context.Context, app.ComparisonRequest) (*app.ComparisonResult, error) {
|
||||||
|
return result, cleanupErr
|
||||||
|
}
|
||||||
|
|
||||||
|
var stdout, stderr bytes.Buffer
|
||||||
|
err := runner.Run(context.Background(), []string{"compare", "daily", "--date", "2026-05-29", "--profile", "weather-light", "--profile", "weather-deep", "--config", configPath}, &stdout, &stderr)
|
||||||
|
if !errors.Is(err, cleanupCause) || stderr.Len() != 0 {
|
||||||
|
t.Fatalf("Run() error/stderr = %v/%q", err, stderr.String())
|
||||||
|
}
|
||||||
|
var summary comparisonSummary
|
||||||
|
if err := json.Unmarshal(stdout.Bytes(), &summary); err != nil {
|
||||||
|
t.Fatalf("decode summary: %v\n%s", err, stdout.String())
|
||||||
|
}
|
||||||
|
if summary.Status != summaryStatusFailed || summary.Error == nil || summary.Error.Category != "publication_cleanup" || summary.Error.Message != "comparison published but cleanup did not complete" || summary.ManifestPath != result.ManifestPath || summary.DataPackagePath != result.DataPackagePath || summary.Results[0].ReportPath == "" {
|
||||||
|
t.Fatalf("summary = %#v", summary)
|
||||||
|
}
|
||||||
|
for _, unsafe := range []string{cleanupCause.Error(), backupPath} {
|
||||||
|
if strings.Contains(stdout.String(), unsafe) {
|
||||||
|
t.Fatalf("summary contains unsafe recovery detail %q: %s", unsafe, stdout.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCompareCommandWritesSuccessForDefaultAndExplicitDestinations(t *testing.T) {
|
||||||
|
workingDir := t.TempDir()
|
||||||
|
configPath := comparisonConfigPath(t, "weather_api:\n base_url: https://weather.api.example.com/\noutput:\n directory: configured-reports\n")
|
||||||
|
for _, test := range []struct {
|
||||||
|
name string
|
||||||
|
outputArgument []string
|
||||||
|
outputDirectory string
|
||||||
|
wantRequestPath string
|
||||||
|
}{
|
||||||
|
{name: "configured default", outputDirectory: filepath.Join(workingDir, "configured-reports", "comparison-daily-2026-05-29")},
|
||||||
|
{name: "explicit directory", outputArgument: []string{"--out-dir", "published"}, outputDirectory: filepath.Join(workingDir, "published"), wantRequestPath: filepath.Join(workingDir, "published")},
|
||||||
|
} {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
var received app.ComparisonRequest
|
||||||
|
runner := comparisonRunner(t, workingDir)
|
||||||
|
runner.compareDetailed = func(_ context.Context, req app.ComparisonRequest) (*app.ComparisonResult, error) {
|
||||||
|
received = req
|
||||||
|
return comparisonResult(test.outputDirectory, []app.ComparisonProfileResult{
|
||||||
|
{Position: 1, ProfileID: "weather-light", ModelName: "light", Status: comparison.StatusSucceeded, ValidationStatus: promptexec.ValidationPassed, ReportPath: filepath.Join(test.outputDirectory, "01-weather-light.md")},
|
||||||
|
{Position: 2, ProfileID: "weather-deep", ModelName: "deep", Status: comparison.StatusSucceeded, ValidationStatus: promptexec.ValidationPassed, ReportPath: filepath.Join(test.outputDirectory, "02-weather-deep.md")},
|
||||||
|
}), nil
|
||||||
|
}
|
||||||
|
args := []string{"compare", "daily", "--date", "2026-05-29", "--profile", "weather-light", "--profile", "weather-deep", "--config", configPath}
|
||||||
|
args = append(args, test.outputArgument...)
|
||||||
|
var stdout, stderr bytes.Buffer
|
||||||
|
if err := runner.Run(context.Background(), args, &stdout, &stderr); err != nil {
|
||||||
|
t.Fatalf("Run() error = %v", err)
|
||||||
|
}
|
||||||
|
var summary comparisonSummary
|
||||||
|
if err := json.Unmarshal(stdout.Bytes(), &summary); err != nil || summary.Status != summaryStatusSucceeded || summary.OutputDirectory != test.outputDirectory || stderr.Len() != 0 || received.OutputDir != test.wantRequestPath {
|
||||||
|
t.Fatalf("summary/error/stderr/request = %#v/%v/%q/%#v", summary, err, stderr.String(), received)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCompareCommandQuietPreservesFailure(t *testing.T) {
|
||||||
|
configPath := comparisonConfigPath(t, "weather_api:\n base_url: https://weather.api.example.com/\n")
|
||||||
|
failure := errors.New("comparison completed with 2 failed profiles")
|
||||||
|
calls := 0
|
||||||
|
runner := comparisonRunner(t, t.TempDir())
|
||||||
|
runner.compareDetailed = func(context.Context, app.ComparisonRequest) (*app.ComparisonResult, error) {
|
||||||
|
calls++
|
||||||
|
return comparisonResult("/reports/comparison-daily", nil), failure
|
||||||
|
}
|
||||||
|
var stdout, stderr bytes.Buffer
|
||||||
|
err := runner.Run(context.Background(), []string{"compare", "daily", "--date", "2026-05-29", "--profile", "weather-light", "--profile", "weather-deep", "--quiet", "--config", configPath}, &stdout, &stderr)
|
||||||
|
if !errors.Is(err, failure) || calls != 1 || stdout.Len() != 0 || stderr.Len() != 0 {
|
||||||
|
t.Fatalf("error/calls/stdout/stderr = %v/%d/%q/%q", err, calls, stdout.String(), stderr.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCompareCommandLeavesPreExecutionFailuresUnstructured(t *testing.T) {
|
||||||
|
runner := comparisonRunner(t, t.TempDir())
|
||||||
|
called := false
|
||||||
|
runner.compareDetailed = func(context.Context, app.ComparisonRequest) (*app.ComparisonResult, error) {
|
||||||
|
called = true
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
var stdout, stderr bytes.Buffer
|
||||||
|
err := runner.Run(context.Background(), []string{"compare", "today", "--profile", "only-one"}, &stdout, &stderr)
|
||||||
|
if err == nil || called || stdout.Len() != 0 || stderr.Len() != 0 {
|
||||||
|
t.Fatalf("error/called/stdout/stderr = %v/%t/%q/%q", err, called, stdout.String(), stderr.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCompareHelpIncludesCommand(t *testing.T) {
|
||||||
|
var stdout, stderr bytes.Buffer
|
||||||
|
if err := (Runner{}).Run(context.Background(), []string{"--help"}, &stdout, &stderr); err != nil || !strings.Contains(stdout.String(), "weatherreporter compare REPORT") || !strings.Contains(stdout.String(), "--profile PROFILE") {
|
||||||
|
t.Fatalf("help/error = %q/%v", stdout.String(), err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCompareHelpDoesNotRequireConfiguration(t *testing.T) {
|
||||||
|
var stdout, stderr bytes.Buffer
|
||||||
|
if err := (Runner{}).Run(context.Background(), []string{"compare", "--help"}, &stdout, &stderr); err != nil || !strings.Contains(stdout.String(), "weatherreporter compare REPORT") || stderr.Len() != 0 {
|
||||||
|
t.Fatalf("help/error/stderr = %q/%v/%q", stdout.String(), err, stderr.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func comparisonRunner(t *testing.T, workingDir string) Runner {
|
||||||
|
t.Helper()
|
||||||
|
return Runner{
|
||||||
|
Clock: timeutil.FixedClock{Time: time.Date(2026, 5, 29, 8, 30, 0, 0, time.UTC)}, WorkingDir: workingDir,
|
||||||
|
ExecutorFactory: func(PromptExecutorConfig) (promptexec.Executor, error) { return &factoryExecutor{}, nil },
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func comparisonConfigPath(t *testing.T, contents string) string {
|
||||||
|
t.Helper()
|
||||||
|
path := filepath.Join(t.TempDir(), "config.yml")
|
||||||
|
if err := os.WriteFile(path, []byte(contents), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return path
|
||||||
|
}
|
||||||
|
|
||||||
|
func comparisonResult(outputDirectory string, results []app.ComparisonProfileResult) *app.ComparisonResult {
|
||||||
|
started := time.Date(2026, 5, 29, 13, 30, 0, 0, time.UTC)
|
||||||
|
result := &app.ComparisonResult{
|
||||||
|
ComparisonID: "comparison_run-123", ReportID: "daily", ReportName: "Daily Report",
|
||||||
|
PromptID: "weather.daily_generated_text", PromptVersion: "2.0.0", PromptHash: strings.Repeat("a", 64),
|
||||||
|
StartedAt: started, FinishedAt: started.Add(time.Minute), Timezone: "America/Chicago",
|
||||||
|
ValidPeriod: timeutil.Period{Start: started, End: started.Add(24 * time.Hour)}, OutputDirectory: outputDirectory,
|
||||||
|
ManifestPath: filepath.Join(outputDirectory, "comparison.json"), DataPackagePath: filepath.Join(outputDirectory, "data-package.yml"),
|
||||||
|
Results: append([]app.ComparisonProfileResult(nil), results...), Total: len(results),
|
||||||
|
}
|
||||||
|
for _, profile := range results {
|
||||||
|
if profile.Status == "succeeded" {
|
||||||
|
result.Succeeded++
|
||||||
|
} else {
|
||||||
|
result.Failed++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return result
|
||||||
|
}
|
||||||
@@ -1,9 +1,16 @@
|
|||||||
package cli
|
package cli
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/app"
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/app"
|
||||||
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/comparison"
|
||||||
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec"
|
||||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/report"
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/report"
|
||||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/timeutil"
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/timeutil"
|
||||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/weatherdata"
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/weatherdata"
|
||||||
@@ -12,6 +19,7 @@ import (
|
|||||||
const (
|
const (
|
||||||
commandGenerate = "generate"
|
commandGenerate = "generate"
|
||||||
commandRun = "run"
|
commandRun = "run"
|
||||||
|
commandCompare = "compare"
|
||||||
|
|
||||||
summaryStatusSucceeded = "succeeded"
|
summaryStatusSucceeded = "succeeded"
|
||||||
summaryStatusFailed = "failed"
|
summaryStatusFailed = "failed"
|
||||||
@@ -67,6 +75,41 @@ type batchSummary struct {
|
|||||||
Error string `json:"error,omitempty"`
|
Error string `json:"error,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type comparisonSummary struct {
|
||||||
|
Command string `json:"command"`
|
||||||
|
ComparisonID string `json:"comparisonId"`
|
||||||
|
ReportID report.ID `json:"reportId"`
|
||||||
|
ReportName string `json:"reportName"`
|
||||||
|
PromptID string `json:"promptId"`
|
||||||
|
PromptVersion string `json:"promptVersion"`
|
||||||
|
PromptHash string `json:"promptHash"`
|
||||||
|
Status string `json:"status"`
|
||||||
|
StartedAt time.Time `json:"startedAt"`
|
||||||
|
FinishedAt time.Time `json:"finishedAt"`
|
||||||
|
Timezone string `json:"timezone"`
|
||||||
|
ValidPeriod timeutil.Period `json:"validPeriod"`
|
||||||
|
OutputDirectory string `json:"outputDirectory"`
|
||||||
|
ManifestPath string `json:"manifestPath,omitempty"`
|
||||||
|
DataPackagePath string `json:"dataPackagePath,omitempty"`
|
||||||
|
Total int `json:"total"`
|
||||||
|
Succeeded int `json:"succeeded"`
|
||||||
|
Failed int `json:"failed"`
|
||||||
|
Results []comparisonProfileSummary `json:"results"`
|
||||||
|
Error *comparison.SafeError `json:"error,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type comparisonProfileSummary struct {
|
||||||
|
Position int `json:"position"`
|
||||||
|
ProfileID string `json:"profileId"`
|
||||||
|
BackendID string `json:"backendId,omitempty"`
|
||||||
|
ModelName string `json:"modelName"`
|
||||||
|
Status string `json:"status"`
|
||||||
|
ValidationStatus string `json:"validationStatus,omitempty"`
|
||||||
|
ReportPath string `json:"reportPath,omitempty"`
|
||||||
|
LLMDebugPath string `json:"llmDebugPath,omitempty"`
|
||||||
|
Error *comparison.SafeError `json:"error,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
func newGenerateSummary(result *app.ReportResult, err error) generateSummary {
|
func newGenerateSummary(result *app.ReportResult, err error) generateSummary {
|
||||||
summary := generateSummary{Command: commandGenerate}
|
summary := generateSummary{Command: commandGenerate}
|
||||||
if result == nil {
|
if result == nil {
|
||||||
@@ -139,6 +182,87 @@ func newBatchSummary(result *app.BatchResult) batchSummary {
|
|||||||
return summary
|
return summary
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func newComparisonSummary(result *app.ComparisonResult, err error) comparisonSummary {
|
||||||
|
summary := comparisonSummary{Command: commandCompare, Results: []comparisonProfileSummary{}}
|
||||||
|
if result == nil {
|
||||||
|
return summary
|
||||||
|
}
|
||||||
|
summary.ComparisonID = result.ComparisonID
|
||||||
|
summary.ReportID, summary.ReportName = result.ReportID, result.ReportName
|
||||||
|
summary.PromptID, summary.PromptVersion, summary.PromptHash = result.PromptID, result.PromptVersion, result.PromptHash
|
||||||
|
summary.StartedAt, summary.FinishedAt = result.StartedAt, result.FinishedAt
|
||||||
|
summary.Timezone, summary.ValidPeriod = result.Timezone, result.ValidPeriod
|
||||||
|
summary.OutputDirectory = result.OutputDirectory
|
||||||
|
summary.ManifestPath, summary.DataPackagePath = result.ManifestPath, result.DataPackagePath
|
||||||
|
summary.Total, summary.Succeeded, summary.Failed = result.Total, result.Succeeded, result.Failed
|
||||||
|
for _, profile := range result.Results {
|
||||||
|
summary.Results = append(summary.Results, comparisonProfileSummary{
|
||||||
|
Position: profile.Position, ProfileID: profile.ProfileID, BackendID: profile.BackendID, ModelName: profile.ModelName,
|
||||||
|
Status: profile.Status, ValidationStatus: string(profile.ValidationStatus), ReportPath: profile.ReportPath,
|
||||||
|
LLMDebugPath: profile.LLMDebugPath, Error: profile.Error,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
summary.Status = comparisonSummaryStatus(result, err)
|
||||||
|
if err != nil {
|
||||||
|
summary.Error = safeComparisonSummaryError(err)
|
||||||
|
}
|
||||||
|
return summary
|
||||||
|
}
|
||||||
|
|
||||||
|
func comparisonSummaryStatus(result *app.ComparisonResult, err error) string {
|
||||||
|
if result == nil || err != nil || result.Total < 2 || result.Succeeded != result.Total || result.Failed != 0 || result.ManifestPath == "" || result.DataPackagePath == "" {
|
||||||
|
return summaryStatusFailed
|
||||||
|
}
|
||||||
|
return summaryStatusSucceeded
|
||||||
|
}
|
||||||
|
|
||||||
|
func safeComparisonSummaryError(err error) *comparison.SafeError {
|
||||||
|
if aggregate, ok := comparisonAggregateErrorMessage(err); ok {
|
||||||
|
safe := comparison.NewSafeError("application", aggregate)
|
||||||
|
return &safe
|
||||||
|
}
|
||||||
|
var cleanupErr *comparison.PublicationCleanupError
|
||||||
|
if errors.As(err, &cleanupErr) {
|
||||||
|
safe := comparison.NewSafeError("publication_cleanup", "comparison published but cleanup did not complete")
|
||||||
|
return &safe
|
||||||
|
}
|
||||||
|
var destinationErr *comparison.DestinationError
|
||||||
|
if errors.As(err, &destinationErr) {
|
||||||
|
safe := comparison.NewSafeError("destination_"+string(destinationErr.Kind), "comparison destination preflight failed")
|
||||||
|
return &safe
|
||||||
|
}
|
||||||
|
if category := promptexec.CategoryOf(err); category != "" {
|
||||||
|
safe := comparison.NewSafeError(string(category), "comparison prompt operation failed")
|
||||||
|
return &safe
|
||||||
|
}
|
||||||
|
if errors.Is(err, context.DeadlineExceeded) {
|
||||||
|
safe := comparison.NewSafeError("deadline_exceeded", "comparison deadline exceeded")
|
||||||
|
return &safe
|
||||||
|
}
|
||||||
|
if errors.Is(err, context.Canceled) {
|
||||||
|
safe := comparison.NewSafeError("canceled", "comparison canceled")
|
||||||
|
return &safe
|
||||||
|
}
|
||||||
|
safe := comparison.NewSafeError("application", "comparison did not complete")
|
||||||
|
return &safe
|
||||||
|
}
|
||||||
|
|
||||||
|
func comparisonAggregateErrorMessage(err error) (string, bool) {
|
||||||
|
const prefix = "comparison completed with "
|
||||||
|
const suffix = " failed profiles"
|
||||||
|
for candidate := err; candidate != nil; candidate = errors.Unwrap(candidate) {
|
||||||
|
value := candidate.Error()
|
||||||
|
if !strings.HasPrefix(value, prefix) || !strings.HasSuffix(value, suffix) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
count, parseErr := strconv.Atoi(strings.TrimSuffix(strings.TrimPrefix(value, prefix), suffix))
|
||||||
|
if parseErr == nil && count > 0 {
|
||||||
|
return fmt.Sprintf("comparison completed with %d failed profiles", count), true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return "", false
|
||||||
|
}
|
||||||
|
|
||||||
func batchSummaryStatus(result *app.BatchResult) string {
|
func batchSummaryStatus(result *app.BatchResult) string {
|
||||||
if result == nil {
|
if result == nil {
|
||||||
return ""
|
return ""
|
||||||
|
|||||||
@@ -1,12 +1,16 @@
|
|||||||
package cli
|
package cli
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"context"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/app"
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/app"
|
||||||
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/comparison"
|
||||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec"
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec"
|
||||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/report"
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/report"
|
||||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/timeutil"
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/timeutil"
|
||||||
@@ -29,3 +33,179 @@ func TestGenerateSummaryUsesActiveResultFields(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestComparisonSummaryUsesLockedOrderAndSafeFields(t *testing.T) {
|
||||||
|
started := time.Date(2026, 5, 29, 13, 30, 0, 0, time.UTC)
|
||||||
|
profileFailure := comparison.NewSafeError("generation", "execute prompt failed")
|
||||||
|
result := &app.ComparisonResult{
|
||||||
|
ComparisonID: "comparison_run-123", ReportID: report.Daily, ReportName: "Daily Report",
|
||||||
|
PromptID: "weather.daily_generated_text", PromptVersion: "2.0.0", PromptHash: strings.Repeat("a", 64),
|
||||||
|
StartedAt: started, FinishedAt: started.Add(time.Minute), Timezone: "America/Chicago",
|
||||||
|
ValidPeriod: timeutil.Period{Start: started, End: started.Add(24 * time.Hour)}, OutputDirectory: "/reports/comparison-daily",
|
||||||
|
ManifestPath: "/reports/comparison-daily/comparison.json", DataPackagePath: "/reports/comparison-daily/data-package.yml",
|
||||||
|
Total: 2, Succeeded: 1, Failed: 1,
|
||||||
|
Results: []app.ComparisonProfileResult{
|
||||||
|
{Position: 1, ProfileID: "weather-light", BackendID: "local", ModelName: "light", Status: comparison.StatusSucceeded, ValidationStatus: promptexec.ValidationPassed, ReportPath: "/reports/comparison-daily/01-weather-light.md"},
|
||||||
|
{Position: 2, ProfileID: "weather-deep", BackendID: "cloud", ModelName: "deep", Status: comparison.StatusFailed, Error: &profileFailure},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
summary := newComparisonSummary(result, errors.New("comparison completed with 1 failed profiles"))
|
||||||
|
if summary.Command != commandCompare || summary.Status != summaryStatusFailed || summary.Error == nil || summary.Error.Message != "comparison completed with 1 failed profiles" || len(summary.Results) != 2 || summary.Results[0].ReportPath == "" || summary.Results[1].Error == nil {
|
||||||
|
t.Fatalf("summary = %#v", summary)
|
||||||
|
}
|
||||||
|
data, err := json.Marshal(summary)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
previous := -1
|
||||||
|
for _, field := range []string{"command", "comparisonId", "reportId", "reportName", "promptId", "promptVersion", "promptHash", "status", "startedAt", "finishedAt", "timezone", "validPeriod", "outputDirectory", "manifestPath", "dataPackagePath", "total", "succeeded", "failed", "results", "error"} {
|
||||||
|
position := strings.Index(string(data), `"`+field+`":`)
|
||||||
|
if field == "error" {
|
||||||
|
position = strings.LastIndex(string(data), `"`+field+`":`)
|
||||||
|
}
|
||||||
|
if position <= previous {
|
||||||
|
t.Fatalf("field order for %q in %s", field, data)
|
||||||
|
}
|
||||||
|
previous = position
|
||||||
|
}
|
||||||
|
for _, unsafe := range []string{"provider response", "rawOutput", "renderedPrompt", "endpoint"} {
|
||||||
|
if strings.Contains(string(data), unsafe) {
|
||||||
|
t.Fatalf("summary includes unsafe content %q: %s", unsafe, data)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestComparisonSummaryOmitsUnpublishedArtifactsAndBoundsErrors(t *testing.T) {
|
||||||
|
result := &app.ComparisonResult{ComparisonID: "comparison_run-123", ReportID: report.Daily, Results: []app.ComparisonProfileResult{{Position: 1, ProfileID: "weather-light", ModelName: "light", Status: comparison.StatusFailed}}}
|
||||||
|
summary := newComparisonSummary(result, context.Canceled)
|
||||||
|
if summary.Status != summaryStatusFailed || summary.Error == nil || summary.Error.Message != "comparison canceled" || summary.Results == nil {
|
||||||
|
t.Fatalf("summary = %#v", summary)
|
||||||
|
}
|
||||||
|
data, err := json.Marshal(summary)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var fields map[string]json.RawMessage
|
||||||
|
if err := json.Unmarshal(data, &fields); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, omitted := range []string{"manifestPath", "dataPackagePath"} {
|
||||||
|
if _, exists := fields[omitted]; exists {
|
||||||
|
t.Fatalf("summary includes unpublished %s: %s", omitted, data)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
publicationResult := &app.ComparisonResult{
|
||||||
|
ComparisonID: "comparison_run-123", ReportID: report.Daily, OutputDirectory: "/reports/comparison-daily", Total: 2, Succeeded: 2,
|
||||||
|
Results: []app.ComparisonProfileResult{
|
||||||
|
{Position: 1, ProfileID: "weather-light", ModelName: "light", Status: comparison.StatusSucceeded},
|
||||||
|
{Position: 2, ProfileID: "weather-deep", ModelName: "deep", Status: comparison.StatusSucceeded},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
unsafe := errors.New("comparison completed with 1 failed profiles; provider response contains sensitive material")
|
||||||
|
publicationSummary := newComparisonSummary(publicationResult, unsafe)
|
||||||
|
if publicationSummary.Status != summaryStatusFailed || publicationSummary.Error == nil || publicationSummary.Error.Message != "comparison did not complete" || strings.Contains(publicationSummary.Error.Message, "sensitive") || publicationSummary.ManifestPath != "" || publicationSummary.DataPackagePath != "" {
|
||||||
|
safe := publicationSummary.Error
|
||||||
|
t.Fatalf("safe error = %#v", safe)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestComparisonSummaryClassifiesCompleteAndAllFailedResults(t *testing.T) {
|
||||||
|
started := time.Date(2026, 5, 29, 13, 30, 0, 0, time.UTC)
|
||||||
|
failure := comparison.NewSafeError("generation", "execute prompt failed")
|
||||||
|
complete := &app.ComparisonResult{
|
||||||
|
ComparisonID: "comparison_run-123", ReportID: report.Daily, PromptID: "weather.daily_generated_text", PromptVersion: "2.0.0", PromptHash: strings.Repeat("a", 64),
|
||||||
|
StartedAt: started, FinishedAt: started, Timezone: "America/Chicago", ValidPeriod: timeutil.Period{Start: started, End: started.Add(time.Hour)},
|
||||||
|
OutputDirectory: "/reports/comparison-daily", ManifestPath: "/reports/comparison-daily/comparison.json", DataPackagePath: "/reports/comparison-daily/data-package.yml",
|
||||||
|
Total: 2, Succeeded: 2,
|
||||||
|
Results: []app.ComparisonProfileResult{
|
||||||
|
{Position: 1, ProfileID: "weather-light", ModelName: "light", Status: comparison.StatusSucceeded, ValidationStatus: promptexec.ValidationPassed, ReportPath: "/reports/comparison-daily/01-weather-light.md"},
|
||||||
|
{Position: 2, ProfileID: "weather-deep", ModelName: "deep", Status: comparison.StatusSucceeded, ValidationStatus: promptexec.ValidationPassed, ReportPath: "/reports/comparison-daily/02-weather-deep.md"},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
if summary := newComparisonSummary(complete, nil); summary.Status != summaryStatusSucceeded || summary.Error != nil {
|
||||||
|
t.Fatalf("complete summary = %#v", summary)
|
||||||
|
}
|
||||||
|
allFailed := *complete
|
||||||
|
allFailed.Succeeded, allFailed.Failed = 0, 2
|
||||||
|
allFailed.Results = []app.ComparisonProfileResult{
|
||||||
|
{Position: 1, ProfileID: "weather-light", ModelName: "light", Status: comparison.StatusFailed, Error: &failure},
|
||||||
|
{Position: 2, ProfileID: "weather-deep", ModelName: "deep", Status: comparison.StatusFailed, Error: &failure},
|
||||||
|
}
|
||||||
|
summary := newComparisonSummary(&allFailed, errors.New("comparison completed with 2 failed profiles"))
|
||||||
|
if summary.Status != summaryStatusFailed || summary.Error == nil || summary.Error.Message != "comparison completed with 2 failed profiles" || summary.Results[0].ReportPath != "" || summary.Results[1].Error == nil {
|
||||||
|
t.Fatalf("all-failed summary = %#v", summary)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSafeComparisonSummaryErrorClassifiesWrappedFailures(t *testing.T) {
|
||||||
|
unsafeDetail := "unsafe filesystem and provider detail"
|
||||||
|
unsafeCause := errors.New(unsafeDetail)
|
||||||
|
for _, test := range []struct {
|
||||||
|
name string
|
||||||
|
err error
|
||||||
|
category string
|
||||||
|
message string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "aggregate profile failure",
|
||||||
|
err: fmt.Errorf("outer wrapper: %w", errors.New("comparison completed with 2 failed profiles")),
|
||||||
|
category: "application",
|
||||||
|
message: "comparison completed with 2 failed profiles",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "canceled",
|
||||||
|
err: fmt.Errorf("outer wrapper: %w", context.Canceled),
|
||||||
|
category: "canceled",
|
||||||
|
message: "comparison canceled",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "deadline exceeded",
|
||||||
|
err: fmt.Errorf("outer wrapper: %w", context.DeadlineExceeded),
|
||||||
|
category: "deadline_exceeded",
|
||||||
|
message: "comparison deadline exceeded",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "prompt operation",
|
||||||
|
err: fmt.Errorf("outer wrapper: %w", promptexec.NewError(promptexec.Generation, "unsafe prompt detail", fmt.Errorf("%w: %s", context.Canceled, unsafeDetail))),
|
||||||
|
category: string(promptexec.Generation),
|
||||||
|
message: "comparison prompt operation failed",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "destination preflight",
|
||||||
|
err: fmt.Errorf("outer wrapper: %w", &comparison.DestinationError{
|
||||||
|
Kind: comparison.DestinationNotEmpty, Target: "/tmp/" + unsafeDetail, Err: fmt.Errorf("%w: %s", context.Canceled, unsafeDetail),
|
||||||
|
}),
|
||||||
|
category: "destination_not_empty",
|
||||||
|
message: "comparison destination preflight failed",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "publication cleanup",
|
||||||
|
err: fmt.Errorf("outer wrapper: %w", &comparison.PublicationCleanupError{
|
||||||
|
RetainedBackupPath: "/tmp/" + unsafeDetail, Err: fmt.Errorf("%w: %s", context.Canceled, unsafeDetail),
|
||||||
|
}),
|
||||||
|
category: "publication_cleanup",
|
||||||
|
message: "comparison published but cleanup did not complete",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "unknown",
|
||||||
|
err: fmt.Errorf("outer wrapper: %w", unsafeCause),
|
||||||
|
category: "application",
|
||||||
|
message: "comparison did not complete",
|
||||||
|
},
|
||||||
|
} {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
result := &app.ComparisonResult{ComparisonID: "comparison_test", ReportID: report.Daily}
|
||||||
|
summary := newComparisonSummary(result, test.err)
|
||||||
|
if summary.Error == nil || summary.Error.Category != test.category || summary.Error.Message != test.message {
|
||||||
|
t.Fatalf("summary error = %#v, want %q/%q", summary.Error, test.category, test.message)
|
||||||
|
}
|
||||||
|
data, err := json.Marshal(summary)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if strings.Contains(string(data), unsafeDetail) {
|
||||||
|
t.Fatalf("summary includes unsafe detail: %s", data)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -10,6 +10,7 @@ import (
|
|||||||
|
|
||||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/app"
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/app"
|
||||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/buildinfo"
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/buildinfo"
|
||||||
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/comparison"
|
||||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/config"
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/config"
|
||||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/report"
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/report"
|
||||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/timeutil"
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/timeutil"
|
||||||
@@ -26,6 +27,7 @@ Usage:
|
|||||||
weatherreporter generate hourly [--config PATH] [--units VALUE] [--tz NAME] [--out PATH] [--llm-debug-dir PATH] [--quiet]
|
weatherreporter generate hourly [--config PATH] [--units VALUE] [--tz NAME] [--out PATH] [--llm-debug-dir PATH] [--quiet]
|
||||||
weatherreporter run morning [--config PATH] [--units VALUE] [--tz NAME] [--out-dir PATH] [--llm-debug-dir PATH] [--quiet]
|
weatherreporter run morning [--config PATH] [--units VALUE] [--tz NAME] [--out-dir PATH] [--llm-debug-dir PATH] [--quiet]
|
||||||
weatherreporter run evening [--config PATH] [--units VALUE] [--tz NAME] [--out-dir PATH] [--llm-debug-dir PATH] [--quiet]
|
weatherreporter run evening [--config PATH] [--units VALUE] [--tz NAME] [--out-dir PATH] [--llm-debug-dir PATH] [--quiet]
|
||||||
|
weatherreporter compare REPORT --profile PROFILE --profile PROFILE [--config PATH] [--units VALUE] [--tz NAME] [--date YYYY-MM-DD] [--out-dir PATH] [--replace] [--llm-debug-dir PATH] [--quiet]
|
||||||
|
|
||||||
Options:
|
Options:
|
||||||
-h, --help Show this help message.
|
-h, --help Show this help message.
|
||||||
@@ -35,8 +37,10 @@ Options:
|
|||||||
--tz NAME Override weather API timezone.
|
--tz NAME Override weather API timezone.
|
||||||
--out PATH Write the generated Markdown report to PATH.
|
--out PATH Write the generated Markdown report to PATH.
|
||||||
--llm-debug-dir PATH Write sensitive prompt debug artifacts under PATH.
|
--llm-debug-dir PATH Write sensitive prompt debug artifacts under PATH.
|
||||||
--out-dir PATH Write generated Markdown reports beneath PATH for run commands.
|
--profile PROFILE Select a prompt profile for compare; repeat for every profile.
|
||||||
--quiet Suppress successful generate and run output.
|
--out-dir PATH Write generated Markdown reports beneath PATH for run commands, or select the exact comparison directory.
|
||||||
|
--replace Authorize replacement of a recognized comparison bundle.
|
||||||
|
--quiet Suppress successful action output.
|
||||||
`
|
`
|
||||||
|
|
||||||
type Runner struct {
|
type Runner struct {
|
||||||
@@ -45,6 +49,7 @@ type Runner struct {
|
|||||||
Version string
|
Version string
|
||||||
WorkingDir string
|
WorkingDir string
|
||||||
runBatchDetailed func(context.Context, app.BatchRequest) (*app.BatchResult, error)
|
runBatchDetailed func(context.Context, app.BatchRequest) (*app.BatchResult, error)
|
||||||
|
compareDetailed func(context.Context, app.ComparisonRequest) (*app.ComparisonResult, error)
|
||||||
}
|
}
|
||||||
|
|
||||||
func Run(ctx context.Context, args []string, stdout io.Writer, stderr io.Writer) error {
|
func Run(ctx context.Context, args []string, stdout io.Writer, stderr io.Writer) error {
|
||||||
@@ -59,6 +64,10 @@ func (r Runner) Run(ctx context.Context, args []string, stdout io.Writer, stderr
|
|||||||
_, err := fmt.Fprint(stdout, helpText)
|
_, err := fmt.Fprint(stdout, helpText)
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
if len(args) == 2 && args[0] == "compare" && (args[1] == "--help" || args[1] == "-h") {
|
||||||
|
_, err := fmt.Fprint(stdout, helpText)
|
||||||
|
return err
|
||||||
|
}
|
||||||
if args[0] == "--version" {
|
if args[0] == "--version" {
|
||||||
if len(args) != 1 {
|
if len(args) != 1 {
|
||||||
return fmt.Errorf("--version does not accept arguments")
|
return fmt.Errorf("--version does not accept arguments")
|
||||||
@@ -107,6 +116,23 @@ func (r Runner) Run(ctx context.Context, args []string, stdout io.Writer, stderr
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
return err
|
return err
|
||||||
|
case "compare":
|
||||||
|
req, opts, err := r.resolveComparisonAction(args[1:])
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
compareDetailed := r.compareDetailed
|
||||||
|
if compareDetailed == nil {
|
||||||
|
compareDetailed = app.CompareDetailed
|
||||||
|
}
|
||||||
|
result, err := compareDetailed(ctx, req)
|
||||||
|
if result != nil {
|
||||||
|
summary := newComparisonSummary(result, err)
|
||||||
|
if encodeErr := writeActionResult(stdout, stderr, summary, outputOptions{Quiet: opts.Quiet}, nil); encodeErr != nil {
|
||||||
|
return encodeErr
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return err
|
||||||
default:
|
default:
|
||||||
return fmt.Errorf("unknown command %q", args[0])
|
return fmt.Errorf("unknown command %q", args[0])
|
||||||
}
|
}
|
||||||
@@ -127,6 +153,24 @@ type generateOptions struct {
|
|||||||
Date string
|
Date string
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type comparisonOptions struct {
|
||||||
|
commonOptions
|
||||||
|
Date string
|
||||||
|
ProfileIDs profileValues
|
||||||
|
Replace bool
|
||||||
|
}
|
||||||
|
|
||||||
|
type profileValues []string
|
||||||
|
|
||||||
|
func (values *profileValues) String() string {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
func (values *profileValues) Set(value string) error {
|
||||||
|
*values = append(*values, value)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
func (r Runner) resolveGenerate(args []string) (app.GenerateRequest, error) {
|
func (r Runner) resolveGenerate(args []string) (app.GenerateRequest, error) {
|
||||||
req, _, err := r.resolveGenerateAction(args)
|
req, _, err := r.resolveGenerateAction(args)
|
||||||
return req, err
|
return req, err
|
||||||
@@ -212,6 +256,76 @@ func (r Runner) resolveRun(args []string) (app.BatchRequest, error) {
|
|||||||
return req, err
|
return req, err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (r Runner) resolveComparisonAction(args []string) (app.ComparisonRequest, commonOptions, error) {
|
||||||
|
if r.Clock == nil {
|
||||||
|
r.Clock = timeutil.SystemClock{}
|
||||||
|
}
|
||||||
|
if len(args) == 0 {
|
||||||
|
return app.ComparisonRequest{}, commonOptions{}, fmt.Errorf("compare requires a report name")
|
||||||
|
}
|
||||||
|
if _, err := report.IDForCommandName(args[0]); err != nil {
|
||||||
|
return app.ComparisonRequest{}, commonOptions{}, fmt.Errorf("unknown compare report %q", args[0])
|
||||||
|
}
|
||||||
|
reportKind := app.ReportKind(args[0])
|
||||||
|
opts, err := parseComparisonFlags(reportKind, args[1:])
|
||||||
|
if err != nil {
|
||||||
|
return app.ComparisonRequest{}, commonOptions{}, err
|
||||||
|
}
|
||||||
|
if err := comparison.ValidateProfileIDs(opts.ProfileIDs); err != nil {
|
||||||
|
return app.ComparisonRequest{}, commonOptions{}, err
|
||||||
|
}
|
||||||
|
cfg, err := config.Load(config.LoadOptions{
|
||||||
|
Path: opts.ConfigPath,
|
||||||
|
Units: opts.Units,
|
||||||
|
Timezone: opts.Timezone,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return app.ComparisonRequest{}, commonOptions{}, err
|
||||||
|
}
|
||||||
|
location, err := timeutil.LoadLocation(cfg.WeatherAPI.Timezone)
|
||||||
|
if err != nil {
|
||||||
|
return app.ComparisonRequest{}, commonOptions{}, err
|
||||||
|
}
|
||||||
|
workingDir, err := r.workingDir()
|
||||||
|
if err != nil {
|
||||||
|
return app.ComparisonRequest{}, commonOptions{}, err
|
||||||
|
}
|
||||||
|
outputDir, err := resolveOutputOverride(workingDir, opts.OutputDir)
|
||||||
|
if err != nil {
|
||||||
|
return app.ComparisonRequest{}, commonOptions{}, err
|
||||||
|
}
|
||||||
|
|
||||||
|
req := app.ComparisonRequest{
|
||||||
|
Config: cfg, Report: reportKind, ProfileIDs: append([]string(nil), opts.ProfileIDs...),
|
||||||
|
WorkingDir: workingDir, OutputDir: outputDir, Replace: opts.Replace, LLMDebugDir: opts.LLMDebugDir, Clock: r.Clock,
|
||||||
|
}
|
||||||
|
switch reportKind {
|
||||||
|
case app.ReportDaily:
|
||||||
|
if opts.Date == "" {
|
||||||
|
return app.ComparisonRequest{}, commonOptions{}, fmt.Errorf("compare daily requires --date YYYY-MM-DD")
|
||||||
|
}
|
||||||
|
req.Date, err = timeutil.ParseLocalDate(opts.Date, location)
|
||||||
|
case app.ReportToday:
|
||||||
|
if opts.Date == "" {
|
||||||
|
req.Date = timeutil.LocalDate(r.Clock.Now(), location)
|
||||||
|
} else {
|
||||||
|
req.Date, err = timeutil.ParseLocalDate(opts.Date, location)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return app.ComparisonRequest{}, commonOptions{}, err
|
||||||
|
}
|
||||||
|
|
||||||
|
promptkitConfig := cfg.Promptkit
|
||||||
|
promptkitConfig.Profile = ""
|
||||||
|
executor, err := r.promptExecutor(promptkitConfig)
|
||||||
|
if err != nil {
|
||||||
|
return app.ComparisonRequest{}, commonOptions{}, err
|
||||||
|
}
|
||||||
|
req.Executor = executor
|
||||||
|
return req, opts.commonOptions, nil
|
||||||
|
}
|
||||||
|
|
||||||
func (r Runner) resolveRunAction(args []string) (app.BatchRequest, commonOptions, error) {
|
func (r Runner) resolveRunAction(args []string) (app.BatchRequest, commonOptions, error) {
|
||||||
if r.Clock == nil {
|
if r.Clock == nil {
|
||||||
r.Clock = timeutil.SystemClock{}
|
r.Clock = timeutil.SystemClock{}
|
||||||
@@ -288,6 +402,27 @@ func parseRunFlags(args []string) (commonOptions, error) {
|
|||||||
return opts, nil
|
return opts, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func parseComparisonFlags(report app.ReportKind, args []string) (comparisonOptions, error) {
|
||||||
|
fs := flag.NewFlagSet("compare "+string(report), flag.ContinueOnError)
|
||||||
|
fs.SetOutput(io.Discard)
|
||||||
|
opts := comparisonOptions{}
|
||||||
|
addCommonFlags(fs, &opts.commonOptions, false)
|
||||||
|
fs.StringVar(&opts.OutputDir, "out-dir", "", "comparison bundle directory")
|
||||||
|
fs.BoolVar(&opts.Replace, "replace", false, "replace a recognized comparison bundle")
|
||||||
|
fs.BoolVar(&opts.Quiet, "quiet", false, "suppress successful action output")
|
||||||
|
fs.Var(&opts.ProfileIDs, "profile", "prompt profile ID")
|
||||||
|
if report == app.ReportDaily || report == app.ReportToday {
|
||||||
|
fs.StringVar(&opts.Date, "date", "", "report date in YYYY-MM-DD")
|
||||||
|
}
|
||||||
|
if err := fs.Parse(args); err != nil {
|
||||||
|
return comparisonOptions{}, err
|
||||||
|
}
|
||||||
|
if fs.NArg() > 0 {
|
||||||
|
return comparisonOptions{}, fmt.Errorf("unexpected argument %q", fs.Arg(0))
|
||||||
|
}
|
||||||
|
return opts, nil
|
||||||
|
}
|
||||||
|
|
||||||
func addCommonFlags(fs *flag.FlagSet, opts *commonOptions, includeOutput bool) {
|
func addCommonFlags(fs *flag.FlagSet, opts *commonOptions, includeOutput bool) {
|
||||||
fs.StringVar(&opts.ConfigPath, "config", "", "configuration file path")
|
fs.StringVar(&opts.ConfigPath, "config", "", "configuration file path")
|
||||||
fs.StringVar(&opts.Units, "units", "", "weather API units")
|
fs.StringVar(&opts.Units, "units", "", "weather API units")
|
||||||
|
|||||||
385
internal/comparison/comparison.go
Normal file
385
internal/comparison/comparison.go
Normal file
@@ -0,0 +1,385 @@
|
|||||||
|
// Package comparison owns logical profile-comparison bundle contracts and
|
||||||
|
// guarded filesystem destination planning and publication.
|
||||||
|
package comparison
|
||||||
|
|
||||||
|
import (
|
||||||
|
"crypto/sha256"
|
||||||
|
"encoding/hex"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"path"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
"unicode/utf8"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
// SchemaVersion identifies the supported comparison manifest schema.
|
||||||
|
SchemaVersion = "weatherreporter.comparison.v1"
|
||||||
|
|
||||||
|
// ManifestFilename is the canonical name of a comparison manifest.
|
||||||
|
ManifestFilename = "comparison.json"
|
||||||
|
// DataPackageFilename is the canonical name of the shared prompt input.
|
||||||
|
DataPackageFilename = "data-package.yml"
|
||||||
|
|
||||||
|
// StatusSucceeded identifies a profile that produced a validated report.
|
||||||
|
StatusSucceeded = "succeeded"
|
||||||
|
// StatusFailed identifies a profile that did not produce a report.
|
||||||
|
StatusFailed = "failed"
|
||||||
|
|
||||||
|
maxProfileSlugBytes = 64
|
||||||
|
maxErrorMessageBytes = 1024
|
||||||
|
)
|
||||||
|
|
||||||
|
// Manifest is the versioned, authoritative index of a comparison bundle.
|
||||||
|
// Field declaration order is the JSON field order.
|
||||||
|
type Manifest struct {
|
||||||
|
SchemaVersion string `json:"schemaVersion"`
|
||||||
|
ComparisonID string `json:"comparisonId"`
|
||||||
|
StartedAt time.Time `json:"startedAt"`
|
||||||
|
FinishedAt time.Time `json:"finishedAt"`
|
||||||
|
ReportID string `json:"reportId"`
|
||||||
|
ValidPeriod ValidPeriod `json:"validPeriod"`
|
||||||
|
Timezone string `json:"timezone"`
|
||||||
|
PromptID string `json:"promptId"`
|
||||||
|
PromptVersion string `json:"promptVersion"`
|
||||||
|
PromptHash string `json:"promptHash"`
|
||||||
|
DataPackage DataPackageReference `json:"dataPackage"`
|
||||||
|
Total int `json:"total"`
|
||||||
|
Succeeded int `json:"succeeded"`
|
||||||
|
Failed int `json:"failed"`
|
||||||
|
Results []Result `json:"results"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// ValidPeriod records the report's resolved half-open period.
|
||||||
|
type ValidPeriod struct {
|
||||||
|
Start time.Time `json:"start"`
|
||||||
|
End time.Time `json:"end"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// DataPackageReference identifies and verifies the shared prompt input.
|
||||||
|
type DataPackageReference struct {
|
||||||
|
Path string `json:"path"`
|
||||||
|
SHA256 string `json:"sha256"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Result records one explicitly selected profile in selection order.
|
||||||
|
type Result struct {
|
||||||
|
Position int `json:"position"`
|
||||||
|
ProfileID string `json:"profileId"`
|
||||||
|
BackendID string `json:"backendId,omitempty"`
|
||||||
|
ModelName string `json:"modelName"`
|
||||||
|
Status string `json:"status"`
|
||||||
|
ValidationStatus string `json:"validationStatus,omitempty"`
|
||||||
|
ReportPath string `json:"reportPath,omitempty"`
|
||||||
|
Error *SafeError `json:"error,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// SafeError contains bounded, operator-safe failure information only.
|
||||||
|
type SafeError struct {
|
||||||
|
Category string `json:"category"`
|
||||||
|
Message string `json:"message"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// LogicalBundle contains every byte required to publish a comparison without
|
||||||
|
// coupling the comparison contract to a filesystem implementation.
|
||||||
|
type LogicalBundle struct {
|
||||||
|
Manifest Manifest
|
||||||
|
DataPackage []byte
|
||||||
|
Reports []BundleReport
|
||||||
|
}
|
||||||
|
|
||||||
|
// BundleReport is one rendered Markdown document in a logical bundle.
|
||||||
|
type BundleReport struct {
|
||||||
|
Position int
|
||||||
|
Path string
|
||||||
|
Markdown []byte
|
||||||
|
}
|
||||||
|
|
||||||
|
// ValidateProfileIDs requires at least two distinct, nonblank profile IDs. It
|
||||||
|
// intentionally preserves accepted IDs unchanged and treats case distinctly.
|
||||||
|
func ValidateProfileIDs(profileIDs []string) error {
|
||||||
|
if len(profileIDs) < 2 {
|
||||||
|
return fmt.Errorf("comparison requires at least two profile IDs")
|
||||||
|
}
|
||||||
|
|
||||||
|
seen := make(map[string]struct{}, len(profileIDs))
|
||||||
|
for _, profileID := range profileIDs {
|
||||||
|
if strings.TrimSpace(profileID) == "" {
|
||||||
|
return fmt.Errorf("profile ID must not be blank")
|
||||||
|
}
|
||||||
|
if _, duplicate := seen[profileID]; duplicate {
|
||||||
|
return fmt.Errorf("duplicate profile ID %q", profileID)
|
||||||
|
}
|
||||||
|
seen[profileID] = struct{}{}
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// ProfileSlug returns a deterministic, filesystem-safe representation of a
|
||||||
|
// logical profile ID. The logical ID remains authoritative in the manifest.
|
||||||
|
func ProfileSlug(profileID string) string {
|
||||||
|
var builder strings.Builder
|
||||||
|
lastReplacement := false
|
||||||
|
for _, r := range profileID {
|
||||||
|
if isSlugRune(r) {
|
||||||
|
builder.WriteRune(r)
|
||||||
|
lastReplacement = false
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if !lastReplacement {
|
||||||
|
builder.WriteByte('-')
|
||||||
|
lastReplacement = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
slug := strings.Trim(builder.String(), "-_")
|
||||||
|
if len(slug) > maxProfileSlugBytes {
|
||||||
|
slug = strings.Trim(slug[:maxProfileSlugBytes], "-_")
|
||||||
|
}
|
||||||
|
if slug == "" {
|
||||||
|
return "profile"
|
||||||
|
}
|
||||||
|
return slug
|
||||||
|
}
|
||||||
|
|
||||||
|
func isSlugRune(r rune) bool {
|
||||||
|
return r >= 'a' && r <= 'z' ||
|
||||||
|
r >= 'A' && r <= 'Z' ||
|
||||||
|
r >= '0' && r <= '9' ||
|
||||||
|
r == '-' || r == '_'
|
||||||
|
}
|
||||||
|
|
||||||
|
// OrdinalWidth returns the zero-padding width for a comparison of count
|
||||||
|
// profiles.
|
||||||
|
func OrdinalWidth(count int) int {
|
||||||
|
width := 2
|
||||||
|
for value := count; value >= 100; value /= 10 {
|
||||||
|
width++
|
||||||
|
}
|
||||||
|
return width
|
||||||
|
}
|
||||||
|
|
||||||
|
// ReportFilename derives a report's deterministic bundle filename.
|
||||||
|
func ReportFilename(position, profileCount int, profileID string) (string, error) {
|
||||||
|
if profileCount < 1 {
|
||||||
|
return "", fmt.Errorf("profile count must be positive")
|
||||||
|
}
|
||||||
|
if position < 1 || position > profileCount {
|
||||||
|
return "", fmt.Errorf("profile position %d is outside 1..%d", position, profileCount)
|
||||||
|
}
|
||||||
|
return fmt.Sprintf("%0*d-%s.md", OrdinalWidth(profileCount), position, ProfileSlug(profileID)), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// BuildComparisonID derives the stable comparison identity for a resolved
|
||||||
|
// report run.
|
||||||
|
func BuildComparisonID(reportRunID string) (string, error) {
|
||||||
|
if strings.TrimSpace(reportRunID) == "" {
|
||||||
|
return "", fmt.Errorf("report run ID must not be blank")
|
||||||
|
}
|
||||||
|
return "comparison_" + reportRunID, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// DefaultDirectoryName derives the bundle directory name for a resolved report
|
||||||
|
// output filename.
|
||||||
|
func DefaultDirectoryName(reportOutputName string) (string, error) {
|
||||||
|
if !strings.HasSuffix(reportOutputName, ".md") {
|
||||||
|
return "", fmt.Errorf("report output name %q must end in .md", reportOutputName)
|
||||||
|
}
|
||||||
|
if !isArtifactBasename(reportOutputName) {
|
||||||
|
return "", fmt.Errorf("report output name %q must be a basename", reportOutputName)
|
||||||
|
}
|
||||||
|
stem := strings.TrimSuffix(reportOutputName, ".md")
|
||||||
|
if stem == "" {
|
||||||
|
return "", fmt.Errorf("report output name %q has an empty stem", reportOutputName)
|
||||||
|
}
|
||||||
|
return "comparison-" + stem, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// SHA256 returns a lowercase hexadecimal SHA-256 digest.
|
||||||
|
func SHA256(content []byte) string {
|
||||||
|
digest := sha256.Sum256(content)
|
||||||
|
return hex.EncodeToString(digest[:])
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewSafeError returns bounded, valid UTF-8 error information suitable for a
|
||||||
|
// comparison manifest.
|
||||||
|
func NewSafeError(category, message string) SafeError {
|
||||||
|
return SafeError{Category: category, Message: TruncateErrorMessage(message)}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TruncateErrorMessage returns a valid UTF-8 message of at most 1,024 bytes.
|
||||||
|
func TruncateErrorMessage(message string) string {
|
||||||
|
message = strings.ToValidUTF8(message, "\uFFFD")
|
||||||
|
if len(message) <= maxErrorMessageBytes {
|
||||||
|
return message
|
||||||
|
}
|
||||||
|
|
||||||
|
end := maxErrorMessageBytes
|
||||||
|
for end > 0 && !utf8.RuneStart(message[end]) {
|
||||||
|
end--
|
||||||
|
}
|
||||||
|
return message[:end]
|
||||||
|
}
|
||||||
|
|
||||||
|
// Validate checks every invariant required for a current comparison manifest.
|
||||||
|
func (manifest Manifest) Validate() error {
|
||||||
|
if manifest.SchemaVersion != SchemaVersion {
|
||||||
|
return fmt.Errorf("unsupported comparison schema version %q", manifest.SchemaVersion)
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(manifest.ComparisonID) == "" {
|
||||||
|
return fmt.Errorf("comparison ID must not be blank")
|
||||||
|
}
|
||||||
|
if manifest.StartedAt.IsZero() || manifest.FinishedAt.IsZero() {
|
||||||
|
return fmt.Errorf("comparison timestamps must be nonzero")
|
||||||
|
}
|
||||||
|
if manifest.StartedAt.Location() != time.UTC || manifest.FinishedAt.Location() != time.UTC {
|
||||||
|
return fmt.Errorf("comparison timestamps must use UTC")
|
||||||
|
}
|
||||||
|
if manifest.FinishedAt.Before(manifest.StartedAt) {
|
||||||
|
return fmt.Errorf("comparison finish time precedes start time")
|
||||||
|
}
|
||||||
|
if manifest.ValidPeriod.Start.IsZero() || manifest.ValidPeriod.End.IsZero() || !manifest.ValidPeriod.End.After(manifest.ValidPeriod.Start) {
|
||||||
|
return fmt.Errorf("valid period must have a nonempty increasing range")
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(manifest.ReportID) == "" || strings.TrimSpace(manifest.Timezone) == "" {
|
||||||
|
return fmt.Errorf("report ID and timezone must not be blank")
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(manifest.PromptID) == "" || strings.TrimSpace(manifest.PromptVersion) == "" || !isSHA256(manifest.PromptHash) {
|
||||||
|
return fmt.Errorf("prompt identity is invalid")
|
||||||
|
}
|
||||||
|
if manifest.DataPackage.Path != DataPackageFilename || !isSHA256(manifest.DataPackage.SHA256) {
|
||||||
|
return fmt.Errorf("data package reference is invalid")
|
||||||
|
}
|
||||||
|
if manifest.Total < 2 || manifest.Total != len(manifest.Results) {
|
||||||
|
return fmt.Errorf("comparison result count is invalid")
|
||||||
|
}
|
||||||
|
if manifest.Succeeded < 0 || manifest.Failed < 0 || manifest.Succeeded+manifest.Failed != manifest.Total {
|
||||||
|
return fmt.Errorf("comparison result totals are inconsistent")
|
||||||
|
}
|
||||||
|
|
||||||
|
profiles := make(map[string]struct{}, len(manifest.Results))
|
||||||
|
reportPaths := make(map[string]struct{}, len(manifest.Results))
|
||||||
|
succeeded, failed := 0, 0
|
||||||
|
for index, result := range manifest.Results {
|
||||||
|
if result.Position != index+1 {
|
||||||
|
return fmt.Errorf("result position %d is not ordered", result.Position)
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(result.ProfileID) == "" {
|
||||||
|
return fmt.Errorf("result %d has a blank profile ID", result.Position)
|
||||||
|
}
|
||||||
|
if _, duplicate := profiles[result.ProfileID]; duplicate {
|
||||||
|
return fmt.Errorf("result %d duplicates profile ID %q", result.Position, result.ProfileID)
|
||||||
|
}
|
||||||
|
profiles[result.ProfileID] = struct{}{}
|
||||||
|
if strings.TrimSpace(result.ModelName) == "" {
|
||||||
|
return fmt.Errorf("result %d has a blank model name", result.Position)
|
||||||
|
}
|
||||||
|
|
||||||
|
switch result.Status {
|
||||||
|
case StatusSucceeded:
|
||||||
|
succeeded++
|
||||||
|
if result.ValidationStatus != "passed" {
|
||||||
|
return fmt.Errorf("successful result %d did not pass validation", result.Position)
|
||||||
|
}
|
||||||
|
if result.Error != nil || !isReportPath(result.ReportPath) {
|
||||||
|
return fmt.Errorf("successful result %d has invalid report details", result.Position)
|
||||||
|
}
|
||||||
|
if _, duplicate := reportPaths[result.ReportPath]; duplicate {
|
||||||
|
return fmt.Errorf("result %d duplicates report path %q", result.Position, result.ReportPath)
|
||||||
|
}
|
||||||
|
reportPaths[result.ReportPath] = struct{}{}
|
||||||
|
case StatusFailed:
|
||||||
|
failed++
|
||||||
|
if result.ReportPath != "" || result.Error == nil || !isValidationStatus(result.ValidationStatus) {
|
||||||
|
return fmt.Errorf("failed result %d has invalid failure details", result.Position)
|
||||||
|
}
|
||||||
|
if err := result.Error.validate(); err != nil {
|
||||||
|
return fmt.Errorf("failed result %d: %w", result.Position, err)
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
return fmt.Errorf("result %d has unsupported status %q", result.Position, result.Status)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if succeeded != manifest.Succeeded || failed != manifest.Failed {
|
||||||
|
return fmt.Errorf("comparison status counts do not match results")
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (safeError SafeError) validate() error {
|
||||||
|
if strings.TrimSpace(safeError.Category) == "" || strings.TrimSpace(safeError.Message) == "" {
|
||||||
|
return fmt.Errorf("safe error category and message must not be blank")
|
||||||
|
}
|
||||||
|
if !utf8.ValidString(safeError.Message) || len(safeError.Message) > maxErrorMessageBytes {
|
||||||
|
return fmt.Errorf("safe error message is not bounded UTF-8")
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func isValidationStatus(status string) bool {
|
||||||
|
return status == "" || status == "failed" || status == "skipped"
|
||||||
|
}
|
||||||
|
|
||||||
|
func isSHA256(value string) bool {
|
||||||
|
if len(value) != sha256.Size*2 {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
for _, r := range value {
|
||||||
|
if (r < '0' || r > '9') && (r < 'a' || r > 'f') {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
func isReportPath(value string) bool {
|
||||||
|
return strings.HasSuffix(value, ".md") && isArtifactBasename(value) && value != ManifestFilename && value != DataPackageFilename
|
||||||
|
}
|
||||||
|
|
||||||
|
func isArtifactBasename(value string) bool {
|
||||||
|
return value != "" && value != "." && value != ".." &&
|
||||||
|
!strings.ContainsAny(value, "/\\") && path.Base(value) == value
|
||||||
|
}
|
||||||
|
|
||||||
|
// Validate checks that the manifest and in-memory bundle payloads agree.
|
||||||
|
func (bundle LogicalBundle) Validate() error {
|
||||||
|
if err := bundle.Manifest.Validate(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if SHA256(bundle.DataPackage) != bundle.Manifest.DataPackage.SHA256 {
|
||||||
|
return fmt.Errorf("data package digest does not match manifest")
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(bundle.Reports) != bundle.Manifest.Succeeded {
|
||||||
|
return fmt.Errorf("bundle report count does not match manifest")
|
||||||
|
}
|
||||||
|
reportIndex := 0
|
||||||
|
for _, result := range bundle.Manifest.Results {
|
||||||
|
if result.Status != StatusSucceeded {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
report := bundle.Reports[reportIndex]
|
||||||
|
if report.Position != result.Position || report.Path != result.ReportPath || !isReportPath(report.Path) {
|
||||||
|
return fmt.Errorf("bundle report for result %d does not match manifest", result.Position)
|
||||||
|
}
|
||||||
|
reportIndex++
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// EncodeManifest validates and deterministically encodes a manifest as the
|
||||||
|
// canonical two-space-indented JSON document with one trailing newline.
|
||||||
|
func EncodeManifest(manifest Manifest) ([]byte, error) {
|
||||||
|
if err := manifest.Validate(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
encoded, err := json.MarshalIndent(manifest, "", " ")
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("encode comparison manifest: %w", err)
|
||||||
|
}
|
||||||
|
return append(encoded, '\n'), nil
|
||||||
|
}
|
||||||
314
internal/comparison/comparison_test.go
Normal file
314
internal/comparison/comparison_test.go
Normal file
@@ -0,0 +1,314 @@
|
|||||||
|
package comparison
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
"unicode/utf8"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestValidateProfileIDs(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
profileIDs []string
|
||||||
|
wantErr bool
|
||||||
|
}{
|
||||||
|
{name: "accepts exact case distinct IDs", profileIDs: []string{"weather-light", "Weather-Light"}},
|
||||||
|
{name: "preserves surrounding whitespace", profileIDs: []string{" weather-light", "weather-deep "}},
|
||||||
|
{name: "rejects one ID", profileIDs: []string{"weather-light"}, wantErr: true},
|
||||||
|
{name: "rejects blank ID", profileIDs: []string{"weather-light", " \t\n"}, wantErr: true},
|
||||||
|
{name: "rejects exact duplicate", profileIDs: []string{"weather-light", "weather-light"}, wantErr: true},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, test := range tests {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
err := ValidateProfileIDs(test.profileIDs)
|
||||||
|
if (err != nil) != test.wantErr {
|
||||||
|
t.Fatalf("ValidateProfileIDs(%q) error = %v, want error %t", test.profileIDs, err, test.wantErr)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestProfileSlug(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
long := strings.Repeat("a", 62) + "-_more"
|
||||||
|
tests := []struct {
|
||||||
|
profileID string
|
||||||
|
want string
|
||||||
|
}{
|
||||||
|
{profileID: "weather-light", want: "weather-light"},
|
||||||
|
{profileID: "model.v1", want: "model-v1"},
|
||||||
|
{profileID: "nested/path\\name", want: "nested-path-name"},
|
||||||
|
{profileID: " weather\tlight\n", want: "weather-light"},
|
||||||
|
{profileID: "\x00\x01", want: "profile"},
|
||||||
|
{profileID: "météo-東京", want: "m-t-o"},
|
||||||
|
{profileID: "___", want: "profile"},
|
||||||
|
{profileID: long, want: strings.Repeat("a", 62)},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, test := range tests {
|
||||||
|
t.Run(test.profileID, func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
if got := ProfileSlug(test.profileID); got != test.want {
|
||||||
|
t.Fatalf("ProfileSlug(%q) = %q, want %q", test.profileID, got, test.want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestReportNaming(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
if got := OrdinalWidth(99); got != 2 {
|
||||||
|
t.Fatalf("OrdinalWidth(99) = %d, want 2", got)
|
||||||
|
}
|
||||||
|
if got := OrdinalWidth(100); got != 3 {
|
||||||
|
t.Fatalf("OrdinalWidth(100) = %d, want 3", got)
|
||||||
|
}
|
||||||
|
if got, err := ReportFilename(1, 3, "weather.light"); err != nil || got != "01-weather-light.md" {
|
||||||
|
t.Fatalf("ReportFilename() = %q, %v, want %q, nil", got, err, "01-weather-light.md")
|
||||||
|
}
|
||||||
|
if got, err := ReportFilename(100, 100, "weather-light"); err != nil || got != "100-weather-light.md" {
|
||||||
|
t.Fatalf("ReportFilename() = %q, %v, want %q, nil", got, err, "100-weather-light.md")
|
||||||
|
}
|
||||||
|
first, err := ReportFilename(1, 2, "model.v1")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("first ReportFilename() error = %v", err)
|
||||||
|
}
|
||||||
|
second, err := ReportFilename(2, 2, "model/v1")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("second ReportFilename() error = %v", err)
|
||||||
|
}
|
||||||
|
if first == second {
|
||||||
|
t.Fatalf("normalized profile collisions produced the same filename %q", first)
|
||||||
|
}
|
||||||
|
if _, err := ReportFilename(0, 2, "weather-light"); err == nil {
|
||||||
|
t.Fatal("ReportFilename accepted position zero")
|
||||||
|
}
|
||||||
|
if _, err := ReportFilename(1, 0, "weather-light"); err == nil {
|
||||||
|
t.Fatal("ReportFilename accepted zero profile count")
|
||||||
|
}
|
||||||
|
if got, err := BuildComparisonID("daily-2026-08-24"); err != nil || got != "comparison_daily-2026-08-24" {
|
||||||
|
t.Fatalf("BuildComparisonID() = %q, %v", got, err)
|
||||||
|
}
|
||||||
|
if _, err := BuildComparisonID(" \t"); err == nil {
|
||||||
|
t.Fatal("BuildComparisonID accepted blank run ID")
|
||||||
|
}
|
||||||
|
if got, err := DefaultDirectoryName("daily-2026-08-24.md"); err != nil || got != "comparison-daily-2026-08-24" {
|
||||||
|
t.Fatalf("DefaultDirectoryName() = %q, %v", got, err)
|
||||||
|
}
|
||||||
|
for _, name := range []string{"daily.txt", "nested/daily.md", ".md"} {
|
||||||
|
if _, err := DefaultDirectoryName(name); err == nil {
|
||||||
|
t.Fatalf("DefaultDirectoryName(%q) accepted invalid name", name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestManifestEncodingAndRoundTrip(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
manifest := validManifest()
|
||||||
|
encoded, err := EncodeManifest(manifest)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("EncodeManifest() error = %v", err)
|
||||||
|
}
|
||||||
|
want := "{\n" +
|
||||||
|
" \"schemaVersion\": \"weatherreporter.comparison.v1\",\n" +
|
||||||
|
" \"comparisonId\": \"comparison_daily-2026-08-24\",\n" +
|
||||||
|
" \"startedAt\": \"2026-08-24T12:00:00Z\",\n" +
|
||||||
|
" \"finishedAt\": \"2026-08-24T12:01:00Z\",\n" +
|
||||||
|
" \"reportId\": \"daily\",\n" +
|
||||||
|
" \"validPeriod\": {\n" +
|
||||||
|
" \"start\": \"2026-08-24T00:00:00-04:00\",\n" +
|
||||||
|
" \"end\": \"2026-08-25T00:00:00-04:00\"\n" +
|
||||||
|
" },\n" +
|
||||||
|
" \"timezone\": \"America/New_York\",\n" +
|
||||||
|
" \"promptId\": \"daily-report\",\n" +
|
||||||
|
" \"promptVersion\": \"2026-08-01\",\n" +
|
||||||
|
" \"promptHash\": \"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\",\n" +
|
||||||
|
" \"dataPackage\": {\n" +
|
||||||
|
" \"path\": \"data-package.yml\",\n" +
|
||||||
|
" \"sha256\": \"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\"\n" +
|
||||||
|
" },\n" +
|
||||||
|
" \"total\": 2,\n" +
|
||||||
|
" \"succeeded\": 1,\n" +
|
||||||
|
" \"failed\": 1,\n" +
|
||||||
|
" \"results\": [\n" +
|
||||||
|
" {\n" +
|
||||||
|
" \"position\": 1,\n" +
|
||||||
|
" \"profileId\": \"weather-light\",\n" +
|
||||||
|
" \"backendId\": \"openai\",\n" +
|
||||||
|
" \"modelName\": \"gpt-5-mini\",\n" +
|
||||||
|
" \"status\": \"succeeded\",\n" +
|
||||||
|
" \"validationStatus\": \"passed\",\n" +
|
||||||
|
" \"reportPath\": \"01-weather-light.md\"\n" +
|
||||||
|
" },\n" +
|
||||||
|
" {\n" +
|
||||||
|
" \"position\": 2,\n" +
|
||||||
|
" \"profileId\": \"weather-deep\",\n" +
|
||||||
|
" \"modelName\": \"gpt-5\",\n" +
|
||||||
|
" \"status\": \"failed\",\n" +
|
||||||
|
" \"error\": {\n" +
|
||||||
|
" \"category\": \"application\",\n" +
|
||||||
|
" \"message\": \"generated text was rejected\"\n" +
|
||||||
|
" }\n" +
|
||||||
|
" }\n" +
|
||||||
|
" ]\n" +
|
||||||
|
"}\n"
|
||||||
|
if string(encoded) != want {
|
||||||
|
t.Fatalf("EncodeManifest() =\n%s\nwant\n%s", encoded, want)
|
||||||
|
}
|
||||||
|
|
||||||
|
var decoded Manifest
|
||||||
|
if err := json.Unmarshal(encoded, &decoded); err != nil {
|
||||||
|
t.Fatalf("json.Unmarshal() error = %v", err)
|
||||||
|
}
|
||||||
|
if err := decoded.Validate(); err != nil {
|
||||||
|
t.Fatalf("decoded manifest validation error = %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestManifestValidateRejectsInvariants(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
mutate func(*Manifest)
|
||||||
|
}{
|
||||||
|
{name: "schema version", mutate: func(manifest *Manifest) { manifest.SchemaVersion = "v0" }},
|
||||||
|
{name: "non UTC timestamp", mutate: func(manifest *Manifest) { manifest.StartedAt = manifest.StartedAt.In(time.FixedZone("UTC", 0)) }},
|
||||||
|
{name: "reversed timestamps", mutate: func(manifest *Manifest) { manifest.FinishedAt = manifest.StartedAt.Add(-time.Second) }},
|
||||||
|
{name: "empty valid period", mutate: func(manifest *Manifest) { manifest.ValidPeriod.End = manifest.ValidPeriod.Start }},
|
||||||
|
{name: "bad prompt hash", mutate: func(manifest *Manifest) { manifest.PromptHash = "ABC" }},
|
||||||
|
{name: "bad data package path", mutate: func(manifest *Manifest) { manifest.DataPackage.Path = "nested/data-package.yml" }},
|
||||||
|
{name: "inconsistent totals", mutate: func(manifest *Manifest) { manifest.Succeeded = 2 }},
|
||||||
|
{name: "unordered position", mutate: func(manifest *Manifest) { manifest.Results[1].Position = 3 }},
|
||||||
|
{name: "duplicate profile", mutate: func(manifest *Manifest) { manifest.Results[1].ProfileID = manifest.Results[0].ProfileID }},
|
||||||
|
{name: "unsupported status", mutate: func(manifest *Manifest) { manifest.Results[1].Status = "skipped" }},
|
||||||
|
{name: "successful result without report", mutate: func(manifest *Manifest) { manifest.Results[0].ReportPath = "" }},
|
||||||
|
{name: "successful result with error", mutate: func(manifest *Manifest) {
|
||||||
|
manifest.Results[0].Error = &SafeError{Category: "application", Message: "bad"}
|
||||||
|
}},
|
||||||
|
{name: "successful result without passed validation", mutate: func(manifest *Manifest) { manifest.Results[0].ValidationStatus = "failed" }},
|
||||||
|
{name: "failed result with report", mutate: func(manifest *Manifest) { manifest.Results[1].ReportPath = "02-weather-deep.md" }},
|
||||||
|
{name: "failed result without error", mutate: func(manifest *Manifest) { manifest.Results[1].Error = nil }},
|
||||||
|
{name: "traversal report path", mutate: func(manifest *Manifest) { manifest.Results[0].ReportPath = "../report.md" }},
|
||||||
|
{name: "duplicate report path", mutate: func(manifest *Manifest) {
|
||||||
|
manifest.Results[1] = Result{Position: 2, ProfileID: "weather-deep", ModelName: "gpt-5", Status: StatusSucceeded, ValidationStatus: "passed", ReportPath: manifest.Results[0].ReportPath}
|
||||||
|
manifest.Succeeded = 2
|
||||||
|
manifest.Failed = 0
|
||||||
|
}},
|
||||||
|
{name: "oversized error", mutate: func(manifest *Manifest) { manifest.Results[1].Error.Message = strings.Repeat("x", 1025) }},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, test := range tests {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
manifest := validManifest()
|
||||||
|
test.mutate(&manifest)
|
||||||
|
if err := manifest.Validate(); err == nil {
|
||||||
|
t.Fatal("Manifest.Validate() succeeded for invalid manifest")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLogicalBundleValidate(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
dataPackage := []byte("report: daily\n")
|
||||||
|
manifest := validManifest()
|
||||||
|
manifest.DataPackage.SHA256 = SHA256(dataPackage)
|
||||||
|
bundle := LogicalBundle{
|
||||||
|
Manifest: manifest,
|
||||||
|
DataPackage: dataPackage,
|
||||||
|
Reports: []BundleReport{{
|
||||||
|
Position: 1,
|
||||||
|
Path: "01-weather-light.md",
|
||||||
|
Markdown: []byte("# Daily\n"),
|
||||||
|
}},
|
||||||
|
}
|
||||||
|
if err := bundle.Validate(); err != nil {
|
||||||
|
t.Fatalf("LogicalBundle.Validate() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
bundle.Reports[0].Path = "other.md"
|
||||||
|
if err := bundle.Validate(); err == nil {
|
||||||
|
t.Fatal("LogicalBundle.Validate() accepted mismatched report path")
|
||||||
|
}
|
||||||
|
bundle.Reports[0].Path = "01-weather-light.md"
|
||||||
|
bundle.Reports[0].Position = 2
|
||||||
|
if err := bundle.Validate(); err == nil {
|
||||||
|
t.Fatal("LogicalBundle.Validate() accepted unordered report position")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSHA256AndTruncateErrorMessage(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
if got, want := SHA256([]byte("weather")), "e5e72beb4e3c6926d3dc9e3e2ef7833ba50cd919c2460a782b244fd071e920de"; got != want {
|
||||||
|
t.Fatalf("SHA256() = %q, want %q", got, want)
|
||||||
|
}
|
||||||
|
message := strings.Repeat("€", 400)
|
||||||
|
got := TruncateErrorMessage(message)
|
||||||
|
if len(got) > 1024 || !utf8.ValidString(got) {
|
||||||
|
t.Fatalf("TruncateErrorMessage() returned %d bytes of valid UTF-8 = %t", len(got), utf8.ValidString(got))
|
||||||
|
}
|
||||||
|
if want := strings.Repeat("€", 341); got != want {
|
||||||
|
t.Fatalf("TruncateErrorMessage() = %q, want %q", got, want)
|
||||||
|
}
|
||||||
|
invalid := string([]byte{'x', 0xff, 'y'})
|
||||||
|
if got := TruncateErrorMessage(invalid); !utf8.ValidString(got) {
|
||||||
|
t.Fatal("TruncateErrorMessage() retained invalid UTF-8")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func validManifest() Manifest {
|
||||||
|
newYork := time.FixedZone("-0400", -4*60*60)
|
||||||
|
return Manifest{
|
||||||
|
SchemaVersion: SchemaVersion,
|
||||||
|
ComparisonID: "comparison_daily-2026-08-24",
|
||||||
|
StartedAt: time.Date(2026, time.August, 24, 12, 0, 0, 0, time.UTC),
|
||||||
|
FinishedAt: time.Date(2026, time.August, 24, 12, 1, 0, 0, time.UTC),
|
||||||
|
ReportID: "daily",
|
||||||
|
ValidPeriod: ValidPeriod{
|
||||||
|
Start: time.Date(2026, time.August, 24, 0, 0, 0, 0, newYork),
|
||||||
|
End: time.Date(2026, time.August, 25, 0, 0, 0, 0, newYork),
|
||||||
|
},
|
||||||
|
Timezone: "America/New_York",
|
||||||
|
PromptID: "daily-report",
|
||||||
|
PromptVersion: "2026-08-01",
|
||||||
|
PromptHash: strings.Repeat("a", 64),
|
||||||
|
DataPackage: DataPackageReference{
|
||||||
|
Path: DataPackageFilename,
|
||||||
|
SHA256: strings.Repeat("a", 64),
|
||||||
|
},
|
||||||
|
Total: 2,
|
||||||
|
Succeeded: 1,
|
||||||
|
Failed: 1,
|
||||||
|
Results: []Result{
|
||||||
|
{
|
||||||
|
Position: 1,
|
||||||
|
ProfileID: "weather-light",
|
||||||
|
BackendID: "openai",
|
||||||
|
ModelName: "gpt-5-mini",
|
||||||
|
Status: StatusSucceeded,
|
||||||
|
ValidationStatus: "passed",
|
||||||
|
ReportPath: "01-weather-light.md",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Position: 2,
|
||||||
|
ProfileID: "weather-deep",
|
||||||
|
ModelName: "gpt-5",
|
||||||
|
Status: StatusFailed,
|
||||||
|
Error: &SafeError{Category: "application", Message: "generated text was rejected"},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
460
internal/comparison/publish.go
Normal file
460
internal/comparison/publish.go
Normal file
@@ -0,0 +1,460 @@
|
|||||||
|
package comparison
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
// DestinationErrorKind identifies a comparison destination safety failure.
|
||||||
|
type DestinationErrorKind string
|
||||||
|
|
||||||
|
const (
|
||||||
|
DestinationInvalidPath DestinationErrorKind = "invalid_path"
|
||||||
|
DestinationFilesystemRoot DestinationErrorKind = "filesystem_root"
|
||||||
|
DestinationWorkingDirectory DestinationErrorKind = "working_directory"
|
||||||
|
DestinationSymlink DestinationErrorKind = "symlink"
|
||||||
|
DestinationNotDirectory DestinationErrorKind = "not_directory"
|
||||||
|
DestinationNotEmpty DestinationErrorKind = "not_empty"
|
||||||
|
DestinationUnrecognized DestinationErrorKind = "unrecognized"
|
||||||
|
DestinationInspection DestinationErrorKind = "inspection"
|
||||||
|
)
|
||||||
|
|
||||||
|
// DestinationError provides inspectable context without making filesystem
|
||||||
|
// implementation details part of a manifest or command-result schema.
|
||||||
|
type DestinationError struct {
|
||||||
|
Kind DestinationErrorKind
|
||||||
|
Target string
|
||||||
|
Err error
|
||||||
|
}
|
||||||
|
|
||||||
|
func (err *DestinationError) Error() string {
|
||||||
|
if err.Err == nil {
|
||||||
|
return fmt.Sprintf("comparison destination %q is %s", err.Target, err.Kind)
|
||||||
|
}
|
||||||
|
return fmt.Sprintf("comparison destination %q is %s: %v", err.Target, err.Kind, err.Err)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (err *DestinationError) Unwrap() error {
|
||||||
|
return err.Err
|
||||||
|
}
|
||||||
|
|
||||||
|
// DestinationPlan records a preflighted, exact bundle directory. Callers must
|
||||||
|
// pass it to Publish rather than recreating destination policy themselves.
|
||||||
|
type DestinationPlan struct {
|
||||||
|
WorkingDirectory string
|
||||||
|
Target string
|
||||||
|
Replace bool
|
||||||
|
state destinationState
|
||||||
|
}
|
||||||
|
|
||||||
|
type destinationState uint8
|
||||||
|
|
||||||
|
const (
|
||||||
|
destinationAbsent destinationState = iota
|
||||||
|
destinationEmpty
|
||||||
|
destinationBundle
|
||||||
|
)
|
||||||
|
|
||||||
|
// ErrUnrecognizedBundle marks a directory that is not a valid current-schema
|
||||||
|
// Weatherreporter comparison bundle.
|
||||||
|
var ErrUnrecognizedBundle = errors.New("unrecognized comparison bundle")
|
||||||
|
|
||||||
|
// PlanDestination performs the read-only comparison destination preflight.
|
||||||
|
func PlanDestination(workingDirectory, target string, replace bool) (DestinationPlan, error) {
|
||||||
|
workingDirectory, err := absoluteCleanPath(workingDirectory)
|
||||||
|
if err != nil {
|
||||||
|
return DestinationPlan{}, newDestinationError(DestinationInvalidPath, workingDirectory, err)
|
||||||
|
}
|
||||||
|
target, err = absoluteCleanPath(target)
|
||||||
|
if err != nil {
|
||||||
|
return DestinationPlan{}, newDestinationError(DestinationInvalidPath, target, err)
|
||||||
|
}
|
||||||
|
if filepath.Dir(target) == target {
|
||||||
|
return DestinationPlan{}, newDestinationError(DestinationFilesystemRoot, target, nil)
|
||||||
|
}
|
||||||
|
if target == workingDirectory {
|
||||||
|
return DestinationPlan{}, newDestinationError(DestinationWorkingDirectory, target, nil)
|
||||||
|
}
|
||||||
|
|
||||||
|
plan := DestinationPlan{WorkingDirectory: workingDirectory, Target: target, Replace: replace}
|
||||||
|
info, err := os.Lstat(target)
|
||||||
|
if err != nil {
|
||||||
|
if !errors.Is(err, os.ErrNotExist) {
|
||||||
|
return DestinationPlan{}, newDestinationError(DestinationInspection, target, err)
|
||||||
|
}
|
||||||
|
if err := inspectMissingDestinationParent(target); err != nil {
|
||||||
|
return DestinationPlan{}, err
|
||||||
|
}
|
||||||
|
return plan, nil
|
||||||
|
}
|
||||||
|
if info.Mode()&os.ModeSymlink != 0 {
|
||||||
|
return DestinationPlan{}, newDestinationError(DestinationSymlink, target, nil)
|
||||||
|
}
|
||||||
|
if !info.IsDir() {
|
||||||
|
return DestinationPlan{}, newDestinationError(DestinationNotDirectory, target, nil)
|
||||||
|
}
|
||||||
|
|
||||||
|
entries, err := os.ReadDir(target)
|
||||||
|
if err != nil {
|
||||||
|
return DestinationPlan{}, newDestinationError(DestinationInspection, target, err)
|
||||||
|
}
|
||||||
|
if len(entries) == 0 {
|
||||||
|
plan.state = destinationEmpty
|
||||||
|
return plan, nil
|
||||||
|
}
|
||||||
|
if !replace {
|
||||||
|
return DestinationPlan{}, newDestinationError(DestinationNotEmpty, target, nil)
|
||||||
|
}
|
||||||
|
if _, err := RecognizeBundle(target); err != nil {
|
||||||
|
return DestinationPlan{}, newDestinationError(DestinationUnrecognized, target, err)
|
||||||
|
}
|
||||||
|
plan.state = destinationBundle
|
||||||
|
return plan, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func absoluteCleanPath(value string) (string, error) {
|
||||||
|
if strings.TrimSpace(value) == "" {
|
||||||
|
return "", fmt.Errorf("path is required")
|
||||||
|
}
|
||||||
|
if !filepath.IsAbs(value) {
|
||||||
|
return "", fmt.Errorf("path %q must be absolute", value)
|
||||||
|
}
|
||||||
|
return filepath.Clean(value), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func inspectMissingDestinationParent(target string) error {
|
||||||
|
for parent := filepath.Dir(target); ; parent = filepath.Dir(parent) {
|
||||||
|
info, err := os.Lstat(parent)
|
||||||
|
if err == nil {
|
||||||
|
if info.Mode()&os.ModeSymlink != 0 {
|
||||||
|
resolved, statErr := os.Stat(parent)
|
||||||
|
if statErr != nil {
|
||||||
|
return newDestinationError(DestinationInspection, target, statErr)
|
||||||
|
}
|
||||||
|
if !resolved.IsDir() {
|
||||||
|
return newDestinationError(DestinationNotDirectory, target, nil)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if !info.IsDir() {
|
||||||
|
return newDestinationError(DestinationNotDirectory, target, nil)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if !errors.Is(err, os.ErrNotExist) {
|
||||||
|
return newDestinationError(DestinationInspection, target, err)
|
||||||
|
}
|
||||||
|
if filepath.Dir(parent) == parent {
|
||||||
|
return newDestinationError(DestinationInspection, target, fmt.Errorf("no existing directory ancestor"))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func newDestinationError(kind DestinationErrorKind, target string, err error) error {
|
||||||
|
return &DestinationError{Kind: kind, Target: target, Err: err}
|
||||||
|
}
|
||||||
|
|
||||||
|
// RecognizeBundle verifies that directory contains exactly one valid current
|
||||||
|
// comparison bundle. It never follows bundle entries through symlinks.
|
||||||
|
func RecognizeBundle(directory string) (Manifest, error) {
|
||||||
|
info, err := os.Lstat(directory)
|
||||||
|
if err != nil {
|
||||||
|
return Manifest{}, unrecognizedBundleError("inspect directory", err)
|
||||||
|
}
|
||||||
|
if info.Mode()&os.ModeSymlink != 0 || !info.IsDir() {
|
||||||
|
return Manifest{}, unrecognizedBundleError("directory is not a real directory", nil)
|
||||||
|
}
|
||||||
|
|
||||||
|
entries, err := os.ReadDir(directory)
|
||||||
|
if err != nil {
|
||||||
|
return Manifest{}, unrecognizedBundleError("read directory", err)
|
||||||
|
}
|
||||||
|
manifestData, err := readBundleFile(directory, ManifestFilename)
|
||||||
|
if err != nil {
|
||||||
|
return Manifest{}, err
|
||||||
|
}
|
||||||
|
manifest, err := decodeManifest(manifestData)
|
||||||
|
if err != nil {
|
||||||
|
return Manifest{}, err
|
||||||
|
}
|
||||||
|
if err := manifest.Validate(); err != nil {
|
||||||
|
return Manifest{}, unrecognizedBundleError("validate manifest", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
expected := map[string]struct{}{
|
||||||
|
ManifestFilename: {},
|
||||||
|
DataPackageFilename: {},
|
||||||
|
}
|
||||||
|
for _, result := range manifest.Results {
|
||||||
|
if result.Status == StatusSucceeded {
|
||||||
|
expected[result.ReportPath] = struct{}{}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(entries) != len(expected) {
|
||||||
|
return Manifest{}, unrecognizedBundleError("directory entries do not match manifest", nil)
|
||||||
|
}
|
||||||
|
for _, entry := range entries {
|
||||||
|
if _, ok := expected[entry.Name()]; !ok {
|
||||||
|
return Manifest{}, unrecognizedBundleError("directory has an undeclared entry", nil)
|
||||||
|
}
|
||||||
|
if _, err := readBundleFile(directory, entry.Name()); err != nil {
|
||||||
|
return Manifest{}, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
dataPackage, err := readBundleFile(directory, DataPackageFilename)
|
||||||
|
if err != nil {
|
||||||
|
return Manifest{}, err
|
||||||
|
}
|
||||||
|
if SHA256(dataPackage) != manifest.DataPackage.SHA256 {
|
||||||
|
return Manifest{}, unrecognizedBundleError("data package digest does not match manifest", nil)
|
||||||
|
}
|
||||||
|
return manifest, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func readBundleFile(directory, name string) ([]byte, error) {
|
||||||
|
if !isArtifactBasename(name) {
|
||||||
|
return nil, unrecognizedBundleError("bundle entry name is unsafe", nil)
|
||||||
|
}
|
||||||
|
filePath := filepath.Join(directory, name)
|
||||||
|
info, err := os.Lstat(filePath)
|
||||||
|
if err != nil {
|
||||||
|
return nil, unrecognizedBundleError("inspect bundle entry", err)
|
||||||
|
}
|
||||||
|
if info.Mode()&os.ModeSymlink != 0 || !info.Mode().IsRegular() {
|
||||||
|
return nil, unrecognizedBundleError("bundle entry is not a regular file", nil)
|
||||||
|
}
|
||||||
|
data, err := os.ReadFile(filePath)
|
||||||
|
if err != nil {
|
||||||
|
return nil, unrecognizedBundleError("read bundle entry", err)
|
||||||
|
}
|
||||||
|
return data, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func decodeManifest(data []byte) (Manifest, error) {
|
||||||
|
decoder := json.NewDecoder(bytes.NewReader(data))
|
||||||
|
decoder.DisallowUnknownFields()
|
||||||
|
var manifest Manifest
|
||||||
|
if err := decoder.Decode(&manifest); err != nil {
|
||||||
|
return Manifest{}, unrecognizedBundleError("decode manifest", err)
|
||||||
|
}
|
||||||
|
var trailing any
|
||||||
|
if err := decoder.Decode(&trailing); err != io.EOF {
|
||||||
|
if err == nil {
|
||||||
|
return Manifest{}, unrecognizedBundleError("manifest has multiple JSON values", nil)
|
||||||
|
}
|
||||||
|
return Manifest{}, unrecognizedBundleError("decode manifest", err)
|
||||||
|
}
|
||||||
|
return manifest, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func unrecognizedBundleError(action string, err error) error {
|
||||||
|
if err == nil {
|
||||||
|
return fmt.Errorf("%w: %s", ErrUnrecognizedBundle, action)
|
||||||
|
}
|
||||||
|
return fmt.Errorf("%w: %s: %v", ErrUnrecognizedBundle, action, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// PublicationResult describes the durable state of a publication attempt.
|
||||||
|
type PublicationResult struct {
|
||||||
|
Committed bool
|
||||||
|
RetainedBackupPath string
|
||||||
|
}
|
||||||
|
|
||||||
|
// PublicationCleanupError reports that a committed bundle could not remove its
|
||||||
|
// prior sibling backup. The new bundle remains installed and the backup path
|
||||||
|
// is retained for operator recovery.
|
||||||
|
type PublicationCleanupError struct {
|
||||||
|
RetainedBackupPath string
|
||||||
|
Err error
|
||||||
|
}
|
||||||
|
|
||||||
|
func (err *PublicationCleanupError) Error() string {
|
||||||
|
return fmt.Sprintf("remove comparison backup %q: %v", err.RetainedBackupPath, err.Err)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (err *PublicationCleanupError) Unwrap() error {
|
||||||
|
return err.Err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Publish writes a complete logical bundle through a private sibling directory
|
||||||
|
// and atomically installs it at a preflighted destination.
|
||||||
|
func Publish(ctx context.Context, plan DestinationPlan, bundle LogicalBundle) (PublicationResult, error) {
|
||||||
|
return publish(ctx, plan, bundle, publishOperations{rename: os.Rename, removeAll: os.RemoveAll})
|
||||||
|
}
|
||||||
|
|
||||||
|
type publishOperations struct {
|
||||||
|
rename func(string, string) error
|
||||||
|
removeAll func(string) error
|
||||||
|
beforeCommit func()
|
||||||
|
}
|
||||||
|
|
||||||
|
func publish(ctx context.Context, plan DestinationPlan, bundle LogicalBundle, operations publishOperations) (PublicationResult, error) {
|
||||||
|
if operations.rename == nil {
|
||||||
|
operations.rename = os.Rename
|
||||||
|
}
|
||||||
|
if operations.removeAll == nil {
|
||||||
|
operations.removeAll = os.RemoveAll
|
||||||
|
}
|
||||||
|
if err := bundle.Validate(); err != nil {
|
||||||
|
return PublicationResult{}, fmt.Errorf("validate comparison bundle: %w", err)
|
||||||
|
}
|
||||||
|
manifestData, err := EncodeManifest(bundle.Manifest)
|
||||||
|
if err != nil {
|
||||||
|
return PublicationResult{}, err
|
||||||
|
}
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return PublicationResult{}, err
|
||||||
|
}
|
||||||
|
plan, err = PlanDestination(plan.WorkingDirectory, plan.Target, plan.Replace)
|
||||||
|
if err != nil {
|
||||||
|
return PublicationResult{}, err
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := os.MkdirAll(filepath.Dir(plan.Target), 0o755); err != nil {
|
||||||
|
return PublicationResult{}, fmt.Errorf("create comparison destination parent %q: %w", filepath.Dir(plan.Target), err)
|
||||||
|
}
|
||||||
|
temporaryDirectory, err := os.MkdirTemp(filepath.Dir(plan.Target), "."+filepath.Base(plan.Target)+".staging-")
|
||||||
|
if err != nil {
|
||||||
|
return PublicationResult{}, fmt.Errorf("create comparison staging directory: %w", err)
|
||||||
|
}
|
||||||
|
if err := os.Chmod(temporaryDirectory, 0o700); err != nil {
|
||||||
|
os.RemoveAll(temporaryDirectory)
|
||||||
|
return PublicationResult{}, fmt.Errorf("secure comparison staging directory: %w", err)
|
||||||
|
}
|
||||||
|
defer func() {
|
||||||
|
if temporaryDirectory != "" {
|
||||||
|
_ = os.RemoveAll(temporaryDirectory)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
if err := writeLogicalBundle(ctx, temporaryDirectory, bundle, manifestData); err != nil {
|
||||||
|
return PublicationResult{}, err
|
||||||
|
}
|
||||||
|
|
||||||
|
currentPlan, err := PlanDestination(plan.WorkingDirectory, plan.Target, plan.Replace)
|
||||||
|
if err != nil {
|
||||||
|
return PublicationResult{}, err
|
||||||
|
}
|
||||||
|
if operations.beforeCommit != nil {
|
||||||
|
operations.beforeCommit()
|
||||||
|
}
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return PublicationResult{}, err
|
||||||
|
}
|
||||||
|
if currentPlan.state == destinationAbsent {
|
||||||
|
if err := operations.rename(temporaryDirectory, currentPlan.Target); err != nil {
|
||||||
|
return PublicationResult{}, fmt.Errorf("publish comparison bundle to %q: %w", currentPlan.Target, err)
|
||||||
|
}
|
||||||
|
temporaryDirectory = ""
|
||||||
|
return PublicationResult{Committed: true}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
backupDirectory, err := uniqueSiblingPath(filepath.Dir(currentPlan.Target), "."+filepath.Base(currentPlan.Target)+".backup-")
|
||||||
|
if err != nil {
|
||||||
|
return PublicationResult{}, err
|
||||||
|
}
|
||||||
|
if err := operations.rename(currentPlan.Target, backupDirectory); err != nil {
|
||||||
|
return PublicationResult{}, fmt.Errorf("back up comparison destination %q: %w", currentPlan.Target, err)
|
||||||
|
}
|
||||||
|
if err := authorizeMovedDestination(currentPlan, backupDirectory); err != nil {
|
||||||
|
return PublicationResult{}, restoreMovedDestination(operations, backupDirectory, currentPlan.Target, err)
|
||||||
|
}
|
||||||
|
if err := operations.rename(temporaryDirectory, currentPlan.Target); err != nil {
|
||||||
|
return PublicationResult{}, restoreMovedDestination(operations, backupDirectory, currentPlan.Target, fmt.Errorf("replace comparison destination %q: %w", currentPlan.Target, err))
|
||||||
|
}
|
||||||
|
temporaryDirectory = ""
|
||||||
|
if err := operations.removeAll(backupDirectory); err != nil {
|
||||||
|
cleanupErr := &PublicationCleanupError{RetainedBackupPath: backupDirectory, Err: err}
|
||||||
|
return PublicationResult{Committed: true, RetainedBackupPath: backupDirectory}, cleanupErr
|
||||||
|
}
|
||||||
|
return PublicationResult{Committed: true}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func authorizeMovedDestination(plan DestinationPlan, backupDirectory string) error {
|
||||||
|
backupPlan, err := PlanDestination(plan.WorkingDirectory, backupDirectory, plan.Replace)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("authorize moved comparison destination %q: %w", backupDirectory, err)
|
||||||
|
}
|
||||||
|
if backupPlan.state != destinationEmpty && backupPlan.state != destinationBundle {
|
||||||
|
return fmt.Errorf("authorize moved comparison destination %q: destination disappeared", backupDirectory)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func restoreMovedDestination(operations publishOperations, backupDirectory, target string, cause error) error {
|
||||||
|
if _, err := os.Lstat(target); err == nil {
|
||||||
|
return errors.Join(
|
||||||
|
cause,
|
||||||
|
fmt.Errorf("restore prior comparison destination from %q: destination %q reappeared", backupDirectory, target),
|
||||||
|
)
|
||||||
|
} else if !errors.Is(err, os.ErrNotExist) {
|
||||||
|
return errors.Join(
|
||||||
|
cause,
|
||||||
|
fmt.Errorf("inspect comparison destination %q before restoring from %q: %w", target, backupDirectory, err),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
if restoreErr := operations.rename(backupDirectory, target); restoreErr != nil {
|
||||||
|
return errors.Join(
|
||||||
|
cause,
|
||||||
|
fmt.Errorf("restore prior comparison destination from %q: %w", backupDirectory, restoreErr),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
return cause
|
||||||
|
}
|
||||||
|
|
||||||
|
func writeLogicalBundle(ctx context.Context, directory string, bundle LogicalBundle, manifestData []byte) error {
|
||||||
|
if err := writeBundleFile(ctx, filepath.Join(directory, DataPackageFilename), bundle.DataPackage); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
for _, report := range bundle.Reports {
|
||||||
|
if err := writeBundleFile(ctx, filepath.Join(directory, report.Path), report.Markdown); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := writeBundleFile(ctx, filepath.Join(directory, ManifestFilename), manifestData); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func writeBundleFile(ctx context.Context, filePath string, content []byte) error {
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
file, err := os.OpenFile(filePath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("create comparison bundle file %q: %w", filePath, err)
|
||||||
|
}
|
||||||
|
if _, err := file.Write(content); err != nil {
|
||||||
|
file.Close()
|
||||||
|
return fmt.Errorf("write comparison bundle file %q: %w", filePath, err)
|
||||||
|
}
|
||||||
|
if err := file.Close(); err != nil {
|
||||||
|
return fmt.Errorf("close comparison bundle file %q: %w", filePath, err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func uniqueSiblingPath(parent, prefix string) (string, error) {
|
||||||
|
file, err := os.CreateTemp(parent, prefix)
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("reserve comparison backup path: %w", err)
|
||||||
|
}
|
||||||
|
path := file.Name()
|
||||||
|
if err := file.Close(); err != nil {
|
||||||
|
os.Remove(path)
|
||||||
|
return "", fmt.Errorf("close comparison backup reservation: %w", err)
|
||||||
|
}
|
||||||
|
if err := os.Remove(path); err != nil {
|
||||||
|
return "", fmt.Errorf("release comparison backup reservation: %w", err)
|
||||||
|
}
|
||||||
|
return path, nil
|
||||||
|
}
|
||||||
754
internal/comparison/publish_test.go
Normal file
754
internal/comparison/publish_test.go
Normal file
@@ -0,0 +1,754 @@
|
|||||||
|
package comparison
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"runtime"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestPlanDestination(t *testing.T) {
|
||||||
|
workingDirectory := t.TempDir()
|
||||||
|
absent := filepath.Join(workingDirectory, "absent")
|
||||||
|
plan, err := PlanDestination(workingDirectory, absent, false)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("PlanDestination() error = %v", err)
|
||||||
|
}
|
||||||
|
if plan.Target != absent || plan.state != destinationAbsent {
|
||||||
|
t.Fatalf("PlanDestination() = %#v, want absent target", plan)
|
||||||
|
}
|
||||||
|
nested := filepath.Join(workingDirectory, "missing-parent", "comparison")
|
||||||
|
if _, err := PlanDestination(workingDirectory, nested, false); err != nil {
|
||||||
|
t.Fatalf("PlanDestination(nested) error = %v", err)
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(filepath.Dir(nested)); !errors.Is(err, os.ErrNotExist) {
|
||||||
|
t.Fatalf("read-only plan created parent: stat error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
empty := filepath.Join(workingDirectory, "empty")
|
||||||
|
if err := os.Mkdir(empty, 0o755); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
plan, err = PlanDestination(workingDirectory, empty, false)
|
||||||
|
if err != nil || plan.state != destinationEmpty {
|
||||||
|
t.Fatalf("PlanDestination(empty) = %#v, %v", plan, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
file := filepath.Join(workingDirectory, "file")
|
||||||
|
if err := os.WriteFile(file, []byte("not a directory"), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
assertDestinationErrorKind(t, workingDirectory, file, false, DestinationNotDirectory)
|
||||||
|
assertDestinationErrorKind(t, workingDirectory, workingDirectory, false, DestinationWorkingDirectory)
|
||||||
|
assertDestinationErrorKind(t, workingDirectory, string(os.PathSeparator), false, DestinationFilesystemRoot)
|
||||||
|
assertDestinationErrorKind(t, workingDirectory, "relative", false, DestinationInvalidPath)
|
||||||
|
|
||||||
|
link := filepath.Join(workingDirectory, "link")
|
||||||
|
if err := os.Symlink(empty, link); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
assertDestinationErrorKind(t, workingDirectory, link, false, DestinationSymlink)
|
||||||
|
|
||||||
|
dangling := filepath.Join(workingDirectory, "dangling")
|
||||||
|
if err := os.Symlink(filepath.Join(workingDirectory, "missing"), dangling); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
assertDestinationErrorKind(t, workingDirectory, dangling, false, DestinationSymlink)
|
||||||
|
assertDestinationErrorKind(t, workingDirectory, filepath.Join(dangling, "child"), false, DestinationInspection)
|
||||||
|
|
||||||
|
nonempty := filepath.Join(workingDirectory, "nonempty")
|
||||||
|
if err := os.Mkdir(nonempty, 0o755); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(filepath.Join(nonempty, "extra"), []byte("x"), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
assertDestinationErrorKind(t, workingDirectory, nonempty, false, DestinationNotEmpty)
|
||||||
|
assertDestinationErrorKind(t, workingDirectory, nonempty, true, DestinationUnrecognized)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPlanDestinationRejectsUnreadableDirectory(t *testing.T) {
|
||||||
|
if os.Geteuid() == 0 {
|
||||||
|
t.Skip("root can inspect directories regardless of mode")
|
||||||
|
}
|
||||||
|
workingDirectory := t.TempDir()
|
||||||
|
target := filepath.Join(workingDirectory, "unreadable")
|
||||||
|
if err := os.Mkdir(target, 0o700); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.Chmod(target, 0); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { _ = os.Chmod(target, 0o700) })
|
||||||
|
assertDestinationErrorKind(t, workingDirectory, target, false, DestinationInspection)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRecognizeBundleRejectsUnsafeAndMalformedContents(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
mutate func(t *testing.T, directory string)
|
||||||
|
}{
|
||||||
|
{name: "extra file", mutate: func(t *testing.T, directory string) {
|
||||||
|
t.Helper()
|
||||||
|
if err := os.WriteFile(filepath.Join(directory, "extra.txt"), []byte("extra"), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}},
|
||||||
|
{name: "subdirectory", mutate: func(t *testing.T, directory string) {
|
||||||
|
t.Helper()
|
||||||
|
if err := os.Mkdir(filepath.Join(directory, "nested"), 0o700); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}},
|
||||||
|
{name: "missing report", mutate: func(t *testing.T, directory string) {
|
||||||
|
t.Helper()
|
||||||
|
if err := os.Remove(filepath.Join(directory, "01-weather-light.md")); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}},
|
||||||
|
{name: "symlink report", mutate: func(t *testing.T, directory string) {
|
||||||
|
t.Helper()
|
||||||
|
path := filepath.Join(directory, "01-weather-light.md")
|
||||||
|
if err := os.Remove(path); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.Symlink(filepath.Join(directory, DataPackageFilename), path); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}},
|
||||||
|
{name: "digest mismatch", mutate: func(t *testing.T, directory string) {
|
||||||
|
t.Helper()
|
||||||
|
if err := os.WriteFile(filepath.Join(directory, DataPackageFilename), []byte("altered"), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}},
|
||||||
|
{name: "unknown manifest field", mutate: func(t *testing.T, directory string) {
|
||||||
|
t.Helper()
|
||||||
|
path := filepath.Join(directory, ManifestFilename)
|
||||||
|
data, err := os.ReadFile(path)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
data = append(data[:len(data)-2], []byte(",\n \"unknown\": true\n}\n")...)
|
||||||
|
if err := os.WriteFile(path, data, 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}},
|
||||||
|
{name: "traversal report path", mutate: func(t *testing.T, directory string) {
|
||||||
|
t.Helper()
|
||||||
|
path := filepath.Join(directory, ManifestFilename)
|
||||||
|
data, err := os.ReadFile(path)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var manifest Manifest
|
||||||
|
if err := json.Unmarshal(data, &manifest); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
manifest.Results[0].ReportPath = "../outside.md"
|
||||||
|
data, err = json.Marshal(manifest)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(path, data, 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, test := range tests {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
directory := publishTestBundle(t, testBundle())
|
||||||
|
test.mutate(t, directory)
|
||||||
|
if _, err := RecognizeBundle(directory); !errors.Is(err, ErrUnrecognizedBundle) {
|
||||||
|
t.Fatalf("RecognizeBundle() error = %v, want ErrUnrecognizedBundle", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPlanDestinationAcceptsRecognizedReplacement(t *testing.T) {
|
||||||
|
directory := publishTestBundle(t, testBundle())
|
||||||
|
plan, err := PlanDestination(filepath.Dir(directory), directory, true)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("PlanDestination() error = %v", err)
|
||||||
|
}
|
||||||
|
if plan.state != destinationBundle {
|
||||||
|
t.Fatal("PlanDestination() did not record recognized existing destination")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPublishReauthorizesMovedDestination(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
replace bool
|
||||||
|
requiresSymlink bool
|
||||||
|
prepare func(t *testing.T, workingDirectory, target string)
|
||||||
|
mutate func(t *testing.T, target string)
|
||||||
|
verify func(t *testing.T, target string)
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "regular file after absent preflight",
|
||||||
|
mutate: func(t *testing.T, target string) {
|
||||||
|
t.Helper()
|
||||||
|
if err := os.WriteFile(target, []byte("unrelated file"), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
},
|
||||||
|
verify: func(t *testing.T, target string) {
|
||||||
|
t.Helper()
|
||||||
|
if data := readFile(t, target); string(data) != "unrelated file" {
|
||||||
|
t.Fatalf("unrelated file data = %q", data)
|
||||||
|
}
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "nonempty directory after empty preflight",
|
||||||
|
prepare: func(t *testing.T, _, target string) {
|
||||||
|
t.Helper()
|
||||||
|
if err := os.Mkdir(target, 0o700); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
},
|
||||||
|
mutate: writeUnrecognizedDirectory,
|
||||||
|
verify: func(t *testing.T, target string) {
|
||||||
|
t.Helper()
|
||||||
|
if data := readFile(t, filepath.Join(target, "unrelated")); string(data) != "unrelated" {
|
||||||
|
t.Fatalf("unrelated data = %q", data)
|
||||||
|
}
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "regular file after empty preflight",
|
||||||
|
prepare: func(t *testing.T, _, target string) {
|
||||||
|
t.Helper()
|
||||||
|
if err := os.Mkdir(target, 0o700); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
},
|
||||||
|
mutate: func(t *testing.T, target string) {
|
||||||
|
t.Helper()
|
||||||
|
if err := os.WriteFile(target, []byte("unrelated file"), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
},
|
||||||
|
verify: func(t *testing.T, target string) {
|
||||||
|
t.Helper()
|
||||||
|
if data := readFile(t, target); string(data) != "unrelated file" {
|
||||||
|
t.Fatalf("unrelated file data = %q", data)
|
||||||
|
}
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "symlink after empty preflight",
|
||||||
|
requiresSymlink: true,
|
||||||
|
prepare: func(t *testing.T, _, target string) {
|
||||||
|
t.Helper()
|
||||||
|
if err := os.Mkdir(target, 0o700); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
},
|
||||||
|
mutate: func(t *testing.T, target string) {
|
||||||
|
t.Helper()
|
||||||
|
if err := os.Symlink("unrelated-target", target); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
},
|
||||||
|
verify: func(t *testing.T, target string) {
|
||||||
|
t.Helper()
|
||||||
|
info, err := os.Lstat(target)
|
||||||
|
if err != nil || info.Mode()&os.ModeSymlink == 0 {
|
||||||
|
t.Fatalf("symlink stat = %v, %v", info, err)
|
||||||
|
}
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "unrecognized directory after bundle preflight",
|
||||||
|
replace: true,
|
||||||
|
prepare: func(t *testing.T, workingDirectory, target string) {
|
||||||
|
t.Helper()
|
||||||
|
plan, err := PlanDestination(workingDirectory, target, false)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := Publish(context.Background(), plan, testBundle()); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
},
|
||||||
|
mutate: writeUnrecognizedDirectory,
|
||||||
|
verify: func(t *testing.T, target string) {
|
||||||
|
t.Helper()
|
||||||
|
if data := readFile(t, filepath.Join(target, "unrelated")); string(data) != "unrelated" {
|
||||||
|
t.Fatalf("unrelated data = %q", data)
|
||||||
|
}
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, test := range tests {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
if test.requiresSymlink && runtime.GOOS == "windows" {
|
||||||
|
t.Skip("symlink replacement coverage requires Unix symlink semantics")
|
||||||
|
}
|
||||||
|
workingDirectory := t.TempDir()
|
||||||
|
target := filepath.Join(workingDirectory, "comparison-daily")
|
||||||
|
if test.prepare != nil {
|
||||||
|
test.prepare(t, workingDirectory, target)
|
||||||
|
}
|
||||||
|
plan, err := PlanDestination(workingDirectory, target, test.replace)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
_, err = publish(context.Background(), plan, testBundle(), publishOperations{
|
||||||
|
rename: os.Rename,
|
||||||
|
beforeCommit: func() {
|
||||||
|
if err := os.RemoveAll(target); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
test.mutate(t, target)
|
||||||
|
},
|
||||||
|
})
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("publish() succeeded despite unauthorized replacement")
|
||||||
|
}
|
||||||
|
test.verify(t, target)
|
||||||
|
assertOnlyDestinationEntry(t, workingDirectory, filepath.Base(target))
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPublishRetainsUnauthorizedMovedDestinationWhenRestoreFails(t *testing.T) {
|
||||||
|
workingDirectory := t.TempDir()
|
||||||
|
target := filepath.Join(workingDirectory, "comparison-daily")
|
||||||
|
if err := os.Mkdir(target, 0o700); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
plan, err := PlanDestination(workingDirectory, target, false)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var backupPath string
|
||||||
|
calls := 0
|
||||||
|
_, err = publish(context.Background(), plan, testBundle(), publishOperations{
|
||||||
|
rename: func(oldPath, newPath string) error {
|
||||||
|
calls++
|
||||||
|
if calls == 1 {
|
||||||
|
backupPath = newPath
|
||||||
|
}
|
||||||
|
if calls == 2 {
|
||||||
|
return errors.New("restore failed")
|
||||||
|
}
|
||||||
|
return os.Rename(oldPath, newPath)
|
||||||
|
},
|
||||||
|
beforeCommit: func() {
|
||||||
|
if err := os.RemoveAll(target); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(target, []byte("unrelated file"), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
},
|
||||||
|
})
|
||||||
|
if err == nil || !strings.Contains(err.Error(), backupPath) {
|
||||||
|
t.Fatalf("publish() error = %v, want retained backup path %q", err, backupPath)
|
||||||
|
}
|
||||||
|
if data := readFile(t, backupPath); string(data) != "unrelated file" {
|
||||||
|
t.Fatalf("retained backup data = %q", data)
|
||||||
|
}
|
||||||
|
if _, err := os.Lstat(target); !errors.Is(err, os.ErrNotExist) {
|
||||||
|
t.Fatalf("target stat error = %v, want not exist", err)
|
||||||
|
}
|
||||||
|
assertOnlyDestinationEntry(t, workingDirectory, filepath.Base(backupPath))
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPublishRetainsMovedDestinationWhenTargetReappears(t *testing.T) {
|
||||||
|
workingDirectory := t.TempDir()
|
||||||
|
target := filepath.Join(workingDirectory, "comparison-daily")
|
||||||
|
if err := os.Mkdir(target, 0o700); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
plan, err := PlanDestination(workingDirectory, target, false)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var backupPath string
|
||||||
|
_, err = publish(context.Background(), plan, testBundle(), publishOperations{
|
||||||
|
rename: func(oldPath, newPath string) error {
|
||||||
|
if backupPath != "" {
|
||||||
|
return os.Rename(oldPath, newPath)
|
||||||
|
}
|
||||||
|
backupPath = newPath
|
||||||
|
if err := os.Rename(oldPath, newPath); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(filepath.Join(backupPath, "unrelated"), []byte("unrelated"), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(target, []byte("reappeared"), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
},
|
||||||
|
})
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "reappeared") || !strings.Contains(err.Error(), backupPath) {
|
||||||
|
t.Fatalf("publish() error = %v, want reappeared target and retained backup path %q", err, backupPath)
|
||||||
|
}
|
||||||
|
if data := readFile(t, target); string(data) != "reappeared" {
|
||||||
|
t.Fatalf("reappeared target data = %q", data)
|
||||||
|
}
|
||||||
|
if data := readFile(t, filepath.Join(backupPath, "unrelated")); string(data) != "unrelated" {
|
||||||
|
t.Fatalf("retained backup data = %q", data)
|
||||||
|
}
|
||||||
|
assertDestinationEntries(t, workingDirectory, filepath.Base(target), filepath.Base(backupPath))
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPublishWritesAndReplacesBundle(t *testing.T) {
|
||||||
|
workingDirectory := t.TempDir()
|
||||||
|
target := filepath.Join(workingDirectory, "comparison-daily")
|
||||||
|
plan, err := PlanDestination(workingDirectory, target, false)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := Publish(context.Background(), plan, testBundle()); err != nil {
|
||||||
|
t.Fatalf("Publish() error = %v", err)
|
||||||
|
}
|
||||||
|
if _, err := RecognizeBundle(target); err != nil {
|
||||||
|
t.Fatalf("RecognizeBundle() error = %v", err)
|
||||||
|
}
|
||||||
|
assertMode(t, target, 0o700)
|
||||||
|
for _, name := range []string{ManifestFilename, DataPackageFilename, "01-weather-light.md"} {
|
||||||
|
assertMode(t, filepath.Join(target, name), 0o600)
|
||||||
|
}
|
||||||
|
|
||||||
|
next := testBundle()
|
||||||
|
next.Manifest.Results[0].ProfileID = "weather-balanced"
|
||||||
|
next.Manifest.Results[0].ReportPath = "01-weather-balanced.md"
|
||||||
|
next.Reports[0].Path = "01-weather-balanced.md"
|
||||||
|
next.Reports[0].Markdown = []byte("# Replacement\n")
|
||||||
|
plan, err = PlanDestination(workingDirectory, target, true)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := Publish(context.Background(), plan, next); err != nil {
|
||||||
|
t.Fatalf("Publish(replace) error = %v", err)
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(filepath.Join(target, "01-weather-light.md")); !errors.Is(err, os.ErrNotExist) {
|
||||||
|
t.Fatalf("stale report stat error = %v, want not exist", err)
|
||||||
|
}
|
||||||
|
if data, err := os.ReadFile(filepath.Join(target, "01-weather-balanced.md")); err != nil || string(data) != "# Replacement\n" {
|
||||||
|
t.Fatalf("replacement report = %q, %v", data, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPublishReplacesEmptyDirectory(t *testing.T) {
|
||||||
|
workingDirectory := t.TempDir()
|
||||||
|
target := filepath.Join(workingDirectory, "comparison-daily")
|
||||||
|
if err := os.Mkdir(target, 0o755); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
plan, err := PlanDestination(workingDirectory, target, false)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := Publish(context.Background(), plan, testBundle()); err != nil {
|
||||||
|
t.Fatalf("Publish() error = %v", err)
|
||||||
|
}
|
||||||
|
if _, err := RecognizeBundle(target); err != nil {
|
||||||
|
t.Fatalf("RecognizeBundle() error = %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPublishReportsCommittedBundleWhenBackupCleanupFails(t *testing.T) {
|
||||||
|
workingDirectory := t.TempDir()
|
||||||
|
target := filepath.Join(workingDirectory, "comparison-daily")
|
||||||
|
initialPlan, err := PlanDestination(workingDirectory, target, false)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := Publish(context.Background(), initialPlan, testBundle()); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
previous := directorySnapshot(t, target)
|
||||||
|
|
||||||
|
next := testBundle()
|
||||||
|
next.Reports[0].Markdown = []byte("# Next\n")
|
||||||
|
plan, err := PlanDestination(workingDirectory, target, true)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
cleanupCause := errors.New("backup removal failed")
|
||||||
|
var backupPath string
|
||||||
|
result, err := publish(context.Background(), plan, next, publishOperations{
|
||||||
|
rename: os.Rename,
|
||||||
|
removeAll: func(path string) error {
|
||||||
|
backupPath = path
|
||||||
|
return cleanupCause
|
||||||
|
},
|
||||||
|
})
|
||||||
|
var cleanupErr *PublicationCleanupError
|
||||||
|
if !result.Committed || result.RetainedBackupPath != backupPath || !filepath.IsAbs(backupPath) || !errors.As(err, &cleanupErr) || cleanupErr.RetainedBackupPath != backupPath || !errors.Is(err, cleanupCause) {
|
||||||
|
t.Fatalf("publish() result/error = %#v/%v", result, err)
|
||||||
|
}
|
||||||
|
if _, err := RecognizeBundle(target); err != nil {
|
||||||
|
t.Fatalf("new bundle recognition error = %v", err)
|
||||||
|
}
|
||||||
|
if retained := directorySnapshot(t, backupPath); !equalSnapshots(previous, retained) {
|
||||||
|
t.Fatalf("retained backup = %#v, want %#v", retained, previous)
|
||||||
|
}
|
||||||
|
manifestData, err := os.ReadFile(filepath.Join(target, ManifestFilename))
|
||||||
|
if err != nil || strings.Contains(string(manifestData), backupPath) {
|
||||||
|
t.Fatalf("manifest/backup path = %q/%q, error = %v", manifestData, backupPath, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPublishRestoresExistingBundleAfterReplacementFailure(t *testing.T) {
|
||||||
|
workingDirectory := t.TempDir()
|
||||||
|
target := filepath.Join(workingDirectory, "comparison-daily")
|
||||||
|
initial := testBundle()
|
||||||
|
initialPlan, err := PlanDestination(workingDirectory, target, false)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := Publish(context.Background(), initialPlan, initial); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
before := directorySnapshot(t, target)
|
||||||
|
|
||||||
|
next := testBundle()
|
||||||
|
next.Reports[0].Markdown = []byte("# New\n")
|
||||||
|
plan, err := PlanDestination(workingDirectory, target, true)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
calls := 0
|
||||||
|
publication, err := publish(context.Background(), plan, next, publishOperations{rename: func(oldPath, newPath string) error {
|
||||||
|
calls++
|
||||||
|
if calls == 2 {
|
||||||
|
return errors.New("replace failed")
|
||||||
|
}
|
||||||
|
return os.Rename(oldPath, newPath)
|
||||||
|
}})
|
||||||
|
if publication.Committed || err == nil {
|
||||||
|
t.Fatal("publish() succeeded despite replacement failure")
|
||||||
|
}
|
||||||
|
after := directorySnapshot(t, target)
|
||||||
|
if !equalSnapshots(before, after) {
|
||||||
|
t.Fatal("failed publication changed the prior bundle")
|
||||||
|
}
|
||||||
|
entries, err := os.ReadDir(workingDirectory)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(entries) != 1 || entries[0].Name() != filepath.Base(target) {
|
||||||
|
t.Fatalf("failed publication left sibling artifacts: %#v", entries)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPublishRetainsBackupWhenRestorationFails(t *testing.T) {
|
||||||
|
workingDirectory := t.TempDir()
|
||||||
|
target := filepath.Join(workingDirectory, "comparison-daily")
|
||||||
|
initialPlan, err := PlanDestination(workingDirectory, target, false)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := Publish(context.Background(), initialPlan, testBundle()); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
plan, err := PlanDestination(workingDirectory, target, true)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var backupPath string
|
||||||
|
calls := 0
|
||||||
|
_, err = publish(context.Background(), plan, testBundle(), publishOperations{rename: func(oldPath, newPath string) error {
|
||||||
|
calls++
|
||||||
|
if calls == 1 {
|
||||||
|
backupPath = newPath
|
||||||
|
}
|
||||||
|
if calls == 2 || calls == 3 {
|
||||||
|
return errors.New("rename failed")
|
||||||
|
}
|
||||||
|
return os.Rename(oldPath, newPath)
|
||||||
|
}})
|
||||||
|
if err == nil || !strings.Contains(err.Error(), backupPath) {
|
||||||
|
t.Fatalf("publish() error = %v, want retained backup path %q", err, backupPath)
|
||||||
|
}
|
||||||
|
if info, statErr := os.Stat(backupPath); statErr != nil || !info.IsDir() {
|
||||||
|
t.Fatalf("backup stat = %v, %v, want retained directory", info, statErr)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPublishCancellationBeforeCommitLeavesNoDestination(t *testing.T) {
|
||||||
|
workingDirectory := t.TempDir()
|
||||||
|
target := filepath.Join(workingDirectory, "comparison-daily")
|
||||||
|
plan, err := PlanDestination(workingDirectory, target, false)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
if publication, err := publish(ctx, plan, testBundle(), publishOperations{rename: os.Rename, beforeCommit: cancel}); publication.Committed || !errors.Is(err, context.Canceled) {
|
||||||
|
t.Fatalf("Publish() error = %v, want context cancellation", err)
|
||||||
|
}
|
||||||
|
if _, err := os.Lstat(target); !errors.Is(err, os.ErrNotExist) {
|
||||||
|
t.Fatalf("destination stat error = %v, want not exist", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func assertDestinationErrorKind(t *testing.T, workingDirectory, target string, replace bool, want DestinationErrorKind) {
|
||||||
|
t.Helper()
|
||||||
|
_, err := PlanDestination(workingDirectory, target, replace)
|
||||||
|
var destinationError *DestinationError
|
||||||
|
if !errors.As(err, &destinationError) || destinationError.Kind != want {
|
||||||
|
t.Fatalf("PlanDestination(%q) error = %v, want destination error %q", target, err, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func publishTestBundle(t *testing.T, bundle LogicalBundle) string {
|
||||||
|
t.Helper()
|
||||||
|
workingDirectory := t.TempDir()
|
||||||
|
target := filepath.Join(workingDirectory, "comparison-daily")
|
||||||
|
plan, err := PlanDestination(workingDirectory, target, false)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := Publish(context.Background(), plan, bundle); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return target
|
||||||
|
}
|
||||||
|
|
||||||
|
func writeUnrecognizedDirectory(t *testing.T, target string) {
|
||||||
|
t.Helper()
|
||||||
|
if err := os.Mkdir(target, 0o700); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(filepath.Join(target, "unrelated"), []byte("unrelated"), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func readFile(t *testing.T, path string) []byte {
|
||||||
|
t.Helper()
|
||||||
|
data, err := os.ReadFile(path)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return data
|
||||||
|
}
|
||||||
|
|
||||||
|
func assertOnlyDestinationEntry(t *testing.T, directory, want string) {
|
||||||
|
t.Helper()
|
||||||
|
assertDestinationEntries(t, directory, want)
|
||||||
|
}
|
||||||
|
|
||||||
|
func assertDestinationEntries(t *testing.T, directory string, wants ...string) {
|
||||||
|
t.Helper()
|
||||||
|
entries, err := os.ReadDir(directory)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(entries) != len(wants) {
|
||||||
|
t.Fatalf("directory entries = %#v, want %#v", entries, wants)
|
||||||
|
}
|
||||||
|
for _, want := range wants {
|
||||||
|
found := false
|
||||||
|
for _, entry := range entries {
|
||||||
|
if entry.Name() == want {
|
||||||
|
found = true
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !found {
|
||||||
|
t.Fatalf("directory entries = %#v, missing %q", entries, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func testBundle() LogicalBundle {
|
||||||
|
dataPackage := []byte("report: daily\n")
|
||||||
|
manifest := validManifest()
|
||||||
|
manifest.DataPackage.SHA256 = SHA256(dataPackage)
|
||||||
|
return LogicalBundle{
|
||||||
|
Manifest: manifest,
|
||||||
|
DataPackage: dataPackage,
|
||||||
|
Reports: []BundleReport{{
|
||||||
|
Position: 1,
|
||||||
|
Path: "01-weather-light.md",
|
||||||
|
Markdown: []byte("# Daily\n"),
|
||||||
|
}},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func assertMode(t *testing.T, path string, want os.FileMode) {
|
||||||
|
t.Helper()
|
||||||
|
info, err := os.Stat(path)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got := info.Mode().Perm(); got != want {
|
||||||
|
t.Fatalf("mode for %q = %#o, want %#o", path, got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func directorySnapshot(t *testing.T, directory string) map[string]string {
|
||||||
|
t.Helper()
|
||||||
|
entries, err := os.ReadDir(directory)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
snapshot := make(map[string]string, len(entries))
|
||||||
|
for _, entry := range entries {
|
||||||
|
data, err := os.ReadFile(filepath.Join(directory, entry.Name()))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
snapshot[entry.Name()] = string(data)
|
||||||
|
}
|
||||||
|
return snapshot
|
||||||
|
}
|
||||||
|
|
||||||
|
func equalSnapshots(left, right map[string]string) bool {
|
||||||
|
if len(left) != len(right) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
for name, value := range left {
|
||||||
|
if right[name] != value {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRecognizeBundleRejectsTrailingJSONValue(t *testing.T) {
|
||||||
|
directory := publishTestBundle(t, testBundle())
|
||||||
|
path := filepath.Join(directory, ManifestFilename)
|
||||||
|
data, err := os.ReadFile(path)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(path, append(data, []byte("{}")...), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := RecognizeBundle(directory); !errors.Is(err, ErrUnrecognizedBundle) {
|
||||||
|
t.Fatalf("RecognizeBundle() error = %v, want ErrUnrecognizedBundle", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPlanDestinationRejectsWhitespaceTarget(t *testing.T) {
|
||||||
|
workingDirectory := t.TempDir()
|
||||||
|
assertDestinationErrorKind(t, workingDirectory, " \t", false, DestinationInvalidPath)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRecognizeBundleDoesNotAcceptNonRegularManifest(t *testing.T) {
|
||||||
|
directory := t.TempDir()
|
||||||
|
if err := os.Mkdir(filepath.Join(directory, ManifestFilename), 0o700); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := RecognizeBundle(directory); !strings.Contains(err.Error(), "regular file") {
|
||||||
|
t.Fatalf("RecognizeBundle() error = %v, want regular file rejection", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -269,7 +269,23 @@ func ensureSecureDirectory(path string) error {
|
|||||||
info, err := os.Lstat(current)
|
info, err := os.Lstat(current)
|
||||||
if os.IsNotExist(err) {
|
if os.IsNotExist(err) {
|
||||||
if err := os.Mkdir(current, debugDirectoryMode); err != nil {
|
if err := os.Mkdir(current, debugDirectoryMode); err != nil {
|
||||||
return err
|
if !os.IsExist(err) {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
info, err = os.Lstat(current)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if info.Mode()&os.ModeSymlink != 0 {
|
||||||
|
return fmt.Errorf("directory component %q must not be a symlink", current)
|
||||||
|
}
|
||||||
|
if !info.IsDir() {
|
||||||
|
return fmt.Errorf("directory component %q is not a directory", current)
|
||||||
|
}
|
||||||
|
if err := os.Chmod(current, debugDirectoryMode); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
continue
|
||||||
}
|
}
|
||||||
if err := os.Chmod(current, debugDirectoryMode); err != nil {
|
if err := os.Chmod(current, debugDirectoryMode); err != nil {
|
||||||
return err
|
return err
|
||||||
|
|||||||
@@ -1,10 +1,12 @@
|
|||||||
package promptdebug
|
package promptdebug
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"fmt"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"runtime"
|
"runtime"
|
||||||
"strings"
|
"strings"
|
||||||
|
"sync"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
@@ -91,6 +93,73 @@ func TestPromptDebugWriterAtomicallyReplacesArtifacts(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestPromptDebugWriterCreatesSharedMissingAncestorsConcurrently(t *testing.T) {
|
||||||
|
root := filepath.Join(t.TempDir(), "debug")
|
||||||
|
writer, err := NewPromptDebugWriter(root)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewPromptDebugWriter() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
const writerCount = 8
|
||||||
|
start := make(chan struct{})
|
||||||
|
type writeResult struct {
|
||||||
|
directory string
|
||||||
|
err error
|
||||||
|
}
|
||||||
|
results := make(chan writeResult, writerCount)
|
||||||
|
var writers sync.WaitGroup
|
||||||
|
for index := 0; index < writerCount; index++ {
|
||||||
|
writers.Add(1)
|
||||||
|
go func(index int) {
|
||||||
|
defer writers.Done()
|
||||||
|
<-start
|
||||||
|
directory, err := writer.WritePreparation(PromptDebugRef{
|
||||||
|
ReportID: report.Daily, ValidDate: "2026-05-29", RunID: fmt.Sprintf("run-%02d", index),
|
||||||
|
}, promptDebugPreparationFixture(), nil)
|
||||||
|
results <- writeResult{directory: directory, err: err}
|
||||||
|
}(index)
|
||||||
|
}
|
||||||
|
close(start)
|
||||||
|
|
||||||
|
finished := make(chan struct{})
|
||||||
|
go func() {
|
||||||
|
writers.Wait()
|
||||||
|
close(finished)
|
||||||
|
}()
|
||||||
|
select {
|
||||||
|
case <-finished:
|
||||||
|
case <-time.After(5 * time.Second):
|
||||||
|
t.Fatal("concurrent prompt debug writes did not finish")
|
||||||
|
}
|
||||||
|
close(results)
|
||||||
|
|
||||||
|
directories := map[string]struct{}{}
|
||||||
|
for result := range results {
|
||||||
|
if result.err != nil {
|
||||||
|
t.Fatalf("WritePreparation() error = %v", result.err)
|
||||||
|
}
|
||||||
|
if _, duplicate := directories[result.directory]; duplicate {
|
||||||
|
t.Fatalf("duplicate debug directory %q", result.directory)
|
||||||
|
}
|
||||||
|
directories[result.directory] = struct{}{}
|
||||||
|
if _, err := os.Stat(filepath.Join(result.directory, "preparation.json")); err != nil {
|
||||||
|
t.Fatalf("preparation artifact %q: %v", result.directory, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(directories) != writerCount {
|
||||||
|
t.Fatalf("debug directories = %#v", directories)
|
||||||
|
}
|
||||||
|
if runtime.GOOS != "windows" {
|
||||||
|
assertPromptDebugMode(t, root, debugDirectoryMode)
|
||||||
|
assertPromptDebugMode(t, filepath.Join(root, "daily"), debugDirectoryMode)
|
||||||
|
assertPromptDebugMode(t, filepath.Join(root, "daily", "2026-05-29"), debugDirectoryMode)
|
||||||
|
for directory := range directories {
|
||||||
|
assertPromptDebugMode(t, directory, debugDirectoryMode)
|
||||||
|
assertPromptDebugMode(t, filepath.Join(directory, "preparation.json"), debugFileMode)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestPromptDebugWriterDisabledDoesNotAccessFilesystem(t *testing.T) {
|
func TestPromptDebugWriterDisabledDoesNotAccessFilesystem(t *testing.T) {
|
||||||
writer, err := NewPromptDebugWriter("")
|
writer, err := NewPromptDebugWriter("")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
Reference in New Issue
Block a user