From 25782447ebe7b72751e13624d1624bc689c2ae4d Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Fri, 31 Jul 2026 16:37:00 +0000 Subject: [PATCH] Correct artifact path bookkeeping --- docs/roadmap/implementation.md | 1192 ++++++----------- internal/app/app.go | 33 +- internal/app/batch_execution_test.go | 24 + internal/app/prompt_artifact_paths_test.go | 272 ++++ internal/app/prompt_generate.go | 48 +- internal/cli/result_test.go | 25 + internal/state/metadata.go | 5 +- internal/state/metadata_reached_paths_test.go | 38 + 8 files changed, 792 insertions(+), 845 deletions(-) create mode 100644 internal/app/prompt_artifact_paths_test.go create mode 100644 internal/state/metadata_reached_paths_test.go diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index a7b1374..21eac71 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -1,476 +1,127 @@ # Promptkit Migration Implementation Plan -Status: Decision-complete implementation plan; unimplemented. +Status: Stages 1–11 completed; audit remediation in Stages 12–19 remains. ## Purpose And Authority -This document defines the ordered implementation procedure for the -[Promptkit migration roadmap](promptkit.md). The feature roadmap is +This document records the completed implementation of the +[Promptkit migration roadmap](promptkit.md) and defines the ordered follow-up +work required by the post-implementation audit. The feature roadmap remains authoritative for scope, user intent, policy choices, and the desired end -state. This plan is authoritative for sequencing, concrete package ownership, -compatibility work, tests, documentation updates, and completion gates. +state. This plan is authoritative for implementation sequence, tests, and +completion gates. -Implement the stages in order. Do not reinterpret a roadmap decision merely -because the current Scriptorium implementation makes another path shorter. -When implementation reveals a conflict with the roadmap, stop and update the -roadmap and this plan deliberately rather than introducing an implicit policy -change. +Follow Stages 12–19 in order, using one implementation prompt per stage. A +stage may fix additional defects exposed by its required tests when those +defects are within the same stated contract. Do not expand a stage into new +product behavior or reinterpret a roadmap decision to accommodate the current +implementation. This plan follows the repository's [architecture](../policy/architecture.md), [documentation](../policy/documentation.md), and -[testing](../policy/testing.md) policies. All stages are parts of one -migration change. Intermediate stages may temporarily retain code needed by a -later cutover, but no intermediate state should be released or documented as -the final current behavior. +[testing](../policy/testing.md) policies. -## Cross-Stage Rules +## Continuing Invariants -- Pin `gitea.maximumdirect.net/eric/promptkit` at exactly `v0.4.0`. Do not - commit a `go.work`, local `replace`, pseudo-version, or unpublished commit. -- Keep Promptkit types inside `internal/adapters/promptkit`, that package's - tests, and the external contract test that verifies `internal/promptassets` - with the real Promptkit inspector. App, CLI, report, state, and domain - packages use project-owned types. -- Keep Promptkit's opaque prepared-execution handle inside its adapter. Do not - expose it through an app interface, serialize it, or make it restartable. -- Construct one Promptkit engine per `generate` or `run` CLI invocation. - Every report in a batch shares that engine. Inspection commands that only - read persisted state do not construct an engine. -- Keep batches sequential. Do not add retries, output repair, direct Markdown - generation, arbitrary backend registration, or live-provider tests. -- Preserve the exact persisted `data_package` bytes as the only model input. - Promptkit receives those bytes through an inline artifact with the managed - YAML path as provenance; it never receives an unrestricted file reference. -- Persist successful preparation before `RunPrepared`. If preparation-record - persistence or enabled debug persistence fails, discard the handle and do - not call the provider. -- Keep normal artifacts, errors, logs, and CLI summaries free of credentials, - rendered messages, schema bodies, data-package bodies, provider response - bodies, and full effective parameter maps. -- Use atomic writes for durable state and debug files. Preserve every - non-secret artifact reached before a later failure when practical. -- Use deterministic Promptkit model-client fakes at the adapter boundary and - project-owned executor fakes at the app boundary. The default test suite - remains offline and credential-free. -- At each stage, update or delete existing tests according to the testing - policy. Protect contracts and failure behavior; do not preserve tests that - exist only to assert the retired subprocess implementation. -- Run the focused checks named by the stage while iterating. Before completing - every stage, run `git diff --check`. Run the full repository gate in the - final stage. +- Keep `gitea.maximumdirect.net/eric/promptkit` pinned at exactly `v0.4.0`. +- Keep Promptkit types inside `internal/adapters/promptkit`, its tests, and the + external prompt-asset contract test. +- Preserve one Promptkit engine per `generate` or `run` invocation and one + shared engine for every sequential report in a batch. +- Preserve exact prompt version `1.0.0`, the exact persisted YAML data-package + bytes, prepared execution, and preparation persistence before provider work. +- Do not add retries, repair attempts, concurrent batch generation, direct + Markdown generation, arbitrary backend registration, or live-provider + tests. +- Keep ordinary artifacts, errors, logs, and summaries free of credentials, + rendered messages, schemas, input bodies, generated bodies, endpoints, and + full effective parameter maps. +- Keep sensitive debug artifacts opt-in, outside normal state, owner-only, + atomic, and free of credentials. +- Treat an artifact path as reached only after the corresponding write or copy + succeeds. Never persist or summarize a merely derivable future path. +- Preserve every safe reached path in partial app and CLI results even when a + later persistence, validation, rendering, copy, or notification step fails. +- Keep v1 metadata read compatibility and write only v2 metadata for new runs. +- Keep the default test suite deterministic, offline, and credential-free. +- Run `git diff --check` before completing every stage. Run the full repository + gate in Stage 19. -## Fixed Package And Contract Decisions +## Completed Migration Summary -Use these ownership boundaries so later stages do not need to choose an -architecture while editing: +Stages 1–11 are implemented and committed. They remain summarized here to +preserve the history and dependencies of the follow-up work. -| Area | Owner | Decision | -| --- | --- | --- | -| Embedded prompt definitions, referenced prompt content, and provider-facing schemas | `internal/promptassets` | Embed one centralized asset tree and expose read-only prompt and schema `fs.FS` sources plus schema lookup by report schema ID. This package does not import Promptkit. | -| Project-owned prompt execution contract | `internal/promptexec` | Own prompt inspection, profile inspection, preparation, execution, validation, usage, debug, and neutral error values. This package does not import Promptkit, app, CLI, report, or state. | -| Promptkit integration | `internal/adapters/promptkit` | Construct and own the Promptkit engine, translate project requests and results, classify public Promptkit errors, and enforce prepared-handle lifecycle. | -| Workflow ordering | `internal/app` | Resolve reports, inspect prompt/profile selections, collect weather, build and save inputs, record preparation, invoke execution, persist results, perform final generated-text validation, render Markdown, and notify. | -| Durable artifacts | `internal/state` | Own v2 paths, preparation and execution schemas, metadata compatibility, atomic writes, and inspection reads. | -| Engine composition and flags | `internal/cli` | Construct one executor per action invocation, pass it into app requests, parse `--llm-debug-dir`, and expose project-owned summary paths. | +| Stage | Completed outcome | +| --- | --- | +| 1 | Removed the unfinished three-day, weekend, and storm product surfaces and retained Daily, Today, Tomorrow, and Hourly with exact prompt version `1.0.0`. | +| 2 | Promoted the four operational prompts and canonical schemas into the embedded `internal/promptassets` source used by Promptkit and generated-text validation. | +| 3 | Added the project-owned `internal/promptexec` inspection, preparation, execution, validation, debug, and error contract. | +| 4 | Added the Promptkit v0.4.0 adapter with prepared execution, explicit value mapping, safe error classification, and offline model-client tests. | +| 5 | Added Promptkit-era preparation and execution artifacts, metadata v2, new paths, and v1 decoding support. | +| 6 | Added explicitly rooted, permission-restricted, atomic LLM debug persistence. | +| 7 | Added Promptkit configuration, executor composition, and pre-collection prompt/profile/credential inspection. | +| 8 | Cut single-report generation over to prepared Promptkit execution and v2 persistence. | +| 9 | Added `--llm-debug-dir` and Promptkit-era single-report summary fields. | +| 10 | Cut morning and evening batches over to one shared Promptkit executor and removed Scriptorium code, configuration, and dependency metadata. | +| 11 | Updated canonical Promptkit documentation, removed the temporary Scriptorium corpus, and ran the available repository checks. | -`internal/promptexec.Executor` must provide three operations: +The post-implementation audit confirmed the principal dependency and package +boundaries, but found incorrect reached-path bookkeeping, incomplete execution +artifact updates, insufficient artifact validation, extensive loss of +behavioral tests during the final cutover, and roadmap lifecycle text that was +not finalized. The remaining stages address those findings without changing +the intended feature scope. -1. inspect an exact prompt ID and version; -2. inspect one explicit profile ID; and -3. prepare and execute one request. - -The execution operation accepts a preparation callback. The adapter calls that -callback exactly once after successful `PrepareExecution` and before -`RunPrepared`. The callback receives a safe project-owned preparation value -and, only when requested, a project-owned sensitive debug value. A callback -error aborts execution. A preparation failure returns a classified -project-owned attempt error without calling the callback; app orchestration -uses that error to persist a failure receipt. - -The adapter execution request contains the exact prompt ID and version, the -optional configured profile override, the exact YAML bytes, the managed -data-package path used only as inline provenance, and whether sensitive debug -capture is enabled. It does not contain a provider output path. - -Project-owned errors use stable categories for invalid configuration, invalid -request, prompt not found/load, profile not found/load, missing credential, -artifact load, prompt render, capacity, generation, operational validation, -completed validation rejection, cancellation, and deadline. A capacity error -also carries the non-secret backend ID. Its `Error` text is -Weatherreporter-owned and safe for CLI output; an unexported or non-serialized -cause may remain available to `errors.Is` and `errors.As`. Never persist -arbitrary dependency error prose. - -## Stage 1: Correct The Implemented Report Surface +## Stage 12: Correct Reached-Artifact Bookkeeping ### Goal -Remove the unfinished three-day, weekend, and storm products before changing -the LLM integration. The repository should expose only Daily, Today, Tomorrow, -and Hourly while continuing to use Scriptorium temporarily for those four -reports. +Make metadata, app results, batch items, and CLI summaries truthful at every +failure boundary: a nonblank path means that artifact was successfully +created. ### Work -1. Remove the three report definitions and their files under `internal/report`. - Remove their IDs, command names, aliases, period inputs, generation modes, - batch flags, registry order, and direct-Markdown-only declarations. -2. Add `PromptVersion string` to `report.Definition` and set it to `1.0.0` on - all four retained definitions. Since every retained report uses generated - text plus a repository template, remove the now-redundant - `GenerationMode` and `Generated` fields and their branches. -3. Remove three-day, weekend, and storm command parsing, help text, app request - fields/constants, resolution branches, Distributor path variables, and - configuration report aliases. -4. Remove report-specific code whose only caller was one of the retired - products. Use graph traces before deletion. This includes the dedicated - three-day/weekend Recent Changes comparators, storm-window fact derivation, - multi-day report derivation used only by these definitions, storm time - parsing, and their focused tests when no retained behavior depends on them. - Do not remove general meteorological behavior merely because a fixture or - prose string contains words such as “storm” or “weekend.” -5. Keep morning and evening planning behavior exactly as implemented: - morning plans Today and Tomorrow, evening plans Tomorrow, and both may add - eligible future Daily reports from complete hourly coverage. -6. Update the current CLI, configuration, report-registry, facts, modules, - changes, operations, troubleshooting, and other affected documentation to - describe only the four implemented reports. Update `docs/roadmap/future.md` - so it no longer claims that a manual Storm Report exists; future versions - of these products remain explicitly unimplemented roadmap work. +1. Change `state.BuildPromptMetadataFromBriefingMetadata` so it initializes + identity, schema, metadata destination, and only artifacts already saved at + the call site. It must not prepopulate raw-output, normalized-text, + render-context, managed-report, preparation, execution, notification, or + output-copy paths. +2. In `generatePromptReport`, assign each metadata and `ReportResult` path + immediately after that artifact write succeeds and before attempting the + next write. In particular: + + - do not initialize `ReportResult.ReportPath` from `Store.Paths`; + - record a saved failed-preparation receipt in the result before saving + metadata; + - record a saved failed or completed execution receipt before saving + metadata; + - retain raw, normalized, context, report, copy, and notification paths + when a later step fails; and + - keep `MetadataPath` unchanged when a metadata rewrite fails, because the + prior successfully written metadata record remains the reached version. + +3. Remove batch-item prepopulation from derived `Store.Paths` values. + `BatchReportResult` receives paths only from the returned `ReportResult` or + from a write that the batch itself successfully completed. +4. Preserve current CLI field names and omission behavior. Human and JSON + summaries must omit every unreached path. +5. Do not change artifact locations, filenames, schemas, report output, or + notification policy in this stage. ### Tests -- Rewrite report registry and CLI parser tests around exactly four commands. -- Remove fake successful-generation tests for nonexistent prompts. -- Preserve Daily/Today/Tomorrow/Hourly period, module, batch-planning, - Distributor path, and output-name coverage. -- Add negative CLI/config tests showing the retired command names and aliases - are rejected. -- Run: - - ```sh - go test ./internal/report ./internal/config ./internal/facts \ - ./internal/changes ./internal/app ./internal/cli - go test ./... - go run ./cmd/weatherreporter --help - git diff --check - ``` - -### Exit Gate - -No implemented registry, CLI, configuration, app, current-state document, or -test claims that three-day, weekend, or storm generation exists. The four -retained reports still pass through the existing generated-text/template -workflow, and each carries exact prompt version `1.0.0`. - -## Stage 2: Promote And Reconcile Runtime Prompt Assets - -### Goal - -Create the single embedded prompt/schema corpus used by Promptkit and by -Weatherreporter's final generated-text boundary. - -### Work - -1. Add Promptkit `v0.4.0` to `go.mod` and `go.sum`. -2. Create `internal/promptassets` with an embedded tree rooted at: - - ```text - assets/ - prompts/ - common/ - daily/ - today/ - tomorrow/ - hourly/ - schemas/ - ``` - - Expose copied/read-only prompt and schema filesystem views and a - `Schema(id)` lookup for the four report schema IDs. Keep path constants - private except where a stable project contract is necessary. -3. Promote only the four `*_generated_text` prompt definitions from - `docs/roadmap/scriptorium`. Do not promote `weather.daily_report`. -4. Use the Scriptorium corpus's common system, data-package guidance, and - report-specific files as the behavioral base. Reconcile rather than - editorially rewrite: - - - preserve its source weighting, hazard, precipitation, and style rules; - - add the optional `confidence` field instruction to Daily, Today, and - Hourly so all prompt text agrees with the canonical domain types and - schemas; - - retain Tomorrow's existing confidence instruction; - - prefer the corpus's correct Tomorrow daily framing over the stale - hourly-style file currently under `internal/reporttemplate/prompts`; - - make no unrelated prompt-tone or product-policy changes. - -5. Every prompt definition must use: - - - its existing prompt ID; - - version `1.0.0`; - - `default_profile: gemini-flash-latest`; - - one required `data_package` input with - `content_type: application/yaml`; - - JSON output with `validation_mode: json_schema`; - - a path inside the embedded schema root; and - - zero or omitted `repair_attempts`. - -6. Move the four canonical schemas from `internal/reporttemplate/schemas` into - `internal/promptassets/assets/schemas`. Preserve their current application - contracts, including optional `confidence`, required summary/discussion, - array discussion for day-style reports, string discussion for Hourly, - `additionalProperties: false`, and the corrected Daily `$id` and title. -7. Change `internal/generatedtext` to load these canonical schemas from - `promptassets`. Remove schema and prompt-fragment ownership from - `internal/reporttemplate`; it should embed and render Markdown templates and - partials only. -8. Retain `docs/roadmap/scriptorium` as migration source evidence until the - final cleanup stage. - -### Tests - -- Add table-driven asset tests for exactly four prompt IDs and versions, - referenced-file resolution, default profiles, YAML input declarations, - output contracts, and schema lookup. -- Construct a Promptkit engine over the embedded sources in an external test - package and call `InspectPrompt` for every report. Do not use a provider or - credentials. -- Retain generated-text schema and typed-validation tests, now reading the - relocated canonical schemas. -- Assert that no active embedded prompt uses `local-heavy`, - `pipeline-weather/`, JSON input metadata, positive repair attempts, or - `weather.daily_report`. -- Run: - - ```sh - go test ./internal/promptassets ./internal/generatedtext \ - ./internal/reporttemplate - git diff --check - ``` - -### Exit Gate - -One embedded source contains exactly four valid Promptkit prompt definitions -and four canonical schemas. Prompt inspection succeeds offline for every -report, and no duplicate runtime provider-facing schema remains. - -## Stage 3: Define The Neutral Prompt Execution Contract - -### Goal - -Define the complete project-owned boundary between Weatherreporter and an LLM -execution adapter. This stage introduces no Promptkit imports and makes no -production orchestration changes. - -### Work - -1. Create `internal/promptexec` with the neutral contract fixed above. Include: - - - exact prompt and profile inspection values; - - safe preparation provenance; - - execution result, validation, token usage, and timing; - - optional sensitive preparation/execution debug values; - - stable error categories and a typed capacity error; and - - small helpers for safe diagnostic bounding and copying. - -2. The safe preparation value includes prompt ID/version/hash, rendered prompt - hash, input hashes, selected profile/backend, effective model name, output - format/validation/schema path, preparation start/end/duration, and the - managed data-package path. It excludes endpoints, API-key environment - names, full parameters, rendered messages, schema bodies, and input bodies. -3. The safe execution value includes Promptkit run ID, repeated prompt and - input provenance, selected profile/backend/model, generated-content hash, - usage, execution start/end/duration, completed validation status, bounded - validation diagnostics, and raw output bytes. It excludes endpoint and full - effective parameters. -4. Bound routine validation diagnostics to at most 10 entries and at most - 1,024 bytes per entry, truncating on a valid UTF-8 boundary. Bound a - persisted safe error message to 2,048 bytes. Prefer stable categories and - fields over diagnostic prose. -5. Specify the execution lifecycle in interface comments and tests: - - - inspection is side-effect-free; - - preparation invokes the callback exactly once only after successful - preparation; - - callback failure prevents provider execution; - - completed validation rejection is a result, not an operational error; - - operational failure does not fabricate a completed result; and - - sensitive debug values are populated only when explicitly requested. -6. Keep copying and truncation helpers internal to `promptexec`. Callers must - not be able to mutate byte slices, maps, or diagnostics retained inside - contract values. - -### Tests - -- Add compile-time test fakes that implement the complete interface without - importing Promptkit. -- Add table-driven tests for every error category, capacity fields, diagnostic - count and byte limits, UTF-8-safe truncation, error-message bounds, and - defensive copying. -- Assert through representative fixtures that safe contract values contain no - endpoint, credential, rendered message, schema body, input body, response - body, or full parameter map. -- Run: - - ```sh - go test ./internal/promptexec - git diff --check - ``` - -### Exit Gate - -`internal/promptexec` provides a stable, documented, offline-tested contract -that can represent every inspection, preparation, execution, validation, -usage, debug, and failure outcome required by the roadmap. It has no Promptkit, -app, CLI, report, or state dependency. - -## Stage 4: Implement The Promptkit Adapter - -### Goal - -Implement and thoroughly test Promptkit behind the Stage 3 contract without -modifying application or CLI orchestration. - -### Work - -1. Create `internal/adapters/promptkit`. Its constructor: - - - uses `promptassets` through `WithPromptFS` and `WithSchemaFS`; - - selects one external profile directory through Promptkit config or one - profile file through `WithProfileFile`; - - optionally registers - `promptkit.LocalBackend(endpoint, concurrencyLimit)`; - - sets the transport timeout; - - accepts an injected Promptkit `LLMClient` only through an adapter-local - test constructor or option; and - - returns project-owned configuration errors. - -2. Implement prompt and profile inspection with exact versions and explicit - field-by-field translation into `promptexec` values. Do not return or embed - Promptkit values. -3. Implement execution: - - - call `PrepareExecution` with the exact requested version and optional - profile override; - - pass the exact YAML as - `InlineWithURI(dataPackagePath, string(bytes))`; - - immediately `defer handle.Discard()` after a successful prepare; - - map safe preparation details and invoke the preparation callback; - - call `RunPrepared` only after the callback succeeds; - - treat `ValidationFailed` as a completed result rather than an operational - error; and - - return no invented execution result for operational errors. - -4. Translate Promptkit's public error sentinels with `errors.Is`, and - `CapacityError` with `errors.As`. Preserve caller cancellation and deadline - identities. Do not parse error strings. -5. Capture sensitive debug values only when requested. Use project-owned debug - structs and explicit field mapping rather than serializing whole Promptkit - values, so a future upstream field cannot silently enter an artifact. -6. Keep the prepared handle entirely within the adapter call. Discard it after - every success or failure path and never make it serializable or reusable. - -### Tests - -- Use an injected deterministic Promptkit model client. Never use a live - endpoint or credential. -- Cover prompt/profile inspection, profile override precedence, built-in and - external profiles, endpoint-only profiles, the conventional local backend, - local concurrency/capacity rejection, missing credentials, and constructor - validation. -- Cover preparation callback ordering, callback failure preventing generation, - exact inline bytes and URI, exact prompt version, successful execution, - completed schema rejection with raw output, operational generation and - validation failures with no partial result, cancellation, timeout, and - handle discard. -- Assert that safe values and errors do not contain rendered messages, - endpoints, schema bodies, input bodies, response bodies, credentials, or - full parameter maps. -- Run: - - ```sh - go test ./internal/promptexec ./internal/adapters/promptkit - go test -race ./internal/adapters/promptkit - git diff --check - ``` - -### Exit Gate - -The Promptkit adapter is a complete offline-tested implementation of the -neutral contract. No Promptkit type is referenced outside the adapter, its -tests, or the external asset contract test, and no production path uses the -adapter yet. - -## Stage 5: Introduce Durable State V2 - -### Goal - -Add the final normal-artifact model and backward-compatible metadata reading -before app orchestration starts writing the new records. Sensitive debug -storage remains out of scope until Stage 6. - -### Work - -1. Add the new-run path members `Preparation` and `Execution`: - - - preparation: - `preflight///prompt_preparation..json`; - - execution: - `snapshots///prompt_execution..json`. - - Keep `workspace.preflight_dir` and its default physical directory unchanged. - Retain `Preflight` and `GeneratedTextResult` only as temporary legacy - write-path members required by the uncut Scriptorium workflows; mark them - for deletion in Stage 10. New Promptkit code must use only the new members. -2. Add `PromptPreparationArtifact` with schema - `weatherreporter.prompt_preparation.v1`. It represents either successful - preparation or a failed preparation receipt and includes schema version, - status, report/RunID, prompt identity, safe preparation provenance when - available, timing, data-package path, and a bounded classified error when - failed. -3. Add `PromptExecutionArtifact` with schema - `weatherreporter.prompt_execution.v1`. It represents success, completed - validation rejection, or operational failure and includes safe execution - provenance, validation, usage, timing, reached artifact paths, and a bounded - classified error when failed. It never embeds generated content. -4. Advance newly written metadata to `weatherreporter.metadata.v2`. Replace - `preflightPath` and `generatedTextResultPath` with `preparationPath` and - `executionPath`. Populate paths only after the corresponding artifact has - actually been saved. -5. Implement explicit v1/v2 metadata decoding: - - - accept only the known v1 and v2 schema versions; - - normalize v1 `preflightPath` and `generatedTextResultPath` internally for - inspection; - - preserve v1 field names when a loaded v1 record is marshaled by - `inspect metadata`; - - write only v2 through the new Promptkit state APIs; the temporary - Scriptorium path may continue writing v1 until its Stage 10 removal; and - - never dual-write legacy aliases. - - Direct inspection of v1 metadata, modules, data packages, sources, and - referenced artifacts remains available for any historical report ID. - Prior-snapshot reconstruction is required only for the four retained report - IDs; do not restore retired definitions solely for legacy comparison. -6. Add typed `SavePromptPreparation`, `SavePromptExecution`, and corresponding - typed load methods where inspection needs them. Temporarily retain the - Scriptorium write methods so the current production paths compile through - Stage 9; delete those methods during Stage 10. Keep long-term legacy support - read-only. -7. Update app/CLI result structs and JSON field names to - `preparationPath`, `executionPath`, and optional `llmDebugPath`; temporarily - adapt old orchestration so the tree compiles until the cutover stages. - Do not write v2 records through the Scriptorium path. - -### Tests - -- Add exact path, schema-defaulting, round-trip, required-field, atomic-write, - and unknown-version tests. -- Add v1 fixtures covering both generated-text and legacy preflight references; - verify list, metadata, modules, data-package, and source inspection. -- Verify that re-marshaled v1 inspection uses v1 field names and that v2 output - contains no deprecated aliases. +- Add focused app tests for one representative report using real temporary + state plus a narrow failure-injecting store wrapper. +- Fail the next persistence step immediately after a successful preparation + receipt, execution receipt, raw output, normalized output, render context, + managed report, output copy, and notification artifact; assert that the + returned result contains every reached path and no future path. +- Include one preparation failure, one operational execution failure, and one + completed validation rejection to cover the three execution outcome shapes. +- Add batch and CLI summary assertions proving unreached paths are omitted. - Run: ```sh @@ -480,429 +131,363 @@ storage remains out of scope until Stage 6. ### Exit Gate -State can read historical v1 runs and write the complete normal v2 artifact -contract through the new APIs. The existing production generation path still -compiles and remains unchanged until cutover; no new Promptkit artifact uses a -Scriptorium-specific filename, schema, or JSON field. +Every nonblank path in newly written metadata, app results, batch items, and +CLI summaries names an artifact that exists. Every safe artifact successfully +written before a later failure remains discoverable from the returned partial +result. -## Stage 6: Add Secure LLM Debug Persistence +## Stage 13: Harden Durable State Contracts And Restore State Coverage ### Goal -Implement the explicitly enabled sensitive-debug store as a small, independently -auditable boundary before any CLI command can request debug capture. +Make the v1/v2 wire boundary and Promptkit-era artifact validation explicit, +strict, and durably tested. ### Work -1. Add a focused debug writer, owned by state or a narrow state-adjacent - package, for an explicit operator root outside normal artifact derivation. - It must: +1. Strengthen `PromptPreparationArtifact.Validate`: - - validate or create the root before collection or provider work; - - use `////`; - - write `preparation.json` and `execution.json` atomically; - - create directories with `0700` and files with `0600`; - - reject symlinks, path escape, non-directory roots, and unsafe - run/report/date segments; and - - return the per-run debug directory as the project-owned summary path. + - require report ID, Weatherreporter RunID, prompt ID, exact prompt version, + data-package path, nonzero start/end times, nonnegative duration, and an + end not earlier than the start; + - for success, require preparation provenance, prohibit an error, and + require its prompt ID/version and data-package path to match the top-level + artifact; + - for failure, require a classified bounded error and prohibit fabricated + preparation provenance. -2. Define explicit project-owned debug wire structs. Preparation debug may - contain rendered messages, structured-output schema, and effective - endpoint/parameters. Execution debug may additionally contain raw generated - output and completed validation details. -3. Exclude direct credentials, resolved environment credential values, and - arbitrary dependency objects. Use field-by-field mapping so future - `promptexec` or Promptkit fields are not serialized implicitly. -4. Make disabled debug behavior a no-op that performs no filesystem access. - A requested debug initialization or write error is returned to the caller; - it is never silently downgraded. -5. Keep this writer independent of normal state path derivation. Normal state - inspection must not discover or serve sensitive debug artifacts. +2. Strengthen `PromptExecutionArtifact.Validate`: + + - require report ID, Weatherreporter RunID, prompt ID, exact prompt version, + nonzero start/end times, nonnegative duration, and an end not earlier than + the start; + - for success and validation rejection, require provenance and completed + validation, prohibit an operational error, and require the provenance + prompt ID/version to match the artifact; + - do not compare the provenance RunID with the Weatherreporter RunID because + the provenance value is Promptkit's run identity; + - for operational failure, require a classified bounded error and prohibit + invented provenance or completed validation. + +3. Validate required provenance fields for completed executions, including + Promptkit RunID, prompt and rendered hashes, selected profile/backend/model, + and data-package path. Permit usage counters and generated hash to be zero + when the provider legitimately reports no value. +4. Restore focused filesystem and metadata tests for: + + - exact v2 paths and filenames; + - preparation/execution round trips and required fields; + - metadata v2 round trips without legacy aliases; + - v1 decoding, normalized internal aliases, and v1-preserving re-marshaling; + - unknown schema rejection; + - report listing, RunID lookup, source/module/data-package inspection, and + retained v1 behavior for historical report IDs; + - atomic writes and unsafe workspace/path rejection; and + - prior-snapshot behavior for the four supported report IDs. + +5. Adapt useful tests from the deleted filesystem suite rather than recreating + redundant low-value cases. Do not restore Scriptorium writes or retired + report behavior. ### Tests -- Verify exact grouping, atomic replacement, disabled behavior, and returned - per-run paths. -- Verify directory `0700` and file `0600` permissions on supported platforms. -- Cover traversal, absolute-segment, symlink-root, symlink-component, - non-directory, and invalid report/date/run segment rejection. -- Marshal representative debug fixtures and verify credentials and resolved - secret values are absent while the explicitly allowed diagnostic fields are - retained. -- Run: +Run: - ```sh - go test ./internal/state - git diff --check - ``` +```sh +go test ./internal/state ./internal/app +git diff --check +``` ### Exit Gate -Sensitive debug persistence is secure, explicitly rooted, atomic, isolated -from normal state, and fully tested. No production CLI path enables it yet. +The state package rejects incomplete or contradictory Promptkit-era artifacts, +reads historical v1 records, writes only valid v2 records, and has focused +offline coverage for its durable compatibility and filesystem contracts. -## Stage 7: Add Promptkit Configuration, Composition, And Inspection +## Stage 14: Complete Execution-Artifact Path Tracking ### Goal -Add the configuration and project-owned composition seams needed for cutover, -and centralize pre-collection prompt/profile validation. Retain Scriptorium as -the production generator until Stage 8 so this stage does not create a -half-cut-over runtime. +Make `PromptExecutionArtifact.Paths` accurately record every downstream +artifact reached after a completed Promptkit run. ### Work -1. Add `config.PromptkitConfig` and nested local config with exactly: +1. Treat the execution artifact as an atomically updated durable record of the + completed Promptkit execution and subsequent artifact destinations. Its + status, provenance, validation, usage, and timing remain the provider-run + outcome; later application failures do not change a successful Promptkit + status into an execution failure. +2. Save the initial execution artifact after raw output is persisted, with + `RawOutputPath` populated. +3. After each later successful write, update and atomically resave the same + execution artifact with the corresponding reached path: - - `profile`; - - `profile_file`; - - `profile_dir`; - - `timeout`, default `2m`; - - `local.endpoint`; and - - `local.concurrency_limit`, default `1`. - - Scriptorium config remains temporarily because production generation has - not yet cut over. Promptkit validation rejects simultaneous profile - sources, non-positive transport timeout, an invalid nonblank local - endpoint, and negative concurrency. A blank local endpoint leaves `local` - unregistered; concurrency zero means unlimited. -2. Add an executor factory seam to `cli.Runner` using only project-owned types. - Production construction delegates to `internal/adapters/promptkit`; CLI - tests inject a fake factory. The factory creates one executor for an action, - not one per report. -3. Add a project-owned app inspection helper that: - - - inspects the exact prompt ID and `report.Definition.PromptVersion`; - - verifies exactly one required `data_package` input with - `application/yaml`; - - verifies the expected JSON Schema output contract and declared default - profile; - - selects `promptkit.profile` when nonblank, otherwise the prompt default; - - inspects that explicit profile; - - rejects a profile requiring a direct API key, because Weatherreporter has - no direct-key configuration; and - - requires a nonblank environment value for any reported `APIKeyEnv`. - -4. Return only safe, project-owned inspection values and classified errors. - The helper performs no collection, provider call, or durable write. -5. Unit-test production factory argument mapping, but do not yet wire it into - `generate` or `run`. Stage 8 performs that atomic production cutover. - -### Tests - -- Cover exact defaults, mutual profile-source exclusion, local endpoint - validation, zero/unlimited and negative concurrency, and timeout validation. -- Cover prompt/profile default and override precedence, prompt contract - mismatch, unsupported direct-key profiles, missing environment credentials, - and safe errors. -- Verify the CLI factory maps embedded assets, external profile source, local - backend, concurrency, and timeout exactly once without exposing Promptkit - types. -- Run: - - ```sh - go test ./internal/config ./internal/app ./internal/cli \ - ./internal/adapters/promptkit - git diff --check - ``` - -### Exit Gate - -Configuration, engine construction, and pre-collection inspection are -decision-complete and offline-tested behind project-owned seams. Production -generation still follows the old path, so there is no dual execution mode. - -## Stage 8: Cut Over Single-Report Execution And Failure Persistence - -### Goal - -Move all four `generate` commands to Promptkit prepared execution while -preserving report output behavior and inspectable partial failure. Debug CLI -enablement and final summary presentation are deferred to Stage 9. - -### Work - -1. Construct exactly one executor after configuration is loaded for a - `generate` action and pass it through `app.GenerateRequest`. -2. Isolate the current shared generation function before changing it: - - - move the Scriptorium implementation behind a temporary, unexported legacy - batch helper; - - make `GenerateDetailed` call the new Promptkit implementation described - below; - - leave `RunBatchDetailed` calling only the legacy helper until Stage 10; + - normalized generated text; + - render context; + - managed Markdown report; + - an explicitly requested extra output copy, only after the copy succeeds; and - - do not add a runtime switch, fallback, or dual invocation for the same - report. + - a Distributor notification artifact, including a persisted failure or + status artifact when notification produced one. - This temporary split is solely a staging seam. Stage 10 moves batches to - the Promptkit implementation and deletes the helper. -3. Reorder `GenerateDetailed`: - - 1. resolve the report and RunID; - 2. require the injected executor; - 3. run the Stage 7 exact prompt/profile/credential inspection; and only then - 4. collect weather. - - Inspection failures occur before managed run artifacts exist. -4. Implement the new project-owned report-generation core with the injected - executor and the Stage 5 artifact contract: - - - build, serialize once, and save the data package; - - use those exact serialized bytes for execution; - - persist a failed preparation receipt and v2 metadata when preparation - fails; - - use the preparation callback to save successful preparation provenance - and metadata before provider work; - - after execution success or completed validation rejection, save exact raw - output, then the execution artifact, then metadata; - - after operational execution failure, save a failure execution receipt and - metadata without inventing raw output; - - on Promptkit validation rejection, return a classified report failure - after preserving raw output and bounded diagnostics; - - on Promptkit validation success, run the existing typed - `internal/generatedtext` validation, normalize/save generated text, build - and save render context, and render managed Markdown as before. - -5. Make partial failures return a non-nil `ReportResult` whenever a RunID and - inspectable paths exist. Preserve preparation/execution/raw/normalized/ - context/report paths reached before preparation, capacity, generation, - Promptkit validation, domain validation, template, copy, or notification - failure. -6. Remove Scriptorium result values from the single-report app requests, - results, and test fakes used by the generate path. App tests depend only on - `promptexec.Executor`. - Retain the old adapter and configuration only for the still-uncut batch - path. -7. Do not expose `--llm-debug-dir` yet. Pass `CaptureDebug: false` during this - stage; Stage 9 adds debug orchestration without changing provider execution - semantics. +4. Keep metadata and execution-artifact path values consistent after every + successful checkpoint. Save the execution artifact before metadata so a + metadata failure does not erase knowledge of a reached downstream artifact. + Failure to update the execution artifact is terminal and returns a partial + result containing the downstream artifact that was already written. +5. Refactor finalization return values only as needed to tell the orchestration + layer which copy and notification paths were actually written. Distributor + must continue uploading only the managed Markdown report. +6. A validation-rejected execution ends after raw output and therefore records + only the raw-output path. An operational execution failure has no completed + provenance and records only safe paths reached before that failure. ### Tests -- Add representative offline app workflows for each of the four - reports, using real state, generated-text validation, contexts, and - templates with a fake executor. -- Verify inspection and credential checks occur before collection. -- Verify exact profile override/default precedence and exact prompt versions. -- Verify persistence ordering by observing that execution is not called until - the successful preparation artifact and metadata exist. -- Cover preparation failure, callback/state failure, capacity rejection, - credential disappearance at execution, cancellation, deadline, generation - failure, operational validation failure, completed schema rejection, - generated-text domain failure, template failure, output-copy failure, and - notification failure. -- Verify partial results retain every reached normal-artifact path without - prompt or response content. -- Verify single-report commands invoke only Promptkit and batch commands invoke - only the temporary legacy helper during this intermediate stage. +- Add table-driven execution-artifact lifecycle tests for success and every + downstream failure point. +- Load the persisted execution artifact after normalized-text, context, + template, copy, metadata, and notification failures and assert its status and + exact reached paths. +- Assert that execution artifacts never contain generated bodies, rendered + prompts, schemas, endpoints, parameters, or credentials. - Run: ```sh - go test ./internal/app ./internal/cli - go test -race ./internal/app ./internal/adapters/promptkit + go test ./internal/state ./internal/app git diff --check ``` ### Exit Gate -Every single-report command uses one Promptkit engine, performs inspection -before collection, persists preparation before provider work, and produces the -same managed Markdown/template and notification outcomes through project-owned -contracts. Sensitive debug capture remains disabled. +For every completed Promptkit run, its execution artifact contains exactly the +safe downstream paths reached by the workflow and remains semantically correct +when a later application stage fails. -## Stage 9: Add Single-Report Debug And CLI Summary Behavior +## Stage 15: Restore Single-Report Behavioral Coverage ### Goal -Expose the opt-in debug workflow and finalize single-report CLI success and -partial-failure summaries without expanding the normal artifact boundary. +Restore the risk-based application coverage removed during final cutover and +prove the complete single-report Promptkit workflow through project-owned +boundaries. ### Work -1. Parse `--llm-debug-dir PATH` for all four generate commands. Do not add a - YAML debug switch. -2. Validate or create the debug root before prompt inspection or weather - collection. A requested initialization failure is terminal and occurs - before managed run artifacts exist. -3. Pass debug intent through project-owned CLI and app request types. When - enabled, request sensitive debug values from the executor: +1. Reintroduce a focused app test harness using real state, prompt-input, + generated-text validation, render contexts, and templates with deterministic + collector, executor, notifier, clock, and filesystem boundaries. +2. Add representative successful workflows for Daily, Today, Tomorrow, and + Hourly. Verify report identity, exact prompt version, one collection, exact + persisted YAML bytes passed to the executor, expected template output, + optional copy behavior, and managed-report notification source. +3. Cover the required failure matrix: - - write preparation debug inside the preparation callback, after the normal - preparation artifact is saved and before metadata is finalized and - provider work begins; - - if the debug write fails, return the callback error so `RunPrepared` is - not called; - - write execution debug immediately after receiving a completed execution - result and before continuing with downstream validation/rendering; and - - treat every requested debug write failure as terminal while preserving - already reached normal artifacts. + - inspection and missing credentials before collection; + - preparation failure and callback persistence failure before provider work; + - execution-time credential disappearance; + - capacity rejection without retry; + - cancellation and deadline; + - generation and operational-validation failure; + - completed Promptkit schema rejection with retained raw output; + - generated-text decode/domain rejection; + - render-context and template failure; + - output-copy failure; and + - notification failure. -4. Set `llmDebugPath` only after the per-run debug directory contains at least - one successfully written debug artifact. Never copy sensitive debug content - into a normal artifact, log, error, or summary. -5. Finalize human and JSON CLI summaries around project-owned - `preparationPath`, `executionPath`, and optional `llmDebugPath`. Include - reached paths on partial failure; omit absent paths instead of inventing - them. -6. Preserve existing output behavior: quiet mode suppresses successful human - output, JSON output remains machine-readable, and failures return a - non-zero status with safe classified text. +4. Verify preparation persistence precedes provider execution, debug-write + failure prevents provider execution, and execution-debug failure preserves + previously reached normal and debug artifacts. +5. Verify Recent Changes, prior-snapshot selection, output naming, and + Distributor template values for all four retained reports. +6. Adapt useful tests from the deleted app suite. Omit Scriptorium mechanics, + subprocess interaction assertions, and retired report products. +7. Fix defects exposed by these tests only when the expected behavior is + already decided by the roadmap or canonical policy. Record any new product + question instead of silently choosing it. ### Tests -- Add CLI parser and help tests for `--llm-debug-dir` on all four generate - commands and its absence from YAML configuration. -- Verify debug initialization precedes inspection and collection. -- Verify preparation debug persistence precedes provider execution and that a - write failure prevents the model-client call. -- Cover execution debug success/failure, partial normal-artifact retention, - `llmDebugPath` population rules, quiet mode, JSON summaries, and safe - failure text. -- Assert that summaries and routine logs contain no rendered prompt, schema - body, input body, generated body, endpoint, full parameters, or credential. -- Run: +Run: - ```sh - go test ./internal/app ./internal/cli ./internal/state - go test -race ./internal/app ./internal/adapters/promptkit - git diff --check - ``` +```sh +go test ./internal/app +go test -race ./internal/app ./internal/adapters/promptkit +git diff --check +``` ### Exit Gate -All single-report commands provide the complete Promptkit, v2 persistence, -opt-in debug, and summary behavior required by the roadmap. Debug failure -ordering is enforced without exposing sensitive content. +The single-report workflow has deterministic behavioral coverage for all four +reports, all consequential failure stages, artifact ordering, partial results, +debug isolation, output copying, and notification behavior. -## Stage 10: Cut Over Batches And Remove Scriptorium +## Stage 16: Refactor Prompt Generation Orchestration ### Goal -Complete the production cutover, share one engine through each sequential -batch, and delete the retired dependency boundary. +Reduce the complexity and duplicated persistence logic in +`generatePromptReport` without changing observable behavior. ### Work -1. Construct one executor in the CLI for each `run morning` or `run evening` - invocation and pass it through `BatchRequest` to every report. -2. Parse and initialize `--llm-debug-dir` for run commands using the same - policy as generate commands. -3. Before collection, inspect the complete candidate set: +1. Use the Stage 12–15 tests as the refactoring safety boundary. Do not weaken + assertions to accommodate structural changes. +2. Split the current orchestration into small app-owned operations with clear + inputs and outcomes for: - - morning: Today, Tomorrow, and Daily; - - evening: Tomorrow and Daily. + - deterministic input and initial state construction; + - preparation callback persistence; + - preparation-failure persistence; + - operational-execution-failure persistence; + - completed execution and raw-output persistence; + - normalized text and render-context persistence; + - managed report, optional copy, metadata, and notification finalization; + and + - reached-path updates shared by success and failure paths. - Inspect exact prompt versions, validate declared input/output contracts, - resolve unique effective profiles, and enforce credential availability. - Daily inspection occurs before its collection-dependent future dates are - known because every eligible Daily run uses the same exact prompt contract. -4. Collect once, plan the batch as before, and execute every planned report - sequentially through the shared executor, Stage 8 generation core, and - store. Delete the temporary legacy batch helper after this call site moves. - Preserve continuation after independent report failures and existing - batch-notification gating. -5. Include preparation, execution, and optional debug paths in each batch - item, including failed items when those paths were reached. -6. Replace Scriptorium configuration entirely with Promptkit configuration. - Update defaults, validation, examples, config tests, and all construction - sites. Explicitly reject a top-level `scriptorium:` key with an actionable - migration error even though the general YAML loader is currently - permissive; silently ignoring a former execution configuration is unsafe. - Do not translate it or add a dual-run mode. The maintained examples contain - only `promptkit:`. -7. Delete: - - - `internal/adapters/scriptorium`; - - the Scriptorium renderer interface and all result/request types; - - the legacy state path members and Scriptorium write methods retained in - Stage 5; - - subprocess fakes and subprocess-specific tests; - - direct-Markdown branches and remaining Scriptorium defaults; and - - dead helpers used only by the old adapter. - -8. Run `go mod tidy` and verify the module graph contains Promptkit `v0.4.0` - and no Scriptorium module or local replacement. +3. Keep workflow order visible in one coordinator. Do not introduce a generic + workflow engine, hidden retry loop, provider-specific app type, or mutable + global state. +4. Centralize the repeated rule that a successful artifact write updates the + result before any following write can fail. +5. Preserve error identities, safe text, atomic writes, exact bytes, debug + ordering, partial results, and notification behavior. ### Tests -- Add batch tests proving one factory call/engine per CLI invocation, one - executor shared across all reports, one collection, sequential ordering, - later-report continuation, per-report path retention, and notification only - after complete success. -- Cover a capacity failure for one report followed by a later independent - report and confirm no retry occurs. -- Cover debug grouping for multiple reports and multiple Daily dates. -- Update config tests for exact defaults, mutual exclusivity, local endpoint, - zero/unlimited and negative concurrency, and maintained examples. -- Run: +Run: - ```sh - go test ./internal/config ./internal/app ./internal/cli - go test -race ./internal/app ./internal/adapters/promptkit - go test ./... - go run ./cmd/weatherreporter --help - git diff --check - ``` +```sh +gofmt -w internal/app/*.go +go test ./internal/app ./internal/state ./internal/cli +go test -race ./internal/app +git diff --check +``` ### Exit Gate -All production generation paths use Promptkit. A batch owns one engine and -continues sequentially under existing failure policy. No Scriptorium code, -configuration field, subprocess path, or dependency remains. +The top-level coordinator communicates the workflow order without containing +the full persistence implementation, duplicate failure branches are reduced, +and every Stage 12–15 behavioral test passes unchanged. -## Stage 11: Update Canonical Documentation And Complete Verification +## Stage 17: Restore Batch Behavioral Coverage ### Goal -Make current-state documentation match the completed implementation, remove -migration-only source material, and perform repository-wide verification. +Re-establish confidence that morning and evening batches preserve their +pre-migration behavior while sharing one Promptkit executor. ### Work -1. Update canonical current-state owners in the same migration change: +1. Add assembled batch tests proving: - - `docs/development.md` for repository orientation, package map, task - routing, and validation language; - - `docs/policy/architecture.md` for the Promptkit adapter boundary, - generated-text-only flow, prepared execution, and four-report product; - - `docs/policy/testing.md` for Promptkit/provider fakes instead of - subprocess fakes and the offline external-boundary rule; - - `docs/cli.md` for four commands, `--llm-debug-dir`, and renamed summary - fields; - - `docs/config.md` and `examples/` for the exact Promptkit contract; - - `docs/operations.md` for v2 paths, sensitive debug retention and - permissions, preparation/execution lifecycle, and v1 inspection; - - `docs/troubleshooting.md` for inspection, profile/credential, - preparation, capacity, execution, validation, and debug failures; - - relevant `docs/internal/` files for app, CLI, report, prompt input, - generated text, templates, state, collection, briefing, facts, changes, - and package boundaries; and - - `docs/templates.md` for Promptkit-generated prose and the relocated schema - owner. + - one executor factory call and one executor per CLI invocation; + - inspection of the full candidate set before collection; + - one weather collection; + - existing morning/evening planning and ordering; + - sequential execution through the shared executor; + - continuation after an independent report failure; + - no retry after capacity rejection; + - distinct identities and debug directories for multiple Daily dates; and + - exact reached paths on successful and failed batch items. -2. Replace `docs/integrations/scriptorium.md` and - `docs/internal/scriptorium-adapter.md` with canonical Promptkit integration - and adapter documents. The integration document owns the logical - prompt/profile/schema and durable compatibility contract; the internal - document owns construction, mapping, lifecycle, and tests. Link rather than - duplicate the full CLI/config/operations references. -3. Search all non-roadmap current-state documentation, examples, Go code, - tests, help output, and module metadata for stale `scriptorium`, - `local-heavy`, retired report commands, old artifact field names, old - filenames, and old metadata versions. Retain old names only in explicit v1 - compatibility code/tests and historical roadmap discussion. -4. Remove `docs/roadmap/scriptorium` after confirming every promoted runtime - asset is represented under `internal/promptassets` and covered by asset - tests. The unused legacy Daily Markdown prompt is deleted with this source - directory. -5. Keep the feature roadmap and this plan as temporary migration documents - while implementation is under review. Once the implementation is accepted, - mark both complete and move every still-useful contract to its canonical - current-state owner before later archival or removal. -6. Review all added links, fenced examples, flags, field names, defaults, - schema IDs, file paths, and version strings against executable sources. - Confirm examples contain no credentials or private infrastructure values. +2. Restore notification coverage for disabled notification, suppressed + per-report notification, all-success batch notification, skipped + notification after report failure, and persisted notification failure/status + artifacts. +3. Restore output-directory, Today/Tomorrow naming, dynamic Daily planning, + prior-snapshot, and managed-Markdown upload-source coverage. +4. Adapt useful tests from the deleted batch portions of the app and CLI suites. + Do not restore retired report cases or Scriptorium fakes. +5. Fix only roadmap-defined batch regressions exposed by the restored tests. + +### Tests + +Run: + +```sh +go test ./internal/app ./internal/cli +go test -race ./internal/app +git diff --check +``` + +### Exit Gate + +Morning and evening batches are covered as assembled sequential workflows and +demonstrably preserve collection, planning, continuation, output, debug, +artifact, and notification contracts with one Promptkit executor. + +## Stage 18: Restore CLI And Inspection Coverage + +### Goal + +Restore the user-facing command, summary, and historical inspection contracts +removed with the old root test suite. + +### Work + +1. Add parser and resolver tests for all four generate commands, both batch + commands, shared flags, report-specific date rules, malformed input, + `--llm-debug-dir`, `--quiet`, output paths, and rejection of retired report + names. +2. Add assembled CLI tests for representative successful single and batch + invocations using injected offline boundaries. Verify exactly one executor + construction per action. +3. Cover pre-run errors with no invented run summary, successful and failed + JSON summaries, quiet-mode behavior, safe human status output, partial paths, + and omission of absent notification/debug fields. +4. Restore inspection tests for report listing and v1/v2 metadata, modules, + data packages, prior snapshots, and sources. Include failed v2 runs and v1 + fixtures using historical report IDs. +5. Assert that routine output never contains rendered prompts, schema bodies, + data packages, generated bodies, endpoints, full parameters, credentials, or + secret-like dependency errors. +6. Keep tests at stable CLI/app boundaries; do not restore assertions about + private parser formatting or Scriptorium subprocess mechanics. + +### Tests + +Run: + +```sh +go test ./internal/cli ./internal/app ./internal/state +go run ./cmd/weatherreporter --help +git diff --check +``` + +### Exit Gate + +The supported CLI surface, summaries, quiet mode, partial failures, executor +composition, and v1/v2 inspection behavior have deterministic offline coverage. + +## Stage 19: Finalize Documentation And Repository Verification + +### Goal + +Close the audit remediation, make roadmap lifecycle state truthful, and verify +the repository against the complete target contract. + +### Work + +1. Update `docs/roadmap/promptkit.md` from future tense and “unimplemented” + statuses to a completed roadmap record. Describe its old seven-report and + Scriptorium material explicitly as the pre-migration baseline rather than + current behavior. +2. Mark Stages 12–19 and this implementation plan complete only after their + exit gates pass. Retain the concise completed-stage history unless the + documentation policy calls for archival in the same change. +3. Review canonical architecture, app, state, CLI, Promptkit integration, + operations, troubleshooting, configuration, and testing documentation + against the corrected implementation. Update only actual current-state + discrepancies; do not duplicate the roadmap. +4. Search current-state code, tests, examples, help, and non-roadmap + documentation for stale Scriptorium terms, retired reports, old artifact + fields, speculative-path descriptions, or claims of missing Promptkit + implementation. +5. Confirm examples contain no credentials or private infrastructure values + and load through config tests. ### Final Verification @@ -911,7 +496,8 @@ Run: ```sh gofmt -w go mod tidy -go test ./... +go vet ./... +go test -count=1 ./... go test -race ./... go run ./cmd/weatherreporter --help git diff --check @@ -922,23 +508,23 @@ Then verify explicitly: - `go list -m gitea.maximumdirect.net/eric/promptkit` reports `v0.4.0`; - no committed `go.work`, `replace`, secret fixture, or live-provider test exists; -- the maintained examples load through config tests; -- all four embedded prompts inspect at exact version `1.0.0`; +- all four prompts inspect at exact version `1.0.0`; - no runtime prompt requests repair attempts; -- no ordinary artifact or CLI summary includes rendered prompts, schema - bodies, input bodies, generated bodies, provider endpoints, or credentials; -- v1 metadata fixtures remain inspectable and new runs write only v2; +- v1 fixtures remain inspectable and new runs write only v2; +- normal artifacts and output contain no sensitive prompt/debug content; +- failed-run metadata, execution artifacts, app results, batch items, and CLI + summaries contain exactly the paths actually reached; - help exposes only Daily, Today, Tomorrow, Hourly, morning, and evening; and - managed Markdown remains the only Distributor upload source. ### Exit Gate -Every completion criterion in the feature roadmap is demonstrably satisfied. -Current-state documentation describes the implementation rather than the -migration, and the repository passes all final verification commands. +Every migration and audit-remediation criterion is demonstrably satisfied, +the restored tests protect the consequential contracts, canonical +documentation describes the corrected implementation, and both roadmap +documents are marked complete. ## Open Questions -None. The feature roadmap and this implementation plan contain all product, -architecture, configuration, compatibility, security, sequencing, and -verification decisions required to begin implementation. +None. The roadmap and this follow-up plan contain the decisions required to +complete the audit remediation. diff --git a/internal/app/app.go b/internal/app/app.go index 60919ef..d28052a 100644 --- a/internal/app/app.go +++ b/internal/app/app.go @@ -343,13 +343,6 @@ func RunBatchDetailed(ctx context.Context, req BatchRequest) (*BatchResult, erro for _, planned := range plannedReports { resolved := planned.Resolved item := batchReportResult(planned) - if paths, err := store.Paths(resolved); err == nil { - item.DataPackagePath = paths.DataPackage - item.PreparationPath = paths.Preparation - item.ExecutionPath = paths.Execution - item.ReportPath = paths.RenderedReport - item.MetadataPath = paths.Metadata - } outputPath := plannedBatchOutputPath(req.OutputDir, planned) reportResult, err := generatePromptReport(ctx, promptReportRequest{ GenerateRequest: GenerateRequest{ @@ -540,6 +533,7 @@ type finalizeRenderedReportRequest struct { Store state.Store Resolved report.Resolved Metadata state.Metadata + MetadataPath string ManagedReportPath string OutputPath string Notifier Notifier @@ -563,27 +557,24 @@ func finalizeRenderedReport(ctx context.Context, req finalizeRenderedReportReque return finalizeRenderedReportResult{}, fmt.Errorf("managed report path is required for report %q", req.Resolved.Definition.ID) } - metadata := req.Metadata - metadata.RenderedReportPath = req.ManagedReportPath - outputPath := req.ManagedReportPath + result := finalizeRenderedReportResult{Metadata: req.Metadata, MetadataPath: req.MetadataPath} if req.OutputPath != "" { - outputPath = req.OutputPath if req.GenerationErr == nil && req.OutputPath != req.ManagedReportPath { if err := fileutil.CopyFileAtomic(req.ManagedReportPath, req.OutputPath); err != nil { - return finalizeRenderedReportResult{}, err + return result, err } } + result.OutputPath = req.OutputPath } + metadata := req.Metadata + metadata.RenderedReportPath = req.ManagedReportPath metadataPath, err := req.Store.SaveMetadata(ctx, metadata) if err != nil { - return finalizeRenderedReportResult{}, err - } - result := finalizeRenderedReportResult{ - OutputPath: outputPath, - Metadata: metadata, - MetadataPath: metadataPath, + return result, err } + result.Metadata = metadata + result.MetadataPath = metadataPath if req.GenerationErr != nil { return result, req.GenerationErr } @@ -593,14 +584,16 @@ func finalizeRenderedReport(ctx context.Context, req finalizeRenderedReportReque notification, notificationPath, err := notifyReport(ctx, req.Config, req.Resolved, req.ManagedReportPath, metadata, req.Notifier, req.Store) if notificationPath != "" { + result.NotificationPath = notificationPath + result.Notification = notification metadata.NotificationPath = notificationPath + result.Metadata = metadata metadataPath, saveErr := req.Store.SaveMetadata(ctx, metadata) if saveErr != nil { - return finalizeRenderedReportResult{}, saveErr + return result, saveErr } result.Metadata = metadata result.MetadataPath = metadataPath - result.NotificationPath = notificationPath } result.Notification = notification if err != nil { diff --git a/internal/app/batch_execution_test.go b/internal/app/batch_execution_test.go index 8d8e88a..b732582 100644 --- a/internal/app/batch_execution_test.go +++ b/internal/app/batch_execution_test.go @@ -2,7 +2,9 @@ package app import ( "context" + "encoding/json" "errors" + "strings" "testing" "gitea.maximumdirect.net/eric/weatherreporter/internal/collect" @@ -39,6 +41,28 @@ func TestRunBatchDetailedInspectsEveryCandidateBeforeCollection(t *testing.T) { } } +func TestCopyBatchReportPathsLeavesUnreachedPathsEmpty(t *testing.T) { + item := BatchReportResult{} + copyBatchReportPaths(&item, &ReportResult{ + DataPackagePath: "/runs/daily/data_package.yaml", + PreparationPath: "/runs/daily/preparation.json", + }) + + data, err := json.Marshal(item) + if err != nil { + t.Fatalf("Marshal() error = %v", err) + } + text := string(data) + for _, omitted := range []string{"executionPath", "reportPath", "outputPath", "metadataPath", "notificationPath"} { + if strings.Contains(text, omitted) { + t.Fatalf("batch item includes unreached field %q:\n%s", omitted, text) + } + } + if !strings.Contains(text, "dataPackagePath") || !strings.Contains(text, "preparationPath") { + t.Fatalf("batch item omits reached paths:\n%s", text) + } +} + type collectorFunc func(context.Context, collect.Request) (*collect.Result, error) func (f collectorFunc) Run(ctx context.Context, req collect.Request) (*collect.Result, error) { diff --git a/internal/app/prompt_artifact_paths_test.go b/internal/app/prompt_artifact_paths_test.go new file mode 100644 index 0000000..468c9ee --- /dev/null +++ b/internal/app/prompt_artifact_paths_test.go @@ -0,0 +1,272 @@ +package app + +import ( + "context" + "encoding/json" + "errors" + "os" + "path/filepath" + "testing" + "time" + + "gitea.maximumdirect.net/eric/weatherreporter/internal/collect" + "gitea.maximumdirect.net/eric/weatherreporter/internal/config" + "gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec" + "gitea.maximumdirect.net/eric/weatherreporter/internal/report" + "gitea.maximumdirect.net/eric/weatherreporter/internal/state" + "gitea.maximumdirect.net/eric/weatherreporter/internal/weatherdata" +) + +const ( + failPromptExecution = "prompt execution" + failMetadata = "metadata" +) + +type failingPersistenceStore struct { + state.Store + failOperation string + failMetadataCall int + metadataCalls int +} + +func (s *failingPersistenceStore) SavePromptExecution(ctx context.Context, resolved report.Resolved, artifact state.PromptExecutionArtifact) (string, error) { + if s.failOperation == failPromptExecution { + return "", errors.New("injected prompt execution persistence failure") + } + return s.Store.SavePromptExecution(ctx, resolved, artifact) +} + +func (s *failingPersistenceStore) SaveMetadata(ctx context.Context, metadata state.Metadata) (string, error) { + s.metadataCalls++ + if s.failOperation == failMetadata && s.metadataCalls == s.failMetadataCall { + return "", errors.New("injected metadata persistence failure") + } + return s.Store.SaveMetadata(ctx, metadata) +} + +type artifactPathExecutor struct { + beforePreparationErr error + afterPreparationErr error + validation promptexec.ValidationStatus +} + +func (e artifactPathExecutor) InspectPrompt(context.Context, string, string) (promptexec.PromptInspection, error) { + return promptexec.PromptInspection{}, errors.New("unexpected inspection") +} + +func (e artifactPathExecutor) InspectProfile(context.Context, string) (promptexec.ProfileInspection, error) { + return promptexec.ProfileInspection{}, errors.New("unexpected inspection") +} + +func (e artifactPathExecutor) Execute(_ context.Context, req promptexec.ExecuteRequest, callback promptexec.PreparationCallback) (*promptexec.Execution, error) { + if e.beforePreparationErr != nil { + return nil, e.beforePreparationErr + } + now := time.Date(2026, 5, 29, 15, 0, 0, 0, time.UTC) + if err := callback(promptexec.Preparation{ + PromptID: req.PromptID, PromptVersion: req.PromptVersion, PromptHash: "prompt-hash", + RenderedPromptHash: "rendered-hash", ProfileID: req.ProfileID, BackendID: "test", + ModelName: "test-model", DataPackagePath: req.DataPackagePath, StartedAt: now, EndedAt: now, + }, nil); err != nil { + return nil, err + } + if e.afterPreparationErr != nil { + return nil, e.afterPreparationErr + } + validation := e.validation + if validation == "" { + validation = promptexec.ValidationPassed + } + return &promptexec.Execution{ + RunID: "provider-run", PromptID: req.PromptID, PromptVersion: req.PromptVersion, + PromptHash: "prompt-hash", RenderedPromptHash: "rendered-hash", ProfileID: req.ProfileID, + BackendID: "test", ModelName: "test-model", GeneratedHash: "generated-hash", + StartedAt: now, EndedAt: now, DataPackagePath: req.DataPackagePath, + RawOutput: []byte(`{"summary":"Showers are possible during the selected day.","forecast_discussion":["A front will keep rain chances in the forecast."],"precipitation_timing":"Rain is most likely during the afternoon.","confidence":"Medium"}`), + Validation: promptexec.NewValidation(validation, "json_schema", "daily.generated_text.schema.json", nil), + }, nil +} + +type successfulNotifier struct{} + +func (successfulNotifier) Notify(context.Context, NotificationRequest) (*NotificationResult, error) { + return &NotificationResult{RunID: "notification-run", Status: "succeeded", UploadStatus: "accepted"}, nil +} + +func TestGeneratePromptReportReturnsOnlyReachedArtifactPaths(t *testing.T) { + tests := []struct { + name string + failOperation string + failMetadataCall int + outputCopy bool + notify bool + want reachedPromptArtifacts + }{ + {name: "preparation then metadata", failOperation: failMetadata, failMetadataCall: 1, want: reachedPromptArtifacts{preparation: true}}, + {name: "raw output then execution", failOperation: failPromptExecution, want: reachedPromptArtifacts{preparation: true, metadata: true, raw: true}}, + {name: "execution then metadata", failOperation: failMetadata, failMetadataCall: 2, want: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true}}, + {name: "normalized output then metadata", failOperation: failMetadata, failMetadataCall: 3, want: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true}}, + {name: "render context then metadata", failOperation: failMetadata, failMetadataCall: 4, want: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true, renderContext: true}}, + {name: "managed report then metadata", failOperation: failMetadata, failMetadataCall: 5, want: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true, renderContext: true, report: true}}, + {name: "output copy then metadata", failOperation: failMetadata, failMetadataCall: 5, outputCopy: true, want: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true, renderContext: true, report: true, output: true}}, + {name: "notification then metadata", failOperation: failMetadata, failMetadataCall: 6, notify: true, want: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true, renderContext: true, report: true, notification: true}}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + req, paths := promptArtifactRequest(t, artifactPathExecutor{}) + store := &failingPersistenceStore{Store: req.Store, failOperation: test.failOperation, failMetadataCall: test.failMetadataCall} + req.Store = store + if test.outputCopy { + req.OutputPath = filepath.Join(t.TempDir(), "daily.md") + paths.output = req.OutputPath + } + if test.notify { + req.Config.Notify.Distributor.Enabled = true + req.Notifier = successfulNotifier{} + req.noNotify = false + } + + result, err := generatePromptReport(context.Background(), req) + if err == nil || result == nil { + t.Fatalf("generatePromptReport() result/error = %#v/%v, want partial result and failure", result, err) + } + assertReachedPromptArtifacts(t, result, paths, test.want) + }) + } +} + +func TestGeneratePromptReportFailureReceiptsExposeReachedPaths(t *testing.T) { + tests := []struct { + name string + executor artifactPathExecutor + want reachedPromptArtifacts + }{ + { + name: "preparation failure", + executor: artifactPathExecutor{beforePreparationErr: promptexec.NewError(promptexec.Generation, "prepare failed", nil)}, + want: reachedPromptArtifacts{preparation: true, metadata: true}, + }, + { + name: "operational execution failure", + executor: artifactPathExecutor{afterPreparationErr: promptexec.NewError(promptexec.Generation, "provider failed", nil)}, + want: reachedPromptArtifacts{preparation: true, execution: true, metadata: true}, + }, + { + name: "completed validation rejection", + executor: artifactPathExecutor{validation: promptexec.ValidationFailed}, + want: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true}, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + req, paths := promptArtifactRequest(t, test.executor) + result, err := generatePromptReport(context.Background(), req) + if err == nil || result == nil { + t.Fatalf("generatePromptReport() result/error = %#v/%v, want partial result and failure", result, err) + } + assertReachedPromptArtifacts(t, result, paths, test.want) + }) + } +} + +type promptArtifactPaths struct { + state.ArtifactPaths + output string +} + +type reachedPromptArtifacts struct { + preparation bool + execution bool + metadata bool + raw bool + normalized bool + renderContext bool + report bool + output bool + notification bool +} + +func promptArtifactRequest(t *testing.T, executor promptexec.Executor) (promptReportRequest, promptArtifactPaths) { + t.Helper() + cfg := config.Defaults() + cfg.Workspace.Root = t.TempDir() + resolved, err := ResolveGenerate(GenerateRequest{ + Config: cfg, Report: ReportDaily, Date: mustParse("2026-05-29T12:00:00-05:00"), + }, mustParse("2026-05-29T05:00:00-05:00")) + if err != nil { + t.Fatalf("ResolveGenerate() error = %v", err) + } + bundleData, err := os.ReadFile(filepath.Join("..", "forecast", "testdata", "daily_bundle.json")) + if err != nil { + t.Fatalf("read daily fixture: %v", err) + } + var bundle weatherdata.Bundle + if err := json.Unmarshal(bundleData, &bundle); err != nil { + t.Fatalf("decode daily fixture: %v", err) + } + filesystemStore, err := state.NewFilesystemStore(cfg.Workspace) + if err != nil { + t.Fatalf("NewFilesystemStore() error = %v", err) + } + paths, err := filesystemStore.Paths(resolved) + if err != nil { + t.Fatalf("Paths() error = %v", err) + } + debugWriter, err := state.NewPromptDebugWriter("") + if err != nil { + t.Fatalf("NewPromptDebugWriter() error = %v", err) + } + return promptReportRequest{ + GenerateRequest: GenerateRequest{Config: cfg, Report: ReportDaily, Executor: executor, Store: filesystemStore}, + Resolved: resolved, Collection: collect.Result{Bundle: &bundle}, + Inspection: PromptInspectionResult{ + PromptID: resolved.Definition.PromptID, PromptVersion: resolved.Definition.PromptVersion, + PromptHash: "prompt-hash", ProfileID: "test-profile", BackendID: "test", ModelName: "test-model", + }, + DebugWriter: debugWriter, noNotify: true, + }, promptArtifactPaths{ArtifactPaths: paths} +} + +func assertReachedPromptArtifacts(t *testing.T, result *ReportResult, paths promptArtifactPaths, want reachedPromptArtifacts) { + t.Helper() + if result.ModuleSnapshotPath != paths.ModuleSnapshot || result.DataPackagePath != paths.DataPackage { + t.Fatalf("base paths = module %q data %q, want %q and %q", result.ModuleSnapshotPath, result.DataPackagePath, paths.ModuleSnapshot, paths.DataPackage) + } + if result.Metadata.ModuleSnapshotPath != paths.ModuleSnapshot || result.Metadata.DataPackagePath != paths.DataPackage || result.Metadata.MetadataPath != paths.Metadata { + t.Fatalf("metadata base paths = %#v, want reached module/data paths and metadata destination", result.Metadata) + } + checks := []struct { + name string + got string + metadataGot string + inMetadata bool + path string + want bool + }{ + {"preparation", result.PreparationPath, result.Metadata.PreparationPath, true, paths.Preparation, want.preparation}, + {"execution", result.ExecutionPath, result.Metadata.ExecutionPath, true, paths.Execution, want.execution}, + {"metadata", result.MetadataPath, "", false, paths.Metadata, want.metadata}, + {"raw", result.GeneratedTextRawPath, result.Metadata.GeneratedTextRawPath, true, paths.GeneratedTextRaw, want.raw}, + {"normalized", result.GeneratedTextPath, result.Metadata.GeneratedTextPath, true, paths.GeneratedText, want.normalized}, + {"render context", result.RenderContextPath, result.Metadata.RenderContextPath, true, paths.RenderContext, want.renderContext}, + {"report", result.ReportPath, result.Metadata.RenderedReportPath, true, paths.RenderedReport, want.report}, + {"output", result.OutputPath, "", false, paths.output, want.output}, + {"notification", result.NotificationPath, result.Metadata.NotificationPath, true, paths.Notification, want.notification}, + } + for _, check := range checks { + if check.want && check.got != check.path { + t.Errorf("%s path = %q, want reached path %q", check.name, check.got, check.path) + } + if check.want && check.inMetadata && check.metadataGot != check.path { + t.Errorf("metadata %s path = %q, want reached path %q", check.name, check.metadataGot, check.path) + } + if !check.want && check.got != "" { + t.Errorf("%s path = %q, want empty because artifact was not reached", check.name, check.got) + } + if !check.want && check.inMetadata && check.metadataGot != "" { + t.Errorf("metadata %s path = %q, want empty because artifact was not reached", check.name, check.metadataGot) + } + } +} diff --git a/internal/app/prompt_generate.go b/internal/app/prompt_generate.go index 94fc395..3e87c8c 100644 --- a/internal/app/prompt_generate.go +++ b/internal/app/prompt_generate.go @@ -41,7 +41,7 @@ func generatePromptReport(ctx context.Context, req promptReportRequest) (*Report if err != nil { return nil, err } - result := &ReportResult{ReportPath: paths.RenderedReport} + result := &ReportResult{} priorSnapshot, err := store.FindPriorSnapshot(ctx, req.Resolved) if err != nil { @@ -70,15 +70,8 @@ func generatePromptReport(ctx context.Context, req promptReportRequest) (*Report result.RecentChanges = recent briefingMetadata := briefing.BuildMetadata(briefingBuildContext(req.Config, req.Resolved, reportFacts.Collected)) metadata := state.BuildPromptMetadataFromBriefingMetadata(req.Resolved, briefingMetadata, state.ArtifactPaths{ - ModuleSnapshot: moduleSnapshotPath, - Metadata: paths.Metadata, - DataPackage: paths.DataPackage, - Preparation: paths.Preparation, - Execution: paths.Execution, - RenderedReport: paths.RenderedReport, - GeneratedTextRaw: paths.GeneratedTextRaw, - GeneratedText: paths.GeneratedText, - RenderContext: paths.RenderContext, + ModuleSnapshot: moduleSnapshotPath, + Metadata: paths.Metadata, }) result.Metadata = metadata dataPackage, err := promptinput.Build(promptinput.BuildRequest{ @@ -179,11 +172,13 @@ func generatePromptReport(ctx context.Context, req promptReportRequest) (*Report return result, saveErr } metadata.PreparationPath = path + result.PreparationPath = path + result.Metadata = metadata metadataPath, saveErr := store.SaveMetadata(ctx, metadata) if saveErr != nil { return result, saveErr } - result.PreparationPath, result.Metadata, result.MetadataPath = path, metadata, metadataPath + result.MetadataPath = metadataPath return result, generatedReportError(req.Resolved, metadata.RunID, "prepare prompt", executeErr) } if promptexec.CategoryOf(executeErr) != "" { @@ -193,11 +188,13 @@ func generatePromptReport(ctx context.Context, req promptReportRequest) (*Report return result, saveErr } metadata.ExecutionPath = path + result.ExecutionPath = path + result.Metadata = metadata metadataPath, saveErr := store.SaveMetadata(ctx, metadata) if saveErr != nil { return result, saveErr } - result.ExecutionPath, result.Metadata, result.MetadataPath = path, metadata, metadataPath + result.MetadataPath = metadataPath } return result, generatedReportError(req.Resolved, metadata.RunID, "execute prompt", executeErr) } @@ -209,11 +206,13 @@ func generatePromptReport(ctx context.Context, req promptReportRequest) (*Report return result, saveErr } metadata.ExecutionPath = executionPath + result.ExecutionPath = executionPath + result.Metadata = metadata metadataPath, saveErr := store.SaveMetadata(ctx, metadata) if saveErr != nil { return result, saveErr } - result.ExecutionPath, result.Metadata, result.MetadataPath = executionPath, metadata, metadataPath + result.MetadataPath = metadataPath return result, generatedReportError(req.Resolved, metadata.RunID, "execute prompt", err) } debugPath, err := req.DebugWriter.WriteExecution(debugRef, *execution) @@ -232,11 +231,13 @@ func generatePromptReport(ctx context.Context, req promptReportRequest) (*Report return result, saveErr } metadata.ExecutionPath = executionPath + result.ExecutionPath = executionPath + result.Metadata = metadata metadataPath, saveErr := store.SaveMetadata(ctx, metadata) if saveErr != nil { return result, saveErr } - result.ExecutionPath, result.Metadata, result.MetadataPath = executionPath, metadata, metadataPath + result.MetadataPath = metadataPath return result, generatedReportError(req.Resolved, metadata.RunID, "validate prompt execution", err) } rawPath, err := store.SaveGeneratedTextRaw(ctx, req.Resolved, execution.RawOutput) @@ -244,6 +245,8 @@ func generatePromptReport(ctx context.Context, req promptReportRequest) (*Report return result, err } result.GeneratedTextRawPath = rawPath + metadata.GeneratedTextRawPath = rawPath + result.Metadata = metadata executionArtifact := state.PromptExecutionArtifact{ SchemaVersion: state.PromptExecutionSchemaVersion, ReportID: req.Resolved.Definition.ID, RunID: metadata.RunID, @@ -262,12 +265,13 @@ func generatePromptReport(ctx context.Context, req promptReportRequest) (*Report return result, err } metadata.ExecutionPath = executionPath - metadata.GeneratedTextRawPath = rawPath + result.ExecutionPath = executionPath + result.Metadata = metadata metadataPath, err := store.SaveMetadata(ctx, metadata) if err != nil { return result, err } - result.ExecutionPath, result.Metadata, result.MetadataPath = executionPath, metadata, metadataPath + result.MetadataPath = metadataPath if execution.Validation.Status == promptexec.ValidationFailed { return result, generatedReportError(req.Resolved, metadata.RunID, "validate prompt execution", promptexec.NewError(promptexec.ValidationRejected, "prompt output did not satisfy its schema", nil)) } @@ -281,11 +285,13 @@ func generatePromptReport(ctx context.Context, req promptReportRequest) (*Report return result, err } metadata.GeneratedTextPath = generatedTextPath + result.GeneratedTextPath = generatedTextPath + result.Metadata = metadata metadataPath, err = store.SaveMetadata(ctx, metadata) if err != nil { return result, err } - result.GeneratedTextPath, result.Metadata, result.MetadataPath = generatedTextPath, metadata, metadataPath + result.MetadataPath = metadataPath renderContext, err := handler.BuildRenderContext(briefingMetadata, moduleSnapshot, reportFacts.Collected, reportFacts.Derived, generatedText) if err != nil { @@ -296,11 +302,13 @@ func generatePromptReport(ctx context.Context, req promptReportRequest) (*Report return result, err } metadata.RenderContextPath = renderContextPath + result.RenderContextPath = renderContextPath + result.Metadata = metadata metadataPath, err = store.SaveMetadata(ctx, metadata) if err != nil { return result, err } - result.RenderContextPath, result.Metadata, result.MetadataPath = renderContextPath, metadata, metadataPath + result.MetadataPath = metadataPath rendered, err := handler.Render(renderContext) if err != nil { @@ -314,8 +322,10 @@ func generatePromptReport(ctx context.Context, req promptReportRequest) (*Report return result, err } result.ReportPath = reportPath + metadata.RenderedReportPath = reportPath + result.Metadata = metadata finalized, err := finalizeRenderedReport(ctx, finalizeRenderedReportRequest{ - Config: req.Config, Store: store, Resolved: req.Resolved, Metadata: metadata, + Config: req.Config, Store: store, Resolved: req.Resolved, Metadata: metadata, MetadataPath: result.MetadataPath, ManagedReportPath: reportPath, OutputPath: req.OutputPath, Notifier: req.Notifier, noNotify: req.noNotify, }) result.OutputPath, result.NotificationPath = finalized.OutputPath, finalized.NotificationPath diff --git a/internal/cli/result_test.go b/internal/cli/result_test.go index 857529e..0e7d2a6 100644 --- a/internal/cli/result_test.go +++ b/internal/cli/result_test.go @@ -113,6 +113,31 @@ func TestNewGenerateSummaryOmitsNotificationWhenNotAttempted(t *testing.T) { } } +func TestNewGenerateSummaryOmitsUnreachedArtifactPaths(t *testing.T) { + result := &app.ReportResult{ + DataPackagePath: "/runs/daily/data_package.yaml", + PreparationPath: "/runs/daily/preparation.json", + Metadata: state.Metadata{ + ReportID: report.Daily, + RunID: "20260529T133000Z_daily", + }, + } + + data, err := json.Marshal(newGenerateSummary(result, errors.New("metadata write failed"))) + if err != nil { + t.Fatalf("Marshal() error = %v", err) + } + text := string(data) + for _, omitted := range []string{"executionPath", "reportPath", "outputPath", "metadataPath", "generatedTextRawPath", "generatedTextPath", "renderContextPath", "notificationPath"} { + if strings.Contains(text, omitted) { + t.Fatalf("partial summary includes unreached field %q:\n%s", omitted, text) + } + } + if !strings.Contains(text, "dataPackagePath") || !strings.Contains(text, "preparationPath") { + t.Fatalf("partial summary omits reached paths:\n%s", text) + } +} + func TestNewGenerateSummaryForNotificationFailure(t *testing.T) { generatedAt := time.Date(2026, 5, 29, 13, 30, 0, 0, time.UTC) result := &app.ReportResult{ diff --git a/internal/state/metadata.go b/internal/state/metadata.go index 930fbda..9828d2f 100644 --- a/internal/state/metadata.go +++ b/internal/state/metadata.go @@ -182,9 +182,8 @@ func BuildPromptMetadataFromBriefingMetadata(resolved report.Resolved, briefingM Location: copyLocation(briefingMetadata.Location), SourceLocationID: briefingMetadata.SourceLocationID, SourceLocation: briefingMetadata.SourceLocation, Sources: briefingMetadata.Sources, SourceWarnings: briefingMetadata.SourceWarnings, ModuleSnapshotPath: paths.ModuleSnapshot, - DataPackagePath: paths.DataPackage, RenderedReportPath: paths.RenderedReport, - GeneratedTextSchemaID: resolved.Definition.GeneratedTextSchemaID, GeneratedTextRawPath: paths.GeneratedTextRaw, - GeneratedTextPath: paths.GeneratedText, RenderContextPath: paths.RenderContext, + DataPackagePath: paths.DataPackage, + GeneratedTextSchemaID: resolved.Definition.GeneratedTextSchemaID, } } diff --git a/internal/state/metadata_reached_paths_test.go b/internal/state/metadata_reached_paths_test.go new file mode 100644 index 0000000..1c35e68 --- /dev/null +++ b/internal/state/metadata_reached_paths_test.go @@ -0,0 +1,38 @@ +package state + +import ( + "testing" + "time" + + "gitea.maximumdirect.net/eric/weatherreporter/internal/briefing" + "gitea.maximumdirect.net/eric/weatherreporter/internal/report" +) + +func TestBuildPromptMetadataIncludesOnlyExistingArtifacts(t *testing.T) { + resolved, err := report.DefaultRegistry().Resolve(report.Daily, report.ResolveRequest{ + Now: time.Date(2026, 5, 29, 12, 0, 0, 0, time.UTC), + Date: time.Date(2026, 5, 29, 0, 0, 0, 0, time.UTC), Location: time.UTC, + }) + if err != nil { + t.Fatalf("Resolve() error = %v", err) + } + metadata := BuildPromptMetadataFromBriefingMetadata(resolved, briefing.Metadata{}, ArtifactPaths{ + ModuleSnapshot: "/saved/modules.json", + Metadata: "/destination/metadata.json", + DataPackage: "/saved/data.yaml", + Preparation: "/future/preparation.json", + Execution: "/future/execution.json", + Notification: "/future/notification.json", + RenderedReport: "/future/report.md", + GeneratedTextRaw: "/future/raw.json", + GeneratedText: "/future/generated.json", + RenderContext: "/future/context.json", + }) + + if metadata.ModuleSnapshotPath != "/saved/modules.json" || metadata.DataPackagePath != "/saved/data.yaml" || metadata.MetadataPath != "/destination/metadata.json" { + t.Fatalf("existing paths = %#v, want module, data package, and metadata destination", metadata) + } + if metadata.PreparationPath != "" || metadata.ExecutionPath != "" || metadata.NotificationPath != "" || metadata.RenderedReportPath != "" || metadata.GeneratedTextRawPath != "" || metadata.GeneratedTextPath != "" || metadata.RenderContextPath != "" { + t.Fatalf("metadata includes paths for unreached artifacts: %#v", metadata) + } +}