From 06b26d5e8860dc898ff77f9abe7c1598a79ece44 Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Fri, 31 Jul 2026 04:41:02 +0000 Subject: [PATCH] Use Promptkit for single report generation --- docs/cli.md | 2 +- docs/config.md | 6 +- docs/development.md | 8 +- docs/internal/app-orchestration.md | 69 ++++--- docs/internal/cli.md | 4 +- docs/operations.md | 27 +-- internal/app/app.go | 197 +++++++++--------- internal/app/app_test.go | 286 +++++++++++++++++-------- internal/app/prompt_generate.go | 321 +++++++++++++++++++++++++++++ internal/cli/result.go | 2 +- internal/cli/root.go | 5 + internal/cli/root_test.go | 113 ++++++---- internal/state/filesystem.go | 12 +- internal/state/store.go | 1 + 14 files changed, 772 insertions(+), 281 deletions(-) create mode 100644 internal/app/prompt_generate.go diff --git a/docs/cli.md b/docs/cli.md index 6c02697..17ea982 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -80,7 +80,7 @@ time, valid period, and status: ``` When available, the summary also includes `reportPath`, `metadataPath`, -`dataPackagePath`, `preflightPath`, `generatedTextRawPath`, `generatedTextResultPath`, +`dataPackagePath`, `preparationPath`, `executionPath`, `generatedTextRawPath`, `generatedTextPath`, and `renderContextPath`. `outputPath` is included only when `--out` wrote an extra copy. Distributor notification, when attempted, adds `notificationPath` and may add a compact `notification` object. diff --git a/docs/config.md b/docs/config.md index e867b90..deaedd9 100644 --- a/docs/config.md +++ b/docs/config.md @@ -147,9 +147,9 @@ source keys are `observations`, `current`, `narrative`, `alerts`, `discussion`, ### `promptkit` -Promptkit configuration prepares the local executor and prompt/profile checks. -Scriptorium remains the active generator until the Promptkit execution workflow -is enabled. +Promptkit configuration selects the executor and prompt/profile checks for +every `generate` command. Scriptorium settings remain in use by batch commands +while their compatibility workflow remains active. | Field | Default | Rules | | --- | --- | --- | diff --git a/docs/development.md b/docs/development.md index c478bda..cf152a3 100644 --- a/docs/development.md +++ b/docs/development.md @@ -5,8 +5,8 @@ Weatherreporter. It provides a concise repository orientation and routes each kind of change to its canonical documentation. Weatherreporter is a Go CLI that collects normalized weather data, derives -deterministic report facts and module snapshots, invokes Scriptorium for -generated text, renders managed Markdown reports, and can upload completed +deterministic report facts and module snapshots, executes Promptkit for +single-report generated text, renders managed Markdown reports, and can upload completed reports through Distributor. Start with the [README](../README.md) for product context and the [architecture policy](policy/architecture.md) for system boundaries and invariants. @@ -27,7 +27,7 @@ boundaries and invariants. | Report definitions, valid periods, report IDs, output naming, or batch composition | [Report registry internals](internal/report-registry.md) and [app orchestration internals](internal/app-orchestration.md) | Report definitions own selection and period rules; orchestration owns execution. | | Module IDs, module composition, briefing values, or prompt-facing exports | [Module contract internals](internal/module.md), [module builder internals](internal/briefing.md), and [prompt-input internals](internal/prompt-input.md) | These own module contracts, value construction, and the curated prompt-package boundary. | | Recent Changes comparison | [Changes internals](internal/changes.md) and [operations guide](operations.md) | The internal guide owns structured comparison; operations owns user-visible artifact behavior. | -| Scriptorium commands, subprocess execution, prompt inputs, or result handling | [Scriptorium integration](integrations/scriptorium.md), [Scriptorium adapter internals](internal/scriptorium-adapter.md), and [prompt-input internals](internal/prompt-input.md) | These separate the external CLI contract, subprocess boundary, and input construction. | +| Prompt execution, profiles, prompt inputs, or result handling | `internal/promptexec`, the Promptkit adapter, and [prompt-input internals](internal/prompt-input.md) | These separate the executor contract and input construction. | | Generated-text schemas, validation, render contexts, templates, or Markdown rendering | [Generated-text internals](internal/generatedtext.md), [report-template internals](internal/reporttemplate.md), and [report template guide](templates.md) | These own structured text, renderer implementation, and the maintainer-facing template surface. | | Workspace paths, metadata, atomic persistence, lookup, inspection, or recovery | [State internals](internal/state.md), [operations guide](operations.md), and [troubleshooting guide](troubleshooting.md) | These separate implementation, operator workflows, and symptom-based recovery. | | Distributor bundles, uploads, notification artifacts, or failures | [Distributor adapter internals](internal/distributor-adapter.md), [Distributor integration contracts](integrations/distributor/), and [operations guide](operations.md) | These separate adapter behavior, external contracts, and operational lifecycle. | @@ -46,7 +46,7 @@ present before introducing a new package or abstraction. | `internal/cli` | Command parsing, flags, help, output, and command wiring. | | `internal/app` | Generation, batches, collection coordination, notification, and inspection orchestration. | | `internal/config` | Configuration defaults, loading, precedence, secrets, and validation. | -| `internal/adapters` | Weather API, Scriptorium, and Distributor boundaries. | +| `internal/adapters` | Weather API, Promptkit, Scriptorium batch compatibility, and Distributor boundaries. | | `internal/weatherdata`, `internal/forecast`, `internal/facts` | Normalized source facts and deterministic derivation. | | `internal/report`, `internal/module`, `internal/briefing`, `internal/changes` | Report registry, module contracts and values, and structured comparison. | | `internal/promptinput`, `internal/generatedtext`, `internal/reporttemplate` | Prompt packages, generated-text validation, render contexts, and Markdown templates. | diff --git a/docs/internal/app-orchestration.md b/docs/internal/app-orchestration.md index 009eb6e..328702c 100644 --- a/docs/internal/app-orchestration.md +++ b/docs/internal/app-orchestration.md @@ -8,12 +8,12 @@ application-facing interfaces used for tests. ## Inputs And Outputs The package accepts generate, resolved-report, batch, explicit-collection, and -inspection requests. Generation and batch requests may supply collector, -renderer, store, and notifier implementations for tests; production defaults -use the focused packages. +inspection requests. Generation and batch requests may supply collaborators for +tests; single-report generation uses a Promptkit executor, while batch requests +retain a renderer for their compatibility workflow. -A report result contains the module snapshot, prompt package, Scriptorium -results, generated-text artifacts, report and metadata +A report result contains the module snapshot, prompt package, prompt +provenance, generated-text artifacts, report and metadata paths, prior snapshot, Recent Changes, and notification information. A batch result contains aggregate counts, per-report outcomes, and an optional batch notification. Inspection returns persisted values only. @@ -31,45 +31,47 @@ safe project-owned identity and provenance values. ## Single-Report Workflow -`GenerateDetailed` first collects weather data, then resolves the requested -report using the configured registry and current time, and finally calls -`GenerateReport` with that explicit collection. It returns no result when -collection or resolution fails. +`GenerateDetailed` resolves the requested report using the configured registry +and current time, verifies the exact Promptkit prompt and selected profile, and +only then collects weather data. Inspection failure produces no collection or +managed artifacts. -`GenerateReport` requires a non-nil normalized bundle and then performs this +Single-report generation requires a non-nil normalized bundle and then performs this ordered work: 1. Select a state store, determine artifact destinations, and locate a prior compatible snapshot. 2. Build report facts and deterministic module snapshots, then save the module snapshot and calculate Recent Changes. -3. Build and save the prompt data package, run Scriptorium render preflight, - save any preflight result, and save initial metadata. -4. Run structured Scriptorium generation, validate and save generated text, - build and save a render context, and render the managed Markdown template. -5. Optionally make an output copy, save final metadata, optionally notify +3. Serialize and save the prompt data package once, then use those exact bytes + for Promptkit execution. +4. Save preparation provenance and V2 metadata from the preparation callback + before provider execution. Save execution provenance after raw output is + persisted, then save updated metadata. +5. Validate and save generated text, build and save a render context, and + render the managed Markdown template. +6. Optionally make an output copy, save final metadata, optionally notify Distributor from the managed report path, and save metadata again when a notification path is produced. -Every report looks up its catalog definition, runs structured Scriptorium -output to the raw artifact, preserves the structured run result, validates and +Every single report looks up its catalog definition, saves raw Promptkit output, +preserves safe preparation and execution provenance separately, validates and saves generated text, builds and saves a render context, then renders the -embedded Markdown template. Schema, template, and subprocess details remain in -their [generated-text](generatedtext.md), -[report-template](reporttemplate.md), and [Scriptorium adapter](scriptorium-adapter.md) +embedded Markdown template. Schema and template details remain in their +[generated-text](generatedtext.md) and [report-template](reporttemplate.md) owners. -If preflight returns a result with an error, the result and initial metadata are -saved before the error returns. If report generation fails after a managed path -is prepared, metadata still records that path; output copies and notification -are skipped. Generated-text failures preserve the latest artifact reached -before failure when it was saved. +Preparation and operational execution failures save classified receipts and +metadata before returning. A completed Promptkit validation rejection saves raw +output, an execution receipt, and metadata before returning. If later report +generation fails, the result retains every reached safe artifact path; output +copies and notification are skipped until rendering succeeds. ## Batch And Inspection Workflows `RunBatchDetailed` collects once, asks the report registry to plan the batch -from that collection, and invokes `GenerateReport` independently for every -planned report using the same collection and state store. Per-report +from that collection, and invokes its isolated Scriptorium compatibility helper +independently for every planned report using the same collection and state store. Per-report notification is suppressed. A failed report is recorded and does not prevent later planned reports from running. @@ -87,13 +89,13 @@ provenance. They neither collect data nor invoke Scriptorium or Distributor. ## Boundaries And Failure Propagation The app layer does not parse flags, load configuration files, implement Weather -API transport, construct Scriptorium argv, or define report registry policy. It +API transport, invoke provider SDKs, or define report registry policy. It coordinates the relevant collaborators and preserves their error context. -- Collection failure stops a single report or batch before resolution or - planning completes. -- State, fact, module, prompt-input, or preflight failures stop that report - before report generation. +- Prompt inspection failure stops a single report before collection or durable + writes. Collection failure stops a single report or batch before planning. +- State, fact, module, prompt-input, preparation, or execution failures stop + that report before later report generation. - A terminal Distributor failure is returned with the saved notification information when available. - Batch failures are represented per report and through aggregate batch status. @@ -108,7 +110,8 @@ Focused tests are in `internal/app/app_test.go` and - Production workflows collect through `internal/collect`. - A report uses one explicit normalized collection throughout its generation. -- Render preflight precedes report generation. +- Prompt preparation provenance and metadata are persisted before provider + execution. - Recent Changes compare structured module snapshots. - Reports render from a validated typed context, never directly from a raw prompt package. diff --git a/docs/internal/cli.md b/docs/internal/cli.md index 27e38c3..e77c347 100644 --- a/docs/internal/cli.md +++ b/docs/internal/cli.md @@ -14,8 +14,8 @@ batch execution to `internal/app`. `Runner` also owns a project-owned prompt-executor factory seam. Its production factory maps `promptkit` configuration to the Promptkit adapter, while tests can -inject a factory without importing dependency types. Construction is retained as -a separate seam until the generation workflow begins using that executor. +inject a factory without importing dependency types. Each `generate` request +constructs one executor after configuration loads and passes it to the app. For inspection, it loads configuration, builds the appropriate app inspection request, and writes the returned value. Inspection is read-only; the inspected diff --git a/docs/operations.md b/docs/operations.md index 84a3f12..bc0d9bd 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -15,10 +15,10 @@ weatherreporter generate today --out ./today.md ``` A generation collects weather data, resolves the report period, builds and -persists the module snapshot and prompt data package, runs Scriptorium -preflight, then persists generated-text artifacts, validates the structured -generated text, and renders the managed Markdown report from the validated text -and deterministic values. +persists the module snapshot and prompt data package, records Promptkit +preparation provenance before provider execution, then persists raw output and +execution provenance, validates the structured generated text, and renders the +managed Markdown report from the validated text and deterministic values. The managed report and its final metadata are saved before single-report Distributor notification is attempted. `--out` writes an extra operator copy; @@ -56,20 +56,21 @@ workspace/ snapshots///modules..json snapshots///metadata..json snapshots///generated_text_raw..json - snapshots///generated_text_result..json + snapshots///prompt_execution..json snapshots///generated_text..json snapshots///render_context..json data-packages///data_package..yaml - preflight///render..json + preflight///prompt_preparation..json notifications///distributor..json notifications/batches///distributor..json ``` -The generated-text and render-context artifacts are written for every report. -A report's metadata links the module snapshot, data package, preflight artifact, -managed report, generated-text artifacts, and any available single-report +The generated-text and render-context artifacts are written for every completed +single-report generation. +A report's metadata links the module snapshot, data package, preparation and +execution receipts, managed report, generated-text artifacts, and any available single-report notification artifact. Batch notification artifacts are separate batch-level records under `notifications/batches`. @@ -128,12 +129,12 @@ RunID is an error; use `inspect reports` to obtain a valid value. Keep the workspace when a run fails: artifacts reached before the failure remain available where they can be safely persisted. -- A preflight failure can leave the preflight artifact and metadata. +- A preparation failure can leave its classified receipt and metadata. - A report-generation failure can leave the managed report, module snapshot, data package, and metadata. -- A generated-text failure can leave raw text, the structured run result, or a - validated generated-text and render-context artifact, depending on where it - stopped. +- A completed prompt validation rejection leaves raw text, an execution receipt, + and metadata. Later generated-text failures can also leave validated text and + a render-context artifact, depending on where they stopped. - A single-report notification failure preserves the report and final metadata, including its notification artifact when it was written. - A batch notification failure preserves each report's artifacts and adds the diff --git a/internal/app/app.go b/internal/app/app.go index 37a239d..61a0e18 100644 --- a/internal/app/app.go +++ b/internal/app/app.go @@ -19,6 +19,7 @@ import ( "gitea.maximumdirect.net/eric/weatherreporter/internal/forecast" "gitea.maximumdirect.net/eric/weatherreporter/internal/generatedtext" "gitea.maximumdirect.net/eric/weatherreporter/internal/module" + "gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec" "gitea.maximumdirect.net/eric/weatherreporter/internal/promptinput" "gitea.maximumdirect.net/eric/weatherreporter/internal/report" "gitea.maximumdirect.net/eric/weatherreporter/internal/state" @@ -50,6 +51,8 @@ type GenerateRequest struct { Date time.Time Collector Collector Notifier Notifier + Executor promptexec.Executor + Store state.Store } type BatchRequest struct { @@ -90,28 +93,25 @@ type ReportRequest struct { } type ReportResult struct { - ModuleSnapshot module.Snapshot - ModuleSnapshotPath string - DataPackage promptinput.Package - DataPackagePath string - PreparationPath string - ExecutionPath string - LLMDebugPath string - PreflightPath string - ReportPath string - OutputPath string - NotificationPath string - Metadata state.Metadata - MetadataPath string - PriorSnapshot *state.PriorSnapshot - RecentChanges []changes.Change - RenderResult *scriptorium.RenderResult - StructuredRunResult *scriptorium.StructuredRunResult - GeneratedTextRawPath string - GeneratedTextResultPath string - GeneratedTextPath string - RenderContextPath string - Notification *NotificationResult + ModuleSnapshot module.Snapshot + ModuleSnapshotPath string + DataPackage promptinput.Package + DataPackagePath string + PreparationPath string + ExecutionPath string + LLMDebugPath string + PreflightPath string + ReportPath string + OutputPath string + NotificationPath string + Metadata state.Metadata + MetadataPath string + PriorSnapshot *state.PriorSnapshot + RecentChanges []changes.Change + GeneratedTextRawPath string + GeneratedTextPath string + RenderContextPath string + Notification *NotificationResult } type BatchResult struct { @@ -275,20 +275,27 @@ func GenerateDetailed(ctx context.Context, req GenerateRequest) (*ReportResult, if now.IsZero() { now = time.Now() } - collection, err := collectWeather(ctx, req.Config, req.Collector) - if err != nil { - return nil, err - } resolved, err := ResolveGenerate(req, now) if err != nil { return nil, err } - return GenerateReport(ctx, ReportRequest{ - Config: req.Config, - Resolved: resolved, - OutputPath: req.OutputPath, - Collection: *collection, - Notifier: req.Notifier, + inspection, err := InspectPromptExecution(ctx, PromptInspectionRequest{ + Resolved: resolved, + Executor: req.Executor, + Promptkit: req.Config.Promptkit, + }) + if err != nil { + return nil, err + } + collection, err := collectWeather(ctx, req.Config, req.Collector) + if err != nil { + return nil, err + } + return generatePromptReport(ctx, promptReportRequest{ + GenerateRequest: req, + Resolved: resolved, + Collection: *collection, + Inspection: inspection, }) } @@ -340,7 +347,7 @@ func RunBatchDetailed(ctx context.Context, req BatchRequest) (*BatchResult, erro item.MetadataPath = paths.Metadata } outputPath := plannedBatchOutputPath(req.OutputDir, planned) - reportResult, err := GenerateReport(ctx, ReportRequest{ + reportResult, err := generateLegacyBatchReport(ctx, ReportRequest{ Config: req.Config, Resolved: resolved, OutputPath: outputPath, @@ -494,7 +501,9 @@ func FetchAndSaveBundle(ctx context.Context, req FetchBundleRequest) (*weatherda return bundle, nil } -func GenerateReport(ctx context.Context, req ReportRequest) (*ReportResult, error) { +// generateLegacyBatchReport is the temporary Scriptorium implementation used +// only by batch commands while their Promptkit migration is deferred. +func generateLegacyBatchReport(ctx context.Context, req ReportRequest) (*ReportResult, error) { bundle := req.Collection.Bundle if bundle == nil { return nil, fmt.Errorf("collected weather bundle is required") @@ -724,40 +733,34 @@ func generateTextTemplateReport(ctx context.Context, req generatedReportRequest) return nil, err } return renderedReportResult(reportResultRequest{ - moduleSnapshot: req.moduleSnapshot, - moduleSnapshotPath: req.moduleSnapshotPath, - dataPackage: req.dataPackage, - dataPackagePath: req.dataPackagePath, - preflightPath: req.preflightPath, - reportPath: reportPath, - finalized: finalized, - priorSnapshot: req.priorSnapshot, - recentChanges: req.recentChanges, - renderResult: req.renderResult, - structuredRunResult: structuredResult, - generatedTextRawPath: req.paths.GeneratedTextRaw, - generatedTextResultPath: generatedTextResultPath, - generatedTextPath: generatedTextPath, - renderContextPath: renderContextPath, + moduleSnapshot: req.moduleSnapshot, + moduleSnapshotPath: req.moduleSnapshotPath, + dataPackage: req.dataPackage, + dataPackagePath: req.dataPackagePath, + preflightPath: req.preflightPath, + reportPath: reportPath, + finalized: finalized, + priorSnapshot: req.priorSnapshot, + recentChanges: req.recentChanges, + generatedTextRawPath: req.paths.GeneratedTextRaw, + generatedTextPath: generatedTextPath, + renderContextPath: renderContextPath, }), err } return renderedReportResult(reportResultRequest{ - moduleSnapshot: req.moduleSnapshot, - moduleSnapshotPath: req.moduleSnapshotPath, - dataPackage: req.dataPackage, - dataPackagePath: req.dataPackagePath, - preflightPath: req.preflightPath, - reportPath: reportPath, - finalized: finalized, - priorSnapshot: req.priorSnapshot, - recentChanges: req.recentChanges, - renderResult: req.renderResult, - structuredRunResult: structuredResult, - generatedTextRawPath: req.paths.GeneratedTextRaw, - generatedTextResultPath: generatedTextResultPath, - generatedTextPath: generatedTextPath, - renderContextPath: renderContextPath, + moduleSnapshot: req.moduleSnapshot, + moduleSnapshotPath: req.moduleSnapshotPath, + dataPackage: req.dataPackage, + dataPackagePath: req.dataPackagePath, + preflightPath: req.preflightPath, + reportPath: reportPath, + finalized: finalized, + priorSnapshot: req.priorSnapshot, + recentChanges: req.recentChanges, + generatedTextRawPath: req.paths.GeneratedTextRaw, + generatedTextPath: generatedTextPath, + renderContextPath: renderContextPath, }), nil } @@ -770,46 +773,40 @@ func finalizeResultEmpty(result finalizeRenderedReportResult) bool { } type reportResultRequest struct { - moduleSnapshot module.Snapshot - moduleSnapshotPath string - dataPackage promptinput.Package - dataPackagePath string - preflightPath string - reportPath string - finalized finalizeRenderedReportResult - priorSnapshot *state.PriorSnapshot - recentChanges []changes.Change - renderResult *scriptorium.RenderResult - structuredRunResult *scriptorium.StructuredRunResult - generatedTextRawPath string - generatedTextResultPath string - generatedTextPath string - renderContextPath string + moduleSnapshot module.Snapshot + moduleSnapshotPath string + dataPackage promptinput.Package + dataPackagePath string + preflightPath string + reportPath string + finalized finalizeRenderedReportResult + priorSnapshot *state.PriorSnapshot + recentChanges []changes.Change + generatedTextRawPath string + generatedTextPath string + renderContextPath string } func renderedReportResult(req reportResultRequest) *ReportResult { return &ReportResult{ - ModuleSnapshot: req.moduleSnapshot, - ModuleSnapshotPath: req.moduleSnapshotPath, - DataPackage: req.dataPackage, - DataPackagePath: req.dataPackagePath, - PreparationPath: req.preflightPath, - ExecutionPath: req.generatedTextResultPath, - PreflightPath: req.preflightPath, - ReportPath: req.reportPath, - OutputPath: req.finalized.OutputPath, - NotificationPath: req.finalized.NotificationPath, - Metadata: req.finalized.Metadata, - MetadataPath: req.finalized.MetadataPath, - PriorSnapshot: req.priorSnapshot, - RecentChanges: req.recentChanges, - RenderResult: req.renderResult, - StructuredRunResult: req.structuredRunResult, - GeneratedTextRawPath: req.generatedTextRawPath, - GeneratedTextResultPath: req.generatedTextResultPath, - GeneratedTextPath: req.generatedTextPath, - RenderContextPath: req.renderContextPath, - Notification: req.finalized.Notification, + ModuleSnapshot: req.moduleSnapshot, + ModuleSnapshotPath: req.moduleSnapshotPath, + DataPackage: req.dataPackage, + DataPackagePath: req.dataPackagePath, + PreparationPath: req.preflightPath, + ExecutionPath: req.finalized.Metadata.GeneratedTextResultPath, + PreflightPath: req.preflightPath, + ReportPath: req.reportPath, + OutputPath: req.finalized.OutputPath, + NotificationPath: req.finalized.NotificationPath, + Metadata: req.finalized.Metadata, + MetadataPath: req.finalized.MetadataPath, + PriorSnapshot: req.priorSnapshot, + RecentChanges: req.recentChanges, + GeneratedTextRawPath: req.generatedTextRawPath, + GeneratedTextPath: req.generatedTextPath, + RenderContextPath: req.renderContextPath, + Notification: req.finalized.Notification, } } diff --git a/internal/app/app_test.go b/internal/app/app_test.go index afe665e..41a00ab 100644 --- a/internal/app/app_test.go +++ b/internal/app/app_test.go @@ -18,6 +18,7 @@ import ( "gitea.maximumdirect.net/eric/weatherreporter/internal/collect" "gitea.maximumdirect.net/eric/weatherreporter/internal/config" "gitea.maximumdirect.net/eric/weatherreporter/internal/module" + "gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec" "gitea.maximumdirect.net/eric/weatherreporter/internal/promptinput" "gitea.maximumdirect.net/eric/weatherreporter/internal/report" "gitea.maximumdirect.net/eric/weatherreporter/internal/state" @@ -72,6 +73,56 @@ func TestFetchAndSaveBundle(t *testing.T) { } } +type promptExecutorTest struct { + err error + afterPreparationErr error + validation promptexec.ValidationStatus +} + +func (e promptExecutorTest) InspectPrompt(_ context.Context, id string, version string) (promptexec.PromptInspection, error) { + name := strings.TrimSuffix(strings.TrimPrefix(id, "weather."), "_generated_text") + return promptexec.PromptInspection{ + PromptID: id, PromptVersion: version, PromptHash: "prompt-hash", DefaultProfileID: "test-profile", + Inputs: []promptexec.InputDefinition{{Name: "data_package", Required: true, ContentType: "application/yaml"}}, + Output: promptexec.OutputContract{Format: "json", ValidationMode: "json_schema", SchemaPath: name + ".generated_text.schema.json"}, + }, nil +} + +func (e promptExecutorTest) InspectProfile(_ context.Context, id string) (promptexec.ProfileInspection, error) { + return promptexec.ProfileInspection{ProfileID: id, BackendID: "test", ModelName: "test-model"}, nil +} + +func (e promptExecutorTest) Execute(_ context.Context, request promptexec.ExecuteRequest, callback promptexec.PreparationCallback) (*promptexec.Execution, error) { + if e.err != nil { + return nil, e.err + } + now := time.Now().UTC() + if err := callback(promptexec.Preparation{ + PromptID: request.PromptID, PromptVersion: request.PromptVersion, PromptHash: "prompt-hash", RenderedPromptHash: "rendered-hash", + ProfileID: request.ProfileID, BackendID: "test", ModelName: "test-model", DataPackagePath: request.DataPackagePath, + StartedAt: now, EndedAt: now, + }, nil); err != nil { + return nil, err + } + if e.afterPreparationErr != nil { + return nil, e.afterPreparationErr + } + raw := []byte(`{"summary":"Showers are possible during the selected day.","forecast_discussion":["A front will keep rain chances in the forecast."],"precipitation_timing":"Rain is most likely during the afternoon.","confidence":"Medium"}`) + if request.PromptID == "weather.hourly_generated_text" { + raw = []byte(`{"summary":"Storm chances increase through late morning.","forecast_discussion":"A front will keep the region unsettled.","precipitation_timing":"A cold front is moving into the region.","confidence":"Medium"}`) + } + validation := e.validation + if validation == "" { + validation = promptexec.ValidationPassed + } + return &promptexec.Execution{ + RunID: "provider-run", PromptID: request.PromptID, PromptVersion: request.PromptVersion, PromptHash: "prompt-hash", RenderedPromptHash: "rendered-hash", + ProfileID: request.ProfileID, BackendID: "test", ModelName: "test-model", GeneratedHash: "generated-hash", + StartedAt: now, EndedAt: now, DataPackagePath: request.DataPackagePath, RawOutput: raw, + Validation: promptexec.NewValidation(validation, "json_schema", "generated_text.schema.json", nil), + }, nil +} + func TestFetchAndSaveBundleRequiresOutputPath(t *testing.T) { _, err := FetchAndSaveBundle(context.Background(), FetchBundleRequest{Config: config.Defaults()}) if err == nil { @@ -94,6 +145,7 @@ func TestGenerateUsesProvidedCollector(t *testing.T) { Date: mustParse("2026-05-29T12:00:00-05:00"), Now: mustParse("2026-05-29T05:00:00-05:00"), Collector: collector, + Executor: promptExecutorTest{}, }) if err == nil { t.Fatal("Generate() error = nil, want collector error") @@ -114,20 +166,13 @@ func TestGenerateCollectsOnceForSingleReport(t *testing.T) { cfg := dailyWorkspaceConfig(t, server) collection := collectionForTest(t, cfg) collector := &recordingCollector{result: &collection} - markerPath := filepath.Join(t.TempDir(), "scriptorium-called") - binaryPath := filepath.Join(t.TempDir(), "scriptorium") - script := fmt.Sprintf("#!/bin/sh\nprintf called > %q\nprintf render failed >&2\nexit 1\n", markerPath) - if err := os.WriteFile(binaryPath, []byte(script), 0o755); err != nil { - t.Fatalf("write scriptorium marker script: %v", err) - } - cfg.Scriptorium.Binary = binaryPath - err := Generate(context.Background(), GenerateRequest{ Config: cfg, Report: ReportDaily, Date: mustParse("2026-05-29T12:00:00-05:00"), Now: mustParse("2026-05-29T05:00:00-05:00"), Collector: collector, + Executor: promptExecutorTest{err: errors.New("prompt execution failed")}, }) if err == nil { t.Fatal("Generate() error = nil, want render error") @@ -135,22 +180,12 @@ func TestGenerateCollectsOnceForSingleReport(t *testing.T) { if len(collector.requests) != 1 { t.Fatalf("collector requests = %d, want 1", len(collector.requests)) } - if _, statErr := os.Stat(markerPath); statErr != nil { - t.Fatalf("scriptorium marker stat error = %v, want report execution reached renderer", statErr) - } } func TestGenerateCollectionFailureStopsBeforeReportExecution(t *testing.T) { cfg := config.Defaults() cfg.WeatherAPI.BaseURL = "" cfg.Workspace.Root = t.TempDir() - markerPath := filepath.Join(t.TempDir(), "scriptorium-called") - binaryPath := filepath.Join(t.TempDir(), "scriptorium") - script := fmt.Sprintf("#!/bin/sh\nprintf called > %q\nexit 0\n", markerPath) - if err := os.WriteFile(binaryPath, []byte(script), 0o755); err != nil { - t.Fatalf("write scriptorium marker script: %v", err) - } - cfg.Scriptorium.Binary = binaryPath collector := &recordingCollector{err: errors.New("collection unavailable")} err := Generate(context.Background(), GenerateRequest{ @@ -159,13 +194,11 @@ func TestGenerateCollectionFailureStopsBeforeReportExecution(t *testing.T) { Date: mustParse("2026-05-29T12:00:00-05:00"), Now: mustParse("2026-05-29T05:00:00-05:00"), Collector: collector, + Executor: promptExecutorTest{}, }) if err == nil { t.Fatal("Generate() error = nil, want collector error") } - if _, statErr := os.Stat(markerPath); !errors.Is(statErr, os.ErrNotExist) { - t.Fatalf("scriptorium marker stat error = %v, want marker absent after collection failure", statErr) - } } func TestGenerateDetailedReturnsReportResult(t *testing.T) { @@ -183,6 +216,7 @@ func TestGenerateDetailedReturnsReportResult(t *testing.T) { Date: mustParse("2026-05-29T12:00:00-05:00"), Now: mustParse("2026-05-29T05:00:00-05:00"), Collector: collector, + Executor: promptExecutorTest{}, }) if err != nil { t.Fatalf("GenerateDetailed() error = %v", err) @@ -196,7 +230,7 @@ func TestGenerateDetailedReturnsReportResult(t *testing.T) { if result.OutputPath != outputPath { t.Fatalf("OutputPath = %q, want requested output copy %q", result.OutputPath, outputPath) } - assertPathsExist(t, result.ModuleSnapshotPath, result.DataPackagePath, result.PreflightPath, result.ReportPath, result.MetadataPath, outputPath) + assertPathsExist(t, result.ModuleSnapshotPath, result.DataPackagePath, result.PreparationPath, result.ExecutionPath, result.ReportPath, result.MetadataPath, outputPath) if len(collector.requests) != 1 { t.Fatalf("collector requests = %d, want one collection", len(collector.requests)) } @@ -214,6 +248,7 @@ func TestGenerateReturnsUnderlyingErrorOnly(t *testing.T) { Date: mustParse("2026-05-29T12:00:00-05:00"), Now: mustParse("2026-05-29T05:00:00-05:00"), Collector: &recordingCollector{err: wantErr}, + Executor: promptExecutorTest{}, }) if !errors.Is(err, wantErr) { t.Fatalf("Generate() error = %v, want underlying collector error", err) @@ -235,6 +270,7 @@ func TestGenerateDetailedNotificationFailureReturnsInspectableResult(t *testing. Now: mustParse("2026-05-29T08:30:00-05:00"), Collector: &recordingCollector{result: &collection}, Notifier: notifier, + Executor: promptExecutorTest{}, }) if err == nil { t.Fatal("GenerateDetailed() error = nil, want notification error") @@ -252,7 +288,7 @@ func TestGenerateDetailedNotificationFailureReturnsInspectableResult(t *testing. if result.NotificationPath == "" || result.ReportPath == "" || result.MetadataPath == "" { t.Fatalf("result paths = report %q metadata %q notification %q, want inspectable artifact paths", result.ReportPath, result.MetadataPath, result.NotificationPath) } - assertPathsExist(t, result.ReportPath, outputPath, result.MetadataPath, result.NotificationPath, result.GeneratedTextRawPath, result.GeneratedTextResultPath, result.GeneratedTextPath, result.RenderContextPath) + assertPathsExist(t, result.ReportPath, outputPath, result.MetadataPath, result.NotificationPath, result.GeneratedTextRawPath, result.GeneratedTextPath, result.RenderContextPath) if result.Notification != nil { t.Fatalf("Notification = %#v, want nil notification result when notifier returned only an error", result.Notification) } @@ -261,6 +297,87 @@ func TestGenerateDetailedNotificationFailureReturnsInspectableResult(t *testing. } } +func TestGenerateDetailedInspectsBeforeCollectionOrArtifactWrites(t *testing.T) { + cfg := config.Defaults() + cfg.Workspace.Root = t.TempDir() + collector := &recordingCollector{err: errors.New("collector must not run")} + _, err := GenerateDetailed(context.Background(), GenerateRequest{ + Config: cfg, Report: ReportDaily, Date: mustParse("2026-05-29T12:00:00-05:00"), + Now: mustParse("2026-05-29T05:00:00-05:00"), Collector: collector, Executor: failingPromptInspectionExecutor{}, + }) + if err == nil || promptexec.CategoryOf(err) != promptexec.InvalidConfiguration { + t.Fatalf("GenerateDetailed() error/category = %v/%q, want invalid configuration", err, promptexec.CategoryOf(err)) + } + if len(collector.requests) != 0 { + t.Fatalf("collector requests = %d, want inspection failure before collection", len(collector.requests)) + } + entries, readErr := os.ReadDir(cfg.Workspace.Root) + if readErr != nil || len(entries) != 0 { + t.Fatalf("workspace entries = %#v, err %v, want none", entries, readErr) + } +} + +func TestGenerateDetailedPersistsPromptFailureReceiptsWithoutRawOutput(t *testing.T) { + server := dailyBundleServer(t) + cfg := dailyWorkspaceConfig(t, server) + collection := collectionForTest(t, cfg) + result, err := GenerateDetailed(context.Background(), GenerateRequest{ + Config: cfg, Report: ReportDaily, Date: mustParse("2026-05-29T12:00:00-05:00"), + Now: mustParse("2026-05-29T05:00:00-05:00"), Collector: &recordingCollector{result: &collection}, + Executor: promptExecutorTest{afterPreparationErr: promptexec.NewError(promptexec.Generation, "provider unavailable", nil)}, + }) + if err == nil || result == nil { + t.Fatalf("GenerateDetailed() result/error = %#v/%v, want partial result and error", result, err) + } + if result.PreparationPath == "" || result.ExecutionPath == "" || result.MetadataPath == "" || result.GeneratedTextRawPath != "" { + t.Fatalf("result paths = %#v, want preparation/execution/metadata without raw output", result) + } + if result.Metadata.SchemaVersion != state.MetadataSchemaVersion || result.Metadata.PreparationPath != result.PreparationPath || result.Metadata.ExecutionPath != result.ExecutionPath { + t.Fatalf("metadata = %#v, want linked V2 prompt artifacts", result.Metadata) + } + store := recordingFilesystemStore(t, cfg) + execution, loadErr := store.LoadPromptExecution(context.Background(), result.ExecutionPath) + if loadErr != nil || execution.Status != state.PromptExecutionFailed || execution.Paths.RawOutputPath != "" { + t.Fatalf("execution/load error = %#v/%v, want failed receipt without raw path", execution, loadErr) + } +} + +func TestGenerateDetailedPersistsRawOutputForValidationRejection(t *testing.T) { + server := dailyBundleServer(t) + cfg := dailyWorkspaceConfig(t, server) + collection := collectionForTest(t, cfg) + result, err := GenerateDetailed(context.Background(), GenerateRequest{ + Config: cfg, Report: ReportDaily, Date: mustParse("2026-05-29T12:00:00-05:00"), + Now: mustParse("2026-05-29T05:00:00-05:00"), Collector: &recordingCollector{result: &collection}, + Executor: promptExecutorTest{validation: promptexec.ValidationFailed}, + }) + if err == nil || result == nil || promptexec.CategoryOf(err) != promptexec.ValidationRejected { + t.Fatalf("GenerateDetailed() result/error/category = %#v/%v/%q, want validation rejection", result, err, promptexec.CategoryOf(err)) + } + if result.GeneratedTextRawPath == "" || result.ExecutionPath == "" || result.GeneratedTextPath != "" { + t.Fatalf("result paths = %#v, want raw and execution paths only", result) + } + store := recordingFilesystemStore(t, cfg) + execution, loadErr := store.LoadPromptExecution(context.Background(), result.ExecutionPath) + if loadErr != nil || execution.Status != state.PromptExecutionValidationRejected || execution.Paths.RawOutputPath != result.GeneratedTextRawPath { + t.Fatalf("execution/load error = %#v/%v, want rejected execution with raw path", execution, loadErr) + } +} + +type failingPromptInspectionExecutor struct{} + +func (failingPromptInspectionExecutor) InspectPrompt(context.Context, string, string) (promptexec.PromptInspection, error) { + return promptexec.PromptInspection{}, errors.New("unavailable") +} + +func (failingPromptInspectionExecutor) InspectProfile(context.Context, string) (promptexec.ProfileInspection, error) { + return promptexec.ProfileInspection{}, errors.New("unexpected") +} + +func (failingPromptInspectionExecutor) Execute(context.Context, promptexec.ExecuteRequest, promptexec.PreparationCallback) (*promptexec.Execution, error) { + return nil, errors.New("unexpected") +} + func TestGenerateReportWritesReportAndPreflight(t *testing.T) { server := dailyBundleServer(t) cfg := dailyWorkspaceConfig(t, server) @@ -283,7 +400,7 @@ func TestGenerateReportWritesReportAndPreflight(t *testing.T) { outputPath := filepath.Join(t.TempDir(), "daily.md") store := recordingFilesystemStore(t, cfg) - result, err := GenerateReport(context.Background(), ReportRequest{ + result, err := generateLegacyBatchReport(context.Background(), ReportRequest{ Config: cfg, Collection: collectionForTest(t, cfg), Resolved: resolved, @@ -292,7 +409,7 @@ func TestGenerateReportWritesReportAndPreflight(t *testing.T) { Store: store, }) if err != nil { - t.Fatalf("GenerateReport() error = %v", err) + t.Fatalf("generateLegacyBatchReport() error = %v", err) } if renderer.renderCalls != 1 { @@ -316,7 +433,7 @@ func TestGenerateReportWritesReportAndPreflight(t *testing.T) { if got, want := strings.Join(store.calls, ","), "module_snapshot,data_package,preflight,metadata,generated_text_result,metadata,generated_text,metadata,render_context,metadata,prepare_report,metadata"; !strings.HasPrefix(got, want) { t.Fatalf("store calls = %v, want prefix %s", store.calls, want) } - assertPathsExist(t, result.ModuleSnapshotPath, result.DataPackagePath, result.PreflightPath, result.GeneratedTextRawPath, result.GeneratedTextResultPath, result.GeneratedTextPath, result.RenderContextPath, result.ReportPath, result.MetadataPath, outputPath) + assertPathsExist(t, result.ModuleSnapshotPath, result.DataPackagePath, result.PreflightPath, result.GeneratedTextRawPath, result.Metadata.GeneratedTextResultPath, result.GeneratedTextPath, result.RenderContextPath, result.ReportPath, result.MetadataPath, outputPath) snapshotData, err := os.ReadFile(result.ModuleSnapshotPath) if err != nil { t.Fatalf("read module snapshot: %v", err) @@ -520,14 +637,14 @@ func TestGeneratedTemplateReportsUseRichArtifactsAndCuratedDataPackages(t *testi }, tt.now.Format(time.RFC3339)) renderer := successfulGeneratedTextRenderer("") - result, err := GenerateReport(context.Background(), ReportRequest{ + result, err := generateLegacyBatchReport(context.Background(), ReportRequest{ Config: cfg, Collection: collectionForTest(t, cfg), Resolved: resolved, Renderer: renderer, }) if err != nil { - t.Fatalf("GenerateReport() error = %v", err) + t.Fatalf("generateLegacyBatchReport() error = %v", err) } if renderer.renderCalls != 1 || renderer.structuredRunCalls != 1 { t.Fatalf("renderer calls render=%d structured=%d, want generated-template workflow", renderer.renderCalls, renderer.structuredRunCalls) @@ -667,7 +784,7 @@ func TestGenerateHourlyReportUsesGeneratedTextTemplateWorkflow(t *testing.T) { }`, } - result, err := GenerateReport(context.Background(), ReportRequest{ + result, err := generateLegacyBatchReport(context.Background(), ReportRequest{ Config: cfg, Collection: collectionForTest(t, cfg), Resolved: resolved, @@ -675,7 +792,7 @@ func TestGenerateHourlyReportUsesGeneratedTextTemplateWorkflow(t *testing.T) { Store: store, }) if err != nil { - t.Fatalf("GenerateReport() error = %v", err) + t.Fatalf("generateLegacyBatchReport() error = %v", err) } if renderer.renderCalls != 1 { @@ -699,7 +816,7 @@ func TestGenerateHourlyReportUsesGeneratedTextTemplateWorkflow(t *testing.T) { result.DataPackagePath, result.PreflightPath, result.GeneratedTextRawPath, - result.GeneratedTextResultPath, + result.Metadata.GeneratedTextResultPath, result.GeneratedTextPath, result.RenderContextPath, result.ReportPath, @@ -762,15 +879,12 @@ func TestGenerateHourlyReportUsesGeneratedTextTemplateWorkflow(t *testing.T) { if result.OutputPath != result.ReportPath { t.Fatalf("OutputPath = %q, want managed report path %q", result.OutputPath, result.ReportPath) } - if result.StructuredRunResult == nil || result.StructuredRunResult.OutputPath != result.GeneratedTextRawPath { - t.Fatalf("StructuredRunResult = %#v, want captured structured run result", result.StructuredRunResult) - } if len(result.RecentChanges) != 0 { t.Fatalf("RecentChanges = %#v, want none for hourly report", result.RecentChanges) } if result.Metadata.GeneratedTextSchemaID != "hourly" || result.Metadata.GeneratedTextRawPath != result.GeneratedTextRawPath || - result.Metadata.GeneratedTextResultPath != result.GeneratedTextResultPath || + result.Metadata.GeneratedTextResultPath == "" || result.Metadata.GeneratedTextPath != result.GeneratedTextPath || result.Metadata.RenderContextPath != result.RenderContextPath || result.Metadata.RenderedReportPath != result.ReportPath { @@ -792,7 +906,7 @@ func TestGenerateHourlyReportCopiesOutputAndNotifiesManagedReport(t *testing.T) notifier.result = successfulNotificationResult() renderer := successfulGeneratedTextRenderer(validHourlyGeneratedTextJSON()) - result, err := GenerateReport(context.Background(), ReportRequest{ + result, err := generateLegacyBatchReport(context.Background(), ReportRequest{ Config: cfg, Collection: collectionForTest(t, cfg), Resolved: resolved, @@ -802,7 +916,7 @@ func TestGenerateHourlyReportCopiesOutputAndNotifiesManagedReport(t *testing.T) Notifier: notifier, }) if err != nil { - t.Fatalf("GenerateReport() error = %v", err) + t.Fatalf("generateLegacyBatchReport() error = %v", err) } if result.OutputPath != outputPath { t.Fatalf("OutputPath = %q, want requested copy %q", result.OutputPath, outputPath) @@ -878,7 +992,7 @@ func TestGenerateTodayReportCopiesOutputAndNotifiesTodayTemplateValues(t *testin } renderer := successfulGeneratedTextRenderer(validTodayGeneratedTextJSON()) - result, err := GenerateReport(context.Background(), ReportRequest{ + result, err := generateLegacyBatchReport(context.Background(), ReportRequest{ Config: cfg, Collection: collectionForTest(t, cfg), Resolved: resolved, @@ -887,7 +1001,7 @@ func TestGenerateTodayReportCopiesOutputAndNotifiesTodayTemplateValues(t *testin Notifier: notifier, }) if err != nil { - t.Fatalf("GenerateReport() error = %v", err) + t.Fatalf("generateLegacyBatchReport() error = %v", err) } if result.OutputPath != outputPath { t.Fatalf("OutputPath = %q, want requested copy %q", result.OutputPath, outputPath) @@ -959,7 +1073,7 @@ func TestGenerateHourlyReportNotificationFailureFailsReport(t *testing.T) { structuredRunBody: validHourlyGeneratedTextJSON(), } - _, err := GenerateReport(context.Background(), ReportRequest{ + _, err := generateLegacyBatchReport(context.Background(), ReportRequest{ Config: cfg, Collection: collectionForTest(t, cfg), Resolved: resolved, @@ -969,11 +1083,11 @@ func TestGenerateHourlyReportNotificationFailureFailsReport(t *testing.T) { Notifier: notifier, }) if err == nil { - t.Fatal("GenerateReport() error = nil, want notification error") + t.Fatal("generateLegacyBatchReport() error = nil, want notification error") } var notificationErr *NotificationError if !errors.As(err, ¬ificationErr) { - t.Fatalf("GenerateReport() error = %T %v, want NotificationError", err, err) + t.Fatalf("generateLegacyBatchReport() error = %T %v, want NotificationError", err, err) } if !strings.Contains(err.Error(), `notify report "hourly"`) || !strings.Contains(err.Error(), "upload rejected") { t.Fatalf("error = %q, want hourly notification context", err.Error()) @@ -1010,7 +1124,7 @@ func TestGenerateReportSavesFinalMetadata(t *testing.T) { structuredRunBody: validHourlyGeneratedTextJSON(), } - result, err := GenerateReport(context.Background(), ReportRequest{ + result, err := generateLegacyBatchReport(context.Background(), ReportRequest{ Config: cfg, Collection: collectionForTest(t, cfg), Resolved: resolved, @@ -1018,7 +1132,7 @@ func TestGenerateReportSavesFinalMetadata(t *testing.T) { Store: store, }) if err != nil { - t.Fatalf("GenerateReport() error = %v", err) + t.Fatalf("generateLegacyBatchReport() error = %v", err) } saved := readMetadataForTest(t, result.MetadataPath) @@ -1027,7 +1141,7 @@ func TestGenerateReportSavesFinalMetadata(t *testing.T) { } if saved.GeneratedTextSchemaID != "hourly" || saved.GeneratedTextRawPath != result.GeneratedTextRawPath || - saved.GeneratedTextResultPath != result.GeneratedTextResultPath || + saved.GeneratedTextResultPath != result.Metadata.GeneratedTextResultPath || saved.GeneratedTextPath != result.GeneratedTextPath || saved.RenderContextPath != result.RenderContextPath { t.Fatalf("saved generated-text metadata = %#v, want generated-text artifact links", saved) @@ -1047,7 +1161,7 @@ func TestGenerateTomorrowReportNotificationUsesTomorrowTemplateValues(t *testing notifier := &recordingNotifier{} renderer := successfulGeneratedTextRenderer(validTomorrowGeneratedTextJSON()) - result, err := GenerateReport(context.Background(), ReportRequest{ + result, err := generateLegacyBatchReport(context.Background(), ReportRequest{ Config: cfg, Collection: collectionForTest(t, cfg), Resolved: resolved, @@ -1055,7 +1169,7 @@ func TestGenerateTomorrowReportNotificationUsesTomorrowTemplateValues(t *testing Notifier: notifier, }) if err != nil { - t.Fatalf("GenerateReport() error = %v", err) + t.Fatalf("generateLegacyBatchReport() error = %v", err) } if len(notifier.requests) != 1 { t.Fatalf("notification requests = %d, want 1", len(notifier.requests)) @@ -1094,7 +1208,7 @@ func TestGenerateHourlyReportPersistsPreflightFailure(t *testing.T) { err: errors.New("scriptorium render exited with code 1: render failed"), } - _, err := GenerateReport(context.Background(), ReportRequest{ + _, err := generateLegacyBatchReport(context.Background(), ReportRequest{ Config: cfg, Collection: collectionForTest(t, cfg), Resolved: resolved, @@ -1141,7 +1255,7 @@ func TestGenerateHourlyReportPersistsStructuredRunFailure(t *testing.T) { structuredRunBody: validHourlyGeneratedTextJSON(), } - _, err := GenerateReport(context.Background(), ReportRequest{ + _, err := generateLegacyBatchReport(context.Background(), ReportRequest{ Config: cfg, Collection: collectionForTest(t, cfg), Resolved: resolved, @@ -1176,7 +1290,7 @@ func TestGenerateHourlyReportPreservesRawTextOnValidationFailure(t *testing.T) { "details": "not allowed" }`) - _, err := GenerateReport(context.Background(), ReportRequest{ + _, err := generateLegacyBatchReport(context.Background(), ReportRequest{ Config: cfg, Collection: collectionForTest(t, cfg), Resolved: resolved, @@ -1205,7 +1319,7 @@ func TestGenerateHourlyReportRejectsUnsupportedTemplateBeforeStructuredRun(t *te resolved.Definition.TemplateID = "missing-template" renderer := successfulGeneratedTextRenderer(validHourlyGeneratedTextJSON()) - _, err := GenerateReport(context.Background(), ReportRequest{ + _, err := generateLegacyBatchReport(context.Background(), ReportRequest{ Config: cfg, Collection: collectionForTest(t, cfg), Resolved: resolved, @@ -1241,7 +1355,7 @@ func TestGenerateReportDisabledNotificationDoesNotCallNotifier(t *testing.T) { }, "2026-05-29T05:00:00-05:00") notifier := &recordingNotifier{} - _, err := GenerateReport(context.Background(), ReportRequest{ + _, err := generateLegacyBatchReport(context.Background(), ReportRequest{ Config: cfg, Collection: collectionForTest(t, cfg), Resolved: resolved, @@ -1249,7 +1363,7 @@ func TestGenerateReportDisabledNotificationDoesNotCallNotifier(t *testing.T) { Notifier: notifier, }) if err != nil { - t.Fatalf("GenerateReport() error = %v", err) + t.Fatalf("generateLegacyBatchReport() error = %v", err) } if len(notifier.requests) != 0 { t.Fatalf("notification requests = %#v, want none when disabled", notifier.requests) @@ -1268,7 +1382,7 @@ func TestGenerateReportNotifiesManagedReportPath(t *testing.T) { } outputPath := filepath.Join(t.TempDir(), "daily-copy.md") - result, err := GenerateReport(context.Background(), ReportRequest{ + result, err := generateLegacyBatchReport(context.Background(), ReportRequest{ Config: cfg, Collection: collectionForTest(t, cfg), Resolved: resolved, @@ -1277,7 +1391,7 @@ func TestGenerateReportNotifiesManagedReportPath(t *testing.T) { Notifier: notifier, }) if err != nil { - t.Fatalf("GenerateReport() error = %v", err) + t.Fatalf("generateLegacyBatchReport() error = %v", err) } if result.Notification == nil { t.Fatal("Notification = nil, want notification result") @@ -1343,7 +1457,7 @@ func TestGenerateReportNotificationFailureFailsReport(t *testing.T) { notifier := &recordingNotifier{err: errors.New("upload rejected")} store := recordingFilesystemStore(t, cfg) - _, err := GenerateReport(context.Background(), ReportRequest{ + _, err := generateLegacyBatchReport(context.Background(), ReportRequest{ Config: cfg, Collection: collectionForTest(t, cfg), Resolved: resolved, @@ -1352,7 +1466,7 @@ func TestGenerateReportNotificationFailureFailsReport(t *testing.T) { Notifier: notifier, }) if err == nil { - t.Fatal("GenerateReport() error = nil, want notification error") + t.Fatal("generateLegacyBatchReport() error = nil, want notification error") } if !strings.Contains(err.Error(), "notify report") || !strings.Contains(err.Error(), "upload rejected") { t.Fatalf("error = %q, want notification context", err.Error()) @@ -1409,7 +1523,7 @@ func TestGenerateReportDoesNotNotifyAfterRenderOrRunFailure(t *testing.T) { }, "2026-05-29T05:00:00-05:00") notifier := &recordingNotifier{} - _, err := GenerateReport(context.Background(), ReportRequest{ + _, err := generateLegacyBatchReport(context.Background(), ReportRequest{ Config: cfg, Collection: collectionForTest(t, cfg), Resolved: resolved, @@ -1417,7 +1531,7 @@ func TestGenerateReportDoesNotNotifyAfterRenderOrRunFailure(t *testing.T) { Notifier: notifier, }) if err == nil { - t.Fatal("GenerateReport() error = nil, want generation error") + t.Fatal("generateLegacyBatchReport() error = nil, want generation error") } if len(notifier.requests) != 0 { t.Fatalf("notification requests = %#v, want none after generation failure", notifier.requests) @@ -1440,7 +1554,7 @@ func TestGenerateReportRequiresCollectedBundleBeforeStateWrites(t *testing.T) { notifier := &recordingNotifier{} store := recordingFilesystemStore(t, cfg) - _, err := GenerateReport(context.Background(), ReportRequest{ + _, err := generateLegacyBatchReport(context.Background(), ReportRequest{ Config: cfg, Resolved: resolved, Renderer: successfulRenderer("# Daily Report\n"), @@ -1448,10 +1562,10 @@ func TestGenerateReportRequiresCollectedBundleBeforeStateWrites(t *testing.T) { Notifier: notifier, }) if err == nil { - t.Fatal("GenerateReport() error = nil, want collected bundle error") + t.Fatal("generateLegacyBatchReport() error = nil, want collected bundle error") } if !strings.Contains(err.Error(), "collected weather bundle is required") { - t.Fatalf("GenerateReport() error = %q, want collected bundle context", err.Error()) + t.Fatalf("generateLegacyBatchReport() error = %q, want collected bundle context", err.Error()) } if len(notifier.requests) != 0 { t.Fatalf("notification requests = %#v, want none without collected data", notifier.requests) @@ -1477,14 +1591,14 @@ func TestGenerateReportPersistsFailedPreflight(t *testing.T) { err: errors.New("scriptorium render exited with code 1: render failed"), } - _, err := GenerateReport(context.Background(), ReportRequest{ + _, err := generateLegacyBatchReport(context.Background(), ReportRequest{ Config: cfg, Collection: collectionForTest(t, cfg), Resolved: resolved, Renderer: renderer, }) if err == nil { - t.Fatal("GenerateReport() error = nil, want render error") + t.Fatal("generateLegacyBatchReport() error = nil, want render error") } store := recordingFilesystemStore(t, cfg) paths, err := store.Paths(resolved) @@ -1523,14 +1637,14 @@ func TestGenerateReportReturnsRunErrorAfterPreflight(t *testing.T) { structuredRunErr: errors.New("scriptorium run exited with code 2: validation failed"), } - _, err := GenerateReport(context.Background(), ReportRequest{ + _, err := generateLegacyBatchReport(context.Background(), ReportRequest{ Config: cfg, Collection: collectionForTest(t, cfg), Resolved: resolved, Renderer: renderer, }) if err == nil { - t.Fatal("GenerateReport() error = nil, want run error") + t.Fatal("generateLegacyBatchReport() error = nil, want run error") } if renderer.renderCalls != 1 || renderer.structuredRunCalls != 1 || renderer.runCalls != 0 { t.Fatalf("calls render=%d structured=%d run=%d, want render and structured run only", renderer.renderCalls, renderer.structuredRunCalls, renderer.runCalls) @@ -1567,7 +1681,7 @@ func TestGenerateReportIncludesRecentChangesFromPriorSnapshot(t *testing.T) { }, "2026-05-29T05:00:00-05:00") renderer := successfulRenderer("") - result, err := GenerateReport(context.Background(), ReportRequest{ + result, err := generateLegacyBatchReport(context.Background(), ReportRequest{ Config: cfg, Collection: collectionForTest(t, cfg), Resolved: currentResolved, @@ -1575,7 +1689,7 @@ func TestGenerateReportIncludesRecentChangesFromPriorSnapshot(t *testing.T) { Store: store, }) if err != nil { - t.Fatalf("GenerateReport() error = %v", err) + t.Fatalf("generateLegacyBatchReport() error = %v", err) } if len(result.RecentChanges) == 0 { t.Fatal("RecentChanges length = 0, want changes from prior snapshot") @@ -1605,7 +1719,7 @@ func TestGenerateTodayReportUsesTodayIdentityAndRecentChanges(t *testing.T) { }, "2026-05-29T05:00:00-05:00") renderer := successfulGeneratedTextRenderer(validTodayGeneratedTextJSON()) - result, err := GenerateReport(context.Background(), ReportRequest{ + result, err := generateLegacyBatchReport(context.Background(), ReportRequest{ Config: cfg, Collection: collectionForTest(t, cfg), Resolved: currentResolved, @@ -1613,7 +1727,7 @@ func TestGenerateTodayReportUsesTodayIdentityAndRecentChanges(t *testing.T) { Store: store, }) if err != nil { - t.Fatalf("GenerateReport() error = %v", err) + t.Fatalf("generateLegacyBatchReport() error = %v", err) } if renderer.runCalls != 0 || renderer.structuredRunCalls != 1 { t.Fatalf("renderer calls run=%d structured=%d, want generated-text flow", renderer.runCalls, renderer.structuredRunCalls) @@ -1667,14 +1781,14 @@ func TestGenerateTomorrowReportUsesTomorrowBriefingDate(t *testing.T) { }, "2026-05-29T18:00:00-05:00") renderer := successfulGeneratedTextRenderer(validTomorrowGeneratedTextJSON()) - result, err := GenerateReport(context.Background(), ReportRequest{ + result, err := generateLegacyBatchReport(context.Background(), ReportRequest{ Config: cfg, Collection: collectionForTest(t, cfg), Resolved: resolved, Renderer: renderer, }) if err != nil { - t.Fatalf("GenerateReport() error = %v", err) + t.Fatalf("generateLegacyBatchReport() error = %v", err) } if renderer.runCalls != 0 { t.Fatalf("markdown run calls = %d, want none", renderer.runCalls) @@ -1705,7 +1819,7 @@ func TestGenerateTomorrowReportUsesTomorrowBriefingDate(t *testing.T) { if !strings.Contains(filepath.Base(result.ReportPath), "tomorrow") { t.Fatalf("ReportPath = %q, want managed tomorrow report path", result.ReportPath) } - assertPathsExist(t, result.GeneratedTextRawPath, result.GeneratedTextResultPath, result.GeneratedTextPath, result.RenderContextPath, result.ReportPath) + assertPathsExist(t, result.GeneratedTextRawPath, result.Metadata.GeneratedTextResultPath, result.GeneratedTextPath, result.RenderContextPath, result.ReportPath) renderContext, err := os.ReadFile(result.RenderContextPath) if err != nil { t.Fatalf("read render context: %v", err) @@ -1740,7 +1854,7 @@ func TestTomorrowReportCanCompareAgainstPriorTomorrowSnapshot(t *testing.T) { }, "2026-05-29T18:00:00-05:00") renderer := successfulGeneratedTextRenderer(validTomorrowGeneratedTextJSON()) - result, err := GenerateReport(context.Background(), ReportRequest{ + result, err := generateLegacyBatchReport(context.Background(), ReportRequest{ Config: cfg, Collection: collectionForTest(t, cfg), Resolved: currentResolved, @@ -1748,7 +1862,7 @@ func TestTomorrowReportCanCompareAgainstPriorTomorrowSnapshot(t *testing.T) { Store: store, }) if err != nil { - t.Fatalf("GenerateReport() error = %v", err) + t.Fatalf("generateLegacyBatchReport() error = %v", err) } if result.PriorSnapshot == nil { t.Fatal("PriorSnapshot = nil, want compatible prior tomorrow snapshot") @@ -1771,7 +1885,7 @@ func TestDailyReportIgnoresPriorTomorrowSnapshot(t *testing.T) { Report: ReportDaily, Date: mustParse("2026-05-29T12:00:00-05:00"), }, "2026-05-29T05:00:00-05:00") - result, err := GenerateReport(context.Background(), ReportRequest{ + result, err := generateLegacyBatchReport(context.Background(), ReportRequest{ Config: cfg, Collection: collectionForTest(t, cfg), Resolved: currentResolved, @@ -1779,7 +1893,7 @@ func TestDailyReportIgnoresPriorTomorrowSnapshot(t *testing.T) { Store: store, }) if err != nil { - t.Fatalf("GenerateReport() error = %v", err) + t.Fatalf("generateLegacyBatchReport() error = %v", err) } if result.PriorSnapshot != nil { t.Fatalf("PriorSnapshot = %#v, want nil for prior tomorrow snapshot", result.PriorSnapshot) @@ -1797,14 +1911,14 @@ func TestInspectGeneratedReportArtifacts(t *testing.T) { Date: mustParse("2026-05-29T12:00:00-05:00"), }, "2026-05-29T05:00:00-05:00") renderer := successfulRenderer("") - result, err := GenerateReport(context.Background(), ReportRequest{ + result, err := generateLegacyBatchReport(context.Background(), ReportRequest{ Config: cfg, Collection: collectionForTest(t, cfg), Resolved: resolved, Renderer: renderer, }) if err != nil { - t.Fatalf("GenerateReport() error = %v", err) + t.Fatalf("generateLegacyBatchReport() error = %v", err) } records, err := InspectReports(context.Background(), InspectReportsRequest{Config: cfg, Limit: 1}) @@ -1860,12 +1974,12 @@ func TestInspectPriorSnapshot(t *testing.T) { Date: mustParse("2026-05-29T12:00:00-05:00"), }, "2026-05-29T05:00:00-05:00") renderer := successfulRenderer("") - if _, err := GenerateReport(context.Background(), ReportRequest{Config: cfg, Collection: collectionForTest(t, cfg), Resolved: priorResolved, Renderer: renderer, Store: store}); err != nil { - t.Fatalf("GenerateReport(prior) error = %v", err) + if _, err := generateLegacyBatchReport(context.Background(), ReportRequest{Config: cfg, Collection: collectionForTest(t, cfg), Resolved: priorResolved, Renderer: renderer, Store: store}); err != nil { + t.Fatalf("generateLegacyBatchReport(prior) error = %v", err) } - current, err := GenerateReport(context.Background(), ReportRequest{Config: cfg, Collection: collectionForTest(t, cfg), Resolved: currentResolved, Renderer: renderer, Store: store}) + current, err := generateLegacyBatchReport(context.Background(), ReportRequest{Config: cfg, Collection: collectionForTest(t, cfg), Resolved: currentResolved, Renderer: renderer, Store: store}) if err != nil { - t.Fatalf("GenerateReport(current) error = %v", err) + t.Fatalf("generateLegacyBatchReport(current) error = %v", err) } prior, err := InspectPriorSnapshot(context.Background(), InspectRunRequest{Config: cfg, RunID: current.Metadata.RunID}) @@ -3373,14 +3487,14 @@ func generateDailyReportForTest(t *testing.T, cfg config.Config) *ReportResult { if err != nil { t.Fatalf("ResolveGenerate() error = %v", err) } - result, err := GenerateReport(context.Background(), ReportRequest{ + result, err := generateLegacyBatchReport(context.Background(), ReportRequest{ Config: cfg, Collection: collectionForTest(t, cfg), Resolved: resolved, Renderer: successfulRenderer("# Daily Report\n"), }) if err != nil { - t.Fatalf("GenerateReport() error = %v", err) + t.Fatalf("generateLegacyBatchReport() error = %v", err) } return result } @@ -3572,7 +3686,7 @@ func readBatchNotificationForTest(t *testing.T, path string) state.BatchDistribu func assertGeneratedReportError(t *testing.T, err error, resolved report.Resolved, operation string) { t.Helper() if err == nil { - t.Fatal("GenerateReport() error = nil, want generated-text report error") + t.Fatal("generateLegacyBatchReport() error = nil, want generated-text report error") } text := err.Error() for _, want := range []string{ diff --git a/internal/app/prompt_generate.go b/internal/app/prompt_generate.go new file mode 100644 index 0000000..0d6a761 --- /dev/null +++ b/internal/app/prompt_generate.go @@ -0,0 +1,321 @@ +package app + +import ( + "context" + "fmt" + "time" + + "gitea.maximumdirect.net/eric/weatherreporter/internal/briefing" + "gitea.maximumdirect.net/eric/weatherreporter/internal/collect" + "gitea.maximumdirect.net/eric/weatherreporter/internal/fileutil" + "gitea.maximumdirect.net/eric/weatherreporter/internal/generatedtext" + "gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec" + "gitea.maximumdirect.net/eric/weatherreporter/internal/promptinput" + "gitea.maximumdirect.net/eric/weatherreporter/internal/report" + "gitea.maximumdirect.net/eric/weatherreporter/internal/state" +) + +type promptReportRequest struct { + GenerateRequest + Resolved report.Resolved + Collection collect.Result + Inspection PromptInspectionResult +} + +func generatePromptReport(ctx context.Context, req promptReportRequest) (*ReportResult, error) { + bundle := req.Collection.Bundle + if bundle == nil { + return nil, fmt.Errorf("collected weather bundle is required") + } + store := req.Store + var err error + if store == nil { + store, err = defaultStore(req.Config) + if err != nil { + return nil, err + } + } + paths, err := store.Paths(req.Resolved) + if err != nil { + return nil, err + } + result := &ReportResult{ReportPath: paths.RenderedReport} + + priorSnapshot, err := store.FindPriorSnapshot(ctx, req.Resolved) + if err != nil { + return result, err + } + reportFacts, err := BuildReportFacts(ModuleSnapshotRequest{Config: req.Config, Resolved: req.Resolved}, bundle) + if err != nil { + return result, generatedReportError(req.Resolved, req.Resolved.Metadata().RunID, "build report facts", err) + } + moduleSnapshot, err := BuildModuleSnapshotFromFacts(ModuleSnapshotRequest{Config: req.Config, Resolved: req.Resolved}, reportFacts) + if err != nil { + return result, generatedReportError(req.Resolved, req.Resolved.Metadata().RunID, "build module snapshot", err) + } + moduleSnapshotPath, err := store.SaveModuleSnapshot(ctx, req.Resolved, moduleSnapshot) + if err != nil { + return result, err + } + result.ModuleSnapshot = moduleSnapshot + result.ModuleSnapshotPath = moduleSnapshotPath + result.PriorSnapshot = priorSnapshot + + recent, err := recentChanges(ctx, store, priorSnapshot, req.Resolved.Definition.ID, moduleSnapshot, req.Config.RecentChange) + if err != nil { + return result, err + } + result.RecentChanges = recent + briefingMetadata := briefing.BuildMetadata(briefingBuildContext(req.Config, req.Resolved, reportFacts.Collected)) + metadata := state.BuildPromptMetadataFromBriefingMetadata(req.Resolved, briefingMetadata, state.ArtifactPaths{ + ModuleSnapshot: moduleSnapshotPath, + Metadata: paths.Metadata, + DataPackage: paths.DataPackage, + Preparation: paths.Preparation, + Execution: paths.Execution, + RenderedReport: paths.RenderedReport, + GeneratedTextRaw: paths.GeneratedTextRaw, + GeneratedText: paths.GeneratedText, + RenderContext: paths.RenderContext, + }) + dataPackage, err := promptinput.Build(promptinput.BuildRequest{ + Metadata: promptMetadata(metadata), + Modules: moduleSnapshot, + RecentChanges: recent, + }) + if err != nil { + return result, generatedReportError(req.Resolved, metadata.RunID, "build data package", err) + } + data, err := promptinput.MarshalYAML(dataPackage) + if err != nil { + return result, err + } + dataPackagePath, err := store.SaveDataPackageBytes(ctx, req.Resolved, data) + if err != nil { + return result, err + } + metadata.DataPackagePath = dataPackagePath + result.DataPackage = dataPackage + result.DataPackagePath = dataPackagePath + + handler, err := generatedtext.LookupDefinition(req.Resolved.Definition) + if err != nil { + return result, generatedReportError(req.Resolved, metadata.RunID, "lookup generated text catalog", err) + } + + prepared := false + callbackFailed := false + callback := func(preparation promptexec.Preparation, _ *promptexec.PreparationDebug) error { + artifact := state.PromptPreparationArtifact{ + SchemaVersion: state.PromptPreparationSchemaVersion, + Status: state.PromptPreparationSucceeded, + ReportID: req.Resolved.Definition.ID, + RunID: metadata.RunID, + PromptID: req.Inspection.PromptID, + PromptVersion: req.Inspection.PromptVersion, + DataPackagePath: dataPackagePath, + Preparation: &preparation, + StartedAt: preparation.StartedAt, + EndedAt: preparation.EndedAt, + Duration: preparation.Duration, + } + path, err := store.SavePromptPreparation(ctx, req.Resolved, artifact) + if err != nil { + callbackFailed = true + return err + } + prepared = true + result.PreparationPath = path + metadata.PreparationPath = path + metadataPath, err := store.SaveMetadata(ctx, metadata) + if err != nil { + callbackFailed = true + return err + } + result.Metadata = metadata + result.MetadataPath = metadataPath + return nil + } + execution, executeErr := req.Executor.Execute(ctx, promptexec.ExecuteRequest{ + PromptID: req.Inspection.PromptID, + PromptVersion: req.Inspection.PromptVersion, + ProfileID: req.Inspection.ProfileID, + DataPackage: data, + DataPackagePath: dataPackagePath, + CaptureDebug: false, + }, callback) + if executeErr != nil { + if callbackFailed { + return result, executeErr + } + executeErr = classifiedPromptError("prompt execution failed", executeErr) + if !prepared { + artifact := state.PromptPreparationArtifact{ + SchemaVersion: state.PromptPreparationSchemaVersion, Status: state.PromptPreparationFailed, + ReportID: req.Resolved.Definition.ID, RunID: metadata.RunID, PromptID: req.Inspection.PromptID, + PromptVersion: req.Inspection.PromptVersion, DataPackagePath: dataPackagePath, + StartedAt: time.Now(), EndedAt: time.Now(), Error: state.NewPromptArtifactError(executeErr), + } + path, saveErr := store.SavePromptPreparation(ctx, req.Resolved, artifact) + if saveErr != nil { + return result, saveErr + } + metadata.PreparationPath = path + metadataPath, saveErr := store.SaveMetadata(ctx, metadata) + if saveErr != nil { + return result, saveErr + } + result.PreparationPath, result.Metadata, result.MetadataPath = path, metadata, metadataPath + return result, generatedReportError(req.Resolved, metadata.RunID, "prepare prompt", executeErr) + } + if promptexec.CategoryOf(executeErr) != "" { + artifact := failedPromptExecutionArtifact(req.Resolved, metadata, req.Inspection, executeErr) + path, saveErr := store.SavePromptExecution(ctx, req.Resolved, artifact) + if saveErr != nil { + return result, saveErr + } + metadata.ExecutionPath = path + metadataPath, saveErr := store.SaveMetadata(ctx, metadata) + if saveErr != nil { + return result, saveErr + } + result.ExecutionPath, result.Metadata, result.MetadataPath = path, metadata, metadataPath + } + return result, generatedReportError(req.Resolved, metadata.RunID, "execute prompt", executeErr) + } + if execution == nil { + err := promptexec.NewError(promptexec.Generation, "prompt executor returned no execution", nil) + artifact := failedPromptExecutionArtifact(req.Resolved, metadata, req.Inspection, err) + executionPath, saveErr := store.SavePromptExecution(ctx, req.Resolved, artifact) + if saveErr != nil { + return result, saveErr + } + metadata.ExecutionPath = executionPath + metadataPath, saveErr := store.SaveMetadata(ctx, metadata) + if saveErr != nil { + return result, saveErr + } + result.ExecutionPath, result.Metadata, result.MetadataPath = executionPath, metadata, metadataPath + return result, generatedReportError(req.Resolved, metadata.RunID, "execute prompt", err) + } + + if execution.Validation.Status != promptexec.ValidationPassed && execution.Validation.Status != promptexec.ValidationFailed { + err := promptexec.NewError(promptexec.OperationalValidation, "prompt execution did not complete validation", nil) + artifact := failedPromptExecutionArtifact(req.Resolved, metadata, req.Inspection, err) + executionPath, saveErr := store.SavePromptExecution(ctx, req.Resolved, artifact) + if saveErr != nil { + return result, saveErr + } + metadata.ExecutionPath = executionPath + metadataPath, saveErr := store.SaveMetadata(ctx, metadata) + if saveErr != nil { + return result, saveErr + } + result.ExecutionPath, result.Metadata, result.MetadataPath = executionPath, metadata, metadataPath + return result, generatedReportError(req.Resolved, metadata.RunID, "validate prompt execution", err) + } + rawPath, err := store.SaveGeneratedTextRaw(ctx, req.Resolved, execution.RawOutput) + if err != nil { + return result, err + } + result.GeneratedTextRawPath = rawPath + executionArtifact := state.PromptExecutionArtifact{ + SchemaVersion: state.PromptExecutionSchemaVersion, + ReportID: req.Resolved.Definition.ID, RunID: metadata.RunID, + PromptID: req.Inspection.PromptID, PromptVersion: req.Inspection.PromptVersion, + Provenance: ptr(state.PromptExecutionProvenanceFrom(*execution)), Validation: &execution.Validation, + Paths: state.PromptExecutionPaths{RawOutputPath: rawPath}, + StartedAt: execution.StartedAt, EndedAt: execution.EndedAt, Duration: execution.Duration, + } + if execution.Validation.Status == promptexec.ValidationPassed { + executionArtifact.Status = state.PromptExecutionSucceeded + } else { + executionArtifact.Status = state.PromptExecutionValidationRejected + } + executionPath, err := store.SavePromptExecution(ctx, req.Resolved, executionArtifact) + if err != nil { + return result, err + } + metadata.ExecutionPath = executionPath + metadata.GeneratedTextRawPath = rawPath + metadataPath, err := store.SaveMetadata(ctx, metadata) + if err != nil { + return result, err + } + result.ExecutionPath, result.Metadata, result.MetadataPath = executionPath, metadata, metadataPath + if execution.Validation.Status == promptexec.ValidationFailed { + return result, generatedReportError(req.Resolved, metadata.RunID, "validate prompt execution", promptexec.NewError(promptexec.ValidationRejected, "prompt output did not satisfy its schema", nil)) + } + + generatedText, normalized, err := handler.Validate(execution.RawOutput) + if err != nil { + return result, generatedReportError(req.Resolved, metadata.RunID, "validate generated text", err) + } + generatedTextPath, err := store.SaveGeneratedText(ctx, req.Resolved, normalized) + if err != nil { + return result, err + } + metadata.GeneratedTextPath = generatedTextPath + metadataPath, err = store.SaveMetadata(ctx, metadata) + if err != nil { + return result, err + } + result.GeneratedTextPath, result.Metadata, result.MetadataPath = generatedTextPath, metadata, metadataPath + + renderContext, err := handler.BuildRenderContext(briefingMetadata, moduleSnapshot, reportFacts.Collected, reportFacts.Derived, generatedText) + if err != nil { + return result, generatedReportError(req.Resolved, metadata.RunID, "build render context", err) + } + renderContextPath, err := store.SaveRenderContext(ctx, req.Resolved, renderContext) + if err != nil { + return result, err + } + metadata.RenderContextPath = renderContextPath + metadataPath, err = store.SaveMetadata(ctx, metadata) + if err != nil { + return result, err + } + result.RenderContextPath, result.Metadata, result.MetadataPath = renderContextPath, metadata, metadataPath + + rendered, err := handler.Render(renderContext) + if err != nil { + return result, generatedReportError(req.Resolved, metadata.RunID, "render template", err) + } + reportPath, err := store.PrepareRenderedReport(ctx, req.Resolved) + if err != nil { + return result, err + } + if err := fileutil.WriteFileAtomic(reportPath, rendered); err != nil { + return result, err + } + result.ReportPath = reportPath + finalized, err := finalizeRenderedReport(ctx, finalizeRenderedReportRequest{ + Config: req.Config, Store: store, Resolved: req.Resolved, Metadata: metadata, + ManagedReportPath: reportPath, OutputPath: req.OutputPath, Notifier: req.Notifier, + }) + result.OutputPath, result.NotificationPath = finalized.OutputPath, finalized.NotificationPath + result.Metadata, result.MetadataPath, result.Notification = finalized.Metadata, finalized.MetadataPath, finalized.Notification + if err != nil { + return result, err + } + return result, nil +} + +func failedPromptExecutionArtifact(resolved report.Resolved, metadata state.Metadata, inspection PromptInspectionResult, err error) state.PromptExecutionArtifact { + now := time.Now() + return state.PromptExecutionArtifact{ + SchemaVersion: state.PromptExecutionSchemaVersion, Status: state.PromptExecutionFailed, + ReportID: resolved.Definition.ID, RunID: metadata.RunID, PromptID: inspection.PromptID, + PromptVersion: inspection.PromptVersion, StartedAt: now, EndedAt: now, + Error: state.NewPromptArtifactError(err), + } +} + +func classifiedPromptError(operation string, err error) error { + if promptexec.CategoryOf(err) != "" { + return err + } + return promptexec.NewError(promptexec.Generation, operation, err) +} + +func ptr[T any](value T) *T { return &value } diff --git a/internal/cli/result.go b/internal/cli/result.go index e7ead63..6f5f19d 100644 --- a/internal/cli/result.go +++ b/internal/cli/result.go @@ -92,7 +92,7 @@ func newGenerateSummary(result *app.ReportResult, err error) generateSummary { } summary.ExecutionPath = result.ExecutionPath if summary.ExecutionPath == "" { - summary.ExecutionPath = result.GeneratedTextResultPath + summary.ExecutionPath = result.Metadata.GeneratedTextResultPath } summary.LLMDebugPath = result.LLMDebugPath summary.GeneratedTextRawPath = result.GeneratedTextRawPath diff --git a/internal/cli/root.go b/internal/cli/root.go index 13c2e63..3f89c35 100644 --- a/internal/cli/root.go +++ b/internal/cli/root.go @@ -214,6 +214,10 @@ func (r Runner) resolveGenerateAction(args []string) (app.GenerateRequest, commo if err != nil { return app.GenerateRequest{}, commonOptions{}, err } + executor, err := r.promptExecutor(cfg.Promptkit) + if err != nil { + return app.GenerateRequest{}, commonOptions{}, err + } location, err := timeutil.LoadLocation(cfg.WeatherAPI.Timezone) if err != nil { return app.GenerateRequest{}, commonOptions{}, err @@ -224,6 +228,7 @@ func (r Runner) resolveGenerateAction(args []string) (app.GenerateRequest, commo Report: reportKind, OutputPath: opts.Output, Now: r.Clock.Now(), + Executor: executor, } switch reportKind { diff --git a/internal/cli/root_test.go b/internal/cli/root_test.go index 2056a70..d0b3205 100644 --- a/internal/cli/root_test.go +++ b/internal/cli/root_test.go @@ -13,10 +13,51 @@ import ( "time" "gitea.maximumdirect.net/eric/weatherreporter/internal/app" + "gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec" "gitea.maximumdirect.net/eric/weatherreporter/internal/report" "gitea.maximumdirect.net/eric/weatherreporter/internal/timeutil" ) +func testRunner() Runner { + return testRunnerWithClock(fixedClock()) +} + +func testRunnerWithClock(clock timeutil.Clock) Runner { + return Runner{Clock: clock, ExecutorFactory: func(PromptExecutorConfig) (promptexec.Executor, error) { + return cliPromptExecutor{}, nil + }} +} + +type cliPromptExecutor struct{} + +func (cliPromptExecutor) InspectPrompt(_ context.Context, id string, version string) (promptexec.PromptInspection, error) { + name := strings.TrimSuffix(strings.TrimPrefix(id, "weather."), "_generated_text") + return promptexec.PromptInspection{ + PromptID: id, PromptVersion: version, PromptHash: "prompt-hash", DefaultProfileID: "test-profile", + Inputs: []promptexec.InputDefinition{{Name: "data_package", Required: true, ContentType: "application/yaml"}}, + Output: promptexec.OutputContract{Format: "json", ValidationMode: "json_schema", SchemaPath: name + ".generated_text.schema.json"}, + }, nil +} + +func (cliPromptExecutor) InspectProfile(_ context.Context, id string) (promptexec.ProfileInspection, error) { + return promptexec.ProfileInspection{ProfileID: id, BackendID: "test", ModelName: "test-model"}, nil +} + +func (cliPromptExecutor) Execute(_ context.Context, request promptexec.ExecuteRequest, callback promptexec.PreparationCallback) (*promptexec.Execution, error) { + now := time.Now().UTC() + if err := callback(promptexec.Preparation{PromptID: request.PromptID, PromptVersion: request.PromptVersion, PromptHash: "prompt-hash", RenderedPromptHash: "rendered-hash", ProfileID: request.ProfileID, BackendID: "test", ModelName: "test-model", DataPackagePath: request.DataPackagePath, StartedAt: now, EndedAt: now}, nil); err != nil { + return nil, err + } + raw := []byte(`{"summary": "Showers are possible during the selected day.", "forecast_discussion": ["A front will keep rain chances in the forecast."], "precipitation_timing": "Rain is most likely during the afternoon."}`) + if request.PromptID == "weather.today_generated_text" { + raw = []byte(`{"summary": "Today starts with showers before improving.", "forecast_discussion": ["Morning showers should taper as drier air arrives.", "Afternoon conditions trend quieter."], "precipitation_timing": "The best rain chance is during the morning."}`) + } + if request.PromptID == "weather.hourly_generated_text" { + raw = []byte(`{"summary":"Storm chances increase through late morning.","forecast_discussion":"A front will keep the region unsettled.","precipitation_timing":"A cold front is moving into the region."}`) + } + return &promptexec.Execution{RunID: "provider-run", PromptID: request.PromptID, PromptVersion: request.PromptVersion, PromptHash: "prompt-hash", RenderedPromptHash: "rendered-hash", ProfileID: request.ProfileID, BackendID: "test", ModelName: "test-model", GeneratedHash: "generated-hash", StartedAt: now, EndedAt: now, DataPackagePath: request.DataPackagePath, RawOutput: raw, Validation: promptexec.NewValidation(promptexec.ValidationPassed, "json_schema", "generated_text.schema.json", nil)}, nil +} + func TestRunHelpLongFlag(t *testing.T) { output, err := runRootCommand(t, "--help") if err != nil { @@ -73,7 +114,7 @@ func TestRunUnknownCommand(t *testing.T) { func TestRunGenerateTomorrowWritesMarkdownReport(t *testing.T) { fixture := newCLIFixture(t, writeFakeScriptorium) outPath := fixture.path("tomorrow.md") - runner := Runner{Clock: fixedClock()} + runner := testRunner() _, err := runTestCommand(t, runner, "generate", "tomorrow", @@ -95,7 +136,7 @@ func TestRunGenerateTomorrowWritesMarkdownReport(t *testing.T) { func TestRunEveningGeneratesTomorrowReport(t *testing.T) { fixture := newCLIFixture(t, writeFakeScriptorium) - runner := Runner{Clock: fixedClock()} + runner := testRunner() _, err := runTestCommand(t, runner, "run", "evening", @@ -113,7 +154,7 @@ func TestRunEveningGeneratesTomorrowReport(t *testing.T) { func TestRunMorningGeneratesTodayAndTomorrow(t *testing.T) { fixture := newCLIFixture(t, writeFakeScriptorium) - runner := Runner{Clock: fixedClock()} + runner := testRunner() _, err := runTestCommand(t, runner, "run", "morning", @@ -129,7 +170,7 @@ func TestRunMorningGeneratesTodayAndTomorrow(t *testing.T) { func TestRunMorningReportsPartialFailureAndContinues(t *testing.T) { fixture := newCLIFixture(t, writeFailingScriptorium) - runner := Runner{Clock: fixedClock()} + runner := testRunner() output, err := runTestCommand(t, runner, "run", "morning", @@ -322,7 +363,7 @@ func TestBatchStatusDoesNotRepeatBatchNotificationErrorPerReport(t *testing.T) { func TestRunEveningUsesOutputDirectoryAndSummary(t *testing.T) { fixture := newCLIFixture(t, writeFakeScriptorium) outputDir := fixture.path("copies") - runner := Runner{Clock: fixedClock()} + runner := testRunner() output, err := runTestCommand(t, runner, "run", "evening", @@ -349,7 +390,7 @@ func TestRunEveningUsesOutputDirectoryAndSummary(t *testing.T) { func TestRunQuietSuppressesSuccessfulOutput(t *testing.T) { fixture := newCLIFixture(t, writeFakeScriptorium) - runner := Runner{Clock: fixedClock()} + runner := testRunner() output, err := runTestCommand(t, runner, "run", "evening", @@ -390,7 +431,7 @@ func TestRunEveningReportsOmitsPerReportNotification(t *testing.T) { t.Setenv("CLI_DISTRIBUTOR_TOKEN", "cli-secret-token") var stdout bytes.Buffer var stderr bytes.Buffer - runner := Runner{Clock: fixedClock()} + runner := testRunner() err := runner.Run(context.Background(), []string{ "run", "evening", @@ -442,7 +483,7 @@ func TestRunEveningReportsDoesNotRequirePerReportDistributorToken(t *testing.T) configPath := writeTestConfigWithDisabledBatchDistributor(t, server, scriptoriumPath, workspaceRoot, distributorServer.URL) var stdout bytes.Buffer var stderr bytes.Buffer - runner := Runner{Clock: fixedClock()} + runner := testRunner() err := runner.Run(context.Background(), []string{ "run", "evening", @@ -490,7 +531,7 @@ func TestRunGenerateDailyWritesMarkdownReport(t *testing.T) { outPath := fixture.path("daily.md") var stdout bytes.Buffer var stderr bytes.Buffer - runner := Runner{Clock: fixedClock()} + runner := testRunner() err := runner.Run(context.Background(), []string{ "generate", "daily", @@ -524,13 +565,13 @@ func TestRunGenerateDailyWritesMarkdownReport(t *testing.T) { !strings.Contains(string(data), "timezone: UTC") { t.Fatalf("data package missing configured location with overridden timezone:\n%s", string(data)) } - preflightPath := oneArtifact(t, fixture.workspaceRoot, "preflight", "daily", "2026-05-29", "render.*.json") - preflight, err := os.ReadFile(preflightPath) + preparationPath := oneArtifact(t, fixture.workspaceRoot, "preflight", "daily", "2026-05-29", "prompt_preparation.*.json") + preparation, err := os.ReadFile(preparationPath) if err != nil { - t.Fatalf("read preflight: %v", err) + t.Fatalf("read preparation: %v", err) } - if !strings.Contains(string(preflight), `ok`) { - t.Fatalf("preflight missing fake render output:\n%s", string(preflight)) + if !strings.Contains(string(preparation), `"status": "succeeded"`) { + t.Fatalf("preparation missing successful status:\n%s", string(preparation)) } _ = oneArtifact(t, fixture.workspaceRoot, "reports", "daily", "2026-05-29", "report.*.md") rawGeneratedTextPath := oneArtifact(t, fixture.workspaceRoot, "snapshots", "daily", "2026-05-29", "generated_text_raw.*.json") @@ -548,7 +589,7 @@ func TestRunGenerateTodayWritesGeneratedTextReport(t *testing.T) { outPath := fixture.path("today.md") var stdout bytes.Buffer var stderr bytes.Buffer - runner := Runner{Clock: fixedClock()} + runner := testRunner() err := runner.Run(context.Background(), []string{ "generate", "today", @@ -612,7 +653,7 @@ func TestRunGenerateHourlyWritesGeneratedTextReport(t *testing.T) { outPath := fixture.path("hourly.md") var stdout bytes.Buffer var stderr bytes.Buffer - runner := Runner{Clock: timeutil.FixedClock{Time: time.Date(2026, 5, 29, 11, 0, 0, 0, time.UTC)}} + runner := testRunnerWithClock(timeutil.FixedClock{Time: time.Date(2026, 5, 29, 11, 0, 0, 0, time.UTC)}) err := runner.Run(context.Background(), []string{ "generate", "hourly", @@ -650,7 +691,7 @@ func TestRunGenerateHourlyWritesGeneratedTextReport(t *testing.T) { validatedGeneratedTextPath := oneArtifact(t, fixture.workspaceRoot, "snapshots", "hourly", "2026-05-29", "generated_text.*.json") renderContextPath := oneArtifact(t, fixture.workspaceRoot, "snapshots", "hourly", "2026-05-29", "render_context.*.json") managedReportPath := oneArtifact(t, fixture.workspaceRoot, "reports", "hourly", "2026-05-29", "report.*.md") - assertFileContains(t, rawGeneratedTextPath, `"summary": " Storm chances increase through late morning. "`) + assertFileContains(t, rawGeneratedTextPath, `"summary":"Storm chances increase through late morning."`) assertFileContains(t, validatedGeneratedTextPath, `"summary":"Storm chances increase through late morning."`) assertFileContains(t, renderContextPath, `"Report": {`) assertFileContains(t, renderContextPath, `"Title": "Hourly Report"`) @@ -663,7 +704,7 @@ func TestRunGenerateHourlyWritesGeneratedTextReport(t *testing.T) { func TestRunGenerateQuietSuppressesSuccessfulOutput(t *testing.T) { fixture := newCLIFixture(t, writeStructuredOutputScriptorium) outPath := fixture.path("today.md") - runner := Runner{Clock: fixedClock()} + runner := testRunner() output, err := runTestCommand(t, runner, "generate", "today", @@ -684,7 +725,7 @@ func TestRunGenerateQuietSuppressesSuccessfulOutput(t *testing.T) { func TestRunGeneratePreRunErrorEmitsNoJSON(t *testing.T) { var stdout bytes.Buffer var stderr bytes.Buffer - runner := Runner{Clock: fixedClock()} + runner := testRunner() err := runner.Run(context.Background(), []string{"generate", "daily"}, &stdout, &stderr) if err == nil { @@ -706,7 +747,7 @@ func TestRunGenerateNotificationFailureEmitsFailureSummary(t *testing.T) { workspaceRoot := filepath.Join(tempDir, "workspace") configPath := writeTestConfigWithDistributor(t, server, scriptoriumPath, workspaceRoot, distributorServer.URL) t.Setenv("CLI_DISTRIBUTOR_TOKEN", "") - runner := Runner{Clock: fixedClock()} + runner := testRunner() output, err := runTestCommand(t, runner, "generate", "daily", @@ -733,7 +774,7 @@ func TestRunGenerateNotificationFailureEmitsFailureSummary(t *testing.T) { func TestRunInspectTodayArtifacts(t *testing.T) { fixture := newCLIFixture(t, writeFakeScriptorium) - runner := Runner{Clock: fixedClock()} + runner := testRunner() var stdout bytes.Buffer var stderr bytes.Buffer @@ -778,7 +819,7 @@ func TestRunInspectTodayArtifacts(t *testing.T) { func TestRunInspectGeneratedArtifacts(t *testing.T) { fixture := newCLIFixture(t, writeFakeScriptorium) - runner := Runner{Clock: fixedClock()} + runner := testRunner() var stdout bytes.Buffer var stderr bytes.Buffer @@ -826,7 +867,7 @@ func TestRunInspectMissingMetadata(t *testing.T) { configPath := writeWorkspaceConfig(t, filepath.Join(tempDir, "workspace")) var stdout bytes.Buffer var stderr bytes.Buffer - runner := Runner{Clock: fixedClock()} + runner := testRunner() err := runner.Run(context.Background(), []string{"inspect", "metadata", "--config", configPath, "missing"}, &stdout, &stderr) if err == nil { @@ -842,7 +883,7 @@ func TestRunInspectRejectsQuiet(t *testing.T) { configPath := writeWorkspaceConfig(t, filepath.Join(tempDir, "workspace")) var stdout bytes.Buffer var stderr bytes.Buffer - runner := Runner{Clock: fixedClock()} + runner := testRunner() err := runner.Run(context.Background(), []string{"inspect", "reports", "--config", configPath, "--quiet"}, &stdout, &stderr) if err == nil { @@ -856,7 +897,7 @@ func TestRunInspectRejectsQuiet(t *testing.T) { func TestRunInspectRunCommandsParseRunIDAndConfig(t *testing.T) { tempDir := t.TempDir() configPath := writeWorkspaceConfig(t, filepath.Join(tempDir, "workspace")) - runner := Runner{Clock: fixedClock()} + runner := testRunner() commands := []string{"metadata", "modules", "data-package", "prior", "sources"} for _, command := range commands { @@ -889,7 +930,7 @@ func TestRunInspectRunCommandsParseRunIDAndConfig(t *testing.T) { } func TestResolveGenerateCommands(t *testing.T) { - runner := Runner{Clock: fixedClock()} + runner := testRunner() tests := []struct { name string args []string @@ -915,7 +956,7 @@ func TestResolveGenerateCommands(t *testing.T) { } func TestResolveGenerateSupportsEveryReportCommandName(t *testing.T) { - runner := Runner{Clock: fixedClock()} + runner := testRunner() for _, name := range report.CommandNames() { t.Run(name, func(t *testing.T) { args := []string{name} @@ -942,7 +983,7 @@ func TestResolveGenerateSupportsEveryReportCommandName(t *testing.T) { } func TestResolveGenerateHourlyAppliesSharedFlags(t *testing.T) { - runner := Runner{Clock: fixedClock()} + runner := testRunner() configPath := writeConfigFile(t, "weather_api:\n units: metric\n timezone: UTC\n") req, err := runner.resolveGenerate([]string{"hourly", "--config", configPath, "--units", "us", "--tz", "America/Chicago", "--out", "./hourly.md"}) @@ -968,7 +1009,7 @@ func TestResolveGenerateHourlyAppliesSharedFlags(t *testing.T) { } func TestResolveGenerateHourlyRejectsDateAndStormBounds(t *testing.T) { - runner := Runner{Clock: fixedClock()} + runner := testRunner() for _, args := range [][]string{ {"hourly", "--date", "2026-05-29"}, @@ -988,7 +1029,7 @@ func TestResolveGenerateHourlyRejectsDateAndStormBounds(t *testing.T) { } func TestResolveGenerateDailyRequiresDate(t *testing.T) { - runner := Runner{Clock: fixedClock()} + runner := testRunner() req, err := runner.resolveGenerate([]string{"daily"}) if err == nil { @@ -1003,7 +1044,7 @@ func TestResolveGenerateDailyRequiresDate(t *testing.T) { } func TestResolveGenerateDailyRejectsMalformedDate(t *testing.T) { - runner := Runner{Clock: fixedClock()} + runner := testRunner() _, err := runner.resolveGenerate([]string{"daily", "--date", "bad-date"}) if err == nil { @@ -1015,7 +1056,7 @@ func TestResolveGenerateDailyRejectsMalformedDate(t *testing.T) { } func TestResolveGenerateDailyParsesDate(t *testing.T) { - runner := Runner{Clock: fixedClock()} + runner := testRunner() req, err := runner.resolveGenerate([]string{"daily", "--date", "2026-05-29"}) if err != nil { @@ -1031,7 +1072,7 @@ func TestResolveGenerateDailyParsesDate(t *testing.T) { } func TestResolveGenerateTodayDate(t *testing.T) { - runner := Runner{Clock: fixedClock()} + runner := testRunner() defaultReq, err := runner.resolveGenerate([]string{"today"}) if err != nil { @@ -1064,7 +1105,7 @@ func TestResolveGenerateTodayDate(t *testing.T) { } func TestResolveGenerateAppliesSharedFlags(t *testing.T) { - runner := Runner{Clock: fixedClock()} + runner := testRunner() req, err := runner.resolveGenerate([]string{"daily", "--date", "2026-05-29", "--units", "metric", "--tz", "UTC", "--out", "./daily.md"}) if err != nil { @@ -1083,7 +1124,7 @@ func TestResolveGenerateAppliesSharedFlags(t *testing.T) { } func TestResolveGenerateRejectsRetiredHourlyCommand(t *testing.T) { - runner := Runner{Clock: fixedClock()} + runner := testRunner() retired := strings.Join([]string{"near", "term"}, "-") _, err := runner.resolveGenerate([]string{retired}) @@ -1096,7 +1137,7 @@ func TestResolveGenerateRejectsRetiredHourlyCommand(t *testing.T) { } func TestResolveGenerateRejectsRetiredReports(t *testing.T) { - runner := Runner{Clock: fixedClock()} + runner := testRunner() for _, name := range []string{"three-day", "weekend", "storm"} { if _, err := runner.resolveGenerate([]string{name}); err == nil { t.Fatalf("resolveGenerate(%q) error = nil, want unknown report", name) diff --git a/internal/state/filesystem.go b/internal/state/filesystem.go index 5a0cc3f..245adef 100644 --- a/internal/state/filesystem.go +++ b/internal/state/filesystem.go @@ -122,12 +122,20 @@ func (s *FilesystemStore) SaveModuleSnapshot(_ context.Context, resolved report. }, snapshot) } -func (s *FilesystemStore) SaveDataPackage(_ context.Context, resolved report.Resolved, pkg promptinput.Package) (string, error) { +func (s *FilesystemStore) SaveDataPackage(ctx context.Context, resolved report.Resolved, pkg promptinput.Package) (string, error) { + data, err := promptinput.MarshalYAML(pkg) + if err != nil { + return "", err + } + return s.SaveDataPackageBytes(ctx, resolved, data) +} + +func (s *FilesystemStore) SaveDataPackageBytes(_ context.Context, resolved report.Resolved, data []byte) (string, error) { paths, err := s.Paths(resolved) if err != nil { return "", err } - if err := promptinput.Save(paths.DataPackage, pkg); err != nil { + if err := fileutil.WriteFileAtomic(paths.DataPackage, data); err != nil { return "", err } return paths.DataPackage, nil diff --git a/internal/state/store.go b/internal/state/store.go index 88cc1ee..c9648ff 100644 --- a/internal/state/store.go +++ b/internal/state/store.go @@ -15,6 +15,7 @@ type Store interface { Paths(report.Resolved) (ArtifactPaths, error) SaveModuleSnapshot(context.Context, report.Resolved, module.Snapshot) (string, error) SaveDataPackage(context.Context, report.Resolved, promptinput.Package) (string, error) + SaveDataPackageBytes(context.Context, report.Resolved, []byte) (string, error) SavePreflight(context.Context, report.Resolved, PreflightArtifact) (string, error) SavePromptPreparation(context.Context, report.Resolved, PromptPreparationArtifact) (string, error) SavePromptExecution(context.Context, report.Resolved, PromptExecutionArtifact) (string, error)