Add Promptkit debug capture for generated reports

This commit is contained in:
2026-07-31 04:48:16 +00:00
parent 06b26d5e88
commit a6d11c01e8
11 changed files with 271 additions and 37 deletions

View File

@@ -17,10 +17,10 @@ required Weather API endpoint.
```text ```text
weatherreporter --help weatherreporter --help
weatherreporter generate daily --date YYYY-MM-DD [--config PATH] [--units VALUE] [--tz NAME] [--out PATH] [--quiet] weatherreporter generate daily --date YYYY-MM-DD [--config PATH] [--units VALUE] [--tz NAME] [--out PATH] [--llm-debug-dir PATH] [--quiet]
weatherreporter generate today [--config PATH] [--units VALUE] [--tz NAME] [--out PATH] [--date YYYY-MM-DD] [--quiet] weatherreporter generate today [--config PATH] [--units VALUE] [--tz NAME] [--out PATH] [--date YYYY-MM-DD] [--llm-debug-dir PATH] [--quiet]
weatherreporter generate tomorrow [--config PATH] [--units VALUE] [--tz NAME] [--out PATH] [--quiet] weatherreporter generate tomorrow [--config PATH] [--units VALUE] [--tz NAME] [--out PATH] [--llm-debug-dir PATH] [--quiet]
weatherreporter generate hourly [--config PATH] [--units VALUE] [--tz NAME] [--out PATH] [--quiet] weatherreporter generate hourly [--config PATH] [--units VALUE] [--tz NAME] [--out PATH] [--llm-debug-dir PATH] [--quiet]
weatherreporter run morning [--config PATH] [--units VALUE] [--tz NAME] [--out-dir PATH] [--quiet] weatherreporter run morning [--config PATH] [--units VALUE] [--tz NAME] [--out-dir PATH] [--quiet]
weatherreporter run evening [--config PATH] [--units VALUE] [--tz NAME] [--out-dir PATH] [--quiet] weatherreporter run evening [--config PATH] [--units VALUE] [--tz NAME] [--out-dir PATH] [--quiet]
weatherreporter inspect reports [--config PATH] [--limit N] weatherreporter inspect reports [--config PATH] [--limit N]
@@ -81,7 +81,7 @@ time, valid period, and status:
When available, the summary also includes `reportPath`, `metadataPath`, When available, the summary also includes `reportPath`, `metadataPath`,
`dataPackagePath`, `preparationPath`, `executionPath`, `generatedTextRawPath`, `dataPackagePath`, `preparationPath`, `executionPath`, `generatedTextRawPath`,
`generatedTextPath`, and `renderContextPath`. `outputPath` is included only `generatedTextPath`, `renderContextPath`, and `llmDebugPath`. `outputPath` is included only
when `--out` wrote an extra copy. Distributor notification, when attempted, when `--out` wrote an extra copy. Distributor notification, when attempted,
adds `notificationPath` and may add a compact `notification` object. adds `notificationPath` and may add a compact `notification` object.
@@ -108,6 +108,7 @@ batch=morning total=2 succeeded=2 failed=0
| `--units VALUE` | `generate`, `run` | Override `weather_api.units` for this command. | | `--units VALUE` | `generate`, `run` | Override `weather_api.units` for this command. |
| `--tz NAME` | `generate`, `run` | Override `weather_api.timezone` for this command. | | `--tz NAME` | `generate`, `run` | Override `weather_api.timezone` for this command. |
| `--out PATH` | every `generate` command | Write an extra Markdown report copy. | | `--out PATH` | every `generate` command | Write an extra Markdown report copy. |
| `--llm-debug-dir PATH` | every `generate` command | Write requested sensitive prompt diagnostics outside the managed workspace. The path must be absolute. |
| `--out-dir PATH` | `run morning`, `run evening` | Write extra Markdown report copies in `PATH`. | | `--out-dir PATH` | `run morning`, `run evening` | Write extra Markdown report copies in `PATH`. |
| `--quiet` | `generate`, `run` | Suppress action summaries and routine batch status output. | | `--quiet` | `generate`, `run` | Suppress action summaries and routine batch status output. |
| `--date YYYY-MM-DD` | `generate daily`, `generate today` | Required for Daily; optional for Today. | | `--date YYYY-MM-DD` | `generate daily`, `generate today` | Required for Daily; optional for Today. |
@@ -122,6 +123,7 @@ no Distributor-specific CLI flags. See the [configuration reference](config.md).
weatherreporter generate daily --date 2026-05-29 --out ./daily.md weatherreporter generate daily --date 2026-05-29 --out ./daily.md
weatherreporter generate today --date 2026-05-29 --out ./today.md weatherreporter generate today --date 2026-05-29 --out ./today.md
weatherreporter generate hourly --out ./hourly.md weatherreporter generate hourly --out ./hourly.md
weatherreporter generate today --llm-debug-dir /var/tmp/weatherreporter-debug
weatherreporter run morning --out-dir ./reports weatherreporter run morning --out-dir ./reports
``` ```

View File

@@ -151,6 +151,9 @@ Promptkit configuration selects the executor and prompt/profile checks for
every `generate` command. Scriptorium settings remain in use by batch commands every `generate` command. Scriptorium settings remain in use by batch commands
while their compatibility workflow remains active. while their compatibility workflow remains active.
Prompt debug capture has no YAML setting. Use `--llm-debug-dir PATH` on an
individual `generate` command when explicitly needed.
| Field | Default | Rules | | Field | Default | Rules |
| --- | --- | --- | | --- | --- | --- |
| `profile` | empty | Optional explicit execution profile. Otherwise the prompt's declared default is used. | | `profile` | empty | Optional explicit execution profile. Otherwise the prompt's declared default is used. |

View File

@@ -32,8 +32,9 @@ safe project-owned identity and provenance values.
## Single-Report Workflow ## Single-Report Workflow
`GenerateDetailed` resolves the requested report using the configured registry `GenerateDetailed` resolves the requested report using the configured registry
and current time, verifies the exact Promptkit prompt and selected profile, and and current time, initializes any requested prompt-debug root, verifies the
only then collects weather data. Inspection failure produces no collection or exact Promptkit prompt and selected profile, and only then collects weather
data. Debug initialization or inspection failure produces no collection or
managed artifacts. managed artifacts.
Single-report generation requires a non-nil normalized bundle and then performs this Single-report generation requires a non-nil normalized bundle and then performs this
@@ -46,8 +47,10 @@ ordered work:
3. Serialize and save the prompt data package once, then use those exact bytes 3. Serialize and save the prompt data package once, then use those exact bytes
for Promptkit execution. for Promptkit execution.
4. Save preparation provenance and V2 metadata from the preparation callback 4. Save preparation provenance and V2 metadata from the preparation callback
before provider execution. Save execution provenance after raw output is before provider execution. When requested, save preparation diagnostics in
persisted, then save updated metadata. the isolated debug store before the callback returns. Save execution
diagnostics immediately after a completed execution result, then persist raw
output and execution provenance before saving updated metadata.
5. Validate and save generated text, build and save a render context, and 5. Validate and save generated text, build and save a render context, and
render the managed Markdown template. render the managed Markdown template.
6. Optionally make an output copy, save final metadata, optionally notify 6. Optionally make an output copy, save final metadata, optionally notify
@@ -67,6 +70,10 @@ output, an execution receipt, and metadata before returning. If later report
generation fails, the result retains every reached safe artifact path; output generation fails, the result retains every reached safe artifact path; output
copies and notification are skipped until rendering succeeds. copies and notification are skipped until rendering succeeds.
The optional debug writer receives sensitive content only when explicitly
enabled. Its path is added to the report result only after a debug artifact is
successfully written; it is never copied into normal state records.
## Batch And Inspection Workflows ## Batch And Inspection Workflows
`RunBatchDetailed` collects once, asks the report registry to plan the batch `RunBatchDetailed` collects once, asks the report registry to plan the batch

View File

@@ -17,6 +17,10 @@ factory maps `promptkit` configuration to the Promptkit adapter, while tests can
inject a factory without importing dependency types. Each `generate` request inject a factory without importing dependency types. Each `generate` request
constructs one executor after configuration loads and passes it to the app. constructs one executor after configuration loads and passes it to the app.
All four `generate` commands also accept `--llm-debug-dir PATH`. The CLI passes
only this explicit request to the app; the app initializes the secure debug
root before prompt inspection. `run` commands do not accept the flag.
For inspection, it loads configuration, builds the appropriate app inspection For inspection, it loads configuration, builds the appropriate app inspection
request, and writes the returned value. Inspection is read-only; the inspected request, and writes the returned value. Inspection is read-only; the inspected
artifact types and user invocation remain owned by the [CLI reference](../cli.md) artifact types and user invocation remain owned by the [CLI reference](../cli.md)
@@ -29,7 +33,7 @@ carry report identity, status, relevant artifact paths, and notification
summary data. Batch summaries carry aggregate counts, per-report outcomes, and summary data. Batch summaries carry aggregate counts, per-report outcomes, and
the optional batch notification result. The translation deliberately excludes the optional batch notification result. The translation deliberately excludes
full module snapshots, prompt packages, raw generated text, Scriptorium output, full module snapshots, prompt packages, raw generated text, Scriptorium output,
and complete Distributor payloads. complete Distributor payloads, and prompt-debug content.
When an action returns both a result and an error, the CLI writes the failed When an action returns both a result and an error, the CLI writes the failed
summary before returning that error. Parse, configuration-load, and other summary before returning that error. Parse, configuration-load, and other

View File

@@ -25,6 +25,22 @@ Distributor notification is attempted. `--out` writes an extra operator copy;
it never changes the managed report or upload source. A successful generate it never changes the managed report or upload source. A successful generate
command prints its summary to stdout unless `--quiet` is used. command prints its summary to stdout unless `--quiet` is used.
## Optional Prompt Debug Capture
Use `--llm-debug-dir` only when content-rich prompt diagnostics are required:
```sh
weatherreporter generate today --llm-debug-dir /var/tmp/weatherreporter-debug
```
The directory must be absolute and is initialized before prompt inspection or
weather collection. Capture files are stored outside the managed workspace,
with restrictive permissions, under the report ID, valid date, and RunID.
They can contain rendered prompts and generated output, so the normal metadata,
CLI summary, and routine logs contain only the optional directory path—not
their content. A capture-write failure stops that run before later work can
continue.
Run a scheduled batch with the same configured collection: Run a scheduled batch with the same configured collection:
```sh ```sh

View File

@@ -47,6 +47,7 @@ type GenerateRequest struct {
Config config.Config Config config.Config
Report ReportKind Report ReportKind
OutputPath string OutputPath string
LLMDebugDir string
Now time.Time Now time.Time
Date time.Time Date time.Time
Collector Collector Collector Collector
@@ -279,6 +280,10 @@ func GenerateDetailed(ctx context.Context, req GenerateRequest) (*ReportResult,
if err != nil { if err != nil {
return nil, err return nil, err
} }
debugWriter, err := state.NewPromptDebugWriter(req.LLMDebugDir)
if err != nil {
return nil, promptexec.NewError(promptexec.InvalidConfiguration, "initialize prompt debug", err)
}
inspection, err := InspectPromptExecution(ctx, PromptInspectionRequest{ inspection, err := InspectPromptExecution(ctx, PromptInspectionRequest{
Resolved: resolved, Resolved: resolved,
Executor: req.Executor, Executor: req.Executor,
@@ -296,6 +301,7 @@ func GenerateDetailed(ctx context.Context, req GenerateRequest) (*ReportResult,
Resolved: resolved, Resolved: resolved,
Collection: *collection, Collection: *collection,
Inspection: inspection, Inspection: inspection,
DebugWriter: debugWriter,
}) })
} }

View File

@@ -77,6 +77,10 @@ type promptExecutorTest struct {
err error err error
afterPreparationErr error afterPreparationErr error
validation promptexec.ValidationStatus validation promptexec.ValidationStatus
preparationDebug *promptexec.PreparationDebug
executionDebug *promptexec.ExecutionDebug
captureDebug *bool
providerCalled *bool
} }
func (e promptExecutorTest) InspectPrompt(_ context.Context, id string, version string) (promptexec.PromptInspection, error) { func (e promptExecutorTest) InspectPrompt(_ context.Context, id string, version string) (promptexec.PromptInspection, error) {
@@ -93,6 +97,9 @@ func (e promptExecutorTest) InspectProfile(_ context.Context, id string) (prompt
} }
func (e promptExecutorTest) Execute(_ context.Context, request promptexec.ExecuteRequest, callback promptexec.PreparationCallback) (*promptexec.Execution, error) { func (e promptExecutorTest) Execute(_ context.Context, request promptexec.ExecuteRequest, callback promptexec.PreparationCallback) (*promptexec.Execution, error) {
if e.captureDebug != nil {
*e.captureDebug = request.CaptureDebug
}
if e.err != nil { if e.err != nil {
return nil, e.err return nil, e.err
} }
@@ -101,9 +108,12 @@ func (e promptExecutorTest) Execute(_ context.Context, request promptexec.Execut
PromptID: request.PromptID, PromptVersion: request.PromptVersion, PromptHash: "prompt-hash", RenderedPromptHash: "rendered-hash", PromptID: request.PromptID, PromptVersion: request.PromptVersion, PromptHash: "prompt-hash", RenderedPromptHash: "rendered-hash",
ProfileID: request.ProfileID, BackendID: "test", ModelName: "test-model", DataPackagePath: request.DataPackagePath, ProfileID: request.ProfileID, BackendID: "test", ModelName: "test-model", DataPackagePath: request.DataPackagePath,
StartedAt: now, EndedAt: now, StartedAt: now, EndedAt: now,
}, nil); err != nil { }, e.preparationDebug); err != nil {
return nil, err return nil, err
} }
if e.providerCalled != nil {
*e.providerCalled = true
}
if e.afterPreparationErr != nil { if e.afterPreparationErr != nil {
return nil, e.afterPreparationErr return nil, e.afterPreparationErr
} }
@@ -119,7 +129,7 @@ func (e promptExecutorTest) Execute(_ context.Context, request promptexec.Execut
RunID: "provider-run", PromptID: request.PromptID, PromptVersion: request.PromptVersion, PromptHash: "prompt-hash", RenderedPromptHash: "rendered-hash", RunID: "provider-run", PromptID: request.PromptID, PromptVersion: request.PromptVersion, PromptHash: "prompt-hash", RenderedPromptHash: "rendered-hash",
ProfileID: request.ProfileID, BackendID: "test", ModelName: "test-model", GeneratedHash: "generated-hash", ProfileID: request.ProfileID, BackendID: "test", ModelName: "test-model", GeneratedHash: "generated-hash",
StartedAt: now, EndedAt: now, DataPackagePath: request.DataPackagePath, RawOutput: raw, StartedAt: now, EndedAt: now, DataPackagePath: request.DataPackagePath, RawOutput: raw,
Validation: promptexec.NewValidation(validation, "json_schema", "generated_text.schema.json", nil), Validation: promptexec.NewValidation(validation, "json_schema", "generated_text.schema.json", nil), Debug: e.executionDebug,
}, nil }, nil
} }
@@ -317,6 +327,22 @@ func TestGenerateDetailedInspectsBeforeCollectionOrArtifactWrites(t *testing.T)
} }
} }
func TestGenerateDetailedRejectsInvalidDebugRootBeforeCollection(t *testing.T) {
cfg := config.Defaults()
cfg.Workspace.Root = t.TempDir()
collector := &recordingCollector{err: errors.New("collector must not run")}
_, err := GenerateDetailed(context.Background(), GenerateRequest{
Config: cfg, Report: ReportDaily, Date: mustParse("2026-05-29T12:00:00-05:00"),
Now: mustParse("2026-05-29T05:00:00-05:00"), Collector: collector, LLMDebugDir: "relative-debug",
})
if err == nil || promptexec.CategoryOf(err) != promptexec.InvalidConfiguration || strings.Contains(err.Error(), "relative-debug") {
t.Fatalf("GenerateDetailed() error/category = %v/%q, want safe debug-root validation", err, promptexec.CategoryOf(err))
}
if len(collector.requests) != 0 {
t.Fatalf("collector requests = %d, want debug initialization before collection", len(collector.requests))
}
}
func TestGenerateDetailedPersistsPromptFailureReceiptsWithoutRawOutput(t *testing.T) { func TestGenerateDetailedPersistsPromptFailureReceiptsWithoutRawOutput(t *testing.T) {
server := dailyBundleServer(t) server := dailyBundleServer(t)
cfg := dailyWorkspaceConfig(t, server) cfg := dailyWorkspaceConfig(t, server)
@@ -364,6 +390,106 @@ func TestGenerateDetailedPersistsRawOutputForValidationRejection(t *testing.T) {
} }
} }
func TestGenerateDetailedWritesRequestedPromptDebugOutsideWorkspace(t *testing.T) {
server := dailyBundleServer(t)
cfg := dailyWorkspaceConfig(t, server)
collection := collectionForTest(t, cfg)
debugRoot := filepath.Join(t.TempDir(), "prompt-debug")
captureDebug := false
result, err := GenerateDetailed(context.Background(), GenerateRequest{
Config: cfg, Report: ReportDaily, Date: mustParse("2026-05-29T12:00:00-05:00"),
Now: mustParse("2026-05-29T05:00:00-05:00"), Collector: &recordingCollector{result: &collection}, LLMDebugDir: debugRoot,
Executor: promptExecutorTest{
captureDebug: &captureDebug,
preparationDebug: &promptexec.PreparationDebug{
RenderedMessages: []promptexec.RenderedMessage{{Role: "system", Content: "sensitive rendered prompt"}},
ParametersJSON: []byte(`{"api_key":"secret-value"}`),
},
executionDebug: &promptexec.ExecutionDebug{ValidationDiagnostics: []string{"provider validation detail"}},
},
})
if err != nil {
t.Fatalf("GenerateDetailed() error = %v", err)
}
if !captureDebug || result.LLMDebugPath == "" || !strings.HasPrefix(result.LLMDebugPath, debugRoot+string(filepath.Separator)) {
t.Fatalf("capture/debug path = %t/%q, want requested isolated debug capture", captureDebug, result.LLMDebugPath)
}
preparationData, readErr := os.ReadFile(filepath.Join(result.LLMDebugPath, "preparation.json"))
if readErr != nil {
t.Fatalf("read preparation debug: %v", readErr)
}
executionData, readErr := os.ReadFile(filepath.Join(result.LLMDebugPath, "execution.json"))
if readErr != nil {
t.Fatalf("read execution debug: %v", readErr)
}
if !strings.Contains(string(preparationData), "sensitive rendered prompt") || strings.Contains(string(preparationData), "secret-value") || !strings.Contains(string(executionData), "provider validation detail") {
t.Fatalf("debug artifacts did not retain/redact expected content:\n%s\n%s", preparationData, executionData)
}
metadataData, readErr := os.ReadFile(result.MetadataPath)
if readErr != nil {
t.Fatalf("read metadata: %v", readErr)
}
if strings.Contains(string(metadataData), "sensitive rendered prompt") || strings.Contains(string(metadataData), "provider validation detail") {
t.Fatalf("normal metadata contains debug content:\n%s", metadataData)
}
}
func TestGenerateDetailedDebugWriteFailureStopsProviderExecution(t *testing.T) {
server := dailyBundleServer(t)
cfg := dailyWorkspaceConfig(t, server)
collection := collectionForTest(t, cfg)
now := mustParse("2026-05-29T05:00:00-05:00")
request := GenerateRequest{Config: cfg, Report: ReportDaily, Date: mustParse("2026-05-29T12:00:00-05:00"), Now: now}
resolved := resolveGenerateForTest(t, cfg, request, now.Format(time.RFC3339))
debugRoot := filepath.Join(t.TempDir(), "prompt-debug")
debugPath := filepath.Join(debugRoot, string(resolved.Definition.ID), resolved.ValidPeriod.Start.Format("2006-01-02"), resolved.Metadata().RunID)
if err := os.MkdirAll(filepath.Dir(debugPath), 0o700); err != nil {
t.Fatalf("create debug parent: %v", err)
}
if err := os.WriteFile(debugPath, []byte("not a directory"), 0o600); err != nil {
t.Fatalf("create debug collision: %v", err)
}
providerCalled := false
request.Collector = &recordingCollector{result: &collection}
request.LLMDebugDir = debugRoot
request.Executor = promptExecutorTest{providerCalled: &providerCalled}
result, err := GenerateDetailed(context.Background(), request)
if err == nil || result == nil || promptexec.CategoryOf(err) != promptexec.InvalidConfiguration {
t.Fatalf("GenerateDetailed() result/error = %#v/%v, want partial result and debug error", result, err)
}
if providerCalled || result.Metadata.RunID == "" || result.PreparationPath == "" || result.ExecutionPath != "" || result.LLMDebugPath != "" {
t.Fatalf("provider/run/preparation/execution/debug = %t/%q/%q/%q/%q, want preparation only before provider", providerCalled, result.Metadata.RunID, result.PreparationPath, result.ExecutionPath, result.LLMDebugPath)
}
}
func TestGenerateDetailedExecutionDebugFailureRetainsPreparationCapture(t *testing.T) {
server := dailyBundleServer(t)
cfg := dailyWorkspaceConfig(t, server)
collection := collectionForTest(t, cfg)
now := mustParse("2026-05-29T05:00:00-05:00")
request := GenerateRequest{Config: cfg, Report: ReportDaily, Date: mustParse("2026-05-29T12:00:00-05:00"), Now: now}
resolved := resolveGenerateForTest(t, cfg, request, now.Format(time.RFC3339))
debugRoot := filepath.Join(t.TempDir(), "prompt-debug")
executionDebugPath := filepath.Join(debugRoot, string(resolved.Definition.ID), resolved.ValidPeriod.Start.Format("2006-01-02"), resolved.Metadata().RunID, "execution.json")
if err := os.MkdirAll(executionDebugPath, 0o700); err != nil {
t.Fatalf("create execution debug collision: %v", err)
}
providerCalled := false
request.Collector = &recordingCollector{result: &collection}
request.LLMDebugDir = debugRoot
request.Executor = promptExecutorTest{providerCalled: &providerCalled}
result, err := GenerateDetailed(context.Background(), request)
if err == nil || result == nil || promptexec.CategoryOf(err) != promptexec.InvalidConfiguration {
t.Fatalf("GenerateDetailed() result/error = %#v/%v, want partial result and debug error", result, err)
}
if !providerCalled || result.LLMDebugPath == "" || result.ExecutionPath != "" || result.GeneratedTextRawPath != "" {
t.Fatalf("provider/debug/execution/raw = %t/%q/%q/%q, want preparation debug only after completed execution", providerCalled, result.LLMDebugPath, result.ExecutionPath, result.GeneratedTextRawPath)
}
if _, statErr := os.Stat(filepath.Join(result.LLMDebugPath, "preparation.json")); statErr != nil {
t.Fatalf("preparation debug artifact: %v", statErr)
}
}
type failingPromptInspectionExecutor struct{} type failingPromptInspectionExecutor struct{}
func (failingPromptInspectionExecutor) InspectPrompt(context.Context, string, string) (promptexec.PromptInspection, error) { func (failingPromptInspectionExecutor) InspectPrompt(context.Context, string, string) (promptexec.PromptInspection, error) {

View File

@@ -20,6 +20,7 @@ type promptReportRequest struct {
Resolved report.Resolved Resolved report.Resolved
Collection collect.Result Collection collect.Result
Inspection PromptInspectionResult Inspection PromptInspectionResult
DebugWriter *state.PromptDebugWriter
} }
func generatePromptReport(ctx context.Context, req promptReportRequest) (*ReportResult, error) { func generatePromptReport(ctx context.Context, req promptReportRequest) (*ReportResult, error) {
@@ -78,6 +79,7 @@ func generatePromptReport(ctx context.Context, req promptReportRequest) (*Report
GeneratedText: paths.GeneratedText, GeneratedText: paths.GeneratedText,
RenderContext: paths.RenderContext, RenderContext: paths.RenderContext,
}) })
result.Metadata = metadata
dataPackage, err := promptinput.Build(promptinput.BuildRequest{ dataPackage, err := promptinput.Build(promptinput.BuildRequest{
Metadata: promptMetadata(metadata), Metadata: promptMetadata(metadata),
Modules: moduleSnapshot, Modules: moduleSnapshot,
@@ -97,6 +99,7 @@ func generatePromptReport(ctx context.Context, req promptReportRequest) (*Report
metadata.DataPackagePath = dataPackagePath metadata.DataPackagePath = dataPackagePath
result.DataPackage = dataPackage result.DataPackage = dataPackage
result.DataPackagePath = dataPackagePath result.DataPackagePath = dataPackagePath
result.Metadata = metadata
handler, err := generatedtext.LookupDefinition(req.Resolved.Definition) handler, err := generatedtext.LookupDefinition(req.Resolved.Definition)
if err != nil { if err != nil {
@@ -105,7 +108,12 @@ func generatePromptReport(ctx context.Context, req promptReportRequest) (*Report
prepared := false prepared := false
callbackFailed := false callbackFailed := false
callback := func(preparation promptexec.Preparation, _ *promptexec.PreparationDebug) error { debugRef := state.PromptDebugRef{
ReportID: req.Resolved.Definition.ID,
ValidDate: req.Resolved.ValidPeriod.Start.Format("2006-01-02"),
RunID: metadata.RunID,
}
callback := func(preparation promptexec.Preparation, debug *promptexec.PreparationDebug) error {
artifact := state.PromptPreparationArtifact{ artifact := state.PromptPreparationArtifact{
SchemaVersion: state.PromptPreparationSchemaVersion, SchemaVersion: state.PromptPreparationSchemaVersion,
Status: state.PromptPreparationSucceeded, Status: state.PromptPreparationSucceeded,
@@ -127,6 +135,15 @@ func generatePromptReport(ctx context.Context, req promptReportRequest) (*Report
prepared = true prepared = true
result.PreparationPath = path result.PreparationPath = path
metadata.PreparationPath = path metadata.PreparationPath = path
result.Metadata = metadata
debugPath, err := req.DebugWriter.WritePreparation(debugRef, preparation, debug)
if err != nil {
callbackFailed = true
return promptDebugWriteError(err)
}
if debugPath != "" {
result.LLMDebugPath = debugPath
}
metadataPath, err := store.SaveMetadata(ctx, metadata) metadataPath, err := store.SaveMetadata(ctx, metadata)
if err != nil { if err != nil {
callbackFailed = true callbackFailed = true
@@ -142,7 +159,7 @@ func generatePromptReport(ctx context.Context, req promptReportRequest) (*Report
ProfileID: req.Inspection.ProfileID, ProfileID: req.Inspection.ProfileID,
DataPackage: data, DataPackage: data,
DataPackagePath: dataPackagePath, DataPackagePath: dataPackagePath,
CaptureDebug: false, CaptureDebug: req.DebugWriter.Enabled(),
}, callback) }, callback)
if executeErr != nil { if executeErr != nil {
if callbackFailed { if callbackFailed {
@@ -198,6 +215,13 @@ func generatePromptReport(ctx context.Context, req promptReportRequest) (*Report
result.ExecutionPath, result.Metadata, result.MetadataPath = executionPath, metadata, metadataPath result.ExecutionPath, result.Metadata, result.MetadataPath = executionPath, metadata, metadataPath
return result, generatedReportError(req.Resolved, metadata.RunID, "execute prompt", err) return result, generatedReportError(req.Resolved, metadata.RunID, "execute prompt", err)
} }
debugPath, err := req.DebugWriter.WriteExecution(debugRef, *execution)
if err != nil {
return result, generatedReportError(req.Resolved, metadata.RunID, "write prompt debug", promptDebugWriteError(err))
}
if debugPath != "" {
result.LLMDebugPath = debugPath
}
if execution.Validation.Status != promptexec.ValidationPassed && execution.Validation.Status != promptexec.ValidationFailed { if execution.Validation.Status != promptexec.ValidationPassed && execution.Validation.Status != promptexec.ValidationFailed {
err := promptexec.NewError(promptexec.OperationalValidation, "prompt execution did not complete validation", nil) err := promptexec.NewError(promptexec.OperationalValidation, "prompt execution did not complete validation", nil)
@@ -318,4 +342,8 @@ func classifiedPromptError(operation string, err error) error {
return promptexec.NewError(promptexec.Generation, operation, err) return promptexec.NewError(promptexec.Generation, operation, err)
} }
func promptDebugWriteError(err error) error {
return promptexec.NewError(promptexec.InvalidConfiguration, "write requested prompt debug artifact", err)
}
func ptr[T any](value T) *T { return &value } func ptr[T any](value T) *T { return &value }

View File

@@ -22,6 +22,7 @@ func TestNewGenerateSummaryForGeneratedTextReport(t *testing.T) {
DataPackagePath: "/runs/hourly/data_package.yaml", DataPackagePath: "/runs/hourly/data_package.yaml",
PreparationPath: "/runs/hourly/preparation.json", PreparationPath: "/runs/hourly/preparation.json",
ExecutionPath: "/runs/hourly/execution.json", ExecutionPath: "/runs/hourly/execution.json",
LLMDebugPath: "/operator-debug/hourly/2026-05-29/run-123",
ReportPath: "/runs/hourly/report.md", ReportPath: "/runs/hourly/report.md",
OutputPath: "/copies/hourly.md", OutputPath: "/copies/hourly.md",
MetadataPath: "/runs/hourly/metadata.json", MetadataPath: "/runs/hourly/metadata.json",
@@ -58,7 +59,7 @@ func TestNewGenerateSummaryForGeneratedTextReport(t *testing.T) {
if summary.ReportID != report.Hourly || summary.ReportName != "Hourly Report" || summary.PromptID != "weather.hourly_generated_text" || summary.RunID != "20260529T133000Z_hourly" { if summary.ReportID != report.Hourly || summary.ReportName != "Hourly Report" || summary.PromptID != "weather.hourly_generated_text" || summary.RunID != "20260529T133000Z_hourly" {
t.Fatalf("summary identity = %#v, want hourly report identity", summary) t.Fatalf("summary identity = %#v, want hourly report identity", summary)
} }
if summary.PreparationPath == "" || summary.ExecutionPath == "" || summary.GeneratedTextRawPath == "" || summary.GeneratedTextPath == "" || summary.RenderContextPath == "" { if summary.PreparationPath == "" || summary.ExecutionPath == "" || summary.LLMDebugPath == "" || summary.GeneratedTextRawPath == "" || summary.GeneratedTextPath == "" || summary.RenderContextPath == "" {
t.Fatalf("generated-text paths = %#v, want generated-text artifact paths", summary) t.Fatalf("generated-text paths = %#v, want generated-text artifact paths", summary)
} }
if summary.Notification == nil || summary.Notification.RunID != "distributor-run" || summary.Notification.AcceptedAt == nil || !summary.Notification.AcceptedAt.Equal(acceptedAt) { if summary.Notification == nil || summary.Notification.RunID != "distributor-run" || summary.Notification.AcceptedAt == nil || !summary.Notification.AcceptedAt.Equal(acceptedAt) {

View File

@@ -16,10 +16,10 @@ const helpText = `weatherreporter prepares weather reports from normalized forec
Usage: Usage:
weatherreporter --help weatherreporter --help
weatherreporter generate daily --date YYYY-MM-DD [--config PATH] [--units VALUE] [--tz NAME] [--out PATH] [--quiet] weatherreporter generate daily --date YYYY-MM-DD [--config PATH] [--units VALUE] [--tz NAME] [--out PATH] [--llm-debug-dir PATH] [--quiet]
weatherreporter generate today [--config PATH] [--units VALUE] [--tz NAME] [--out PATH] [--date YYYY-MM-DD] [--quiet] weatherreporter generate today [--config PATH] [--units VALUE] [--tz NAME] [--out PATH] [--date YYYY-MM-DD] [--llm-debug-dir PATH] [--quiet]
weatherreporter generate tomorrow [--config PATH] [--units VALUE] [--tz NAME] [--out PATH] [--quiet] weatherreporter generate tomorrow [--config PATH] [--units VALUE] [--tz NAME] [--out PATH] [--llm-debug-dir PATH] [--quiet]
weatherreporter generate hourly [--config PATH] [--units VALUE] [--tz NAME] [--out PATH] [--quiet] weatherreporter generate hourly [--config PATH] [--units VALUE] [--tz NAME] [--out PATH] [--llm-debug-dir PATH] [--quiet]
weatherreporter run morning [--config PATH] [--units VALUE] [--tz NAME] [--out-dir PATH] [--quiet] weatherreporter run morning [--config PATH] [--units VALUE] [--tz NAME] [--out-dir PATH] [--quiet]
weatherreporter run evening [--config PATH] [--units VALUE] [--tz NAME] [--out-dir PATH] [--quiet] weatherreporter run evening [--config PATH] [--units VALUE] [--tz NAME] [--out-dir PATH] [--quiet]
weatherreporter inspect reports [--config PATH] [--limit N] weatherreporter inspect reports [--config PATH] [--limit N]
@@ -35,6 +35,7 @@ Options:
--units VALUE Override weather API units. --units VALUE Override weather API units.
--tz NAME Override weather API timezone. --tz NAME Override weather API timezone.
--out PATH Write an extra Markdown report copy where supported by the generate command. --out PATH Write an extra Markdown report copy where supported by the generate command.
--llm-debug-dir PATH Write sensitive prompt debug artifacts outside the managed workspace.
--out-dir PATH Write extra Markdown report copies for run commands. --out-dir PATH Write extra Markdown report copies for run commands.
--quiet Suppress successful generate and run output. --quiet Suppress successful generate and run output.
` `
@@ -108,6 +109,7 @@ type commonOptions struct {
type generateOptions struct { type generateOptions struct {
commonOptions commonOptions
Date string Date string
LLMDebugDir string
} }
type inspectOptions struct { type inspectOptions struct {
@@ -227,6 +229,7 @@ func (r Runner) resolveGenerateAction(args []string) (app.GenerateRequest, commo
Config: cfg, Config: cfg,
Report: reportKind, Report: reportKind,
OutputPath: opts.Output, OutputPath: opts.Output,
LLMDebugDir: opts.LLMDebugDir,
Now: r.Clock.Now(), Now: r.Clock.Now(),
Executor: executor, Executor: executor,
} }
@@ -295,6 +298,7 @@ func parseGenerateFlags(report app.ReportKind, args []string) (generateOptions,
opts := generateOptions{} opts := generateOptions{}
addCommonFlags(fs, &opts.commonOptions, true) addCommonFlags(fs, &opts.commonOptions, true)
fs.BoolVar(&opts.Quiet, "quiet", false, "suppress successful action output") fs.BoolVar(&opts.Quiet, "quiet", false, "suppress successful action output")
fs.StringVar(&opts.LLMDebugDir, "llm-debug-dir", "", "write sensitive prompt debug artifacts under PATH")
if report == app.ReportDaily || report == app.ReportToday { if report == app.ReportDaily || report == app.ReportToday {
fs.StringVar(&opts.Date, "date", "", "report date in YYYY-MM-DD") fs.StringVar(&opts.Date, "date", "", "report date in YYYY-MM-DD")
} }

View File

@@ -73,6 +73,9 @@ func TestRunHelpLongFlag(t *testing.T) {
if !strings.Contains(output.stdout, "weatherreporter generate hourly") { if !strings.Contains(output.stdout, "weatherreporter generate hourly") {
t.Fatalf("help output missing hourly generate command:\n%s", output.stdout) t.Fatalf("help output missing hourly generate command:\n%s", output.stdout)
} }
if strings.Count(output.stdout, "--llm-debug-dir PATH") != 5 {
t.Fatalf("help output = %q, want debug flag for four generate commands and its option", output.stdout)
}
if !strings.Contains(output.stdout, "generate today") || !strings.Contains(output.stdout, "[--quiet]") { if !strings.Contains(output.stdout, "generate today") || !strings.Contains(output.stdout, "[--quiet]") {
t.Fatalf("help output missing quiet generate usage:\n%s", output.stdout) t.Fatalf("help output missing quiet generate usage:\n%s", output.stdout)
} }
@@ -134,6 +137,37 @@ func TestRunGenerateTomorrowWritesMarkdownReport(t *testing.T) {
} }
} }
func TestRunGenerateWritesRequestedDebugAndSafeSummary(t *testing.T) {
fixture := newCLIFixture(t, writeFakeScriptorium)
debugDir := fixture.path("prompt-debug")
output, err := runTestCommand(t, testRunner(),
"generate", "today", "--config", fixture.configPath, "--llm-debug-dir", debugDir,
)
if err != nil {
t.Fatalf("Run() error = %v", err)
}
summary := decodeGenerateSummary(t, output.stdout)
if summary.LLMDebugPath == "" || !strings.HasPrefix(summary.LLMDebugPath, debugDir+string(filepath.Separator)) {
t.Fatalf("debug path = %q, want isolated requested directory", summary.LLMDebugPath)
}
if strings.Contains(output.stdout, "Showers are possible during the selected day") || strings.Contains(output.stdout, "Today starts with showers before improving") || strings.Contains(output.stdout, "rendered-hash") {
t.Fatalf("summary contains prompt or generated content:\n%s", output.stdout)
}
assertFileContains(t, filepath.Join(summary.LLMDebugPath, "preparation.json"), "weather.today_generated_text")
assertFileContains(t, filepath.Join(summary.LLMDebugPath, "execution.json"), "Today starts with showers before improving.")
}
func TestParseGenerateFlagsAcceptsDebugDirectoryForEveryReport(t *testing.T) {
for _, reportKind := range []app.ReportKind{app.ReportDaily, app.ReportToday, app.ReportTomorrow, app.ReportHourly} {
t.Run(string(reportKind), func(t *testing.T) {
opts, err := parseGenerateFlags(reportKind, []string{"--llm-debug-dir", "/tmp/prompt-debug"})
if err != nil || opts.LLMDebugDir != "/tmp/prompt-debug" {
t.Fatalf("parseGenerateFlags() options/error = %#v/%v", opts, err)
}
})
}
}
func TestRunEveningGeneratesTomorrowReport(t *testing.T) { func TestRunEveningGeneratesTomorrowReport(t *testing.T) {
fixture := newCLIFixture(t, writeFakeScriptorium) fixture := newCLIFixture(t, writeFakeScriptorium)
runner := testRunner() runner := testRunner()
@@ -704,6 +738,7 @@ func TestRunGenerateHourlyWritesGeneratedTextReport(t *testing.T) {
func TestRunGenerateQuietSuppressesSuccessfulOutput(t *testing.T) { func TestRunGenerateQuietSuppressesSuccessfulOutput(t *testing.T) {
fixture := newCLIFixture(t, writeStructuredOutputScriptorium) fixture := newCLIFixture(t, writeStructuredOutputScriptorium)
outPath := fixture.path("today.md") outPath := fixture.path("today.md")
debugDir := fixture.path("prompt-debug")
runner := testRunner() runner := testRunner()
output, err := runTestCommand(t, runner, output, err := runTestCommand(t, runner,
@@ -711,6 +746,7 @@ func TestRunGenerateQuietSuppressesSuccessfulOutput(t *testing.T) {
"--config", fixture.configPath, "--config", fixture.configPath,
"--date", "2026-05-29", "--date", "2026-05-29",
"--out", outPath, "--out", outPath,
"--llm-debug-dir", debugDir,
"--quiet", "--quiet",
) )
if err != nil { if err != nil {
@@ -720,6 +756,7 @@ func TestRunGenerateQuietSuppressesSuccessfulOutput(t *testing.T) {
t.Fatalf("stdout/stderr = %q/%q, want quiet success output", output.stdout, output.stderr) t.Fatalf("stdout/stderr = %q/%q, want quiet success output", output.stdout, output.stderr)
} }
assertFileContains(t, outPath, "# Today's Weather") assertFileContains(t, outPath, "# Today's Weather")
_ = oneArtifact(t, debugDir, "today", "2026-05-29", "*", "preparation.json")
} }
func TestRunGeneratePreRunErrorEmitsNoJSON(t *testing.T) { func TestRunGeneratePreRunErrorEmitsNoJSON(t *testing.T) {