diff --git a/docs/cli.md b/docs/cli.md index 17ea982..6d795b5 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -17,10 +17,10 @@ required Weather API endpoint. ```text weatherreporter --help -weatherreporter generate daily --date YYYY-MM-DD [--config PATH] [--units VALUE] [--tz NAME] [--out PATH] [--quiet] -weatherreporter generate today [--config PATH] [--units VALUE] [--tz NAME] [--out PATH] [--date YYYY-MM-DD] [--quiet] -weatherreporter generate tomorrow [--config PATH] [--units VALUE] [--tz NAME] [--out PATH] [--quiet] -weatherreporter generate hourly [--config PATH] [--units VALUE] [--tz NAME] [--out PATH] [--quiet] +weatherreporter generate daily --date YYYY-MM-DD [--config PATH] [--units VALUE] [--tz NAME] [--out PATH] [--llm-debug-dir PATH] [--quiet] +weatherreporter generate today [--config PATH] [--units VALUE] [--tz NAME] [--out PATH] [--date YYYY-MM-DD] [--llm-debug-dir PATH] [--quiet] +weatherreporter generate tomorrow [--config PATH] [--units VALUE] [--tz NAME] [--out PATH] [--llm-debug-dir PATH] [--quiet] +weatherreporter generate hourly [--config PATH] [--units VALUE] [--tz NAME] [--out PATH] [--llm-debug-dir PATH] [--quiet] weatherreporter run morning [--config PATH] [--units VALUE] [--tz NAME] [--out-dir PATH] [--quiet] weatherreporter run evening [--config PATH] [--units VALUE] [--tz NAME] [--out-dir PATH] [--quiet] weatherreporter inspect reports [--config PATH] [--limit N] @@ -81,7 +81,7 @@ time, valid period, and status: When available, the summary also includes `reportPath`, `metadataPath`, `dataPackagePath`, `preparationPath`, `executionPath`, `generatedTextRawPath`, -`generatedTextPath`, and `renderContextPath`. `outputPath` is included only +`generatedTextPath`, `renderContextPath`, and `llmDebugPath`. `outputPath` is included only when `--out` wrote an extra copy. Distributor notification, when attempted, adds `notificationPath` and may add a compact `notification` object. @@ -108,6 +108,7 @@ batch=morning total=2 succeeded=2 failed=0 | `--units VALUE` | `generate`, `run` | Override `weather_api.units` for this command. | | `--tz NAME` | `generate`, `run` | Override `weather_api.timezone` for this command. | | `--out PATH` | every `generate` command | Write an extra Markdown report copy. | +| `--llm-debug-dir PATH` | every `generate` command | Write requested sensitive prompt diagnostics outside the managed workspace. The path must be absolute. | | `--out-dir PATH` | `run morning`, `run evening` | Write extra Markdown report copies in `PATH`. | | `--quiet` | `generate`, `run` | Suppress action summaries and routine batch status output. | | `--date YYYY-MM-DD` | `generate daily`, `generate today` | Required for Daily; optional for Today. | @@ -122,6 +123,7 @@ no Distributor-specific CLI flags. See the [configuration reference](config.md). weatherreporter generate daily --date 2026-05-29 --out ./daily.md weatherreporter generate today --date 2026-05-29 --out ./today.md weatherreporter generate hourly --out ./hourly.md +weatherreporter generate today --llm-debug-dir /var/tmp/weatherreporter-debug weatherreporter run morning --out-dir ./reports ``` diff --git a/docs/config.md b/docs/config.md index deaedd9..1b42609 100644 --- a/docs/config.md +++ b/docs/config.md @@ -151,6 +151,9 @@ Promptkit configuration selects the executor and prompt/profile checks for every `generate` command. Scriptorium settings remain in use by batch commands while their compatibility workflow remains active. +Prompt debug capture has no YAML setting. Use `--llm-debug-dir PATH` on an +individual `generate` command when explicitly needed. + | Field | Default | Rules | | --- | --- | --- | | `profile` | empty | Optional explicit execution profile. Otherwise the prompt's declared default is used. | diff --git a/docs/internal/app-orchestration.md b/docs/internal/app-orchestration.md index 328702c..d129677 100644 --- a/docs/internal/app-orchestration.md +++ b/docs/internal/app-orchestration.md @@ -32,8 +32,9 @@ safe project-owned identity and provenance values. ## Single-Report Workflow `GenerateDetailed` resolves the requested report using the configured registry -and current time, verifies the exact Promptkit prompt and selected profile, and -only then collects weather data. Inspection failure produces no collection or +and current time, initializes any requested prompt-debug root, verifies the +exact Promptkit prompt and selected profile, and only then collects weather +data. Debug initialization or inspection failure produces no collection or managed artifacts. Single-report generation requires a non-nil normalized bundle and then performs this @@ -46,8 +47,10 @@ ordered work: 3. Serialize and save the prompt data package once, then use those exact bytes for Promptkit execution. 4. Save preparation provenance and V2 metadata from the preparation callback - before provider execution. Save execution provenance after raw output is - persisted, then save updated metadata. + before provider execution. When requested, save preparation diagnostics in + the isolated debug store before the callback returns. Save execution + diagnostics immediately after a completed execution result, then persist raw + output and execution provenance before saving updated metadata. 5. Validate and save generated text, build and save a render context, and render the managed Markdown template. 6. Optionally make an output copy, save final metadata, optionally notify @@ -67,6 +70,10 @@ output, an execution receipt, and metadata before returning. If later report generation fails, the result retains every reached safe artifact path; output copies and notification are skipped until rendering succeeds. +The optional debug writer receives sensitive content only when explicitly +enabled. Its path is added to the report result only after a debug artifact is +successfully written; it is never copied into normal state records. + ## Batch And Inspection Workflows `RunBatchDetailed` collects once, asks the report registry to plan the batch diff --git a/docs/internal/cli.md b/docs/internal/cli.md index e77c347..26a5e3f 100644 --- a/docs/internal/cli.md +++ b/docs/internal/cli.md @@ -17,6 +17,10 @@ factory maps `promptkit` configuration to the Promptkit adapter, while tests can inject a factory without importing dependency types. Each `generate` request constructs one executor after configuration loads and passes it to the app. +All four `generate` commands also accept `--llm-debug-dir PATH`. The CLI passes +only this explicit request to the app; the app initializes the secure debug +root before prompt inspection. `run` commands do not accept the flag. + For inspection, it loads configuration, builds the appropriate app inspection request, and writes the returned value. Inspection is read-only; the inspected artifact types and user invocation remain owned by the [CLI reference](../cli.md) @@ -29,7 +33,7 @@ carry report identity, status, relevant artifact paths, and notification summary data. Batch summaries carry aggregate counts, per-report outcomes, and the optional batch notification result. The translation deliberately excludes full module snapshots, prompt packages, raw generated text, Scriptorium output, -and complete Distributor payloads. +complete Distributor payloads, and prompt-debug content. When an action returns both a result and an error, the CLI writes the failed summary before returning that error. Parse, configuration-load, and other diff --git a/docs/operations.md b/docs/operations.md index bc0d9bd..857c77c 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -25,6 +25,22 @@ Distributor notification is attempted. `--out` writes an extra operator copy; it never changes the managed report or upload source. A successful generate command prints its summary to stdout unless `--quiet` is used. +## Optional Prompt Debug Capture + +Use `--llm-debug-dir` only when content-rich prompt diagnostics are required: + +```sh +weatherreporter generate today --llm-debug-dir /var/tmp/weatherreporter-debug +``` + +The directory must be absolute and is initialized before prompt inspection or +weather collection. Capture files are stored outside the managed workspace, +with restrictive permissions, under the report ID, valid date, and RunID. +They can contain rendered prompts and generated output, so the normal metadata, +CLI summary, and routine logs contain only the optional directory path—not +their content. A capture-write failure stops that run before later work can +continue. + Run a scheduled batch with the same configured collection: ```sh diff --git a/internal/app/app.go b/internal/app/app.go index 61a0e18..5663e73 100644 --- a/internal/app/app.go +++ b/internal/app/app.go @@ -44,15 +44,16 @@ const ( ) type GenerateRequest struct { - Config config.Config - Report ReportKind - OutputPath string - Now time.Time - Date time.Time - Collector Collector - Notifier Notifier - Executor promptexec.Executor - Store state.Store + Config config.Config + Report ReportKind + OutputPath string + LLMDebugDir string + Now time.Time + Date time.Time + Collector Collector + Notifier Notifier + Executor promptexec.Executor + Store state.Store } type BatchRequest struct { @@ -279,6 +280,10 @@ func GenerateDetailed(ctx context.Context, req GenerateRequest) (*ReportResult, if err != nil { return nil, err } + debugWriter, err := state.NewPromptDebugWriter(req.LLMDebugDir) + if err != nil { + return nil, promptexec.NewError(promptexec.InvalidConfiguration, "initialize prompt debug", err) + } inspection, err := InspectPromptExecution(ctx, PromptInspectionRequest{ Resolved: resolved, Executor: req.Executor, @@ -296,6 +301,7 @@ func GenerateDetailed(ctx context.Context, req GenerateRequest) (*ReportResult, Resolved: resolved, Collection: *collection, Inspection: inspection, + DebugWriter: debugWriter, }) } diff --git a/internal/app/app_test.go b/internal/app/app_test.go index 41a00ab..2aed3d8 100644 --- a/internal/app/app_test.go +++ b/internal/app/app_test.go @@ -77,6 +77,10 @@ type promptExecutorTest struct { err error afterPreparationErr error validation promptexec.ValidationStatus + preparationDebug *promptexec.PreparationDebug + executionDebug *promptexec.ExecutionDebug + captureDebug *bool + providerCalled *bool } func (e promptExecutorTest) InspectPrompt(_ context.Context, id string, version string) (promptexec.PromptInspection, error) { @@ -93,6 +97,9 @@ func (e promptExecutorTest) InspectProfile(_ context.Context, id string) (prompt } func (e promptExecutorTest) Execute(_ context.Context, request promptexec.ExecuteRequest, callback promptexec.PreparationCallback) (*promptexec.Execution, error) { + if e.captureDebug != nil { + *e.captureDebug = request.CaptureDebug + } if e.err != nil { return nil, e.err } @@ -101,9 +108,12 @@ func (e promptExecutorTest) Execute(_ context.Context, request promptexec.Execut PromptID: request.PromptID, PromptVersion: request.PromptVersion, PromptHash: "prompt-hash", RenderedPromptHash: "rendered-hash", ProfileID: request.ProfileID, BackendID: "test", ModelName: "test-model", DataPackagePath: request.DataPackagePath, StartedAt: now, EndedAt: now, - }, nil); err != nil { + }, e.preparationDebug); err != nil { return nil, err } + if e.providerCalled != nil { + *e.providerCalled = true + } if e.afterPreparationErr != nil { return nil, e.afterPreparationErr } @@ -119,7 +129,7 @@ func (e promptExecutorTest) Execute(_ context.Context, request promptexec.Execut RunID: "provider-run", PromptID: request.PromptID, PromptVersion: request.PromptVersion, PromptHash: "prompt-hash", RenderedPromptHash: "rendered-hash", ProfileID: request.ProfileID, BackendID: "test", ModelName: "test-model", GeneratedHash: "generated-hash", StartedAt: now, EndedAt: now, DataPackagePath: request.DataPackagePath, RawOutput: raw, - Validation: promptexec.NewValidation(validation, "json_schema", "generated_text.schema.json", nil), + Validation: promptexec.NewValidation(validation, "json_schema", "generated_text.schema.json", nil), Debug: e.executionDebug, }, nil } @@ -317,6 +327,22 @@ func TestGenerateDetailedInspectsBeforeCollectionOrArtifactWrites(t *testing.T) } } +func TestGenerateDetailedRejectsInvalidDebugRootBeforeCollection(t *testing.T) { + cfg := config.Defaults() + cfg.Workspace.Root = t.TempDir() + collector := &recordingCollector{err: errors.New("collector must not run")} + _, err := GenerateDetailed(context.Background(), GenerateRequest{ + Config: cfg, Report: ReportDaily, Date: mustParse("2026-05-29T12:00:00-05:00"), + Now: mustParse("2026-05-29T05:00:00-05:00"), Collector: collector, LLMDebugDir: "relative-debug", + }) + if err == nil || promptexec.CategoryOf(err) != promptexec.InvalidConfiguration || strings.Contains(err.Error(), "relative-debug") { + t.Fatalf("GenerateDetailed() error/category = %v/%q, want safe debug-root validation", err, promptexec.CategoryOf(err)) + } + if len(collector.requests) != 0 { + t.Fatalf("collector requests = %d, want debug initialization before collection", len(collector.requests)) + } +} + func TestGenerateDetailedPersistsPromptFailureReceiptsWithoutRawOutput(t *testing.T) { server := dailyBundleServer(t) cfg := dailyWorkspaceConfig(t, server) @@ -364,6 +390,106 @@ func TestGenerateDetailedPersistsRawOutputForValidationRejection(t *testing.T) { } } +func TestGenerateDetailedWritesRequestedPromptDebugOutsideWorkspace(t *testing.T) { + server := dailyBundleServer(t) + cfg := dailyWorkspaceConfig(t, server) + collection := collectionForTest(t, cfg) + debugRoot := filepath.Join(t.TempDir(), "prompt-debug") + captureDebug := false + result, err := GenerateDetailed(context.Background(), GenerateRequest{ + Config: cfg, Report: ReportDaily, Date: mustParse("2026-05-29T12:00:00-05:00"), + Now: mustParse("2026-05-29T05:00:00-05:00"), Collector: &recordingCollector{result: &collection}, LLMDebugDir: debugRoot, + Executor: promptExecutorTest{ + captureDebug: &captureDebug, + preparationDebug: &promptexec.PreparationDebug{ + RenderedMessages: []promptexec.RenderedMessage{{Role: "system", Content: "sensitive rendered prompt"}}, + ParametersJSON: []byte(`{"api_key":"secret-value"}`), + }, + executionDebug: &promptexec.ExecutionDebug{ValidationDiagnostics: []string{"provider validation detail"}}, + }, + }) + if err != nil { + t.Fatalf("GenerateDetailed() error = %v", err) + } + if !captureDebug || result.LLMDebugPath == "" || !strings.HasPrefix(result.LLMDebugPath, debugRoot+string(filepath.Separator)) { + t.Fatalf("capture/debug path = %t/%q, want requested isolated debug capture", captureDebug, result.LLMDebugPath) + } + preparationData, readErr := os.ReadFile(filepath.Join(result.LLMDebugPath, "preparation.json")) + if readErr != nil { + t.Fatalf("read preparation debug: %v", readErr) + } + executionData, readErr := os.ReadFile(filepath.Join(result.LLMDebugPath, "execution.json")) + if readErr != nil { + t.Fatalf("read execution debug: %v", readErr) + } + if !strings.Contains(string(preparationData), "sensitive rendered prompt") || strings.Contains(string(preparationData), "secret-value") || !strings.Contains(string(executionData), "provider validation detail") { + t.Fatalf("debug artifacts did not retain/redact expected content:\n%s\n%s", preparationData, executionData) + } + metadataData, readErr := os.ReadFile(result.MetadataPath) + if readErr != nil { + t.Fatalf("read metadata: %v", readErr) + } + if strings.Contains(string(metadataData), "sensitive rendered prompt") || strings.Contains(string(metadataData), "provider validation detail") { + t.Fatalf("normal metadata contains debug content:\n%s", metadataData) + } +} + +func TestGenerateDetailedDebugWriteFailureStopsProviderExecution(t *testing.T) { + server := dailyBundleServer(t) + cfg := dailyWorkspaceConfig(t, server) + collection := collectionForTest(t, cfg) + now := mustParse("2026-05-29T05:00:00-05:00") + request := GenerateRequest{Config: cfg, Report: ReportDaily, Date: mustParse("2026-05-29T12:00:00-05:00"), Now: now} + resolved := resolveGenerateForTest(t, cfg, request, now.Format(time.RFC3339)) + debugRoot := filepath.Join(t.TempDir(), "prompt-debug") + debugPath := filepath.Join(debugRoot, string(resolved.Definition.ID), resolved.ValidPeriod.Start.Format("2006-01-02"), resolved.Metadata().RunID) + if err := os.MkdirAll(filepath.Dir(debugPath), 0o700); err != nil { + t.Fatalf("create debug parent: %v", err) + } + if err := os.WriteFile(debugPath, []byte("not a directory"), 0o600); err != nil { + t.Fatalf("create debug collision: %v", err) + } + providerCalled := false + request.Collector = &recordingCollector{result: &collection} + request.LLMDebugDir = debugRoot + request.Executor = promptExecutorTest{providerCalled: &providerCalled} + result, err := GenerateDetailed(context.Background(), request) + if err == nil || result == nil || promptexec.CategoryOf(err) != promptexec.InvalidConfiguration { + t.Fatalf("GenerateDetailed() result/error = %#v/%v, want partial result and debug error", result, err) + } + if providerCalled || result.Metadata.RunID == "" || result.PreparationPath == "" || result.ExecutionPath != "" || result.LLMDebugPath != "" { + t.Fatalf("provider/run/preparation/execution/debug = %t/%q/%q/%q/%q, want preparation only before provider", providerCalled, result.Metadata.RunID, result.PreparationPath, result.ExecutionPath, result.LLMDebugPath) + } +} + +func TestGenerateDetailedExecutionDebugFailureRetainsPreparationCapture(t *testing.T) { + server := dailyBundleServer(t) + cfg := dailyWorkspaceConfig(t, server) + collection := collectionForTest(t, cfg) + now := mustParse("2026-05-29T05:00:00-05:00") + request := GenerateRequest{Config: cfg, Report: ReportDaily, Date: mustParse("2026-05-29T12:00:00-05:00"), Now: now} + resolved := resolveGenerateForTest(t, cfg, request, now.Format(time.RFC3339)) + debugRoot := filepath.Join(t.TempDir(), "prompt-debug") + executionDebugPath := filepath.Join(debugRoot, string(resolved.Definition.ID), resolved.ValidPeriod.Start.Format("2006-01-02"), resolved.Metadata().RunID, "execution.json") + if err := os.MkdirAll(executionDebugPath, 0o700); err != nil { + t.Fatalf("create execution debug collision: %v", err) + } + providerCalled := false + request.Collector = &recordingCollector{result: &collection} + request.LLMDebugDir = debugRoot + request.Executor = promptExecutorTest{providerCalled: &providerCalled} + result, err := GenerateDetailed(context.Background(), request) + if err == nil || result == nil || promptexec.CategoryOf(err) != promptexec.InvalidConfiguration { + t.Fatalf("GenerateDetailed() result/error = %#v/%v, want partial result and debug error", result, err) + } + if !providerCalled || result.LLMDebugPath == "" || result.ExecutionPath != "" || result.GeneratedTextRawPath != "" { + t.Fatalf("provider/debug/execution/raw = %t/%q/%q/%q, want preparation debug only after completed execution", providerCalled, result.LLMDebugPath, result.ExecutionPath, result.GeneratedTextRawPath) + } + if _, statErr := os.Stat(filepath.Join(result.LLMDebugPath, "preparation.json")); statErr != nil { + t.Fatalf("preparation debug artifact: %v", statErr) + } +} + type failingPromptInspectionExecutor struct{} func (failingPromptInspectionExecutor) InspectPrompt(context.Context, string, string) (promptexec.PromptInspection, error) { diff --git a/internal/app/prompt_generate.go b/internal/app/prompt_generate.go index 0d6a761..1f4cc59 100644 --- a/internal/app/prompt_generate.go +++ b/internal/app/prompt_generate.go @@ -17,9 +17,10 @@ import ( type promptReportRequest struct { GenerateRequest - Resolved report.Resolved - Collection collect.Result - Inspection PromptInspectionResult + Resolved report.Resolved + Collection collect.Result + Inspection PromptInspectionResult + DebugWriter *state.PromptDebugWriter } func generatePromptReport(ctx context.Context, req promptReportRequest) (*ReportResult, error) { @@ -78,6 +79,7 @@ func generatePromptReport(ctx context.Context, req promptReportRequest) (*Report GeneratedText: paths.GeneratedText, RenderContext: paths.RenderContext, }) + result.Metadata = metadata dataPackage, err := promptinput.Build(promptinput.BuildRequest{ Metadata: promptMetadata(metadata), Modules: moduleSnapshot, @@ -97,6 +99,7 @@ func generatePromptReport(ctx context.Context, req promptReportRequest) (*Report metadata.DataPackagePath = dataPackagePath result.DataPackage = dataPackage result.DataPackagePath = dataPackagePath + result.Metadata = metadata handler, err := generatedtext.LookupDefinition(req.Resolved.Definition) if err != nil { @@ -105,7 +108,12 @@ func generatePromptReport(ctx context.Context, req promptReportRequest) (*Report prepared := false callbackFailed := false - callback := func(preparation promptexec.Preparation, _ *promptexec.PreparationDebug) error { + debugRef := state.PromptDebugRef{ + ReportID: req.Resolved.Definition.ID, + ValidDate: req.Resolved.ValidPeriod.Start.Format("2006-01-02"), + RunID: metadata.RunID, + } + callback := func(preparation promptexec.Preparation, debug *promptexec.PreparationDebug) error { artifact := state.PromptPreparationArtifact{ SchemaVersion: state.PromptPreparationSchemaVersion, Status: state.PromptPreparationSucceeded, @@ -127,6 +135,15 @@ func generatePromptReport(ctx context.Context, req promptReportRequest) (*Report prepared = true result.PreparationPath = path metadata.PreparationPath = path + result.Metadata = metadata + debugPath, err := req.DebugWriter.WritePreparation(debugRef, preparation, debug) + if err != nil { + callbackFailed = true + return promptDebugWriteError(err) + } + if debugPath != "" { + result.LLMDebugPath = debugPath + } metadataPath, err := store.SaveMetadata(ctx, metadata) if err != nil { callbackFailed = true @@ -142,7 +159,7 @@ func generatePromptReport(ctx context.Context, req promptReportRequest) (*Report ProfileID: req.Inspection.ProfileID, DataPackage: data, DataPackagePath: dataPackagePath, - CaptureDebug: false, + CaptureDebug: req.DebugWriter.Enabled(), }, callback) if executeErr != nil { if callbackFailed { @@ -198,6 +215,13 @@ func generatePromptReport(ctx context.Context, req promptReportRequest) (*Report result.ExecutionPath, result.Metadata, result.MetadataPath = executionPath, metadata, metadataPath return result, generatedReportError(req.Resolved, metadata.RunID, "execute prompt", err) } + debugPath, err := req.DebugWriter.WriteExecution(debugRef, *execution) + if err != nil { + return result, generatedReportError(req.Resolved, metadata.RunID, "write prompt debug", promptDebugWriteError(err)) + } + if debugPath != "" { + result.LLMDebugPath = debugPath + } if execution.Validation.Status != promptexec.ValidationPassed && execution.Validation.Status != promptexec.ValidationFailed { err := promptexec.NewError(promptexec.OperationalValidation, "prompt execution did not complete validation", nil) @@ -318,4 +342,8 @@ func classifiedPromptError(operation string, err error) error { return promptexec.NewError(promptexec.Generation, operation, err) } +func promptDebugWriteError(err error) error { + return promptexec.NewError(promptexec.InvalidConfiguration, "write requested prompt debug artifact", err) +} + func ptr[T any](value T) *T { return &value } diff --git a/internal/cli/result_test.go b/internal/cli/result_test.go index cd2a19a..857529e 100644 --- a/internal/cli/result_test.go +++ b/internal/cli/result_test.go @@ -22,6 +22,7 @@ func TestNewGenerateSummaryForGeneratedTextReport(t *testing.T) { DataPackagePath: "/runs/hourly/data_package.yaml", PreparationPath: "/runs/hourly/preparation.json", ExecutionPath: "/runs/hourly/execution.json", + LLMDebugPath: "/operator-debug/hourly/2026-05-29/run-123", ReportPath: "/runs/hourly/report.md", OutputPath: "/copies/hourly.md", MetadataPath: "/runs/hourly/metadata.json", @@ -58,7 +59,7 @@ func TestNewGenerateSummaryForGeneratedTextReport(t *testing.T) { if summary.ReportID != report.Hourly || summary.ReportName != "Hourly Report" || summary.PromptID != "weather.hourly_generated_text" || summary.RunID != "20260529T133000Z_hourly" { t.Fatalf("summary identity = %#v, want hourly report identity", summary) } - if summary.PreparationPath == "" || summary.ExecutionPath == "" || summary.GeneratedTextRawPath == "" || summary.GeneratedTextPath == "" || summary.RenderContextPath == "" { + if summary.PreparationPath == "" || summary.ExecutionPath == "" || summary.LLMDebugPath == "" || summary.GeneratedTextRawPath == "" || summary.GeneratedTextPath == "" || summary.RenderContextPath == "" { t.Fatalf("generated-text paths = %#v, want generated-text artifact paths", summary) } if summary.Notification == nil || summary.Notification.RunID != "distributor-run" || summary.Notification.AcceptedAt == nil || !summary.Notification.AcceptedAt.Equal(acceptedAt) { diff --git a/internal/cli/root.go b/internal/cli/root.go index 3f89c35..40a9627 100644 --- a/internal/cli/root.go +++ b/internal/cli/root.go @@ -16,10 +16,10 @@ const helpText = `weatherreporter prepares weather reports from normalized forec Usage: weatherreporter --help - weatherreporter generate daily --date YYYY-MM-DD [--config PATH] [--units VALUE] [--tz NAME] [--out PATH] [--quiet] - weatherreporter generate today [--config PATH] [--units VALUE] [--tz NAME] [--out PATH] [--date YYYY-MM-DD] [--quiet] - weatherreporter generate tomorrow [--config PATH] [--units VALUE] [--tz NAME] [--out PATH] [--quiet] - weatherreporter generate hourly [--config PATH] [--units VALUE] [--tz NAME] [--out PATH] [--quiet] + weatherreporter generate daily --date YYYY-MM-DD [--config PATH] [--units VALUE] [--tz NAME] [--out PATH] [--llm-debug-dir PATH] [--quiet] + weatherreporter generate today [--config PATH] [--units VALUE] [--tz NAME] [--out PATH] [--date YYYY-MM-DD] [--llm-debug-dir PATH] [--quiet] + weatherreporter generate tomorrow [--config PATH] [--units VALUE] [--tz NAME] [--out PATH] [--llm-debug-dir PATH] [--quiet] + weatherreporter generate hourly [--config PATH] [--units VALUE] [--tz NAME] [--out PATH] [--llm-debug-dir PATH] [--quiet] weatherreporter run morning [--config PATH] [--units VALUE] [--tz NAME] [--out-dir PATH] [--quiet] weatherreporter run evening [--config PATH] [--units VALUE] [--tz NAME] [--out-dir PATH] [--quiet] weatherreporter inspect reports [--config PATH] [--limit N] @@ -35,6 +35,7 @@ Options: --units VALUE Override weather API units. --tz NAME Override weather API timezone. --out PATH Write an extra Markdown report copy where supported by the generate command. + --llm-debug-dir PATH Write sensitive prompt debug artifacts outside the managed workspace. --out-dir PATH Write extra Markdown report copies for run commands. --quiet Suppress successful generate and run output. ` @@ -107,7 +108,8 @@ type commonOptions struct { type generateOptions struct { commonOptions - Date string + Date string + LLMDebugDir string } type inspectOptions struct { @@ -224,11 +226,12 @@ func (r Runner) resolveGenerateAction(args []string) (app.GenerateRequest, commo } req := app.GenerateRequest{ - Config: cfg, - Report: reportKind, - OutputPath: opts.Output, - Now: r.Clock.Now(), - Executor: executor, + Config: cfg, + Report: reportKind, + OutputPath: opts.Output, + LLMDebugDir: opts.LLMDebugDir, + Now: r.Clock.Now(), + Executor: executor, } switch reportKind { @@ -295,6 +298,7 @@ func parseGenerateFlags(report app.ReportKind, args []string) (generateOptions, opts := generateOptions{} addCommonFlags(fs, &opts.commonOptions, true) fs.BoolVar(&opts.Quiet, "quiet", false, "suppress successful action output") + fs.StringVar(&opts.LLMDebugDir, "llm-debug-dir", "", "write sensitive prompt debug artifacts under PATH") if report == app.ReportDaily || report == app.ReportToday { fs.StringVar(&opts.Date, "date", "", "report date in YYYY-MM-DD") } diff --git a/internal/cli/root_test.go b/internal/cli/root_test.go index d0b3205..6bd4dd6 100644 --- a/internal/cli/root_test.go +++ b/internal/cli/root_test.go @@ -73,6 +73,9 @@ func TestRunHelpLongFlag(t *testing.T) { if !strings.Contains(output.stdout, "weatherreporter generate hourly") { t.Fatalf("help output missing hourly generate command:\n%s", output.stdout) } + if strings.Count(output.stdout, "--llm-debug-dir PATH") != 5 { + t.Fatalf("help output = %q, want debug flag for four generate commands and its option", output.stdout) + } if !strings.Contains(output.stdout, "generate today") || !strings.Contains(output.stdout, "[--quiet]") { t.Fatalf("help output missing quiet generate usage:\n%s", output.stdout) } @@ -134,6 +137,37 @@ func TestRunGenerateTomorrowWritesMarkdownReport(t *testing.T) { } } +func TestRunGenerateWritesRequestedDebugAndSafeSummary(t *testing.T) { + fixture := newCLIFixture(t, writeFakeScriptorium) + debugDir := fixture.path("prompt-debug") + output, err := runTestCommand(t, testRunner(), + "generate", "today", "--config", fixture.configPath, "--llm-debug-dir", debugDir, + ) + if err != nil { + t.Fatalf("Run() error = %v", err) + } + summary := decodeGenerateSummary(t, output.stdout) + if summary.LLMDebugPath == "" || !strings.HasPrefix(summary.LLMDebugPath, debugDir+string(filepath.Separator)) { + t.Fatalf("debug path = %q, want isolated requested directory", summary.LLMDebugPath) + } + if strings.Contains(output.stdout, "Showers are possible during the selected day") || strings.Contains(output.stdout, "Today starts with showers before improving") || strings.Contains(output.stdout, "rendered-hash") { + t.Fatalf("summary contains prompt or generated content:\n%s", output.stdout) + } + assertFileContains(t, filepath.Join(summary.LLMDebugPath, "preparation.json"), "weather.today_generated_text") + assertFileContains(t, filepath.Join(summary.LLMDebugPath, "execution.json"), "Today starts with showers before improving.") +} + +func TestParseGenerateFlagsAcceptsDebugDirectoryForEveryReport(t *testing.T) { + for _, reportKind := range []app.ReportKind{app.ReportDaily, app.ReportToday, app.ReportTomorrow, app.ReportHourly} { + t.Run(string(reportKind), func(t *testing.T) { + opts, err := parseGenerateFlags(reportKind, []string{"--llm-debug-dir", "/tmp/prompt-debug"}) + if err != nil || opts.LLMDebugDir != "/tmp/prompt-debug" { + t.Fatalf("parseGenerateFlags() options/error = %#v/%v", opts, err) + } + }) + } +} + func TestRunEveningGeneratesTomorrowReport(t *testing.T) { fixture := newCLIFixture(t, writeFakeScriptorium) runner := testRunner() @@ -704,6 +738,7 @@ func TestRunGenerateHourlyWritesGeneratedTextReport(t *testing.T) { func TestRunGenerateQuietSuppressesSuccessfulOutput(t *testing.T) { fixture := newCLIFixture(t, writeStructuredOutputScriptorium) outPath := fixture.path("today.md") + debugDir := fixture.path("prompt-debug") runner := testRunner() output, err := runTestCommand(t, runner, @@ -711,6 +746,7 @@ func TestRunGenerateQuietSuppressesSuccessfulOutput(t *testing.T) { "--config", fixture.configPath, "--date", "2026-05-29", "--out", outPath, + "--llm-debug-dir", debugDir, "--quiet", ) if err != nil { @@ -720,6 +756,7 @@ func TestRunGenerateQuietSuppressesSuccessfulOutput(t *testing.T) { t.Fatalf("stdout/stderr = %q/%q, want quiet success output", output.stdout, output.stderr) } assertFileContains(t, outPath, "# Today's Weather") + _ = oneArtifact(t, debugDir, "today", "2026-05-29", "*", "preparation.json") } func TestRunGeneratePreRunErrorEmitsNoJSON(t *testing.T) {