eric/weatherreporter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

Add Promptkit debug capture for generated reports

Browse Source
This commit is contained in:
Eric Rakestraw
2026-07-31 04:48:16 +00:00
parent 06b26d5e88
commit a6d11c01e8
11 changed files with 271 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
internal/cli/result_test.go
View File

@@ -22,6 +22,7 @@ func TestNewGenerateSummaryForGeneratedTextReport(t *testing.T) {
DataPackagePath: "/runs/hourly/data_package.yaml",
PreparationPath: "/runs/hourly/preparation.json",
ExecutionPath: "/runs/hourly/execution.json",
LLMDebugPath: "/operator-debug/hourly/2026-05-29/run-123",
ReportPath: "/runs/hourly/report.md",
OutputPath: "/copies/hourly.md",
MetadataPath: "/runs/hourly/metadata.json",
@@ -58,7 +59,7 @@ func TestNewGenerateSummaryForGeneratedTextReport(t *testing.T) {
if summary.ReportID != report.Hourly || summary.ReportName != "Hourly Report" || summary.PromptID != "weather.hourly_generated_text" || summary.RunID != "20260529T133000Z_hourly" {
t.Fatalf("summary identity = %#v, want hourly report identity", summary)
}
if summary.PreparationPath == "" || summary.ExecutionPath == "" || summary.GeneratedTextRawPath == "" || summary.GeneratedTextPath == "" || summary.RenderContextPath == "" {
if summary.PreparationPath == "" || summary.ExecutionPath == "" || summary.LLMDebugPath == "" || summary.GeneratedTextRawPath == "" || summary.GeneratedTextPath == "" || summary.RenderContextPath == "" {
t.Fatalf("generated-text paths = %#v, want generated-text artifact paths", summary)
}
if summary.Notification == nil || summary.Notification.RunID != "distributor-run" || summary.Notification.AcceptedAt == nil || !summary.Notification.AcceptedAt.Equal(acceptedAt) {

24
internal/cli/root.go
View File

@@ -16,10 +16,10 @@ const helpText = `weatherreporter prepares weather reports from normalized forec
Usage:
weatherreporter --help
weatherreporter generate daily --date YYYY-MM-DD [--config PATH] [--units VALUE] [--tz NAME] [--out PATH] [--quiet]
weatherreporter generate today [--config PATH] [--units VALUE] [--tz NAME] [--out PATH] [--date YYYY-MM-DD] [--quiet]
weatherreporter generate tomorrow [--config PATH] [--units VALUE] [--tz NAME] [--out PATH] [--quiet]
weatherreporter generate hourly [--config PATH] [--units VALUE] [--tz NAME] [--out PATH] [--quiet]
weatherreporter generate daily --date YYYY-MM-DD [--config PATH] [--units VALUE] [--tz NAME] [--out PATH] [--llm-debug-dir PATH] [--quiet]
weatherreporter generate today [--config PATH] [--units VALUE] [--tz NAME] [--out PATH] [--date YYYY-MM-DD] [--llm-debug-dir PATH] [--quiet]
weatherreporter generate tomorrow [--config PATH] [--units VALUE] [--tz NAME] [--out PATH] [--llm-debug-dir PATH] [--quiet]
weatherreporter generate hourly [--config PATH] [--units VALUE] [--tz NAME] [--out PATH] [--llm-debug-dir PATH] [--quiet]
weatherreporter run morning [--config PATH] [--units VALUE] [--tz NAME] [--out-dir PATH] [--quiet]
weatherreporter run evening [--config PATH] [--units VALUE] [--tz NAME] [--out-dir PATH] [--quiet]
weatherreporter inspect reports [--config PATH] [--limit N]
@@ -35,6 +35,7 @@ Options:
--units VALUE Override weather API units.
--tz NAME Override weather API timezone.
--out PATH Write an extra Markdown report copy where supported by the generate command.
--llm-debug-dir PATH Write sensitive prompt debug artifacts outside the managed workspace.
--out-dir PATH Write extra Markdown report copies for run commands.
--quiet Suppress successful generate and run output.
`
@@ -107,7 +108,8 @@ type commonOptions struct {
type generateOptions struct {
commonOptions
Date string
Date string
LLMDebugDir string
}
type inspectOptions struct {
@@ -224,11 +226,12 @@ func (r Runner) resolveGenerateAction(args []string) (app.GenerateRequest, commo
}
req := app.GenerateRequest{
Config: cfg,
Report: reportKind,
OutputPath: opts.Output,
Now: r.Clock.Now(),
Executor: executor,
Config: cfg,
Report: reportKind,
OutputPath: opts.Output,
LLMDebugDir: opts.LLMDebugDir,
Now: r.Clock.Now(),
Executor: executor,
}
switch reportKind {
@@ -295,6 +298,7 @@ func parseGenerateFlags(report app.ReportKind, args []string) (generateOptions,
opts := generateOptions{}
addCommonFlags(fs, &opts.commonOptions, true)
fs.BoolVar(&opts.Quiet, "quiet", false, "suppress successful action output")
fs.StringVar(&opts.LLMDebugDir, "llm-debug-dir", "", "write sensitive prompt debug artifacts under PATH")
if report == app.ReportDaily || report == app.ReportToday {
fs.StringVar(&opts.Date, "date", "", "report date in YYYY-MM-DD")
}

37
internal/cli/root_test.go
View File

@@ -73,6 +73,9 @@ func TestRunHelpLongFlag(t *testing.T) {
if !strings.Contains(output.stdout, "weatherreporter generate hourly") {
t.Fatalf("help output missing hourly generate command:\n%s", output.stdout)
}
if strings.Count(output.stdout, "--llm-debug-dir PATH") != 5 {
t.Fatalf("help output = %q, want debug flag for four generate commands and its option", output.stdout)
}
if !strings.Contains(output.stdout, "generate today") || !strings.Contains(output.stdout, "[--quiet]") {
t.Fatalf("help output missing quiet generate usage:\n%s", output.stdout)
}
@@ -134,6 +137,37 @@ func TestRunGenerateTomorrowWritesMarkdownReport(t *testing.T) {
}
}
func TestRunGenerateWritesRequestedDebugAndSafeSummary(t *testing.T) {
fixture := newCLIFixture(t, writeFakeScriptorium)
debugDir := fixture.path("prompt-debug")
output, err := runTestCommand(t, testRunner(),
"generate", "today", "--config", fixture.configPath, "--llm-debug-dir", debugDir,
)
if err != nil {
t.Fatalf("Run() error = %v", err)
}
summary := decodeGenerateSummary(t, output.stdout)
if summary.LLMDebugPath == "" || !strings.HasPrefix(summary.LLMDebugPath, debugDir+string(filepath.Separator)) {
t.Fatalf("debug path = %q, want isolated requested directory", summary.LLMDebugPath)
}
if strings.Contains(output.stdout, "Showers are possible during the selected day") || strings.Contains(output.stdout, "Today starts with showers before improving") || strings.Contains(output.stdout, "rendered-hash") {
t.Fatalf("summary contains prompt or generated content:\n%s", output.stdout)
}
assertFileContains(t, filepath.Join(summary.LLMDebugPath, "preparation.json"), "weather.today_generated_text")
assertFileContains(t, filepath.Join(summary.LLMDebugPath, "execution.json"), "Today starts with showers before improving.")
}
func TestParseGenerateFlagsAcceptsDebugDirectoryForEveryReport(t *testing.T) {
for _, reportKind := range []app.ReportKind{app.ReportDaily, app.ReportToday, app.ReportTomorrow, app.ReportHourly} {
t.Run(string(reportKind), func(t *testing.T) {
opts, err := parseGenerateFlags(reportKind, []string{"--llm-debug-dir", "/tmp/prompt-debug"})
if err != nil || opts.LLMDebugDir != "/tmp/prompt-debug" {
t.Fatalf("parseGenerateFlags() options/error = %#v/%v", opts, err)
}
})
}
}
func TestRunEveningGeneratesTomorrowReport(t *testing.T) {
fixture := newCLIFixture(t, writeFakeScriptorium)
runner := testRunner()
@@ -704,6 +738,7 @@ func TestRunGenerateHourlyWritesGeneratedTextReport(t *testing.T) {
func TestRunGenerateQuietSuppressesSuccessfulOutput(t *testing.T) {
fixture := newCLIFixture(t, writeStructuredOutputScriptorium)
outPath := fixture.path("today.md")
debugDir := fixture.path("prompt-debug")
runner := testRunner()
output, err := runTestCommand(t, runner,
@@ -711,6 +746,7 @@ func TestRunGenerateQuietSuppressesSuccessfulOutput(t *testing.T) {
"--config", fixture.configPath,
"--date", "2026-05-29",
"--out", outPath,
"--llm-debug-dir", debugDir,
"--quiet",
)
if err != nil {
@@ -720,6 +756,7 @@ func TestRunGenerateQuietSuppressesSuccessfulOutput(t *testing.T) {
t.Fatalf("stdout/stderr = %q/%q, want quiet success output", output.stdout, output.stderr)
}
assertFileContains(t, outPath, "# Today's Weather")
_ = oneArtifact(t, debugDir, "today", "2026-05-29", "*", "preparation.json")
}
func TestRunGeneratePreRunErrorEmitsNoJSON(t *testing.T) {