eric/weatherreporter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

Add Promptkit debug capture for generated reports

Browse Source
This commit is contained in:
Eric Rakestraw
2026-07-31 04:48:16 +00:00
parent 06b26d5e88
commit a6d11c01e8
11 changed files with 271 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files

37
internal/cli/root_test.go
View File

@@ -73,6 +73,9 @@ func TestRunHelpLongFlag(t *testing.T) {
if !strings.Contains(output.stdout, "weatherreporter generate hourly") {
t.Fatalf("help output missing hourly generate command:\n%s", output.stdout)
}
if strings.Count(output.stdout, "--llm-debug-dir PATH") != 5 {
t.Fatalf("help output = %q, want debug flag for four generate commands and its option", output.stdout)
}
if !strings.Contains(output.stdout, "generate today") || !strings.Contains(output.stdout, "[--quiet]") {
t.Fatalf("help output missing quiet generate usage:\n%s", output.stdout)
}
@@ -134,6 +137,37 @@ func TestRunGenerateTomorrowWritesMarkdownReport(t *testing.T) {
}
}
func TestRunGenerateWritesRequestedDebugAndSafeSummary(t *testing.T) {
fixture := newCLIFixture(t, writeFakeScriptorium)
debugDir := fixture.path("prompt-debug")
output, err := runTestCommand(t, testRunner(),
"generate", "today", "--config", fixture.configPath, "--llm-debug-dir", debugDir,
)
if err != nil {
t.Fatalf("Run() error = %v", err)
}
summary := decodeGenerateSummary(t, output.stdout)
if summary.LLMDebugPath == "" || !strings.HasPrefix(summary.LLMDebugPath, debugDir+string(filepath.Separator)) {
t.Fatalf("debug path = %q, want isolated requested directory", summary.LLMDebugPath)
}
if strings.Contains(output.stdout, "Showers are possible during the selected day") || strings.Contains(output.stdout, "Today starts with showers before improving") || strings.Contains(output.stdout, "rendered-hash") {
t.Fatalf("summary contains prompt or generated content:\n%s", output.stdout)
}
assertFileContains(t, filepath.Join(summary.LLMDebugPath, "preparation.json"), "weather.today_generated_text")
assertFileContains(t, filepath.Join(summary.LLMDebugPath, "execution.json"), "Today starts with showers before improving.")
}
func TestParseGenerateFlagsAcceptsDebugDirectoryForEveryReport(t *testing.T) {
for _, reportKind := range []app.ReportKind{app.ReportDaily, app.ReportToday, app.ReportTomorrow, app.ReportHourly} {
t.Run(string(reportKind), func(t *testing.T) {
opts, err := parseGenerateFlags(reportKind, []string{"--llm-debug-dir", "/tmp/prompt-debug"})
if err != nil || opts.LLMDebugDir != "/tmp/prompt-debug" {
t.Fatalf("parseGenerateFlags() options/error = %#v/%v", opts, err)
}
})
}
}
func TestRunEveningGeneratesTomorrowReport(t *testing.T) {
fixture := newCLIFixture(t, writeFakeScriptorium)
runner := testRunner()
@@ -704,6 +738,7 @@ func TestRunGenerateHourlyWritesGeneratedTextReport(t *testing.T) {
func TestRunGenerateQuietSuppressesSuccessfulOutput(t *testing.T) {
fixture := newCLIFixture(t, writeStructuredOutputScriptorium)
outPath := fixture.path("today.md")
debugDir := fixture.path("prompt-debug")
runner := testRunner()
output, err := runTestCommand(t, runner,
@@ -711,6 +746,7 @@ func TestRunGenerateQuietSuppressesSuccessfulOutput(t *testing.T) {
"--config", fixture.configPath,
"--date", "2026-05-29",
"--out", outPath,
"--llm-debug-dir", debugDir,
"--quiet",
)
if err != nil {
@@ -720,6 +756,7 @@ func TestRunGenerateQuietSuppressesSuccessfulOutput(t *testing.T) {
t.Fatalf("stdout/stderr = %q/%q, want quiet success output", output.stdout, output.stderr)
}
assertFileContains(t, outPath, "# Today's Weather")
_ = oneArtifact(t, debugDir, "today", "2026-05-29", "*", "preparation.json")
}
func TestRunGeneratePreRunErrorEmitsNoJSON(t *testing.T) {