eric/weatherreporter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

Add Promptkit debug capture for generated reports

Browse Source
This commit is contained in:
Eric Rakestraw
2026-07-31 04:48:16 +00:00
parent 06b26d5e88
commit a6d11c01e8
11 changed files with 271 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
internal/app/app.go
View File

@@ -44,15 +44,16 @@ const (
)
type GenerateRequest struct {
Config config.Config
Report ReportKind
OutputPath string
Now time.Time
Date time.Time
Collector Collector
Notifier Notifier
Executor promptexec.Executor
Store state.Store
Config config.Config
Report ReportKind
OutputPath string
LLMDebugDir string
Now time.Time
Date time.Time
Collector Collector
Notifier Notifier
Executor promptexec.Executor
Store state.Store
}
type BatchRequest struct {
@@ -279,6 +280,10 @@ func GenerateDetailed(ctx context.Context, req GenerateRequest) (*ReportResult,
if err != nil {
return nil, err
}
debugWriter, err := state.NewPromptDebugWriter(req.LLMDebugDir)
if err != nil {
return nil, promptexec.NewError(promptexec.InvalidConfiguration, "initialize prompt debug", err)
}
inspection, err := InspectPromptExecution(ctx, PromptInspectionRequest{
Resolved: resolved,
Executor: req.Executor,
@@ -296,6 +301,7 @@ func GenerateDetailed(ctx context.Context, req GenerateRequest) (*ReportResult,
Resolved: resolved,
Collection: *collection,
Inspection: inspection,
DebugWriter: debugWriter,
})
}

130
internal/app/app_test.go
View File

@@ -77,6 +77,10 @@ type promptExecutorTest struct {
err error
afterPreparationErr error
validation promptexec.ValidationStatus
preparationDebug *promptexec.PreparationDebug
executionDebug *promptexec.ExecutionDebug
captureDebug *bool
providerCalled *bool
}
func (e promptExecutorTest) InspectPrompt(_ context.Context, id string, version string) (promptexec.PromptInspection, error) {
@@ -93,6 +97,9 @@ func (e promptExecutorTest) InspectProfile(_ context.Context, id string) (prompt
}
func (e promptExecutorTest) Execute(_ context.Context, request promptexec.ExecuteRequest, callback promptexec.PreparationCallback) (*promptexec.Execution, error) {
if e.captureDebug != nil {
*e.captureDebug = request.CaptureDebug
}
if e.err != nil {
return nil, e.err
}
@@ -101,9 +108,12 @@ func (e promptExecutorTest) Execute(_ context.Context, request promptexec.Execut
PromptID: request.PromptID, PromptVersion: request.PromptVersion, PromptHash: "prompt-hash", RenderedPromptHash: "rendered-hash",
ProfileID: request.ProfileID, BackendID: "test", ModelName: "test-model", DataPackagePath: request.DataPackagePath,
StartedAt: now, EndedAt: now,
}, nil); err != nil {
}, e.preparationDebug); err != nil {
return nil, err
}
if e.providerCalled != nil {
*e.providerCalled = true
}
if e.afterPreparationErr != nil {
return nil, e.afterPreparationErr
}
@@ -119,7 +129,7 @@ func (e promptExecutorTest) Execute(_ context.Context, request promptexec.Execut
RunID: "provider-run", PromptID: request.PromptID, PromptVersion: request.PromptVersion, PromptHash: "prompt-hash", RenderedPromptHash: "rendered-hash",
ProfileID: request.ProfileID, BackendID: "test", ModelName: "test-model", GeneratedHash: "generated-hash",
StartedAt: now, EndedAt: now, DataPackagePath: request.DataPackagePath, RawOutput: raw,
Validation: promptexec.NewValidation(validation, "json_schema", "generated_text.schema.json", nil),
Validation: promptexec.NewValidation(validation, "json_schema", "generated_text.schema.json", nil), Debug: e.executionDebug,
}, nil
}
@@ -317,6 +327,22 @@ func TestGenerateDetailedInspectsBeforeCollectionOrArtifactWrites(t *testing.T)
}
}
func TestGenerateDetailedRejectsInvalidDebugRootBeforeCollection(t *testing.T) {
cfg := config.Defaults()
cfg.Workspace.Root = t.TempDir()
collector := &recordingCollector{err: errors.New("collector must not run")}
_, err := GenerateDetailed(context.Background(), GenerateRequest{
Config: cfg, Report: ReportDaily, Date: mustParse("2026-05-29T12:00:00-05:00"),
Now: mustParse("2026-05-29T05:00:00-05:00"), Collector: collector, LLMDebugDir: "relative-debug",
})
if err == nil || promptexec.CategoryOf(err) != promptexec.InvalidConfiguration || strings.Contains(err.Error(), "relative-debug") {
t.Fatalf("GenerateDetailed() error/category = %v/%q, want safe debug-root validation", err, promptexec.CategoryOf(err))
}
if len(collector.requests) != 0 {
t.Fatalf("collector requests = %d, want debug initialization before collection", len(collector.requests))
}
}
func TestGenerateDetailedPersistsPromptFailureReceiptsWithoutRawOutput(t *testing.T) {
server := dailyBundleServer(t)
cfg := dailyWorkspaceConfig(t, server)
@@ -364,6 +390,106 @@ func TestGenerateDetailedPersistsRawOutputForValidationRejection(t *testing.T) {
}
}
func TestGenerateDetailedWritesRequestedPromptDebugOutsideWorkspace(t *testing.T) {
server := dailyBundleServer(t)
cfg := dailyWorkspaceConfig(t, server)
collection := collectionForTest(t, cfg)
debugRoot := filepath.Join(t.TempDir(), "prompt-debug")
captureDebug := false
result, err := GenerateDetailed(context.Background(), GenerateRequest{
Config: cfg, Report: ReportDaily, Date: mustParse("2026-05-29T12:00:00-05:00"),
Now: mustParse("2026-05-29T05:00:00-05:00"), Collector: &recordingCollector{result: &collection}, LLMDebugDir: debugRoot,
Executor: promptExecutorTest{
captureDebug: &captureDebug,
preparationDebug: &promptexec.PreparationDebug{
RenderedMessages: []promptexec.RenderedMessage{{Role: "system", Content: "sensitive rendered prompt"}},
ParametersJSON: []byte(`{"api_key":"secret-value"}`),
},
executionDebug: &promptexec.ExecutionDebug{ValidationDiagnostics: []string{"provider validation detail"}},
},
})
if err != nil {
t.Fatalf("GenerateDetailed() error = %v", err)
}
if !captureDebug || result.LLMDebugPath == "" || !strings.HasPrefix(result.LLMDebugPath, debugRoot+string(filepath.Separator)) {
t.Fatalf("capture/debug path = %t/%q, want requested isolated debug capture", captureDebug, result.LLMDebugPath)
}
preparationData, readErr := os.ReadFile(filepath.Join(result.LLMDebugPath, "preparation.json"))
if readErr != nil {
t.Fatalf("read preparation debug: %v", readErr)
}
executionData, readErr := os.ReadFile(filepath.Join(result.LLMDebugPath, "execution.json"))
if readErr != nil {
t.Fatalf("read execution debug: %v", readErr)
}
if !strings.Contains(string(preparationData), "sensitive rendered prompt") || strings.Contains(string(preparationData), "secret-value") || !strings.Contains(string(executionData), "provider validation detail") {
t.Fatalf("debug artifacts did not retain/redact expected content:\n%s\n%s", preparationData, executionData)
}
metadataData, readErr := os.ReadFile(result.MetadataPath)
if readErr != nil {
t.Fatalf("read metadata: %v", readErr)
}
if strings.Contains(string(metadataData), "sensitive rendered prompt") || strings.Contains(string(metadataData), "provider validation detail") {
t.Fatalf("normal metadata contains debug content:\n%s", metadataData)
}
}
func TestGenerateDetailedDebugWriteFailureStopsProviderExecution(t *testing.T) {
server := dailyBundleServer(t)
cfg := dailyWorkspaceConfig(t, server)
collection := collectionForTest(t, cfg)
now := mustParse("2026-05-29T05:00:00-05:00")
request := GenerateRequest{Config: cfg, Report: ReportDaily, Date: mustParse("2026-05-29T12:00:00-05:00"), Now: now}
resolved := resolveGenerateForTest(t, cfg, request, now.Format(time.RFC3339))
debugRoot := filepath.Join(t.TempDir(), "prompt-debug")
debugPath := filepath.Join(debugRoot, string(resolved.Definition.ID), resolved.ValidPeriod.Start.Format("2006-01-02"), resolved.Metadata().RunID)
if err := os.MkdirAll(filepath.Dir(debugPath), 0o700); err != nil {
t.Fatalf("create debug parent: %v", err)
}
if err := os.WriteFile(debugPath, []byte("not a directory"), 0o600); err != nil {
t.Fatalf("create debug collision: %v", err)
}
providerCalled := false
request.Collector = &recordingCollector{result: &collection}
request.LLMDebugDir = debugRoot
request.Executor = promptExecutorTest{providerCalled: &providerCalled}
result, err := GenerateDetailed(context.Background(), request)
if err == nil || result == nil || promptexec.CategoryOf(err) != promptexec.InvalidConfiguration {
t.Fatalf("GenerateDetailed() result/error = %#v/%v, want partial result and debug error", result, err)
}
if providerCalled || result.Metadata.RunID == "" || result.PreparationPath == "" || result.ExecutionPath != "" || result.LLMDebugPath != "" {
t.Fatalf("provider/run/preparation/execution/debug = %t/%q/%q/%q/%q, want preparation only before provider", providerCalled, result.Metadata.RunID, result.PreparationPath, result.ExecutionPath, result.LLMDebugPath)
}
}
func TestGenerateDetailedExecutionDebugFailureRetainsPreparationCapture(t *testing.T) {
server := dailyBundleServer(t)
cfg := dailyWorkspaceConfig(t, server)
collection := collectionForTest(t, cfg)
now := mustParse("2026-05-29T05:00:00-05:00")
request := GenerateRequest{Config: cfg, Report: ReportDaily, Date: mustParse("2026-05-29T12:00:00-05:00"), Now: now}
resolved := resolveGenerateForTest(t, cfg, request, now.Format(time.RFC3339))
debugRoot := filepath.Join(t.TempDir(), "prompt-debug")
executionDebugPath := filepath.Join(debugRoot, string(resolved.Definition.ID), resolved.ValidPeriod.Start.Format("2006-01-02"), resolved.Metadata().RunID, "execution.json")
if err := os.MkdirAll(executionDebugPath, 0o700); err != nil {
t.Fatalf("create execution debug collision: %v", err)
}
providerCalled := false
request.Collector = &recordingCollector{result: &collection}
request.LLMDebugDir = debugRoot
request.Executor = promptExecutorTest{providerCalled: &providerCalled}
result, err := GenerateDetailed(context.Background(), request)
if err == nil || result == nil || promptexec.CategoryOf(err) != promptexec.InvalidConfiguration {
t.Fatalf("GenerateDetailed() result/error = %#v/%v, want partial result and debug error", result, err)
}
if !providerCalled || result.LLMDebugPath == "" || result.ExecutionPath != "" || result.GeneratedTextRawPath != "" {
t.Fatalf("provider/debug/execution/raw = %t/%q/%q/%q, want preparation debug only after completed execution", providerCalled, result.LLMDebugPath, result.ExecutionPath, result.GeneratedTextRawPath)
}
if _, statErr := os.Stat(filepath.Join(result.LLMDebugPath, "preparation.json")); statErr != nil {
t.Fatalf("preparation debug artifact: %v", statErr)
}
}
type failingPromptInspectionExecutor struct{}
func (failingPromptInspectionExecutor) InspectPrompt(context.Context, string, string) (promptexec.PromptInspection, error) {

38
internal/app/prompt_generate.go
View File

@@ -17,9 +17,10 @@ import (
type promptReportRequest struct {
GenerateRequest
Resolved report.Resolved
Collection collect.Result
Inspection PromptInspectionResult
Resolved report.Resolved
Collection collect.Result
Inspection PromptInspectionResult
DebugWriter *state.PromptDebugWriter
}
func generatePromptReport(ctx context.Context, req promptReportRequest) (*ReportResult, error) {
@@ -78,6 +79,7 @@ func generatePromptReport(ctx context.Context, req promptReportRequest) (*Report
GeneratedText: paths.GeneratedText,
RenderContext: paths.RenderContext,
})
result.Metadata = metadata
dataPackage, err := promptinput.Build(promptinput.BuildRequest{
Metadata: promptMetadata(metadata),
Modules: moduleSnapshot,
@@ -97,6 +99,7 @@ func generatePromptReport(ctx context.Context, req promptReportRequest) (*Report
metadata.DataPackagePath = dataPackagePath
result.DataPackage = dataPackage
result.DataPackagePath = dataPackagePath
result.Metadata = metadata
handler, err := generatedtext.LookupDefinition(req.Resolved.Definition)
if err != nil {
@@ -105,7 +108,12 @@ func generatePromptReport(ctx context.Context, req promptReportRequest) (*Report
prepared := false
callbackFailed := false
callback := func(preparation promptexec.Preparation, _ *promptexec.PreparationDebug) error {
debugRef := state.PromptDebugRef{
ReportID: req.Resolved.Definition.ID,
ValidDate: req.Resolved.ValidPeriod.Start.Format("2006-01-02"),
RunID: metadata.RunID,
}
callback := func(preparation promptexec.Preparation, debug *promptexec.PreparationDebug) error {
artifact := state.PromptPreparationArtifact{
SchemaVersion: state.PromptPreparationSchemaVersion,
Status: state.PromptPreparationSucceeded,
@@ -127,6 +135,15 @@ func generatePromptReport(ctx context.Context, req promptReportRequest) (*Report
prepared = true
result.PreparationPath = path
metadata.PreparationPath = path
result.Metadata = metadata
debugPath, err := req.DebugWriter.WritePreparation(debugRef, preparation, debug)
if err != nil {
callbackFailed = true
return promptDebugWriteError(err)
}
if debugPath != "" {
result.LLMDebugPath = debugPath
}
metadataPath, err := store.SaveMetadata(ctx, metadata)
if err != nil {
callbackFailed = true
@@ -142,7 +159,7 @@ func generatePromptReport(ctx context.Context, req promptReportRequest) (*Report
ProfileID: req.Inspection.ProfileID,
DataPackage: data,
DataPackagePath: dataPackagePath,
CaptureDebug: false,
CaptureDebug: req.DebugWriter.Enabled(),
}, callback)
if executeErr != nil {
if callbackFailed {
@@ -198,6 +215,13 @@ func generatePromptReport(ctx context.Context, req promptReportRequest) (*Report
result.ExecutionPath, result.Metadata, result.MetadataPath = executionPath, metadata, metadataPath
return result, generatedReportError(req.Resolved, metadata.RunID, "execute prompt", err)
}
debugPath, err := req.DebugWriter.WriteExecution(debugRef, *execution)
if err != nil {
return result, generatedReportError(req.Resolved, metadata.RunID, "write prompt debug", promptDebugWriteError(err))
}
if debugPath != "" {
result.LLMDebugPath = debugPath
}
if execution.Validation.Status != promptexec.ValidationPassed && execution.Validation.Status != promptexec.ValidationFailed {
err := promptexec.NewError(promptexec.OperationalValidation, "prompt execution did not complete validation", nil)
@@ -318,4 +342,8 @@ func classifiedPromptError(operation string, err error) error {
return promptexec.NewError(promptexec.Generation, operation, err)
}
func promptDebugWriteError(err error) error {
return promptexec.NewError(promptexec.InvalidConfiguration, "write requested prompt debug artifact", err)
}
func ptr[T any](value T) *T { return &value }

3
internal/cli/result_test.go
View File

@@ -22,6 +22,7 @@ func TestNewGenerateSummaryForGeneratedTextReport(t *testing.T) {
DataPackagePath: "/runs/hourly/data_package.yaml",
PreparationPath: "/runs/hourly/preparation.json",
ExecutionPath: "/runs/hourly/execution.json",
LLMDebugPath: "/operator-debug/hourly/2026-05-29/run-123",
ReportPath: "/runs/hourly/report.md",
OutputPath: "/copies/hourly.md",
MetadataPath: "/runs/hourly/metadata.json",
@@ -58,7 +59,7 @@ func TestNewGenerateSummaryForGeneratedTextReport(t *testing.T) {
if summary.ReportID != report.Hourly || summary.ReportName != "Hourly Report" || summary.PromptID != "weather.hourly_generated_text" || summary.RunID != "20260529T133000Z_hourly" {
t.Fatalf("summary identity = %#v, want hourly report identity", summary)
}
if summary.PreparationPath == "" || summary.ExecutionPath == "" || summary.GeneratedTextRawPath == "" || summary.GeneratedTextPath == "" || summary.RenderContextPath == "" {
if summary.PreparationPath == "" || summary.ExecutionPath == "" || summary.LLMDebugPath == "" || summary.GeneratedTextRawPath == "" || summary.GeneratedTextPath == "" || summary.RenderContextPath == "" {
t.Fatalf("generated-text paths = %#v, want generated-text artifact paths", summary)
}
if summary.Notification == nil || summary.Notification.RunID != "distributor-run" || summary.Notification.AcceptedAt == nil || !summary.Notification.AcceptedAt.Equal(acceptedAt) {

24
internal/cli/root.go
View File

@@ -16,10 +16,10 @@ const helpText = `weatherreporter prepares weather reports from normalized forec
Usage:
weatherreporter --help
weatherreporter generate daily --date YYYY-MM-DD [--config PATH] [--units VALUE] [--tz NAME] [--out PATH] [--quiet]
weatherreporter generate today [--config PATH] [--units VALUE] [--tz NAME] [--out PATH] [--date YYYY-MM-DD] [--quiet]
weatherreporter generate tomorrow [--config PATH] [--units VALUE] [--tz NAME] [--out PATH] [--quiet]
weatherreporter generate hourly [--config PATH] [--units VALUE] [--tz NAME] [--out PATH] [--quiet]
weatherreporter generate daily --date YYYY-MM-DD [--config PATH] [--units VALUE] [--tz NAME] [--out PATH] [--llm-debug-dir PATH] [--quiet]
weatherreporter generate today [--config PATH] [--units VALUE] [--tz NAME] [--out PATH] [--date YYYY-MM-DD] [--llm-debug-dir PATH] [--quiet]
weatherreporter generate tomorrow [--config PATH] [--units VALUE] [--tz NAME] [--out PATH] [--llm-debug-dir PATH] [--quiet]
weatherreporter generate hourly [--config PATH] [--units VALUE] [--tz NAME] [--out PATH] [--llm-debug-dir PATH] [--quiet]
weatherreporter run morning [--config PATH] [--units VALUE] [--tz NAME] [--out-dir PATH] [--quiet]
weatherreporter run evening [--config PATH] [--units VALUE] [--tz NAME] [--out-dir PATH] [--quiet]
weatherreporter inspect reports [--config PATH] [--limit N]
@@ -35,6 +35,7 @@ Options:
--units VALUE Override weather API units.
--tz NAME Override weather API timezone.
--out PATH Write an extra Markdown report copy where supported by the generate command.
--llm-debug-dir PATH Write sensitive prompt debug artifacts outside the managed workspace.
--out-dir PATH Write extra Markdown report copies for run commands.
--quiet Suppress successful generate and run output.
`
@@ -107,7 +108,8 @@ type commonOptions struct {
type generateOptions struct {
commonOptions
Date string
Date string
LLMDebugDir string
}
type inspectOptions struct {
@@ -224,11 +226,12 @@ func (r Runner) resolveGenerateAction(args []string) (app.GenerateRequest, commo
}
req := app.GenerateRequest{
Config: cfg,
Report: reportKind,
OutputPath: opts.Output,
Now: r.Clock.Now(),
Executor: executor,
Config: cfg,
Report: reportKind,
OutputPath: opts.Output,
LLMDebugDir: opts.LLMDebugDir,
Now: r.Clock.Now(),
Executor: executor,
}
switch reportKind {
@@ -295,6 +298,7 @@ func parseGenerateFlags(report app.ReportKind, args []string) (generateOptions,
opts := generateOptions{}
addCommonFlags(fs, &opts.commonOptions, true)
fs.BoolVar(&opts.Quiet, "quiet", false, "suppress successful action output")
fs.StringVar(&opts.LLMDebugDir, "llm-debug-dir", "", "write sensitive prompt debug artifacts under PATH")
if report == app.ReportDaily || report == app.ReportToday {
fs.StringVar(&opts.Date, "date", "", "report date in YYYY-MM-DD")
}

37
internal/cli/root_test.go
View File

@@ -73,6 +73,9 @@ func TestRunHelpLongFlag(t *testing.T) {
if !strings.Contains(output.stdout, "weatherreporter generate hourly") {
t.Fatalf("help output missing hourly generate command:\n%s", output.stdout)
}
if strings.Count(output.stdout, "--llm-debug-dir PATH") != 5 {
t.Fatalf("help output = %q, want debug flag for four generate commands and its option", output.stdout)
}
if !strings.Contains(output.stdout, "generate today") || !strings.Contains(output.stdout, "[--quiet]") {
t.Fatalf("help output missing quiet generate usage:\n%s", output.stdout)
}
@@ -134,6 +137,37 @@ func TestRunGenerateTomorrowWritesMarkdownReport(t *testing.T) {
}
}
func TestRunGenerateWritesRequestedDebugAndSafeSummary(t *testing.T) {
fixture := newCLIFixture(t, writeFakeScriptorium)
debugDir := fixture.path("prompt-debug")
output, err := runTestCommand(t, testRunner(),
"generate", "today", "--config", fixture.configPath, "--llm-debug-dir", debugDir,
)
if err != nil {
t.Fatalf("Run() error = %v", err)
}
summary := decodeGenerateSummary(t, output.stdout)
if summary.LLMDebugPath == "" || !strings.HasPrefix(summary.LLMDebugPath, debugDir+string(filepath.Separator)) {
t.Fatalf("debug path = %q, want isolated requested directory", summary.LLMDebugPath)
}
if strings.Contains(output.stdout, "Showers are possible during the selected day") || strings.Contains(output.stdout, "Today starts with showers before improving") || strings.Contains(output.stdout, "rendered-hash") {
t.Fatalf("summary contains prompt or generated content:\n%s", output.stdout)
}
assertFileContains(t, filepath.Join(summary.LLMDebugPath, "preparation.json"), "weather.today_generated_text")
assertFileContains(t, filepath.Join(summary.LLMDebugPath, "execution.json"), "Today starts with showers before improving.")
}
func TestParseGenerateFlagsAcceptsDebugDirectoryForEveryReport(t *testing.T) {
for _, reportKind := range []app.ReportKind{app.ReportDaily, app.ReportToday, app.ReportTomorrow, app.ReportHourly} {
t.Run(string(reportKind), func(t *testing.T) {
opts, err := parseGenerateFlags(reportKind, []string{"--llm-debug-dir", "/tmp/prompt-debug"})
if err != nil || opts.LLMDebugDir != "/tmp/prompt-debug" {
t.Fatalf("parseGenerateFlags() options/error = %#v/%v", opts, err)
}
})
}
}
func TestRunEveningGeneratesTomorrowReport(t *testing.T) {
fixture := newCLIFixture(t, writeFakeScriptorium)
runner := testRunner()
@@ -704,6 +738,7 @@ func TestRunGenerateHourlyWritesGeneratedTextReport(t *testing.T) {
func TestRunGenerateQuietSuppressesSuccessfulOutput(t *testing.T) {
fixture := newCLIFixture(t, writeStructuredOutputScriptorium)
outPath := fixture.path("today.md")
debugDir := fixture.path("prompt-debug")
runner := testRunner()
output, err := runTestCommand(t, runner,
@@ -711,6 +746,7 @@ func TestRunGenerateQuietSuppressesSuccessfulOutput(t *testing.T) {
"--config", fixture.configPath,
"--date", "2026-05-29",
"--out", outPath,
"--llm-debug-dir", debugDir,
"--quiet",
)
if err != nil {
@@ -720,6 +756,7 @@ func TestRunGenerateQuietSuppressesSuccessfulOutput(t *testing.T) {
t.Fatalf("stdout/stderr = %q/%q, want quiet success output", output.stdout, output.stderr)
}
assertFileContains(t, outPath, "# Today's Weather")
_ = oneArtifact(t, debugDir, "today", "2026-05-29", "*", "preparation.json")
}
func TestRunGeneratePreRunErrorEmitsNoJSON(t *testing.T) {