eric/weatherreporter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

Add Promptkit debug capture for generated reports

Browse Source
This commit is contained in:
Eric Rakestraw
2026-07-31 04:48:16 +00:00
parent 06b26d5e88
commit a6d11c01e8
11 changed files with 271 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files

130
internal/app/app_test.go
View File

@@ -77,6 +77,10 @@ type promptExecutorTest struct {
err error
afterPreparationErr error
validation promptexec.ValidationStatus
preparationDebug *promptexec.PreparationDebug
executionDebug *promptexec.ExecutionDebug
captureDebug *bool
providerCalled *bool
}
func (e promptExecutorTest) InspectPrompt(_ context.Context, id string, version string) (promptexec.PromptInspection, error) {
@@ -93,6 +97,9 @@ func (e promptExecutorTest) InspectProfile(_ context.Context, id string) (prompt
}
func (e promptExecutorTest) Execute(_ context.Context, request promptexec.ExecuteRequest, callback promptexec.PreparationCallback) (*promptexec.Execution, error) {
if e.captureDebug != nil {
*e.captureDebug = request.CaptureDebug
}
if e.err != nil {
return nil, e.err
}
@@ -101,9 +108,12 @@ func (e promptExecutorTest) Execute(_ context.Context, request promptexec.Execut
PromptID: request.PromptID, PromptVersion: request.PromptVersion, PromptHash: "prompt-hash", RenderedPromptHash: "rendered-hash",
ProfileID: request.ProfileID, BackendID: "test", ModelName: "test-model", DataPackagePath: request.DataPackagePath,
StartedAt: now, EndedAt: now,
}, nil); err != nil {
}, e.preparationDebug); err != nil {
return nil, err
}
if e.providerCalled != nil {
*e.providerCalled = true
}
if e.afterPreparationErr != nil {
return nil, e.afterPreparationErr
}
@@ -119,7 +129,7 @@ func (e promptExecutorTest) Execute(_ context.Context, request promptexec.Execut
RunID: "provider-run", PromptID: request.PromptID, PromptVersion: request.PromptVersion, PromptHash: "prompt-hash", RenderedPromptHash: "rendered-hash",
ProfileID: request.ProfileID, BackendID: "test", ModelName: "test-model", GeneratedHash: "generated-hash",
StartedAt: now, EndedAt: now, DataPackagePath: request.DataPackagePath, RawOutput: raw,
Validation: promptexec.NewValidation(validation, "json_schema", "generated_text.schema.json", nil),
Validation: promptexec.NewValidation(validation, "json_schema", "generated_text.schema.json", nil), Debug: e.executionDebug,
}, nil
}
@@ -317,6 +327,22 @@ func TestGenerateDetailedInspectsBeforeCollectionOrArtifactWrites(t *testing.T)
}
}
func TestGenerateDetailedRejectsInvalidDebugRootBeforeCollection(t *testing.T) {
cfg := config.Defaults()
cfg.Workspace.Root = t.TempDir()
collector := &recordingCollector{err: errors.New("collector must not run")}
_, err := GenerateDetailed(context.Background(), GenerateRequest{
Config: cfg, Report: ReportDaily, Date: mustParse("2026-05-29T12:00:00-05:00"),
Now: mustParse("2026-05-29T05:00:00-05:00"), Collector: collector, LLMDebugDir: "relative-debug",
})
if err == nil || promptexec.CategoryOf(err) != promptexec.InvalidConfiguration || strings.Contains(err.Error(), "relative-debug") {
t.Fatalf("GenerateDetailed() error/category = %v/%q, want safe debug-root validation", err, promptexec.CategoryOf(err))
}
if len(collector.requests) != 0 {
t.Fatalf("collector requests = %d, want debug initialization before collection", len(collector.requests))
}
}
func TestGenerateDetailedPersistsPromptFailureReceiptsWithoutRawOutput(t *testing.T) {
server := dailyBundleServer(t)
cfg := dailyWorkspaceConfig(t, server)
@@ -364,6 +390,106 @@ func TestGenerateDetailedPersistsRawOutputForValidationRejection(t *testing.T) {
}
}
func TestGenerateDetailedWritesRequestedPromptDebugOutsideWorkspace(t *testing.T) {
server := dailyBundleServer(t)
cfg := dailyWorkspaceConfig(t, server)
collection := collectionForTest(t, cfg)
debugRoot := filepath.Join(t.TempDir(), "prompt-debug")
captureDebug := false
result, err := GenerateDetailed(context.Background(), GenerateRequest{
Config: cfg, Report: ReportDaily, Date: mustParse("2026-05-29T12:00:00-05:00"),
Now: mustParse("2026-05-29T05:00:00-05:00"), Collector: &recordingCollector{result: &collection}, LLMDebugDir: debugRoot,
Executor: promptExecutorTest{
captureDebug: &captureDebug,
preparationDebug: &promptexec.PreparationDebug{
RenderedMessages: []promptexec.RenderedMessage{{Role: "system", Content: "sensitive rendered prompt"}},
ParametersJSON: []byte(`{"api_key":"secret-value"}`),
},
executionDebug: &promptexec.ExecutionDebug{ValidationDiagnostics: []string{"provider validation detail"}},
},
})
if err != nil {
t.Fatalf("GenerateDetailed() error = %v", err)
}
if !captureDebug || result.LLMDebugPath == "" || !strings.HasPrefix(result.LLMDebugPath, debugRoot+string(filepath.Separator)) {
t.Fatalf("capture/debug path = %t/%q, want requested isolated debug capture", captureDebug, result.LLMDebugPath)
}
preparationData, readErr := os.ReadFile(filepath.Join(result.LLMDebugPath, "preparation.json"))
if readErr != nil {
t.Fatalf("read preparation debug: %v", readErr)
}
executionData, readErr := os.ReadFile(filepath.Join(result.LLMDebugPath, "execution.json"))
if readErr != nil {
t.Fatalf("read execution debug: %v", readErr)
}
if !strings.Contains(string(preparationData), "sensitive rendered prompt") || strings.Contains(string(preparationData), "secret-value") || !strings.Contains(string(executionData), "provider validation detail") {
t.Fatalf("debug artifacts did not retain/redact expected content:\n%s\n%s", preparationData, executionData)
}
metadataData, readErr := os.ReadFile(result.MetadataPath)
if readErr != nil {
t.Fatalf("read metadata: %v", readErr)
}
if strings.Contains(string(metadataData), "sensitive rendered prompt") || strings.Contains(string(metadataData), "provider validation detail") {
t.Fatalf("normal metadata contains debug content:\n%s", metadataData)
}
}
func TestGenerateDetailedDebugWriteFailureStopsProviderExecution(t *testing.T) {
server := dailyBundleServer(t)
cfg := dailyWorkspaceConfig(t, server)
collection := collectionForTest(t, cfg)
now := mustParse("2026-05-29T05:00:00-05:00")
request := GenerateRequest{Config: cfg, Report: ReportDaily, Date: mustParse("2026-05-29T12:00:00-05:00"), Now: now}
resolved := resolveGenerateForTest(t, cfg, request, now.Format(time.RFC3339))
debugRoot := filepath.Join(t.TempDir(), "prompt-debug")
debugPath := filepath.Join(debugRoot, string(resolved.Definition.ID), resolved.ValidPeriod.Start.Format("2006-01-02"), resolved.Metadata().RunID)
if err := os.MkdirAll(filepath.Dir(debugPath), 0o700); err != nil {
t.Fatalf("create debug parent: %v", err)
}
if err := os.WriteFile(debugPath, []byte("not a directory"), 0o600); err != nil {
t.Fatalf("create debug collision: %v", err)
}
providerCalled := false
request.Collector = &recordingCollector{result: &collection}
request.LLMDebugDir = debugRoot
request.Executor = promptExecutorTest{providerCalled: &providerCalled}
result, err := GenerateDetailed(context.Background(), request)
if err == nil || result == nil || promptexec.CategoryOf(err) != promptexec.InvalidConfiguration {
t.Fatalf("GenerateDetailed() result/error = %#v/%v, want partial result and debug error", result, err)
}
if providerCalled || result.Metadata.RunID == "" || result.PreparationPath == "" || result.ExecutionPath != "" || result.LLMDebugPath != "" {
t.Fatalf("provider/run/preparation/execution/debug = %t/%q/%q/%q/%q, want preparation only before provider", providerCalled, result.Metadata.RunID, result.PreparationPath, result.ExecutionPath, result.LLMDebugPath)
}
}
func TestGenerateDetailedExecutionDebugFailureRetainsPreparationCapture(t *testing.T) {
server := dailyBundleServer(t)
cfg := dailyWorkspaceConfig(t, server)
collection := collectionForTest(t, cfg)
now := mustParse("2026-05-29T05:00:00-05:00")
request := GenerateRequest{Config: cfg, Report: ReportDaily, Date: mustParse("2026-05-29T12:00:00-05:00"), Now: now}
resolved := resolveGenerateForTest(t, cfg, request, now.Format(time.RFC3339))
debugRoot := filepath.Join(t.TempDir(), "prompt-debug")
executionDebugPath := filepath.Join(debugRoot, string(resolved.Definition.ID), resolved.ValidPeriod.Start.Format("2006-01-02"), resolved.Metadata().RunID, "execution.json")
if err := os.MkdirAll(executionDebugPath, 0o700); err != nil {
t.Fatalf("create execution debug collision: %v", err)
}
providerCalled := false
request.Collector = &recordingCollector{result: &collection}
request.LLMDebugDir = debugRoot
request.Executor = promptExecutorTest{providerCalled: &providerCalled}
result, err := GenerateDetailed(context.Background(), request)
if err == nil || result == nil || promptexec.CategoryOf(err) != promptexec.InvalidConfiguration {
t.Fatalf("GenerateDetailed() result/error = %#v/%v, want partial result and debug error", result, err)
}
if !providerCalled || result.LLMDebugPath == "" || result.ExecutionPath != "" || result.GeneratedTextRawPath != "" {
t.Fatalf("provider/debug/execution/raw = %t/%q/%q/%q, want preparation debug only after completed execution", providerCalled, result.LLMDebugPath, result.ExecutionPath, result.GeneratedTextRawPath)
}
if _, statErr := os.Stat(filepath.Join(result.LLMDebugPath, "preparation.json")); statErr != nil {
t.Fatalf("preparation debug artifact: %v", statErr)
}
}
type failingPromptInspectionExecutor struct{}
func (failingPromptInspectionExecutor) InspectPrompt(context.Context, string, string) (promptexec.PromptInspection, error) {