eric/weatherreporter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

Capture provider failures in secure debug artifacts

Browse Source
This commit is contained in:
Eric Rakestraw
2026-08-25 19:55:09 +00:00
parent b3b23fb381
commit 4cd5f505df
7 changed files with 91 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
internal/promptdebug/debug_writer_test.go
View File

@@ -44,13 +44,13 @@ func TestPromptDebugWriterWritesIsolatedArtifacts(t *testing.T) {
t.Fatalf("WriteExecution() directory = %q, want %q", executionDir, preparationDir)
}
preparationData := readPromptDebugFile(t, filepath.Join(preparationDir, "preparation.json"))
for _, want := range []string{"weatherreporter.prompt_preparation_debug.v2", "Use the supplied weather facts.", `"type": "object"`, "https://llm.example.test", `"temperature": 0.2`} {
for _, want := range []string{"weatherreporter.prompt_preparation_debug.v3", "Use the supplied weather facts.", `"type": "object"`, "https://llm.example.test", `"temperature": 0.2`, `"repairAttempts": 0`} {
if !strings.Contains(string(preparationData), want) {
t.Fatalf("preparation debug artifact missing %q:\n%s", want, preparationData)
}
}
executionData := readPromptDebugFile(t, filepath.Join(executionDir, "execution.json"))
for _, want := range []string{"weatherreporter.prompt_execution_debug.v2", "Generated forecast prose.", "validation details", `"status": "passed"`} {
for _, want := range []string{"weatherreporter.prompt_execution_debug.v3", "Generated forecast prose.", "validation details", `"status": "passed"`, `"repairAttempts": 0`} {
if !strings.Contains(string(executionData), want) {
t.Fatalf("execution debug artifact missing %q:\n%s", want, executionData)
}
@@ -66,6 +66,25 @@ func TestPromptDebugWriterWritesIsolatedArtifacts(t *testing.T) {
assertPromptDebugMode(t, filepath.Join(executionDir, "execution.json"), debugFileMode)
}
func TestPromptDebugWriterWritesProviderFailureOnlyToDebugStore(t *testing.T) {
writer, err := NewPromptDebugWriter(filepath.Join(t.TempDir(), "operator-debug"))
if err != nil {
t.Fatalf("NewPromptDebugWriter() error = %v", err)
}
marker := "provider-private-marker"
directory, err := writer.WriteFailure(promptDebugRef(), promptexec.NewGenerationError(429, "rate_limit", "provider_error", marker, nil))
if err != nil {
t.Fatalf("WriteFailure() error = %v", err)
}
data := readPromptDebugFile(t, filepath.Join(directory, "failure.json"))
for _, want := range []string{"weatherreporter.prompt_failure_debug.v1", `"category": "generation"`, `"statusCode": 429`, marker} {
if !strings.Contains(string(data), want) {
t.Fatalf("failure artifact missing %q: %s", want, data)
}
}
assertPromptDebugMode(t, filepath.Join(directory, "failure.json"), debugFileMode)
}
func TestPromptDebugWriterProjectsProviderConfigurationSafely(t *testing.T) {
const marker = "private-debug-marker"
tests := []struct {