From 4cd5f505df950b06e73c90b3ad55bab7bc89ca52 Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Tue, 25 Aug 2026 19:55:09 +0000 Subject: [PATCH] Capture provider failures in secure debug artifacts --- docs/integrations/promptkit.md | 2 +- docs/operations.md | 6 +++ docs/policy/architecture.md | 2 + docs/roadmap/implementation.md | 2 + internal/app/profile_execution.go | 13 ++++++ internal/promptdebug/debug_writer.go | 54 +++++++++++++++++++---- internal/promptdebug/debug_writer_test.go | 23 +++++++++- 7 files changed, 91 insertions(+), 11 deletions(-) diff --git a/docs/integrations/promptkit.md b/docs/integrations/promptkit.md index 035cfe3..819d339 100644 --- a/docs/integrations/promptkit.md +++ b/docs/integrations/promptkit.md @@ -42,7 +42,7 @@ logical profile ID and resolved backend and model. Ordinary errors, summaries, logs, and outputs exclude endpoints, credentials, rendered messages, schemas, request bodies, response bodies, and complete parameter maps. -Promptkit receives the YAML data package as an inline input and returns structured JSON that Weatherreporter validates before rendering its own Markdown template. Before accepting that JSON, Weatherreporter requires exactly one preparation callback and reconciles its prompt/profile/backend/model and rendered/input hashes with the inspected identity and completed result. The callback output contract and completed validation must use the report's expected JSON Schema mode and path. The package contains only reviewed prompt-facing warning summaries, never source transport or provenance details. Safe active provenance remains in memory. Content-rich diagnostics are opt-in through `--llm-debug-dir`; see [operations](../operations.md) for retention and permissions. +Promptkit receives the YAML data package as an inline input and returns structured JSON that Weatherreporter validates before rendering its own Markdown template. Before accepting that JSON, Weatherreporter requires exactly one preparation callback and reconciles its prompt/profile/backend/model and rendered/input hashes with the inspected identity and completed result. The callback output contract and completed validation must use the report's expected JSON Schema mode and path. The package contains only reviewed prompt-facing warning summaries, never source transport or provenance details. Safe active provenance remains in memory. Content-rich diagnostics are opt-in through `--llm-debug-dir`; see [operations](../operations.md) for retention and permissions. Ordinary generation errors disclose only the safe Weatherreporter category and optional HTTP status; provider code, type, and message are written only to the explicit secure failure-debug artifact. Each embedded prompt permits one Promptkit-owned corrective generation after an eligible failed or explicitly empty result. This is not an application retry: diff --git a/docs/operations.md b/docs/operations.md index 890bb0a..2e0b423 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -178,6 +178,12 @@ Preparation captures retain only the provider endpoint origin and reviewed execution settings. URL user information, paths, queries, fragments, and unrecognized provider parameters are omitted. +Each run directory may contain `preparation.json` (v3), `execution.json` (v3), +and, for a provider generation failure, `failure.json` (v1). The failure +artifact retains the safe category, HTTP status, and provider code, type, and +message for trusted debugging only. Ordinary command output never includes +those provider details. + Capture writes are confined to the requested root and fail if an unsafe filesystem component prevents secure artifact creation. diff --git a/docs/policy/architecture.md b/docs/policy/architecture.md index bdc8190..b7a3339 100644 --- a/docs/policy/architecture.md +++ b/docs/policy/architecture.md @@ -66,6 +66,8 @@ directly. - Sensitive rendered prompts, schemas, input bodies, provider endpoints, and credentials never enter normal summaries or logs. They are written only to an explicit secure debug root when requested. +- Provider-controlled diagnostics never enter ordinary outputs; they are + retained only in explicit secure failure-debug artifacts. ## Output, Notification, And Testing Invariants diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index be0e112..b7e2b75 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -439,6 +439,8 @@ GOWORK=off go test -race -count=1 ./internal/comparison ./internal/app ### Stage 8: Add Secure Provider-Failure Debug Capture +Status: Complete. + Purpose: expose useful PromptKit v0.7.0 provider diagnostics only through the existing explicit secure debug boundary while keeping ordinary errors safe. diff --git a/internal/app/profile_execution.go b/internal/app/profile_execution.go index 84cf272..1e9ad75 100644 --- a/internal/app/profile_execution.go +++ b/internal/app/profile_execution.go @@ -2,6 +2,7 @@ package app import ( "context" + "errors" "fmt" "reflect" @@ -95,6 +96,18 @@ func executePreparedProfile(ctx context.Context, req profileExecutionRequest) (p if callbackFailed { return outcome, nil, &profileExecutionError{operation: "execute prompt", err: err, callbackFailure: true} } + if req.DebugWriter != nil && req.DebugWriter.Enabled() && req.DebugRef != nil { + var generationError *promptexec.GenerationError + if errors.As(err, &generationError) { + path, debugErr := req.DebugWriter.WriteFailure(*req.DebugRef, generationError) + if path != "" { + outcome.LLMDebugPath = path + } + if debugErr != nil { + err = errors.Join(err, promptDebugWriteError(debugErr)) + } + } + } return outcome, nil, &profileExecutionError{operation: "execute prompt", err: classifiedPromptError("prompt execution failed", err)} } if execution == nil { diff --git a/internal/promptdebug/debug_writer.go b/internal/promptdebug/debug_writer.go index 94210ed..5094c13 100644 --- a/internal/promptdebug/debug_writer.go +++ b/internal/promptdebug/debug_writer.go @@ -21,8 +21,9 @@ import ( var ErrSecureCaptureUnsupported = errors.New("secure prompt debug capture is unavailable on this platform") const ( - promptPreparationDebugSchemaVersion = "weatherreporter.prompt_preparation_debug.v2" - promptExecutionDebugSchemaVersion = "weatherreporter.prompt_execution_debug.v2" + promptPreparationDebugSchemaVersion = "weatherreporter.prompt_preparation_debug.v3" + promptExecutionDebugSchemaVersion = "weatherreporter.prompt_execution_debug.v3" + promptFailureDebugSchemaVersion = "weatherreporter.prompt_failure_debug.v1" debugDirectoryMode = 0o700 debugFileMode = 0o600 ) @@ -53,6 +54,7 @@ type PromptDebugOutput struct { Format string `json:"format"` ValidationMode string `json:"validationMode"` SchemaPath string `json:"schemaPath"` + RepairAttempts int `json:"repairAttempts"` } // PromptDebugPreparation is the explicit, content-safe mapping of preparation @@ -97,10 +99,28 @@ type PromptDebugUsage struct { // PromptDebugValidation is the completed validation detail retained in the // explicitly enabled debug store. type PromptDebugValidation struct { - Status string `json:"status"` - Mode string `json:"mode"` - SchemaPath string `json:"schemaPath"` - Diagnostics []string `json:"diagnostics,omitempty"` + Status string `json:"status"` + Mode string `json:"mode"` + SchemaPath string `json:"schemaPath"` + RepairAttempts int `json:"repairAttempts"` + Diagnostics []string `json:"diagnostics,omitempty"` +} + +// PromptFailureDebugArtifact records provider detail only in explicit debug storage. +type PromptFailureDebugArtifact struct { + SchemaVersion string `json:"schemaVersion"` + ReportID report.ID `json:"reportId"` + ValidDate string `json:"validDate"` + RunID string `json:"runId"` + Failure PromptFailure `json:"failure"` +} + +type PromptFailure struct { + Category string `json:"category"` + StatusCode int `json:"statusCode,omitempty"` + ProviderCode string `json:"providerCode,omitempty"` + ProviderType string `json:"providerType,omitempty"` + ProviderMessage string `json:"providerMessage,omitempty"` } // PromptDebugExecution is the explicit mapping of execution provenance. @@ -235,6 +255,24 @@ func (w *PromptDebugWriter) WriteExecution(ref PromptDebugRef, execution prompte return directory, nil } +// WriteFailure stores the project-owned structured provider failure. +func (w *PromptDebugWriter) WriteFailure(ref PromptDebugRef, failure *promptexec.GenerationError) (string, error) { + if !w.Enabled() || failure == nil { + return "", nil + } + directory, secureDirectory, err := w.runDirectory(ref) + if err != nil { + return "", err + } + defer secureDirectory.Close() + artifact := PromptFailureDebugArtifact{SchemaVersion: promptFailureDebugSchemaVersion, ReportID: ref.ReportID, ValidDate: ref.ValidDate, RunID: ref.RunID, + Failure: PromptFailure{Category: string(failure.Category()), StatusCode: failure.StatusCode(), ProviderCode: failure.ProviderCode(), ProviderType: failure.ProviderType(), ProviderMessage: failure.ProviderMessage()}} + if err := secureDirectory.writeJSON("failure.json", artifact); err != nil { + return "", err + } + return directory, nil +} + func (w *PromptDebugWriter) runDirectory(ref PromptDebugRef) (string, *secureDirectory, error) { if err := validatePromptDebugRef(ref); err != nil { return "", nil, err @@ -283,7 +321,7 @@ func promptDebugPreparation(value promptexec.Preparation) PromptDebugPreparation PromptID: value.PromptID, PromptVersion: value.PromptVersion, PromptHash: value.PromptHash, RenderedPromptHash: value.RenderedPromptHash, InputHashes: copyPromptDebugMap(value.InputHashes), ProfileID: value.ProfileID, BackendID: value.BackendID, ModelName: value.ModelName, - Output: PromptDebugOutput{Format: value.Output.Format, ValidationMode: value.Output.ValidationMode, SchemaPath: value.Output.SchemaPath}, + Output: PromptDebugOutput{Format: value.Output.Format, ValidationMode: value.Output.ValidationMode, SchemaPath: value.Output.SchemaPath, RepairAttempts: value.Output.RepairAttempts}, StartedAt: value.StartedAt, EndedAt: value.EndedAt, Duration: value.Duration, } } @@ -300,7 +338,7 @@ func promptDebugExecution(value promptexec.Execution) PromptDebugExecution { } func promptDebugValidation(value promptexec.Validation) PromptDebugValidation { - return PromptDebugValidation{Status: string(value.Status), Mode: value.Mode, SchemaPath: value.SchemaPath, Diagnostics: append([]string(nil), value.Diagnostics...)} + return PromptDebugValidation{Status: string(value.Status), Mode: value.Mode, SchemaPath: value.SchemaPath, RepairAttempts: value.RepairAttempts, Diagnostics: append([]string(nil), value.Diagnostics...)} } func promptDebugMessages(values []promptexec.RenderedMessage) []PromptDebugMessage { diff --git a/internal/promptdebug/debug_writer_test.go b/internal/promptdebug/debug_writer_test.go index 54b86e7..866794e 100644 --- a/internal/promptdebug/debug_writer_test.go +++ b/internal/promptdebug/debug_writer_test.go @@ -44,13 +44,13 @@ func TestPromptDebugWriterWritesIsolatedArtifacts(t *testing.T) { t.Fatalf("WriteExecution() directory = %q, want %q", executionDir, preparationDir) } preparationData := readPromptDebugFile(t, filepath.Join(preparationDir, "preparation.json")) - for _, want := range []string{"weatherreporter.prompt_preparation_debug.v2", "Use the supplied weather facts.", `"type": "object"`, "https://llm.example.test", `"temperature": 0.2`} { + for _, want := range []string{"weatherreporter.prompt_preparation_debug.v3", "Use the supplied weather facts.", `"type": "object"`, "https://llm.example.test", `"temperature": 0.2`, `"repairAttempts": 0`} { if !strings.Contains(string(preparationData), want) { t.Fatalf("preparation debug artifact missing %q:\n%s", want, preparationData) } } executionData := readPromptDebugFile(t, filepath.Join(executionDir, "execution.json")) - for _, want := range []string{"weatherreporter.prompt_execution_debug.v2", "Generated forecast prose.", "validation details", `"status": "passed"`} { + for _, want := range []string{"weatherreporter.prompt_execution_debug.v3", "Generated forecast prose.", "validation details", `"status": "passed"`, `"repairAttempts": 0`} { if !strings.Contains(string(executionData), want) { t.Fatalf("execution debug artifact missing %q:\n%s", want, executionData) } @@ -66,6 +66,25 @@ func TestPromptDebugWriterWritesIsolatedArtifacts(t *testing.T) { assertPromptDebugMode(t, filepath.Join(executionDir, "execution.json"), debugFileMode) } +func TestPromptDebugWriterWritesProviderFailureOnlyToDebugStore(t *testing.T) { + writer, err := NewPromptDebugWriter(filepath.Join(t.TempDir(), "operator-debug")) + if err != nil { + t.Fatalf("NewPromptDebugWriter() error = %v", err) + } + marker := "provider-private-marker" + directory, err := writer.WriteFailure(promptDebugRef(), promptexec.NewGenerationError(429, "rate_limit", "provider_error", marker, nil)) + if err != nil { + t.Fatalf("WriteFailure() error = %v", err) + } + data := readPromptDebugFile(t, filepath.Join(directory, "failure.json")) + for _, want := range []string{"weatherreporter.prompt_failure_debug.v1", `"category": "generation"`, `"statusCode": 429`, marker} { + if !strings.Contains(string(data), want) { + t.Fatalf("failure artifact missing %q: %s", want, data) + } + } + assertPromptDebugMode(t, filepath.Join(directory, "failure.json"), debugFileMode) +} + func TestPromptDebugWriterProjectsProviderConfigurationSafely(t *testing.T) { const marker = "private-debug-marker" tests := []struct {