Capture provider failures in secure debug artifacts

docs/integrations/promptkit.md

@@ -42,7 +42,7 @@ logical profile ID and resolved backend and model. Ordinary errors, summaries,
 logs, and outputs exclude endpoints, credentials, rendered messages, schemas,
 request bodies, response bodies, and complete parameter maps.
 
-Promptkit receives the YAML data package as an inline input and returns structured JSON that Weatherreporter validates before rendering its own Markdown template. Before accepting that JSON, Weatherreporter requires exactly one preparation callback and reconciles its prompt/profile/backend/model and rendered/input hashes with the inspected identity and completed result. The callback output contract and completed validation must use the report's expected JSON Schema mode and path. The package contains only reviewed prompt-facing warning summaries, never source transport or provenance details. Safe active provenance remains in memory. Content-rich diagnostics are opt-in through `--llm-debug-dir`; see [operations](../operations.md) for retention and permissions.
+Promptkit receives the YAML data package as an inline input and returns structured JSON that Weatherreporter validates before rendering its own Markdown template. Before accepting that JSON, Weatherreporter requires exactly one preparation callback and reconciles its prompt/profile/backend/model and rendered/input hashes with the inspected identity and completed result. The callback output contract and completed validation must use the report's expected JSON Schema mode and path. The package contains only reviewed prompt-facing warning summaries, never source transport or provenance details. Safe active provenance remains in memory. Content-rich diagnostics are opt-in through `--llm-debug-dir`; see [operations](../operations.md) for retention and permissions. Ordinary generation errors disclose only the safe Weatherreporter category and optional HTTP status; provider code, type, and message are written only to the explicit secure failure-debug artifact.
 
 Each embedded prompt permits one Promptkit-owned corrective generation after an
 eligible failed or explicitly empty result. This is not an application retry:

docs/operations.md

@@ -178,6 +178,12 @@ Preparation captures retain only the provider endpoint origin and reviewed
 execution settings. URL user information, paths, queries, fragments, and
 unrecognized provider parameters are omitted.
 
+Each run directory may contain `preparation.json` (v3), `execution.json` (v3),
+and, for a provider generation failure, `failure.json` (v1). The failure
+artifact retains the safe category, HTTP status, and provider code, type, and
+message for trusted debugging only. Ordinary command output never includes
+those provider details.
+
 Capture writes are confined to the requested root and fail if an unsafe
 filesystem component prevents secure artifact creation.
 

docs/policy/architecture.md

@@ -66,6 +66,8 @@ directly.
 - Sensitive rendered prompts, schemas, input bodies, provider endpoints, and
   credentials never enter normal summaries or logs. They are written only to
   an explicit secure debug root when requested.
+- Provider-controlled diagnostics never enter ordinary outputs; they are
+  retained only in explicit secure failure-debug artifacts.
 
 ## Output, Notification, And Testing Invariants
 

docs/roadmap/implementation.md

@@ -439,6 +439,8 @@ GOWORK=off go test -race -count=1 ./internal/comparison ./internal/app
 
 ### Stage 8: Add Secure Provider-Failure Debug Capture
 
+Status: Complete.
+
 Purpose: expose useful PromptKit v0.7.0 provider diagnostics only through the
 existing explicit secure debug boundary while keeping ordinary errors safe.
 

internal/app/profile_execution.go

@@ -2,6 +2,7 @@ package app
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"reflect"
 
@@ -95,6 +96,18 @@ func executePreparedProfile(ctx context.Context, req profileExecutionRequest) (p
 		if callbackFailed {
 			return outcome, nil, &profileExecutionError{operation: "execute prompt", err: err, callbackFailure: true}
 		}
+		if req.DebugWriter != nil && req.DebugWriter.Enabled() && req.DebugRef != nil {
+			var generationError *promptexec.GenerationError
+			if errors.As(err, &generationError) {
+				path, debugErr := req.DebugWriter.WriteFailure(*req.DebugRef, generationError)
+				if path != "" {
+					outcome.LLMDebugPath = path
+				}
+				if debugErr != nil {
+					err = errors.Join(err, promptDebugWriteError(debugErr))
+				}
+			}
+		}
 		return outcome, nil, &profileExecutionError{operation: "execute prompt", err: classifiedPromptError("prompt execution failed", err)}
 	}
 	if execution == nil {

internal/promptdebug/debug_writer.go

@@ -21,8 +21,9 @@ import (
 var ErrSecureCaptureUnsupported = errors.New("secure prompt debug capture is unavailable on this platform")
 
 const (
-	promptPreparationDebugSchemaVersion = "weatherreporter.prompt_preparation_debug.v2"
+	promptPreparationDebugSchemaVersion = "weatherreporter.prompt_preparation_debug.v3"
-	promptExecutionDebugSchemaVersion   = "weatherreporter.prompt_execution_debug.v2"
+	promptExecutionDebugSchemaVersion   = "weatherreporter.prompt_execution_debug.v3"
+	promptFailureDebugSchemaVersion     = "weatherreporter.prompt_failure_debug.v1"
 	debugDirectoryMode                  = 0o700
 	debugFileMode                       = 0o600
 )
@@ -53,6 +54,7 @@ type PromptDebugOutput struct {
 	Format         string `json:"format"`
 	ValidationMode string `json:"validationMode"`
 	SchemaPath     string `json:"schemaPath"`
+	RepairAttempts int    `json:"repairAttempts"`
 }
 
 // PromptDebugPreparation is the explicit, content-safe mapping of preparation
@@ -97,10 +99,28 @@ type PromptDebugUsage struct {
 // PromptDebugValidation is the completed validation detail retained in the
 // explicitly enabled debug store.
 type PromptDebugValidation struct {
-	Status      string   `json:"status"`
+	Status         string   `json:"status"`
-	Mode        string   `json:"mode"`
+	Mode           string   `json:"mode"`
-	SchemaPath  string   `json:"schemaPath"`
+	SchemaPath     string   `json:"schemaPath"`
-	Diagnostics []string `json:"diagnostics,omitempty"`
+	RepairAttempts int      `json:"repairAttempts"`
+	Diagnostics    []string `json:"diagnostics,omitempty"`
+}
+
+// PromptFailureDebugArtifact records provider detail only in explicit debug storage.
+type PromptFailureDebugArtifact struct {
+	SchemaVersion string        `json:"schemaVersion"`
+	ReportID      report.ID     `json:"reportId"`
+	ValidDate     string        `json:"validDate"`
+	RunID         string        `json:"runId"`
+	Failure       PromptFailure `json:"failure"`
+}
+
+type PromptFailure struct {
+	Category        string `json:"category"`
+	StatusCode      int    `json:"statusCode,omitempty"`
+	ProviderCode    string `json:"providerCode,omitempty"`
+	ProviderType    string `json:"providerType,omitempty"`
+	ProviderMessage string `json:"providerMessage,omitempty"`
 }
 
 // PromptDebugExecution is the explicit mapping of execution provenance.
@@ -235,6 +255,24 @@ func (w *PromptDebugWriter) WriteExecution(ref PromptDebugRef, execution prompte
 	return directory, nil
 }
 
+// WriteFailure stores the project-owned structured provider failure.
+func (w *PromptDebugWriter) WriteFailure(ref PromptDebugRef, failure *promptexec.GenerationError) (string, error) {
+	if !w.Enabled() || failure == nil {
+		return "", nil
+	}
+	directory, secureDirectory, err := w.runDirectory(ref)
+	if err != nil {
+		return "", err
+	}
+	defer secureDirectory.Close()
+	artifact := PromptFailureDebugArtifact{SchemaVersion: promptFailureDebugSchemaVersion, ReportID: ref.ReportID, ValidDate: ref.ValidDate, RunID: ref.RunID,
+		Failure: PromptFailure{Category: string(failure.Category()), StatusCode: failure.StatusCode(), ProviderCode: failure.ProviderCode(), ProviderType: failure.ProviderType(), ProviderMessage: failure.ProviderMessage()}}
+	if err := secureDirectory.writeJSON("failure.json", artifact); err != nil {
+		return "", err
+	}
+	return directory, nil
+}
+
 func (w *PromptDebugWriter) runDirectory(ref PromptDebugRef) (string, *secureDirectory, error) {
 	if err := validatePromptDebugRef(ref); err != nil {
 		return "", nil, err
@@ -283,7 +321,7 @@ func promptDebugPreparation(value promptexec.Preparation) PromptDebugPreparation
 		PromptID: value.PromptID, PromptVersion: value.PromptVersion, PromptHash: value.PromptHash,
 		RenderedPromptHash: value.RenderedPromptHash, InputHashes: copyPromptDebugMap(value.InputHashes),
 		ProfileID: value.ProfileID, BackendID: value.BackendID, ModelName: value.ModelName,
-		Output:    PromptDebugOutput{Format: value.Output.Format, ValidationMode: value.Output.ValidationMode, SchemaPath: value.Output.SchemaPath},
+		Output:    PromptDebugOutput{Format: value.Output.Format, ValidationMode: value.Output.ValidationMode, SchemaPath: value.Output.SchemaPath, RepairAttempts: value.Output.RepairAttempts},
 		StartedAt: value.StartedAt, EndedAt: value.EndedAt, Duration: value.Duration,
 	}
 }
@@ -300,7 +338,7 @@ func promptDebugExecution(value promptexec.Execution) PromptDebugExecution {
 }
 
 func promptDebugValidation(value promptexec.Validation) PromptDebugValidation {
-	return PromptDebugValidation{Status: string(value.Status), Mode: value.Mode, SchemaPath: value.SchemaPath, Diagnostics: append([]string(nil), value.Diagnostics...)}
+	return PromptDebugValidation{Status: string(value.Status), Mode: value.Mode, SchemaPath: value.SchemaPath, RepairAttempts: value.RepairAttempts, Diagnostics: append([]string(nil), value.Diagnostics...)}
 }
 
 func promptDebugMessages(values []promptexec.RenderedMessage) []PromptDebugMessage {

internal/promptdebug/debug_writer_test.go

@@ -44,13 +44,13 @@ func TestPromptDebugWriterWritesIsolatedArtifacts(t *testing.T) {
 		t.Fatalf("WriteExecution() directory = %q, want %q", executionDir, preparationDir)
 	}
 	preparationData := readPromptDebugFile(t, filepath.Join(preparationDir, "preparation.json"))
-	for _, want := range []string{"weatherreporter.prompt_preparation_debug.v2", "Use the supplied weather facts.", `"type": "object"`, "https://llm.example.test", `"temperature": 0.2`} {
+	for _, want := range []string{"weatherreporter.prompt_preparation_debug.v3", "Use the supplied weather facts.", `"type": "object"`, "https://llm.example.test", `"temperature": 0.2`, `"repairAttempts": 0`} {
 		if !strings.Contains(string(preparationData), want) {
 			t.Fatalf("preparation debug artifact missing %q:\n%s", want, preparationData)
 		}
 	}
 	executionData := readPromptDebugFile(t, filepath.Join(executionDir, "execution.json"))
-	for _, want := range []string{"weatherreporter.prompt_execution_debug.v2", "Generated forecast prose.", "validation details", `"status": "passed"`} {
+	for _, want := range []string{"weatherreporter.prompt_execution_debug.v3", "Generated forecast prose.", "validation details", `"status": "passed"`, `"repairAttempts": 0`} {
 		if !strings.Contains(string(executionData), want) {
 			t.Fatalf("execution debug artifact missing %q:\n%s", want, executionData)
 		}
@@ -66,6 +66,25 @@ func TestPromptDebugWriterWritesIsolatedArtifacts(t *testing.T) {
 	assertPromptDebugMode(t, filepath.Join(executionDir, "execution.json"), debugFileMode)
 }
 
+func TestPromptDebugWriterWritesProviderFailureOnlyToDebugStore(t *testing.T) {
+	writer, err := NewPromptDebugWriter(filepath.Join(t.TempDir(), "operator-debug"))
+	if err != nil {
+		t.Fatalf("NewPromptDebugWriter() error = %v", err)
+	}
+	marker := "provider-private-marker"
+	directory, err := writer.WriteFailure(promptDebugRef(), promptexec.NewGenerationError(429, "rate_limit", "provider_error", marker, nil))
+	if err != nil {
+		t.Fatalf("WriteFailure() error = %v", err)
+	}
+	data := readPromptDebugFile(t, filepath.Join(directory, "failure.json"))
+	for _, want := range []string{"weatherreporter.prompt_failure_debug.v1", `"category": "generation"`, `"statusCode": 429`, marker} {
+		if !strings.Contains(string(data), want) {
+			t.Fatalf("failure artifact missing %q: %s", want, data)
+		}
+	}
+	assertPromptDebugMode(t, filepath.Join(directory, "failure.json"), debugFileMode)
+}
+
 func TestPromptDebugWriterProjectsProviderConfigurationSafely(t *testing.T) {
 	const marker = "private-debug-marker"
 	tests := []struct {