eric/weatherreporter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

Capture provider failures in secure debug artifacts

Browse Source
This commit is contained in:
Eric Rakestraw
2026-08-25 19:55:09 +00:00
parent b3b23fb381
commit 4cd5f505df
7 changed files with 91 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
docs/integrations/promptkit.md
View File

@@ -42,7 +42,7 @@ logical profile ID and resolved backend and model. Ordinary errors, summaries,
logs, and outputs exclude endpoints, credentials, rendered messages, schemas,
request bodies, response bodies, and complete parameter maps.
Promptkit receives the YAML data package as an inline input and returns structured JSON that Weatherreporter validates before rendering its own Markdown template. Before accepting that JSON, Weatherreporter requires exactly one preparation callback and reconciles its prompt/profile/backend/model and rendered/input hashes with the inspected identity and completed result. The callback output contract and completed validation must use the report's expected JSON Schema mode and path. The package contains only reviewed prompt-facing warning summaries, never source transport or provenance details. Safe active provenance remains in memory. Content-rich diagnostics are opt-in through `--llm-debug-dir`; see [operations](../operations.md) for retention and permissions.
Promptkit receives the YAML data package as an inline input and returns structured JSON that Weatherreporter validates before rendering its own Markdown template. Before accepting that JSON, Weatherreporter requires exactly one preparation callback and reconciles its prompt/profile/backend/model and rendered/input hashes with the inspected identity and completed result. The callback output contract and completed validation must use the report's expected JSON Schema mode and path. The package contains only reviewed prompt-facing warning summaries, never source transport or provenance details. Safe active provenance remains in memory. Content-rich diagnostics are opt-in through `--llm-debug-dir`; see [operations](../operations.md) for retention and permissions. Ordinary generation errors disclose only the safe Weatherreporter category and optional HTTP status; provider code, type, and message are written only to the explicit secure failure-debug artifact.
Each embedded prompt permits one Promptkit-owned corrective generation after an
eligible failed or explicitly empty result. This is not an application retry:

6
docs/operations.md
View File

@@ -178,6 +178,12 @@ Preparation captures retain only the provider endpoint origin and reviewed
execution settings. URL user information, paths, queries, fragments, and
unrecognized provider parameters are omitted.
Each run directory may contain `preparation.json` (v3), `execution.json` (v3),
and, for a provider generation failure, `failure.json` (v1). The failure
artifact retains the safe category, HTTP status, and provider code, type, and
message for trusted debugging only. Ordinary command output never includes
those provider details.
Capture writes are confined to the requested root and fail if an unsafe
filesystem component prevents secure artifact creation.

2
docs/policy/architecture.md
View File

@@ -66,6 +66,8 @@ directly.
- Sensitive rendered prompts, schemas, input bodies, provider endpoints, and
credentials never enter normal summaries or logs. They are written only to
an explicit secure debug root when requested.
- Provider-controlled diagnostics never enter ordinary outputs; they are
retained only in explicit secure failure-debug artifacts.
## Output, Notification, And Testing Invariants

2
docs/roadmap/implementation.md
View File

@@ -439,6 +439,8 @@ GOWORK=off go test -race -count=1 ./internal/comparison ./internal/app
### Stage 8: Add Secure Provider-Failure Debug Capture
Status: Complete.
Purpose: expose useful PromptKit v0.7.0 provider diagnostics only through the
existing explicit secure debug boundary while keeping ordinary errors safe.

13
internal/app/profile_execution.go
View File

@@ -2,6 +2,7 @@ package app
import (
"context"
"errors"
"fmt"
"reflect"
@@ -95,6 +96,18 @@ func executePreparedProfile(ctx context.Context, req profileExecutionRequest) (p
if callbackFailed {
return outcome, nil, &profileExecutionError{operation: "execute prompt", err: err, callbackFailure: true}
}
if req.DebugWriter != nil && req.DebugWriter.Enabled() && req.DebugRef != nil {
var generationError *promptexec.GenerationError
if errors.As(err, &generationError) {
path, debugErr := req.DebugWriter.WriteFailure(*req.DebugRef, generationError)
if path != "" {
outcome.LLMDebugPath = path
}
if debugErr != nil {
err = errors.Join(err, promptDebugWriteError(debugErr))
}
}
}
return outcome, nil, &profileExecutionError{operation: "execute prompt", err: classifiedPromptError("prompt execution failed", err)}
}
if execution == nil {

54
internal/promptdebug/debug_writer.go
View File

@@ -21,8 +21,9 @@ import (
var ErrSecureCaptureUnsupported = errors.New("secure prompt debug capture is unavailable on this platform")
const (
promptPreparationDebugSchemaVersion = "weatherreporter.prompt_preparation_debug.v2"
promptExecutionDebugSchemaVersion = "weatherreporter.prompt_execution_debug.v2"
promptPreparationDebugSchemaVersion = "weatherreporter.prompt_preparation_debug.v3"
promptExecutionDebugSchemaVersion = "weatherreporter.prompt_execution_debug.v3"
promptFailureDebugSchemaVersion = "weatherreporter.prompt_failure_debug.v1"
debugDirectoryMode = 0o700
debugFileMode = 0o600
)
@@ -53,6 +54,7 @@ type PromptDebugOutput struct {
Format string `json:"format"`
ValidationMode string `json:"validationMode"`
SchemaPath string `json:"schemaPath"`
RepairAttempts int `json:"repairAttempts"`
}
// PromptDebugPreparation is the explicit, content-safe mapping of preparation
@@ -97,10 +99,28 @@ type PromptDebugUsage struct {
// PromptDebugValidation is the completed validation detail retained in the
// explicitly enabled debug store.
type PromptDebugValidation struct {
Status string `json:"status"`
Mode string `json:"mode"`
SchemaPath string `json:"schemaPath"`
Diagnostics []string `json:"diagnostics,omitempty"`
Status string `json:"status"`
Mode string `json:"mode"`
SchemaPath string `json:"schemaPath"`
RepairAttempts int `json:"repairAttempts"`
Diagnostics []string `json:"diagnostics,omitempty"`
}
// PromptFailureDebugArtifact records provider detail only in explicit debug storage.
type PromptFailureDebugArtifact struct {
SchemaVersion string `json:"schemaVersion"`
ReportID report.ID `json:"reportId"`
ValidDate string `json:"validDate"`
RunID string `json:"runId"`
Failure PromptFailure `json:"failure"`
}
type PromptFailure struct {
Category string `json:"category"`
StatusCode int `json:"statusCode,omitempty"`
ProviderCode string `json:"providerCode,omitempty"`
ProviderType string `json:"providerType,omitempty"`
ProviderMessage string `json:"providerMessage,omitempty"`
}
// PromptDebugExecution is the explicit mapping of execution provenance.
@@ -235,6 +255,24 @@ func (w *PromptDebugWriter) WriteExecution(ref PromptDebugRef, execution prompte
return directory, nil
}
// WriteFailure stores the project-owned structured provider failure.
func (w *PromptDebugWriter) WriteFailure(ref PromptDebugRef, failure *promptexec.GenerationError) (string, error) {
if !w.Enabled() || failure == nil {
return "", nil
}
directory, secureDirectory, err := w.runDirectory(ref)
if err != nil {
return "", err
}
defer secureDirectory.Close()
artifact := PromptFailureDebugArtifact{SchemaVersion: promptFailureDebugSchemaVersion, ReportID: ref.ReportID, ValidDate: ref.ValidDate, RunID: ref.RunID,
Failure: PromptFailure{Category: string(failure.Category()), StatusCode: failure.StatusCode(), ProviderCode: failure.ProviderCode(), ProviderType: failure.ProviderType(), ProviderMessage: failure.ProviderMessage()}}
if err := secureDirectory.writeJSON("failure.json", artifact); err != nil {
return "", err
}
return directory, nil
}
func (w *PromptDebugWriter) runDirectory(ref PromptDebugRef) (string, *secureDirectory, error) {
if err := validatePromptDebugRef(ref); err != nil {
return "", nil, err
@@ -283,7 +321,7 @@ func promptDebugPreparation(value promptexec.Preparation) PromptDebugPreparation
PromptID: value.PromptID, PromptVersion: value.PromptVersion, PromptHash: value.PromptHash,
RenderedPromptHash: value.RenderedPromptHash, InputHashes: copyPromptDebugMap(value.InputHashes),
ProfileID: value.ProfileID, BackendID: value.BackendID, ModelName: value.ModelName,
Output: PromptDebugOutput{Format: value.Output.Format, ValidationMode: value.Output.ValidationMode, SchemaPath: value.Output.SchemaPath},
Output: PromptDebugOutput{Format: value.Output.Format, ValidationMode: value.Output.ValidationMode, SchemaPath: value.Output.SchemaPath, RepairAttempts: value.Output.RepairAttempts},
StartedAt: value.StartedAt, EndedAt: value.EndedAt, Duration: value.Duration,
}
}
@@ -300,7 +338,7 @@ func promptDebugExecution(value promptexec.Execution) PromptDebugExecution {
}
func promptDebugValidation(value promptexec.Validation) PromptDebugValidation {
return PromptDebugValidation{Status: string(value.Status), Mode: value.Mode, SchemaPath: value.SchemaPath, Diagnostics: append([]string(nil), value.Diagnostics...)}
return PromptDebugValidation{Status: string(value.Status), Mode: value.Mode, SchemaPath: value.SchemaPath, RepairAttempts: value.RepairAttempts, Diagnostics: append([]string(nil), value.Diagnostics...)}
}
func promptDebugMessages(values []promptexec.RenderedMessage) []PromptDebugMessage {

23
internal/promptdebug/debug_writer_test.go
View File

@@ -44,13 +44,13 @@ func TestPromptDebugWriterWritesIsolatedArtifacts(t *testing.T) {
t.Fatalf("WriteExecution() directory = %q, want %q", executionDir, preparationDir)
}
preparationData := readPromptDebugFile(t, filepath.Join(preparationDir, "preparation.json"))
for _, want := range []string{"weatherreporter.prompt_preparation_debug.v2", "Use the supplied weather facts.", `"type": "object"`, "https://llm.example.test", `"temperature": 0.2`} {
for _, want := range []string{"weatherreporter.prompt_preparation_debug.v3", "Use the supplied weather facts.", `"type": "object"`, "https://llm.example.test", `"temperature": 0.2`, `"repairAttempts": 0`} {
if !strings.Contains(string(preparationData), want) {
t.Fatalf("preparation debug artifact missing %q:\n%s", want, preparationData)
}
}
executionData := readPromptDebugFile(t, filepath.Join(executionDir, "execution.json"))
for _, want := range []string{"weatherreporter.prompt_execution_debug.v2", "Generated forecast prose.", "validation details", `"status": "passed"`} {
for _, want := range []string{"weatherreporter.prompt_execution_debug.v3", "Generated forecast prose.", "validation details", `"status": "passed"`, `"repairAttempts": 0`} {
if !strings.Contains(string(executionData), want) {
t.Fatalf("execution debug artifact missing %q:\n%s", want, executionData)
}
@@ -66,6 +66,25 @@ func TestPromptDebugWriterWritesIsolatedArtifacts(t *testing.T) {
assertPromptDebugMode(t, filepath.Join(executionDir, "execution.json"), debugFileMode)
}
func TestPromptDebugWriterWritesProviderFailureOnlyToDebugStore(t *testing.T) {
writer, err := NewPromptDebugWriter(filepath.Join(t.TempDir(), "operator-debug"))
if err != nil {
t.Fatalf("NewPromptDebugWriter() error = %v", err)
}
marker := "provider-private-marker"
directory, err := writer.WriteFailure(promptDebugRef(), promptexec.NewGenerationError(429, "rate_limit", "provider_error", marker, nil))
if err != nil {
t.Fatalf("WriteFailure() error = %v", err)
}
data := readPromptDebugFile(t, filepath.Join(directory, "failure.json"))
for _, want := range []string{"weatherreporter.prompt_failure_debug.v1", `"category": "generation"`, `"statusCode": 429`, marker} {
if !strings.Contains(string(data), want) {
t.Fatalf("failure artifact missing %q: %s", want, data)
}
}
assertPromptDebugMode(t, filepath.Join(directory, "failure.json"), debugFileMode)
}
func TestPromptDebugWriterProjectsProviderConfigurationSafely(t *testing.T) {
const marker = "private-debug-marker"
tests := []struct {