eric/weatherreporter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

Capture provider failures in secure debug artifacts

Browse Source
This commit is contained in:
Eric Rakestraw
2026-08-25 19:55:09 +00:00
parent b3b23fb381
commit 4cd5f505df
7 changed files with 91 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

54
internal/promptdebug/debug_writer.go
View File

@@ -21,8 +21,9 @@ import (
var ErrSecureCaptureUnsupported = errors.New("secure prompt debug capture is unavailable on this platform")
const (
promptPreparationDebugSchemaVersion = "weatherreporter.prompt_preparation_debug.v2"
promptExecutionDebugSchemaVersion = "weatherreporter.prompt_execution_debug.v2"
promptPreparationDebugSchemaVersion = "weatherreporter.prompt_preparation_debug.v3"
promptExecutionDebugSchemaVersion = "weatherreporter.prompt_execution_debug.v3"
promptFailureDebugSchemaVersion = "weatherreporter.prompt_failure_debug.v1"
debugDirectoryMode = 0o700
debugFileMode = 0o600
)
@@ -53,6 +54,7 @@ type PromptDebugOutput struct {
Format string `json:"format"`
ValidationMode string `json:"validationMode"`
SchemaPath string `json:"schemaPath"`
RepairAttempts int `json:"repairAttempts"`
}
// PromptDebugPreparation is the explicit, content-safe mapping of preparation
@@ -97,10 +99,28 @@ type PromptDebugUsage struct {
// PromptDebugValidation is the completed validation detail retained in the
// explicitly enabled debug store.
type PromptDebugValidation struct {
Status string `json:"status"`
Mode string `json:"mode"`
SchemaPath string `json:"schemaPath"`
Diagnostics []string `json:"diagnostics,omitempty"`
Status string `json:"status"`
Mode string `json:"mode"`
SchemaPath string `json:"schemaPath"`
RepairAttempts int `json:"repairAttempts"`
Diagnostics []string `json:"diagnostics,omitempty"`
}
// PromptFailureDebugArtifact records provider detail only in explicit debug storage.
type PromptFailureDebugArtifact struct {
SchemaVersion string `json:"schemaVersion"`
ReportID report.ID `json:"reportId"`
ValidDate string `json:"validDate"`
RunID string `json:"runId"`
Failure PromptFailure `json:"failure"`
}
type PromptFailure struct {
Category string `json:"category"`
StatusCode int `json:"statusCode,omitempty"`
ProviderCode string `json:"providerCode,omitempty"`
ProviderType string `json:"providerType,omitempty"`
ProviderMessage string `json:"providerMessage,omitempty"`
}
// PromptDebugExecution is the explicit mapping of execution provenance.
@@ -235,6 +255,24 @@ func (w *PromptDebugWriter) WriteExecution(ref PromptDebugRef, execution prompte
return directory, nil
}
// WriteFailure stores the project-owned structured provider failure.
func (w *PromptDebugWriter) WriteFailure(ref PromptDebugRef, failure *promptexec.GenerationError) (string, error) {
if !w.Enabled() || failure == nil {
return "", nil
}
directory, secureDirectory, err := w.runDirectory(ref)
if err != nil {
return "", err
}
defer secureDirectory.Close()
artifact := PromptFailureDebugArtifact{SchemaVersion: promptFailureDebugSchemaVersion, ReportID: ref.ReportID, ValidDate: ref.ValidDate, RunID: ref.RunID,
Failure: PromptFailure{Category: string(failure.Category()), StatusCode: failure.StatusCode(), ProviderCode: failure.ProviderCode(), ProviderType: failure.ProviderType(), ProviderMessage: failure.ProviderMessage()}}
if err := secureDirectory.writeJSON("failure.json", artifact); err != nil {
return "", err
}
return directory, nil
}
func (w *PromptDebugWriter) runDirectory(ref PromptDebugRef) (string, *secureDirectory, error) {
if err := validatePromptDebugRef(ref); err != nil {
return "", nil, err
@@ -283,7 +321,7 @@ func promptDebugPreparation(value promptexec.Preparation) PromptDebugPreparation
PromptID: value.PromptID, PromptVersion: value.PromptVersion, PromptHash: value.PromptHash,
RenderedPromptHash: value.RenderedPromptHash, InputHashes: copyPromptDebugMap(value.InputHashes),
ProfileID: value.ProfileID, BackendID: value.BackendID, ModelName: value.ModelName,
Output: PromptDebugOutput{Format: value.Output.Format, ValidationMode: value.Output.ValidationMode, SchemaPath: value.Output.SchemaPath},
Output: PromptDebugOutput{Format: value.Output.Format, ValidationMode: value.Output.ValidationMode, SchemaPath: value.Output.SchemaPath, RepairAttempts: value.Output.RepairAttempts},
StartedAt: value.StartedAt, EndedAt: value.EndedAt, Duration: value.Duration,
}
}
@@ -300,7 +338,7 @@ func promptDebugExecution(value promptexec.Execution) PromptDebugExecution {
}
func promptDebugValidation(value promptexec.Validation) PromptDebugValidation {
return PromptDebugValidation{Status: string(value.Status), Mode: value.Mode, SchemaPath: value.SchemaPath, Diagnostics: append([]string(nil), value.Diagnostics...)}
return PromptDebugValidation{Status: string(value.Status), Mode: value.Mode, SchemaPath: value.SchemaPath, RepairAttempts: value.RepairAttempts, Diagnostics: append([]string(nil), value.Diagnostics...)}
}
func promptDebugMessages(values []promptexec.RenderedMessage) []PromptDebugMessage {