eric/weatherreporter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

Add Promptkit debug capture for generated reports

Browse Source
This commit is contained in:
Eric Rakestraw
2026-07-31 04:48:16 +00:00
parent 06b26d5e88
commit a6d11c01e8
11 changed files with 271 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files

38
internal/app/prompt_generate.go
View File

@@ -17,9 +17,10 @@ import (
type promptReportRequest struct {
GenerateRequest
Resolved report.Resolved
Collection collect.Result
Inspection PromptInspectionResult
Resolved report.Resolved
Collection collect.Result
Inspection PromptInspectionResult
DebugWriter *state.PromptDebugWriter
}
func generatePromptReport(ctx context.Context, req promptReportRequest) (*ReportResult, error) {
@@ -78,6 +79,7 @@ func generatePromptReport(ctx context.Context, req promptReportRequest) (*Report
GeneratedText: paths.GeneratedText,
RenderContext: paths.RenderContext,
})
result.Metadata = metadata
dataPackage, err := promptinput.Build(promptinput.BuildRequest{
Metadata: promptMetadata(metadata),
Modules: moduleSnapshot,
@@ -97,6 +99,7 @@ func generatePromptReport(ctx context.Context, req promptReportRequest) (*Report
metadata.DataPackagePath = dataPackagePath
result.DataPackage = dataPackage
result.DataPackagePath = dataPackagePath
result.Metadata = metadata
handler, err := generatedtext.LookupDefinition(req.Resolved.Definition)
if err != nil {
@@ -105,7 +108,12 @@ func generatePromptReport(ctx context.Context, req promptReportRequest) (*Report
prepared := false
callbackFailed := false
callback := func(preparation promptexec.Preparation, _ *promptexec.PreparationDebug) error {
debugRef := state.PromptDebugRef{
ReportID: req.Resolved.Definition.ID,
ValidDate: req.Resolved.ValidPeriod.Start.Format("2006-01-02"),
RunID: metadata.RunID,
}
callback := func(preparation promptexec.Preparation, debug *promptexec.PreparationDebug) error {
artifact := state.PromptPreparationArtifact{
SchemaVersion: state.PromptPreparationSchemaVersion,
Status: state.PromptPreparationSucceeded,
@@ -127,6 +135,15 @@ func generatePromptReport(ctx context.Context, req promptReportRequest) (*Report
prepared = true
result.PreparationPath = path
metadata.PreparationPath = path
result.Metadata = metadata
debugPath, err := req.DebugWriter.WritePreparation(debugRef, preparation, debug)
if err != nil {
callbackFailed = true
return promptDebugWriteError(err)
}
if debugPath != "" {
result.LLMDebugPath = debugPath
}
metadataPath, err := store.SaveMetadata(ctx, metadata)
if err != nil {
callbackFailed = true
@@ -142,7 +159,7 @@ func generatePromptReport(ctx context.Context, req promptReportRequest) (*Report
ProfileID: req.Inspection.ProfileID,
DataPackage: data,
DataPackagePath: dataPackagePath,
CaptureDebug: false,
CaptureDebug: req.DebugWriter.Enabled(),
}, callback)
if executeErr != nil {
if callbackFailed {
@@ -198,6 +215,13 @@ func generatePromptReport(ctx context.Context, req promptReportRequest) (*Report
result.ExecutionPath, result.Metadata, result.MetadataPath = executionPath, metadata, metadataPath
return result, generatedReportError(req.Resolved, metadata.RunID, "execute prompt", err)
}
debugPath, err := req.DebugWriter.WriteExecution(debugRef, *execution)
if err != nil {
return result, generatedReportError(req.Resolved, metadata.RunID, "write prompt debug", promptDebugWriteError(err))
}
if debugPath != "" {
result.LLMDebugPath = debugPath
}
if execution.Validation.Status != promptexec.ValidationPassed && execution.Validation.Status != promptexec.ValidationFailed {
err := promptexec.NewError(promptexec.OperationalValidation, "prompt execution did not complete validation", nil)
@@ -318,4 +342,8 @@ func classifiedPromptError(operation string, err error) error {
return promptexec.NewError(promptexec.Generation, operation, err)
}
func promptDebugWriteError(err error) error {
return promptexec.NewError(promptexec.InvalidConfiguration, "write requested prompt debug artifact", err)
}
func ptr[T any](value T) *T { return &value }