eric/notarius
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Record execution summaries in debug artifacts

Browse Source
This commit is contained in:
Eric Rakestraw
2026-09-25 22:02:56 +00:00
parent 6b8a4408c1
commit b09ce47d4a
4 changed files with 47 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files

56
internal/framework/pipeline/debug.go
View File

@@ -113,13 +113,14 @@ type debugSerializedOutput struct {
type debugStructuredCompletionRequest = contracts.DebugStructuredCompletionRequest
type debugStructuredCompletionResponse struct {
Content string `json:"content,omitempty"`
Provider string `json:"provider,omitempty"`
Model string `json:"model,omitempty"`
ProfileID string `json:"profile_id,omitempty"`
PromptTokens int `json:"prompt_tokens,omitempty"`
CompletionTokens int `json:"completion_tokens,omitempty"`
TotalTokens int `json:"total_tokens,omitempty"`
Content string `json:"content,omitempty"`
Provider string `json:"provider,omitempty"`
Model string `json:"model,omitempty"`
ProfileID string `json:"profile_id,omitempty"`
PromptTokens int `json:"prompt_tokens,omitempty"`
CompletionTokens int `json:"completion_tokens,omitempty"`
TotalTokens int `json:"total_tokens,omitempty"`
Execution *contracts.LLMExecutionSummary `json:"execution,omitempty"`
}
type debugLLMPromptArtifact struct {
@@ -133,21 +134,23 @@ type debugLLMResponseArtifact struct {
Fallback *debugStructuredCompletionResponse `json:"fallback,omitempty"`
ContentPath string `json:"content_path,omitempty"`
Error string `json:"error,omitempty"`
Execution *contracts.LLMExecutionSummary `json:"execution,omitempty"`
}
type debugLLMCallReference struct {
CallID string `json:"call_id"`
PromptPath string `json:"prompt_path,omitempty"`
ResponsePath string `json:"response_path"`
ResponseContentPath string `json:"response_content_path,omitempty"`
PromptID string `json:"prompt_id,omitempty"`
ProfileID string `json:"profile_id,omitempty"`
Model string `json:"model,omitempty"`
PromptTokens int `json:"prompt_tokens,omitempty"`
CompletionTokens int `json:"completion_tokens,omitempty"`
TotalTokens int `json:"total_tokens,omitempty"`
RepairAttempts int `json:"repair_attempts,omitempty"`
Error bool `json:"error,omitempty"`
CallID string `json:"call_id"`
PromptPath string `json:"prompt_path,omitempty"`
ResponsePath string `json:"response_path"`
ResponseContentPath string `json:"response_content_path,omitempty"`
PromptID string `json:"prompt_id,omitempty"`
ProfileID string `json:"profile_id,omitempty"`
Model string `json:"model,omitempty"`
PromptTokens int `json:"prompt_tokens,omitempty"`
CompletionTokens int `json:"completion_tokens,omitempty"`
TotalTokens int `json:"total_tokens,omitempty"`
RepairAttempts int `json:"repair_attempts,omitempty"`
Error bool `json:"error,omitempty"`
Execution *contracts.LLMExecutionSummary `json:"execution,omitempty"`
}
type debugValidationOutcome struct {
@@ -267,6 +270,7 @@ func (client *debugLLMClient) CompleteStructured(ctx context.Context, req contra
Fallback: fallbackForArtifact,
ContentPath: responseContentPath,
Error: errorText,
Execution: cloneLLMExecutionSummary(response.Execution),
},
Error: errorText,
}))
@@ -283,6 +287,7 @@ func (client *debugLLMClient) CompleteStructured(ctx context.Context, req contra
TotalTokens: response.TotalTokens,
RepairAttempts: response.RepairAttempts,
Error: err != nil,
Execution: cloneLLMExecutionSummary(response.Execution),
}
if scope := debugLLMScopeFromContext(ctx); scope != nil {
scope.record(callRef)
@@ -637,6 +642,7 @@ func debugCompletionResponse(response contracts.StructuredCompletionResponse) de
PromptTokens: response.PromptTokens,
CompletionTokens: response.CompletionTokens,
TotalTokens: response.TotalTokens,
Execution: cloneLLMExecutionSummary(response.Execution),
}
}
@@ -651,7 +657,8 @@ func debugCompletionFallback(response contracts.StructuredCompletionResponse) *d
fallback.ProfileID == "" &&
fallback.PromptTokens == 0 &&
fallback.CompletionTokens == 0 &&
fallback.TotalTokens == 0 {
fallback.TotalTokens == 0 &&
fallback.Execution == nil {
return nil
}
return &fallback
@@ -696,6 +703,15 @@ func cloneDebugFallbackWithoutContent(fallback *debugStructuredCompletionRespons
}
clone := *fallback
clone.Content = ""
clone.Execution = cloneLLMExecutionSummary(fallback.Execution)
return &clone
}
func cloneLLMExecutionSummary(summary *contracts.LLMExecutionSummary) *contracts.LLMExecutionSummary {
if summary == nil {
return nil
}
clone := *summary
return &clone
}