From b09ce47d4a19ceafb7bcde39a9b0ce6c42798dfb Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Fri, 25 Sep 2026 22:02:56 +0000 Subject: [PATCH] Record execution summaries in debug artifacts --- docs/internal/state.md | 4 ++ docs/operations.md | 5 +++ docs/roadmap/implementation.md | 2 + internal/framework/pipeline/debug.go | 56 ++++++++++++++++++---------- 4 files changed, 47 insertions(+), 20 deletions(-) diff --git a/docs/internal/state.md b/docs/internal/state.md index 86492735..e7d02886 100644 --- a/docs/internal/state.md +++ b/docs/internal/state.md @@ -1,5 +1,9 @@ # Run State Internals +LLM debug response artifacts and their safe per-call references retain an +optional terminal execution summary when one is available. This metadata is +diagnostic only and does not change receipts, checkpoints, or aggregate usage. + This document describes the implementation collaborators behind output, cache, and debug state. User-visible fields belong in [Configuration](../config.md), and physical layout, retention, recovery, reason codes, and cleanup belong in diff --git a/docs/operations.md b/docs/operations.md index d3454d62..cf9790be 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -147,6 +147,11 @@ their exit classes are defined in the [CLI reference](cli.md#output-streams-and- ## Validation Retries And Terminal Outcomes +When debug recording is enabled, per-call response artifacts include the +terminal execution summary for retry, repair, timing, and presence-aware usage +diagnosis. Normal output, receipts, and warnings do not aggregate or publish +this diagnostic metadata. + Each producer binding has one outer **retries** budget. It covers complete producer attempts for operational failures, invalid structured output, normalizer fallback retry, and semantic correction. It is independent from diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index 98675c6c..9f4ecf93 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -270,6 +270,8 @@ and `GOWORK=off go test -race ./internal/framework/llm`. ## Stage 6 — Carry Summaries Through Existing Debug Paths +**Status:** Completed 2026-09-25. + **Goal:** Make observations useful to operators without a new output channel. One prompt, limited to debug propagation and terminal allowlists. diff --git a/internal/framework/pipeline/debug.go b/internal/framework/pipeline/debug.go index bb8ef355..d4aefb78 100644 --- a/internal/framework/pipeline/debug.go +++ b/internal/framework/pipeline/debug.go @@ -113,13 +113,14 @@ type debugSerializedOutput struct { type debugStructuredCompletionRequest = contracts.DebugStructuredCompletionRequest type debugStructuredCompletionResponse struct { - Content string `json:"content,omitempty"` - Provider string `json:"provider,omitempty"` - Model string `json:"model,omitempty"` - ProfileID string `json:"profile_id,omitempty"` - PromptTokens int `json:"prompt_tokens,omitempty"` - CompletionTokens int `json:"completion_tokens,omitempty"` - TotalTokens int `json:"total_tokens,omitempty"` + Content string `json:"content,omitempty"` + Provider string `json:"provider,omitempty"` + Model string `json:"model,omitempty"` + ProfileID string `json:"profile_id,omitempty"` + PromptTokens int `json:"prompt_tokens,omitempty"` + CompletionTokens int `json:"completion_tokens,omitempty"` + TotalTokens int `json:"total_tokens,omitempty"` + Execution *contracts.LLMExecutionSummary `json:"execution,omitempty"` } type debugLLMPromptArtifact struct { @@ -133,21 +134,23 @@ type debugLLMResponseArtifact struct { Fallback *debugStructuredCompletionResponse `json:"fallback,omitempty"` ContentPath string `json:"content_path,omitempty"` Error string `json:"error,omitempty"` + Execution *contracts.LLMExecutionSummary `json:"execution,omitempty"` } type debugLLMCallReference struct { - CallID string `json:"call_id"` - PromptPath string `json:"prompt_path,omitempty"` - ResponsePath string `json:"response_path"` - ResponseContentPath string `json:"response_content_path,omitempty"` - PromptID string `json:"prompt_id,omitempty"` - ProfileID string `json:"profile_id,omitempty"` - Model string `json:"model,omitempty"` - PromptTokens int `json:"prompt_tokens,omitempty"` - CompletionTokens int `json:"completion_tokens,omitempty"` - TotalTokens int `json:"total_tokens,omitempty"` - RepairAttempts int `json:"repair_attempts,omitempty"` - Error bool `json:"error,omitempty"` + CallID string `json:"call_id"` + PromptPath string `json:"prompt_path,omitempty"` + ResponsePath string `json:"response_path"` + ResponseContentPath string `json:"response_content_path,omitempty"` + PromptID string `json:"prompt_id,omitempty"` + ProfileID string `json:"profile_id,omitempty"` + Model string `json:"model,omitempty"` + PromptTokens int `json:"prompt_tokens,omitempty"` + CompletionTokens int `json:"completion_tokens,omitempty"` + TotalTokens int `json:"total_tokens,omitempty"` + RepairAttempts int `json:"repair_attempts,omitempty"` + Error bool `json:"error,omitempty"` + Execution *contracts.LLMExecutionSummary `json:"execution,omitempty"` } type debugValidationOutcome struct { @@ -267,6 +270,7 @@ func (client *debugLLMClient) CompleteStructured(ctx context.Context, req contra Fallback: fallbackForArtifact, ContentPath: responseContentPath, Error: errorText, + Execution: cloneLLMExecutionSummary(response.Execution), }, Error: errorText, })) @@ -283,6 +287,7 @@ func (client *debugLLMClient) CompleteStructured(ctx context.Context, req contra TotalTokens: response.TotalTokens, RepairAttempts: response.RepairAttempts, Error: err != nil, + Execution: cloneLLMExecutionSummary(response.Execution), } if scope := debugLLMScopeFromContext(ctx); scope != nil { scope.record(callRef) @@ -637,6 +642,7 @@ func debugCompletionResponse(response contracts.StructuredCompletionResponse) de PromptTokens: response.PromptTokens, CompletionTokens: response.CompletionTokens, TotalTokens: response.TotalTokens, + Execution: cloneLLMExecutionSummary(response.Execution), } } @@ -651,7 +657,8 @@ func debugCompletionFallback(response contracts.StructuredCompletionResponse) *d fallback.ProfileID == "" && fallback.PromptTokens == 0 && fallback.CompletionTokens == 0 && - fallback.TotalTokens == 0 { + fallback.TotalTokens == 0 && + fallback.Execution == nil { return nil } return &fallback @@ -696,6 +703,15 @@ func cloneDebugFallbackWithoutContent(fallback *debugStructuredCompletionRespons } clone := *fallback clone.Content = "" + clone.Execution = cloneLLMExecutionSummary(fallback.Execution) + return &clone +} + +func cloneLLMExecutionSummary(summary *contracts.LLMExecutionSummary) *contracts.LLMExecutionSummary { + if summary == nil { + return nil + } + clone := *summary return &clone }