eric/notarius
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Record execution summaries in debug artifacts

Browse Source
This commit is contained in:
Eric Rakestraw
2026-09-25 22:02:56 +00:00
parent 6b8a4408c1
commit b09ce47d4a
4 changed files with 47 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
docs/internal/state.md
View File

@@ -1,5 +1,9 @@
# Run State Internals # Run State Internals
LLM debug response artifacts and their safe per-call references retain an
optional terminal execution summary when one is available. This metadata is
diagnostic only and does not change receipts, checkpoints, or aggregate usage.
This document describes the implementation collaborators behind output, cache, This document describes the implementation collaborators behind output, cache,
and debug state. User-visible fields belong in [Configuration](../config.md), and debug state. User-visible fields belong in [Configuration](../config.md),
and physical layout, retention, recovery, reason codes, and cleanup belong in and physical layout, retention, recovery, reason codes, and cleanup belong in

5
docs/operations.md
View File

@@ -147,6 +147,11 @@ their exit classes are defined in the [CLI reference](cli.md#output-streams-and-
## Validation Retries And Terminal Outcomes ## Validation Retries And Terminal Outcomes
When debug recording is enabled, per-call response artifacts include the
terminal execution summary for retry, repair, timing, and presence-aware usage
diagnosis. Normal output, receipts, and warnings do not aggregate or publish
this diagnostic metadata.
Each producer binding has one outer **retries** budget. It covers complete Each producer binding has one outer **retries** budget. It covers complete
producer attempts for operational failures, invalid structured output, producer attempts for operational failures, invalid structured output,
normalizer fallback retry, and semantic correction. It is independent from normalizer fallback retry, and semantic correction. It is independent from

2
docs/roadmap/implementation.md
View File

@@ -270,6 +270,8 @@ and `GOWORK=off go test -race ./internal/framework/llm`.
## Stage 6 — Carry Summaries Through Existing Debug Paths ## Stage 6 — Carry Summaries Through Existing Debug Paths
**Status:** Completed 2026-09-25.
**Goal:** Make observations useful to operators without a new output channel. **Goal:** Make observations useful to operators without a new output channel.
One prompt, limited to debug propagation and terminal allowlists. One prompt, limited to debug propagation and terminal allowlists.

56
internal/framework/pipeline/debug.go
View File

@@ -113,13 +113,14 @@ type debugSerializedOutput struct {
type debugStructuredCompletionRequest = contracts.DebugStructuredCompletionRequest type debugStructuredCompletionRequest = contracts.DebugStructuredCompletionRequest
type debugStructuredCompletionResponse struct { type debugStructuredCompletionResponse struct {
Content string `json:"content,omitempty"` Content string `json:"content,omitempty"`
Provider string `json:"provider,omitempty"` Provider string `json:"provider,omitempty"`
Model string `json:"model,omitempty"` Model string `json:"model,omitempty"`
ProfileID string `json:"profile_id,omitempty"` ProfileID string `json:"profile_id,omitempty"`
PromptTokens int `json:"prompt_tokens,omitempty"` PromptTokens int `json:"prompt_tokens,omitempty"`
CompletionTokens int `json:"completion_tokens,omitempty"` CompletionTokens int `json:"completion_tokens,omitempty"`
TotalTokens int `json:"total_tokens,omitempty"` TotalTokens int `json:"total_tokens,omitempty"`
Execution *contracts.LLMExecutionSummary `json:"execution,omitempty"`
} }
type debugLLMPromptArtifact struct { type debugLLMPromptArtifact struct {
@@ -133,21 +134,23 @@ type debugLLMResponseArtifact struct {
Fallback *debugStructuredCompletionResponse `json:"fallback,omitempty"` Fallback *debugStructuredCompletionResponse `json:"fallback,omitempty"`
ContentPath string `json:"content_path,omitempty"` ContentPath string `json:"content_path,omitempty"`
Error string `json:"error,omitempty"` Error string `json:"error,omitempty"`
Execution *contracts.LLMExecutionSummary `json:"execution,omitempty"`
} }
type debugLLMCallReference struct { type debugLLMCallReference struct {
CallID string `json:"call_id"` CallID string `json:"call_id"`
PromptPath string `json:"prompt_path,omitempty"` PromptPath string `json:"prompt_path,omitempty"`
ResponsePath string `json:"response_path"` ResponsePath string `json:"response_path"`
ResponseContentPath string `json:"response_content_path,omitempty"` ResponseContentPath string `json:"response_content_path,omitempty"`
PromptID string `json:"prompt_id,omitempty"` PromptID string `json:"prompt_id,omitempty"`
ProfileID string `json:"profile_id,omitempty"` ProfileID string `json:"profile_id,omitempty"`
Model string `json:"model,omitempty"` Model string `json:"model,omitempty"`
PromptTokens int `json:"prompt_tokens,omitempty"` PromptTokens int `json:"prompt_tokens,omitempty"`
CompletionTokens int `json:"completion_tokens,omitempty"` CompletionTokens int `json:"completion_tokens,omitempty"`
TotalTokens int `json:"total_tokens,omitempty"` TotalTokens int `json:"total_tokens,omitempty"`
RepairAttempts int `json:"repair_attempts,omitempty"` RepairAttempts int `json:"repair_attempts,omitempty"`
Error bool `json:"error,omitempty"` Error bool `json:"error,omitempty"`
Execution *contracts.LLMExecutionSummary `json:"execution,omitempty"`
} }
type debugValidationOutcome struct { type debugValidationOutcome struct {
@@ -267,6 +270,7 @@ func (client *debugLLMClient) CompleteStructured(ctx context.Context, req contra
Fallback: fallbackForArtifact, Fallback: fallbackForArtifact,
ContentPath: responseContentPath, ContentPath: responseContentPath,
Error: errorText, Error: errorText,
Execution: cloneLLMExecutionSummary(response.Execution),
}, },
Error: errorText, Error: errorText,
})) }))
@@ -283,6 +287,7 @@ func (client *debugLLMClient) CompleteStructured(ctx context.Context, req contra
TotalTokens: response.TotalTokens, TotalTokens: response.TotalTokens,
RepairAttempts: response.RepairAttempts, RepairAttempts: response.RepairAttempts,
Error: err != nil, Error: err != nil,
Execution: cloneLLMExecutionSummary(response.Execution),
} }
if scope := debugLLMScopeFromContext(ctx); scope != nil { if scope := debugLLMScopeFromContext(ctx); scope != nil {
scope.record(callRef) scope.record(callRef)
@@ -637,6 +642,7 @@ func debugCompletionResponse(response contracts.StructuredCompletionResponse) de
PromptTokens: response.PromptTokens, PromptTokens: response.PromptTokens,
CompletionTokens: response.CompletionTokens, CompletionTokens: response.CompletionTokens,
TotalTokens: response.TotalTokens, TotalTokens: response.TotalTokens,
Execution: cloneLLMExecutionSummary(response.Execution),
} }
} }
@@ -651,7 +657,8 @@ func debugCompletionFallback(response contracts.StructuredCompletionResponse) *d
fallback.ProfileID == "" && fallback.ProfileID == "" &&
fallback.PromptTokens == 0 && fallback.PromptTokens == 0 &&
fallback.CompletionTokens == 0 && fallback.CompletionTokens == 0 &&
fallback.TotalTokens == 0 { fallback.TotalTokens == 0 &&
fallback.Execution == nil {
return nil return nil
} }
return &fallback return &fallback
@@ -696,6 +703,15 @@ func cloneDebugFallbackWithoutContent(fallback *debugStructuredCompletionRespons
} }
clone := *fallback clone := *fallback
clone.Content = "" clone.Content = ""
clone.Execution = cloneLLMExecutionSummary(fallback.Execution)
return &clone
}
func cloneLLMExecutionSummary(summary *contracts.LLMExecutionSummary) *contracts.LLMExecutionSummary {
if summary == nil {
return nil
}
clone := *summary
return &clone return &clone
} }