eric/notarius
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Record execution summaries in debug artifacts

Browse Source
This commit is contained in:
Eric Rakestraw
2026-09-25 22:02:56 +00:00
parent 6b8a4408c1
commit b09ce47d4a
4 changed files with 47 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
docs/internal/state.md
View File

@@ -1,5 +1,9 @@
# Run State Internals
LLM debug response artifacts and their safe per-call references retain an
optional terminal execution summary when one is available. This metadata is
diagnostic only and does not change receipts, checkpoints, or aggregate usage.
This document describes the implementation collaborators behind output, cache,
and debug state. User-visible fields belong in [Configuration](../config.md),
and physical layout, retention, recovery, reason codes, and cleanup belong in

5
docs/operations.md
View File

@@ -147,6 +147,11 @@ their exit classes are defined in the [CLI reference](cli.md#output-streams-and-
## Validation Retries And Terminal Outcomes
When debug recording is enabled, per-call response artifacts include the
terminal execution summary for retry, repair, timing, and presence-aware usage
diagnosis. Normal output, receipts, and warnings do not aggregate or publish
this diagnostic metadata.
Each producer binding has one outer **retries** budget. It covers complete
producer attempts for operational failures, invalid structured output,
normalizer fallback retry, and semantic correction. It is independent from

2
docs/roadmap/implementation.md
View File

@@ -270,6 +270,8 @@ and `GOWORK=off go test -race ./internal/framework/llm`.
## Stage 6 — Carry Summaries Through Existing Debug Paths
**Status:** Completed 2026-09-25.
**Goal:** Make observations useful to operators without a new output channel.
One prompt, limited to debug propagation and terminal allowlists.

56
internal/framework/pipeline/debug.go
View File

@@ -113,13 +113,14 @@ type debugSerializedOutput struct {
type debugStructuredCompletionRequest = contracts.DebugStructuredCompletionRequest
type debugStructuredCompletionResponse struct {
Content string `json:"content,omitempty"`
Provider string `json:"provider,omitempty"`
Model string `json:"model,omitempty"`
ProfileID string `json:"profile_id,omitempty"`
PromptTokens int `json:"prompt_tokens,omitempty"`
CompletionTokens int `json:"completion_tokens,omitempty"`
TotalTokens int `json:"total_tokens,omitempty"`
Content string `json:"content,omitempty"`
Provider string `json:"provider,omitempty"`
Model string `json:"model,omitempty"`
ProfileID string `json:"profile_id,omitempty"`
PromptTokens int `json:"prompt_tokens,omitempty"`
CompletionTokens int `json:"completion_tokens,omitempty"`
TotalTokens int `json:"total_tokens,omitempty"`
Execution *contracts.LLMExecutionSummary `json:"execution,omitempty"`
}
type debugLLMPromptArtifact struct {
@@ -133,21 +134,23 @@ type debugLLMResponseArtifact struct {
Fallback *debugStructuredCompletionResponse `json:"fallback,omitempty"`
ContentPath string `json:"content_path,omitempty"`
Error string `json:"error,omitempty"`
Execution *contracts.LLMExecutionSummary `json:"execution,omitempty"`
}
type debugLLMCallReference struct {
CallID string `json:"call_id"`
PromptPath string `json:"prompt_path,omitempty"`
ResponsePath string `json:"response_path"`
ResponseContentPath string `json:"response_content_path,omitempty"`
PromptID string `json:"prompt_id,omitempty"`
ProfileID string `json:"profile_id,omitempty"`
Model string `json:"model,omitempty"`
PromptTokens int `json:"prompt_tokens,omitempty"`
CompletionTokens int `json:"completion_tokens,omitempty"`
TotalTokens int `json:"total_tokens,omitempty"`
RepairAttempts int `json:"repair_attempts,omitempty"`
Error bool `json:"error,omitempty"`
CallID string `json:"call_id"`
PromptPath string `json:"prompt_path,omitempty"`
ResponsePath string `json:"response_path"`
ResponseContentPath string `json:"response_content_path,omitempty"`
PromptID string `json:"prompt_id,omitempty"`
ProfileID string `json:"profile_id,omitempty"`
Model string `json:"model,omitempty"`
PromptTokens int `json:"prompt_tokens,omitempty"`
CompletionTokens int `json:"completion_tokens,omitempty"`
TotalTokens int `json:"total_tokens,omitempty"`
RepairAttempts int `json:"repair_attempts,omitempty"`
Error bool `json:"error,omitempty"`
Execution *contracts.LLMExecutionSummary `json:"execution,omitempty"`
}
type debugValidationOutcome struct {
@@ -267,6 +270,7 @@ func (client *debugLLMClient) CompleteStructured(ctx context.Context, req contra
Fallback: fallbackForArtifact,
ContentPath: responseContentPath,
Error: errorText,
Execution: cloneLLMExecutionSummary(response.Execution),
},
Error: errorText,
}))
@@ -283,6 +287,7 @@ func (client *debugLLMClient) CompleteStructured(ctx context.Context, req contra
TotalTokens: response.TotalTokens,
RepairAttempts: response.RepairAttempts,
Error: err != nil,
Execution: cloneLLMExecutionSummary(response.Execution),
}
if scope := debugLLMScopeFromContext(ctx); scope != nil {
scope.record(callRef)
@@ -637,6 +642,7 @@ func debugCompletionResponse(response contracts.StructuredCompletionResponse) de
PromptTokens: response.PromptTokens,
CompletionTokens: response.CompletionTokens,
TotalTokens: response.TotalTokens,
Execution: cloneLLMExecutionSummary(response.Execution),
}
}
@@ -651,7 +657,8 @@ func debugCompletionFallback(response contracts.StructuredCompletionResponse) *d
fallback.ProfileID == "" &&
fallback.PromptTokens == 0 &&
fallback.CompletionTokens == 0 &&
fallback.TotalTokens == 0 {
fallback.TotalTokens == 0 &&
fallback.Execution == nil {
return nil
}
return &fallback
@@ -696,6 +703,15 @@ func cloneDebugFallbackWithoutContent(fallback *debugStructuredCompletionRespons
}
clone := *fallback
clone.Content = ""
clone.Execution = cloneLLMExecutionSummary(fallback.Execution)
return &clone
}
func cloneLLMExecutionSummary(summary *contracts.LLMExecutionSummary) *contracts.LLMExecutionSummary {
if summary == nil {
return nil
}
clone := *summary
return &clone
}