eric/notarius
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add LLM execution summaries

Browse Source
This commit is contained in:
Eric Rakestraw
2026-09-25 22:01:16 +00:00
parent bd83255a8e
commit 6b8a4408c1
5 changed files with 108 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
docs/internal/llm.md
View File

@@ -223,6 +223,12 @@ also records that exhaustion and the attempts in that generation. Those fields
do not include repair generations, and zero attempts means unavailable rather
than proof that no request was sent.
Every execution that reaches PromptKit returns an optional request-local
`execution` summary. It contains only terminal aggregate attempts, retry and
repair counts, durations, outcome and failure classification, status, and
presence-aware token usage. Preparation failures have no summary; later local
decode failures retain the upstream terminal summary without changing it.
When PromptKit rejects backend admission before generation, the adapter maps
`promptkit.ErrCapacityExceeded` to
`contracts.ErrLLMCapacityExceeded`, retaining prompt context and a redacted

2
docs/roadmap/implementation.md
View File

@@ -218,6 +218,8 @@ and `git diff --check`.
## Stage 5 — Project Request-Local Execution Observations
**Status:** Completed 2026-09-25.
**Goal:** Return a lightweight terminal summary even when generation fails.
One prompt focused on contracts and adapter mapping, without persistence changes.

48
internal/framework/contracts/contracts.go
View File

@@ -21,15 +21,45 @@ type StructuredCompletionRequest struct {
}
type StructuredCompletionResponse struct {
Content json.RawMessage `json:"content"`
Provider string `json:"provider,omitempty"`
Model string `json:"model,omitempty"`
ProfileID string `json:"profile_id,omitempty"`
PromptTokens int `json:"prompt_tokens,omitempty"`
CompletionTokens int `json:"completion_tokens,omitempty"`
TotalTokens int `json:"total_tokens,omitempty"`
RepairAttempts int `json:"repair_attempts,omitempty"`
Debug *LLMDebugMaterial `json:"debug,omitempty"`
Content json.RawMessage `json:"content"`
Provider string `json:"provider,omitempty"`
Model string `json:"model,omitempty"`
ProfileID string `json:"profile_id,omitempty"`
PromptTokens int `json:"prompt_tokens,omitempty"`
CompletionTokens int `json:"completion_tokens,omitempty"`
TotalTokens int `json:"total_tokens,omitempty"`
RepairAttempts int `json:"repair_attempts,omitempty"`
Debug *LLMDebugMaterial `json:"debug,omitempty"`
Execution *LLMExecutionSummary `json:"execution,omitempty"`
}
type LLMObservedTokenCount struct {
Value int `json:"value"`
Known bool `json:"known"`
Complete bool `json:"complete"`
}
type LLMObservedTokenUsage struct {
PromptTokens LLMObservedTokenCount `json:"prompt_tokens"`
CompletionTokens LLMObservedTokenCount `json:"completion_tokens"`
TotalTokens LLMObservedTokenCount `json:"total_tokens"`
CachedTokens LLMObservedTokenCount `json:"cached_tokens"`
CacheWriteTokens LLMObservedTokenCount `json:"cache_write_tokens"`
}
type LLMExecutionSummary struct {
Attempts uint64 `json:"attempts"`
TransientRetries uint64 `json:"transient_retries"`
RepairCalls uint64 `json:"repair_calls"`
CapacityWaitMS int64 `json:"capacity_wait_ms"`
GenerationMS int64 `json:"generation_ms"`
RetryDelayMS int64 `json:"retry_delay_ms"`
DurationMS int64 `json:"duration_ms"`
Outcome string `json:"outcome"`
FailureKind string `json:"failure_kind"`
StatusCode int `json:"status_code"`
RetriesExhausted bool `json:"retries_exhausted"`
Usage LLMObservedTokenUsage `json:"usage"`
}
type StructuredLLMClient interface {

67
internal/framework/llm/promptkit_client.go
View File

@@ -130,10 +130,10 @@ func (c *PromptKitClient) CompleteStructured(ctx context.Context, req contracts.
return contracts.StructuredCompletionResponse{}, fmt.Errorf("structured completion prompt_id must not be empty")
}
sessionID := strings.TrimSpace(req.SessionID)
var execution *promptkit.ExecutionTargetOverride
var targetOverride *promptkit.ExecutionTargetOverride
if c.reasoningEffort != nil {
reasoningEffort := *c.reasoningEffort
execution = &promptkit.ExecutionTargetOverride{
targetOverride = &promptkit.ExecutionTargetOverride{
ReasoningEffort: &reasoningEffort,
}
}
@@ -145,9 +145,15 @@ func (c *PromptKitClient) CompleteStructured(ctx context.Context, req contracts.
SessionID: sessionID,
Inputs: promptKitInputs(req.Inputs),
Vars: promptKitVars(req, sessionID),
Execution: execution,
Execution: targetOverride,
AppendedMessages: appendedMessages,
}
var executionSummary *contracts.LLMExecutionSummary
runReq.Observer = func(event promptkit.ExecutionEvent) {
if event.Kind == promptkit.ExecutionFinished {
executionSummary = promptKitExecutionSummary(event)
}
}
if req.StructuredOutputRepairAttempts != nil {
inspection, err := c.engine.InspectPrompt(ctx, promptID, strings.TrimSpace(req.PromptVersion))
if err != nil {
@@ -172,12 +178,12 @@ func (c *PromptKitClient) CompleteStructured(ctx context.Context, req contracts.
result, err := c.engine.RunPrepared(ctx, prepared)
if err != nil {
if ctxErr := ctx.Err(); ctxErr != nil {
return contracts.StructuredCompletionResponse{}, ctxErr
return promptKitResponseWithExecution(contracts.StructuredCompletionResponse{}, executionSummary), ctxErr
}
if errors.Is(err, promptkit.ErrCapacityExceeded) {
var capacityErr *promptkit.CapacityError
if errors.As(err, &capacityErr) && strings.TrimSpace(capacityErr.BackendID) != "" {
return contracts.StructuredCompletionResponse{}, fmt.Errorf(
return promptKitResponseWithExecution(contracts.StructuredCompletionResponse{}, executionSummary), fmt.Errorf(
"run PromptKit prompt %q on backend %q: %w: %v",
promptID,
strings.TrimSpace(capacityErr.BackendID),
@@ -185,7 +191,7 @@ func (c *PromptKitClient) CompleteStructured(ctx context.Context, req contracts.
redactPromptKitError(err),
)
}
return contracts.StructuredCompletionResponse{}, fmt.Errorf(
return promptKitResponseWithExecution(contracts.StructuredCompletionResponse{}, executionSummary), fmt.Errorf(
"run PromptKit prompt %q: %w: %v",
promptID,
contracts.ErrLLMCapacityExceeded,
@@ -206,14 +212,14 @@ func (c *PromptKitClient) CompleteStructured(ctx context.Context, req contracts.
status = generationErr.StatusCode()
response.Debug.Response = promptKitDebugGenerationError(&preparedDetails, generationErr)
}
return response, contracts.NewLLMGenerationErrorWithMetadata(status, fmt.Sprintf("run PromptKit prompt %q: %v", promptID, redactPromptKitError(err)), metadata)
return promptKitResponseWithExecution(response, executionSummary), contracts.NewLLMGenerationErrorWithMetadata(status, fmt.Sprintf("run PromptKit prompt %q: %v", promptID, redactPromptKitError(err)), metadata)
}
return contracts.StructuredCompletionResponse{}, fmt.Errorf("run PromptKit prompt %q: %v", promptID, redactPromptKitError(err))
return promptKitResponseWithExecution(contracts.StructuredCompletionResponse{}, executionSummary), fmt.Errorf("run PromptKit prompt %q: %v", promptID, redactPromptKitError(err))
}
if result == nil {
return contracts.StructuredCompletionResponse{}, fmt.Errorf("run PromptKit prompt %q: %w: empty result", promptID, contracts.ErrInvalidStructuredOutput)
return promptKitResponseWithExecution(contracts.StructuredCompletionResponse{}, executionSummary), fmt.Errorf("run PromptKit prompt %q: %w: empty result", promptID, contracts.ErrInvalidStructuredOutput)
}
response := c.responseFromResult(result, &preparedDetails)
response := promptKitResponseWithExecution(c.responseFromResult(result, &preparedDetails), executionSummary)
if result.Validation.Status == promptkit.ValidationFailed || !result.Validation.IsValid {
return response, fmt.Errorf("run PromptKit prompt %q: %w: validation failed: %s", promptID, contracts.ErrInvalidStructuredOutput, strings.Join(result.Validation.Errors, "; "))
}
@@ -226,6 +232,47 @@ func (c *PromptKitClient) CompleteStructured(ctx context.Context, req contracts.
return response, nil
}
func promptKitResponseWithExecution(response contracts.StructuredCompletionResponse, execution *contracts.LLMExecutionSummary) contracts.StructuredCompletionResponse {
response.Execution = execution
return response
}
func promptKitExecutionSummary(event promptkit.ExecutionEvent) *contracts.LLMExecutionSummary {
count := func(value promptkit.ObservedTokenCount) contracts.LLMObservedTokenCount {
return contracts.LLMObservedTokenCount{Value: value.Value, Known: value.Known, Complete: value.Complete}
}
return &contracts.LLMExecutionSummary{
Attempts: event.Attempts, TransientRetries: event.TransientRetries, RepairCalls: event.RepairCalls,
CapacityWaitMS: event.CapacityWait.Milliseconds(), GenerationMS: event.GenerationDuration.Milliseconds(), RetryDelayMS: event.RetryDelay.Milliseconds(), DurationMS: event.Duration.Milliseconds(),
Outcome: promptKitExecutionOutcome(event.Outcome), FailureKind: promptKitExecutionFailure(event.Failure), StatusCode: event.StatusCode, RetriesExhausted: event.RetriesExhausted,
Usage: contracts.LLMObservedTokenUsage{PromptTokens: count(event.Usage.PromptTokens), CompletionTokens: count(event.Usage.CompletionTokens), TotalTokens: count(event.Usage.TotalTokens), CachedTokens: count(event.Usage.CachedTokens), CacheWriteTokens: count(event.Usage.CacheWriteTokens)},
}
}
func promptKitExecutionOutcome(value promptkit.ExecutionOutcome) string {
switch value {
case promptkit.ExecutionOutcomeSuccess:
return "success"
case promptkit.ExecutionOutcomeInvalidOutput:
return "invalid_output"
case promptkit.ExecutionOutcomeError:
return "error"
default:
return "other"
}
}
func promptKitExecutionFailure(value promptkit.ExecutionFailureKind) string {
switch value {
case promptkit.ExecutionFailureNone:
return ""
case promptkit.ExecutionFailureCanceled, promptkit.ExecutionFailureDeadline, promptkit.ExecutionFailureTransport, promptkit.ExecutionFailureProviderStatus, promptkit.ExecutionFailureMalformedResponse, promptkit.ExecutionFailureCapacity, promptkit.ExecutionFailureInvalidConfig, promptkit.ExecutionFailureInvalidRequest, promptkit.ExecutionFailurePromptNotFound, promptkit.ExecutionFailureProfileNotFound, promptkit.ExecutionFailurePromptLoad, promptkit.ExecutionFailureProfileLoad, promptkit.ExecutionFailureArtifactLoad, promptkit.ExecutionFailurePromptRender, promptkit.ExecutionFailureValidation, promptkit.ExecutionFailureGeneration:
return string(value)
default:
return "other"
}
}
func promptKitGenerationFailureKind(err error) contracts.LLMGenerationFailureKind {
switch {
case errors.Is(err, promptkit.ErrTransport):

4
internal/framework/llm/promptkit_client_test.go
View File

@@ -55,6 +55,10 @@ func TestPromptKitClientMapsPromptRequestAndUnmarshalsOutput(t *testing.T) {
if resp.PromptTokens != 11 || resp.CompletionTokens != 7 || resp.TotalTokens != 18 {
t.Fatalf("usage = %#v, want mapped token counts", resp)
}
if resp.Execution == nil || resp.Execution.Outcome != "success" || resp.Execution.Attempts != 1 ||
!resp.Execution.Usage.PromptTokens.Known || resp.Execution.Usage.PromptTokens.Value != 11 {
t.Fatalf("execution summary = %#v", resp.Execution)
}
if resp.Debug == nil || resp.Debug.Prompt == nil {
t.Fatalf("debug prompt = nil, want prepared prompt material")
}