Add LLM execution summaries

docs/internal/llm.md

@@ -223,6 +223,12 @@ also records that exhaustion and the attempts in that generation. Those fields
 do not include repair generations, and zero attempts means unavailable rather
 than proof that no request was sent.
 
+Every execution that reaches PromptKit returns an optional request-local
+`execution` summary. It contains only terminal aggregate attempts, retry and
+repair counts, durations, outcome and failure classification, status, and
+presence-aware token usage. Preparation failures have no summary; later local
+decode failures retain the upstream terminal summary without changing it.
+
 When PromptKit rejects backend admission before generation, the adapter maps
 `promptkit.ErrCapacityExceeded` to
 `contracts.ErrLLMCapacityExceeded`, retaining prompt context and a redacted

docs/roadmap/implementation.md

@@ -218,6 +218,8 @@ and `git diff --check`.
 
 ## Stage 5 — Project Request-Local Execution Observations
 
+**Status:** Completed 2026-09-25.
+
 **Goal:** Return a lightweight terminal summary even when generation fails.
 One prompt focused on contracts and adapter mapping, without persistence changes.
 

internal/framework/contracts/contracts.go

@@ -21,15 +21,45 @@ type StructuredCompletionRequest struct {
 }
 
 type StructuredCompletionResponse struct {
-	Content          json.RawMessage   `json:"content"`
+	Content          json.RawMessage      `json:"content"`
-	Provider         string            `json:"provider,omitempty"`
+	Provider         string               `json:"provider,omitempty"`
-	Model            string            `json:"model,omitempty"`
+	Model            string               `json:"model,omitempty"`
-	ProfileID        string            `json:"profile_id,omitempty"`
+	ProfileID        string               `json:"profile_id,omitempty"`
-	PromptTokens     int               `json:"prompt_tokens,omitempty"`
+	PromptTokens     int                  `json:"prompt_tokens,omitempty"`
-	CompletionTokens int               `json:"completion_tokens,omitempty"`
+	CompletionTokens int                  `json:"completion_tokens,omitempty"`
-	TotalTokens      int               `json:"total_tokens,omitempty"`
+	TotalTokens      int                  `json:"total_tokens,omitempty"`
-	RepairAttempts   int               `json:"repair_attempts,omitempty"`
+	RepairAttempts   int                  `json:"repair_attempts,omitempty"`
-	Debug            *LLMDebugMaterial `json:"debug,omitempty"`
+	Debug            *LLMDebugMaterial    `json:"debug,omitempty"`
+	Execution        *LLMExecutionSummary `json:"execution,omitempty"`
+}
+
+type LLMObservedTokenCount struct {
+	Value    int  `json:"value"`
+	Known    bool `json:"known"`
+	Complete bool `json:"complete"`
+}
+
+type LLMObservedTokenUsage struct {
+	PromptTokens     LLMObservedTokenCount `json:"prompt_tokens"`
+	CompletionTokens LLMObservedTokenCount `json:"completion_tokens"`
+	TotalTokens      LLMObservedTokenCount `json:"total_tokens"`
+	CachedTokens     LLMObservedTokenCount `json:"cached_tokens"`
+	CacheWriteTokens LLMObservedTokenCount `json:"cache_write_tokens"`
+}
+
+type LLMExecutionSummary struct {
+	Attempts         uint64                `json:"attempts"`
+	TransientRetries uint64                `json:"transient_retries"`
+	RepairCalls      uint64                `json:"repair_calls"`
+	CapacityWaitMS   int64                 `json:"capacity_wait_ms"`
+	GenerationMS     int64                 `json:"generation_ms"`
+	RetryDelayMS     int64                 `json:"retry_delay_ms"`
+	DurationMS       int64                 `json:"duration_ms"`
+	Outcome          string                `json:"outcome"`
+	FailureKind      string                `json:"failure_kind"`
+	StatusCode       int                   `json:"status_code"`
+	RetriesExhausted bool                  `json:"retries_exhausted"`
+	Usage            LLMObservedTokenUsage `json:"usage"`
 }
 
 type StructuredLLMClient interface {

internal/framework/llm/promptkit_client.go

@@ -130,10 +130,10 @@ func (c *PromptKitClient) CompleteStructured(ctx context.Context, req contracts.
 		return contracts.StructuredCompletionResponse{}, fmt.Errorf("structured completion prompt_id must not be empty")
 	}
 	sessionID := strings.TrimSpace(req.SessionID)
-	var execution *promptkit.ExecutionTargetOverride
+	var targetOverride *promptkit.ExecutionTargetOverride
 	if c.reasoningEffort != nil {
 		reasoningEffort := *c.reasoningEffort
-		execution = &promptkit.ExecutionTargetOverride{
+		targetOverride = &promptkit.ExecutionTargetOverride{
 			ReasoningEffort: &reasoningEffort,
 		}
 	}
@@ -145,9 +145,15 @@ func (c *PromptKitClient) CompleteStructured(ctx context.Context, req contracts.
 		SessionID:        sessionID,
 		Inputs:           promptKitInputs(req.Inputs),
 		Vars:             promptKitVars(req, sessionID),
-		Execution:        execution,
+		Execution:        targetOverride,
 		AppendedMessages: appendedMessages,
 	}
+	var executionSummary *contracts.LLMExecutionSummary
+	runReq.Observer = func(event promptkit.ExecutionEvent) {
+		if event.Kind == promptkit.ExecutionFinished {
+			executionSummary = promptKitExecutionSummary(event)
+		}
+	}
 	if req.StructuredOutputRepairAttempts != nil {
 		inspection, err := c.engine.InspectPrompt(ctx, promptID, strings.TrimSpace(req.PromptVersion))
 		if err != nil {
@@ -172,12 +178,12 @@ func (c *PromptKitClient) CompleteStructured(ctx context.Context, req contracts.
 	result, err := c.engine.RunPrepared(ctx, prepared)
 	if err != nil {
 		if ctxErr := ctx.Err(); ctxErr != nil {
-			return contracts.StructuredCompletionResponse{}, ctxErr
+			return promptKitResponseWithExecution(contracts.StructuredCompletionResponse{}, executionSummary), ctxErr
 		}
 		if errors.Is(err, promptkit.ErrCapacityExceeded) {
 			var capacityErr *promptkit.CapacityError
 			if errors.As(err, &capacityErr) && strings.TrimSpace(capacityErr.BackendID) != "" {
-				return contracts.StructuredCompletionResponse{}, fmt.Errorf(
+				return promptKitResponseWithExecution(contracts.StructuredCompletionResponse{}, executionSummary), fmt.Errorf(
 					"run PromptKit prompt %q on backend %q: %w: %v",
 					promptID,
 					strings.TrimSpace(capacityErr.BackendID),
@@ -185,7 +191,7 @@ func (c *PromptKitClient) CompleteStructured(ctx context.Context, req contracts.
 					redactPromptKitError(err),
 				)
 			}
-			return contracts.StructuredCompletionResponse{}, fmt.Errorf(
+			return promptKitResponseWithExecution(contracts.StructuredCompletionResponse{}, executionSummary), fmt.Errorf(
 				"run PromptKit prompt %q: %w: %v",
 				promptID,
 				contracts.ErrLLMCapacityExceeded,
@@ -206,14 +212,14 @@ func (c *PromptKitClient) CompleteStructured(ctx context.Context, req contracts.
 				status = generationErr.StatusCode()
 				response.Debug.Response = promptKitDebugGenerationError(&preparedDetails, generationErr)
 			}
-			return response, contracts.NewLLMGenerationErrorWithMetadata(status, fmt.Sprintf("run PromptKit prompt %q: %v", promptID, redactPromptKitError(err)), metadata)
+			return promptKitResponseWithExecution(response, executionSummary), contracts.NewLLMGenerationErrorWithMetadata(status, fmt.Sprintf("run PromptKit prompt %q: %v", promptID, redactPromptKitError(err)), metadata)
 		}
-		return contracts.StructuredCompletionResponse{}, fmt.Errorf("run PromptKit prompt %q: %v", promptID, redactPromptKitError(err))
+		return promptKitResponseWithExecution(contracts.StructuredCompletionResponse{}, executionSummary), fmt.Errorf("run PromptKit prompt %q: %v", promptID, redactPromptKitError(err))
 	}
 	if result == nil {
-		return contracts.StructuredCompletionResponse{}, fmt.Errorf("run PromptKit prompt %q: %w: empty result", promptID, contracts.ErrInvalidStructuredOutput)
+		return promptKitResponseWithExecution(contracts.StructuredCompletionResponse{}, executionSummary), fmt.Errorf("run PromptKit prompt %q: %w: empty result", promptID, contracts.ErrInvalidStructuredOutput)
 	}
-	response := c.responseFromResult(result, &preparedDetails)
+	response := promptKitResponseWithExecution(c.responseFromResult(result, &preparedDetails), executionSummary)
 	if result.Validation.Status == promptkit.ValidationFailed || !result.Validation.IsValid {
 		return response, fmt.Errorf("run PromptKit prompt %q: %w: validation failed: %s", promptID, contracts.ErrInvalidStructuredOutput, strings.Join(result.Validation.Errors, "; "))
 	}
@@ -226,6 +232,47 @@ func (c *PromptKitClient) CompleteStructured(ctx context.Context, req contracts.
 	return response, nil
 }
 
+func promptKitResponseWithExecution(response contracts.StructuredCompletionResponse, execution *contracts.LLMExecutionSummary) contracts.StructuredCompletionResponse {
+	response.Execution = execution
+	return response
+}
+
+func promptKitExecutionSummary(event promptkit.ExecutionEvent) *contracts.LLMExecutionSummary {
+	count := func(value promptkit.ObservedTokenCount) contracts.LLMObservedTokenCount {
+		return contracts.LLMObservedTokenCount{Value: value.Value, Known: value.Known, Complete: value.Complete}
+	}
+	return &contracts.LLMExecutionSummary{
+		Attempts: event.Attempts, TransientRetries: event.TransientRetries, RepairCalls: event.RepairCalls,
+		CapacityWaitMS: event.CapacityWait.Milliseconds(), GenerationMS: event.GenerationDuration.Milliseconds(), RetryDelayMS: event.RetryDelay.Milliseconds(), DurationMS: event.Duration.Milliseconds(),
+		Outcome: promptKitExecutionOutcome(event.Outcome), FailureKind: promptKitExecutionFailure(event.Failure), StatusCode: event.StatusCode, RetriesExhausted: event.RetriesExhausted,
+		Usage: contracts.LLMObservedTokenUsage{PromptTokens: count(event.Usage.PromptTokens), CompletionTokens: count(event.Usage.CompletionTokens), TotalTokens: count(event.Usage.TotalTokens), CachedTokens: count(event.Usage.CachedTokens), CacheWriteTokens: count(event.Usage.CacheWriteTokens)},
+	}
+}
+
+func promptKitExecutionOutcome(value promptkit.ExecutionOutcome) string {
+	switch value {
+	case promptkit.ExecutionOutcomeSuccess:
+		return "success"
+	case promptkit.ExecutionOutcomeInvalidOutput:
+		return "invalid_output"
+	case promptkit.ExecutionOutcomeError:
+		return "error"
+	default:
+		return "other"
+	}
+}
+
+func promptKitExecutionFailure(value promptkit.ExecutionFailureKind) string {
+	switch value {
+	case promptkit.ExecutionFailureNone:
+		return ""
+	case promptkit.ExecutionFailureCanceled, promptkit.ExecutionFailureDeadline, promptkit.ExecutionFailureTransport, promptkit.ExecutionFailureProviderStatus, promptkit.ExecutionFailureMalformedResponse, promptkit.ExecutionFailureCapacity, promptkit.ExecutionFailureInvalidConfig, promptkit.ExecutionFailureInvalidRequest, promptkit.ExecutionFailurePromptNotFound, promptkit.ExecutionFailureProfileNotFound, promptkit.ExecutionFailurePromptLoad, promptkit.ExecutionFailureProfileLoad, promptkit.ExecutionFailureArtifactLoad, promptkit.ExecutionFailurePromptRender, promptkit.ExecutionFailureValidation, promptkit.ExecutionFailureGeneration:
+		return string(value)
+	default:
+		return "other"
+	}
+}
+
 func promptKitGenerationFailureKind(err error) contracts.LLMGenerationFailureKind {
 	switch {
 	case errors.Is(err, promptkit.ErrTransport):

internal/framework/llm/promptkit_client_test.go

@@ -55,6 +55,10 @@ func TestPromptKitClientMapsPromptRequestAndUnmarshalsOutput(t *testing.T) {
 	if resp.PromptTokens != 11 || resp.CompletionTokens != 7 || resp.TotalTokens != 18 {
 		t.Fatalf("usage = %#v, want mapped token counts", resp)
 	}
+	if resp.Execution == nil || resp.Execution.Outcome != "success" || resp.Execution.Attempts != 1 ||
+		!resp.Execution.Usage.PromptTokens.Known || resp.Execution.Usage.PromptTokens.Value != 11 {
+		t.Fatalf("execution summary = %#v", resp.Execution)
+	}
 	if resp.Debug == nil || resp.Debug.Prompt == nil {
 		t.Fatalf("debug prompt = nil, want prepared prompt material")
 	}