diff --git a/docs/internal/llm.md b/docs/internal/llm.md index a3318ce6..6c71401e 100644 --- a/docs/internal/llm.md +++ b/docs/internal/llm.md @@ -223,6 +223,12 @@ also records that exhaustion and the attempts in that generation. Those fields do not include repair generations, and zero attempts means unavailable rather than proof that no request was sent. +Every execution that reaches PromptKit returns an optional request-local +`execution` summary. It contains only terminal aggregate attempts, retry and +repair counts, durations, outcome and failure classification, status, and +presence-aware token usage. Preparation failures have no summary; later local +decode failures retain the upstream terminal summary without changing it. + When PromptKit rejects backend admission before generation, the adapter maps `promptkit.ErrCapacityExceeded` to `contracts.ErrLLMCapacityExceeded`, retaining prompt context and a redacted diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index b5be18c2..98675c6c 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -218,6 +218,8 @@ and `git diff --check`. ## Stage 5 — Project Request-Local Execution Observations +**Status:** Completed 2026-09-25. + **Goal:** Return a lightweight terminal summary even when generation fails. One prompt focused on contracts and adapter mapping, without persistence changes. diff --git a/internal/framework/contracts/contracts.go b/internal/framework/contracts/contracts.go index 4712965e..61725862 100644 --- a/internal/framework/contracts/contracts.go +++ b/internal/framework/contracts/contracts.go @@ -21,15 +21,45 @@ type StructuredCompletionRequest struct { } type StructuredCompletionResponse struct { - Content json.RawMessage `json:"content"` - Provider string `json:"provider,omitempty"` - Model string `json:"model,omitempty"` - ProfileID string `json:"profile_id,omitempty"` - PromptTokens int `json:"prompt_tokens,omitempty"` - CompletionTokens int `json:"completion_tokens,omitempty"` - TotalTokens int `json:"total_tokens,omitempty"` - RepairAttempts int `json:"repair_attempts,omitempty"` - Debug *LLMDebugMaterial `json:"debug,omitempty"` + Content json.RawMessage `json:"content"` + Provider string `json:"provider,omitempty"` + Model string `json:"model,omitempty"` + ProfileID string `json:"profile_id,omitempty"` + PromptTokens int `json:"prompt_tokens,omitempty"` + CompletionTokens int `json:"completion_tokens,omitempty"` + TotalTokens int `json:"total_tokens,omitempty"` + RepairAttempts int `json:"repair_attempts,omitempty"` + Debug *LLMDebugMaterial `json:"debug,omitempty"` + Execution *LLMExecutionSummary `json:"execution,omitempty"` +} + +type LLMObservedTokenCount struct { + Value int `json:"value"` + Known bool `json:"known"` + Complete bool `json:"complete"` +} + +type LLMObservedTokenUsage struct { + PromptTokens LLMObservedTokenCount `json:"prompt_tokens"` + CompletionTokens LLMObservedTokenCount `json:"completion_tokens"` + TotalTokens LLMObservedTokenCount `json:"total_tokens"` + CachedTokens LLMObservedTokenCount `json:"cached_tokens"` + CacheWriteTokens LLMObservedTokenCount `json:"cache_write_tokens"` +} + +type LLMExecutionSummary struct { + Attempts uint64 `json:"attempts"` + TransientRetries uint64 `json:"transient_retries"` + RepairCalls uint64 `json:"repair_calls"` + CapacityWaitMS int64 `json:"capacity_wait_ms"` + GenerationMS int64 `json:"generation_ms"` + RetryDelayMS int64 `json:"retry_delay_ms"` + DurationMS int64 `json:"duration_ms"` + Outcome string `json:"outcome"` + FailureKind string `json:"failure_kind"` + StatusCode int `json:"status_code"` + RetriesExhausted bool `json:"retries_exhausted"` + Usage LLMObservedTokenUsage `json:"usage"` } type StructuredLLMClient interface { diff --git a/internal/framework/llm/promptkit_client.go b/internal/framework/llm/promptkit_client.go index 3c5e39bd..1c6f4e8d 100644 --- a/internal/framework/llm/promptkit_client.go +++ b/internal/framework/llm/promptkit_client.go @@ -130,10 +130,10 @@ func (c *PromptKitClient) CompleteStructured(ctx context.Context, req contracts. return contracts.StructuredCompletionResponse{}, fmt.Errorf("structured completion prompt_id must not be empty") } sessionID := strings.TrimSpace(req.SessionID) - var execution *promptkit.ExecutionTargetOverride + var targetOverride *promptkit.ExecutionTargetOverride if c.reasoningEffort != nil { reasoningEffort := *c.reasoningEffort - execution = &promptkit.ExecutionTargetOverride{ + targetOverride = &promptkit.ExecutionTargetOverride{ ReasoningEffort: &reasoningEffort, } } @@ -145,9 +145,15 @@ func (c *PromptKitClient) CompleteStructured(ctx context.Context, req contracts. SessionID: sessionID, Inputs: promptKitInputs(req.Inputs), Vars: promptKitVars(req, sessionID), - Execution: execution, + Execution: targetOverride, AppendedMessages: appendedMessages, } + var executionSummary *contracts.LLMExecutionSummary + runReq.Observer = func(event promptkit.ExecutionEvent) { + if event.Kind == promptkit.ExecutionFinished { + executionSummary = promptKitExecutionSummary(event) + } + } if req.StructuredOutputRepairAttempts != nil { inspection, err := c.engine.InspectPrompt(ctx, promptID, strings.TrimSpace(req.PromptVersion)) if err != nil { @@ -172,12 +178,12 @@ func (c *PromptKitClient) CompleteStructured(ctx context.Context, req contracts. result, err := c.engine.RunPrepared(ctx, prepared) if err != nil { if ctxErr := ctx.Err(); ctxErr != nil { - return contracts.StructuredCompletionResponse{}, ctxErr + return promptKitResponseWithExecution(contracts.StructuredCompletionResponse{}, executionSummary), ctxErr } if errors.Is(err, promptkit.ErrCapacityExceeded) { var capacityErr *promptkit.CapacityError if errors.As(err, &capacityErr) && strings.TrimSpace(capacityErr.BackendID) != "" { - return contracts.StructuredCompletionResponse{}, fmt.Errorf( + return promptKitResponseWithExecution(contracts.StructuredCompletionResponse{}, executionSummary), fmt.Errorf( "run PromptKit prompt %q on backend %q: %w: %v", promptID, strings.TrimSpace(capacityErr.BackendID), @@ -185,7 +191,7 @@ func (c *PromptKitClient) CompleteStructured(ctx context.Context, req contracts. redactPromptKitError(err), ) } - return contracts.StructuredCompletionResponse{}, fmt.Errorf( + return promptKitResponseWithExecution(contracts.StructuredCompletionResponse{}, executionSummary), fmt.Errorf( "run PromptKit prompt %q: %w: %v", promptID, contracts.ErrLLMCapacityExceeded, @@ -206,14 +212,14 @@ func (c *PromptKitClient) CompleteStructured(ctx context.Context, req contracts. status = generationErr.StatusCode() response.Debug.Response = promptKitDebugGenerationError(&preparedDetails, generationErr) } - return response, contracts.NewLLMGenerationErrorWithMetadata(status, fmt.Sprintf("run PromptKit prompt %q: %v", promptID, redactPromptKitError(err)), metadata) + return promptKitResponseWithExecution(response, executionSummary), contracts.NewLLMGenerationErrorWithMetadata(status, fmt.Sprintf("run PromptKit prompt %q: %v", promptID, redactPromptKitError(err)), metadata) } - return contracts.StructuredCompletionResponse{}, fmt.Errorf("run PromptKit prompt %q: %v", promptID, redactPromptKitError(err)) + return promptKitResponseWithExecution(contracts.StructuredCompletionResponse{}, executionSummary), fmt.Errorf("run PromptKit prompt %q: %v", promptID, redactPromptKitError(err)) } if result == nil { - return contracts.StructuredCompletionResponse{}, fmt.Errorf("run PromptKit prompt %q: %w: empty result", promptID, contracts.ErrInvalidStructuredOutput) + return promptKitResponseWithExecution(contracts.StructuredCompletionResponse{}, executionSummary), fmt.Errorf("run PromptKit prompt %q: %w: empty result", promptID, contracts.ErrInvalidStructuredOutput) } - response := c.responseFromResult(result, &preparedDetails) + response := promptKitResponseWithExecution(c.responseFromResult(result, &preparedDetails), executionSummary) if result.Validation.Status == promptkit.ValidationFailed || !result.Validation.IsValid { return response, fmt.Errorf("run PromptKit prompt %q: %w: validation failed: %s", promptID, contracts.ErrInvalidStructuredOutput, strings.Join(result.Validation.Errors, "; ")) } @@ -226,6 +232,47 @@ func (c *PromptKitClient) CompleteStructured(ctx context.Context, req contracts. return response, nil } +func promptKitResponseWithExecution(response contracts.StructuredCompletionResponse, execution *contracts.LLMExecutionSummary) contracts.StructuredCompletionResponse { + response.Execution = execution + return response +} + +func promptKitExecutionSummary(event promptkit.ExecutionEvent) *contracts.LLMExecutionSummary { + count := func(value promptkit.ObservedTokenCount) contracts.LLMObservedTokenCount { + return contracts.LLMObservedTokenCount{Value: value.Value, Known: value.Known, Complete: value.Complete} + } + return &contracts.LLMExecutionSummary{ + Attempts: event.Attempts, TransientRetries: event.TransientRetries, RepairCalls: event.RepairCalls, + CapacityWaitMS: event.CapacityWait.Milliseconds(), GenerationMS: event.GenerationDuration.Milliseconds(), RetryDelayMS: event.RetryDelay.Milliseconds(), DurationMS: event.Duration.Milliseconds(), + Outcome: promptKitExecutionOutcome(event.Outcome), FailureKind: promptKitExecutionFailure(event.Failure), StatusCode: event.StatusCode, RetriesExhausted: event.RetriesExhausted, + Usage: contracts.LLMObservedTokenUsage{PromptTokens: count(event.Usage.PromptTokens), CompletionTokens: count(event.Usage.CompletionTokens), TotalTokens: count(event.Usage.TotalTokens), CachedTokens: count(event.Usage.CachedTokens), CacheWriteTokens: count(event.Usage.CacheWriteTokens)}, + } +} + +func promptKitExecutionOutcome(value promptkit.ExecutionOutcome) string { + switch value { + case promptkit.ExecutionOutcomeSuccess: + return "success" + case promptkit.ExecutionOutcomeInvalidOutput: + return "invalid_output" + case promptkit.ExecutionOutcomeError: + return "error" + default: + return "other" + } +} + +func promptKitExecutionFailure(value promptkit.ExecutionFailureKind) string { + switch value { + case promptkit.ExecutionFailureNone: + return "" + case promptkit.ExecutionFailureCanceled, promptkit.ExecutionFailureDeadline, promptkit.ExecutionFailureTransport, promptkit.ExecutionFailureProviderStatus, promptkit.ExecutionFailureMalformedResponse, promptkit.ExecutionFailureCapacity, promptkit.ExecutionFailureInvalidConfig, promptkit.ExecutionFailureInvalidRequest, promptkit.ExecutionFailurePromptNotFound, promptkit.ExecutionFailureProfileNotFound, promptkit.ExecutionFailurePromptLoad, promptkit.ExecutionFailureProfileLoad, promptkit.ExecutionFailureArtifactLoad, promptkit.ExecutionFailurePromptRender, promptkit.ExecutionFailureValidation, promptkit.ExecutionFailureGeneration: + return string(value) + default: + return "other" + } +} + func promptKitGenerationFailureKind(err error) contracts.LLMGenerationFailureKind { switch { case errors.Is(err, promptkit.ErrTransport): diff --git a/internal/framework/llm/promptkit_client_test.go b/internal/framework/llm/promptkit_client_test.go index 2cb7f5a5..b4056cf7 100644 --- a/internal/framework/llm/promptkit_client_test.go +++ b/internal/framework/llm/promptkit_client_test.go @@ -55,6 +55,10 @@ func TestPromptKitClientMapsPromptRequestAndUnmarshalsOutput(t *testing.T) { if resp.PromptTokens != 11 || resp.CompletionTokens != 7 || resp.TotalTokens != 18 { t.Fatalf("usage = %#v, want mapped token counts", resp) } + if resp.Execution == nil || resp.Execution.Outcome != "success" || resp.Execution.Attempts != 1 || + !resp.Execution.Usage.PromptTokens.Known || resp.Execution.Usage.PromptTokens.Value != 11 { + t.Fatalf("execution summary = %#v", resp.Execution) + } if resp.Debug == nil || resp.Debug.Prompt == nil { t.Fatalf("debug prompt = nil, want prepared prompt material") }