eric/notarius
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add LLM execution summaries

Browse Source
This commit is contained in:
Eric Rakestraw
2026-09-25 22:01:16 +00:00
parent bd83255a8e
commit 6b8a4408c1
5 changed files with 108 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files

67
internal/framework/llm/promptkit_client.go
View File

@@ -130,10 +130,10 @@ func (c *PromptKitClient) CompleteStructured(ctx context.Context, req contracts.
return contracts.StructuredCompletionResponse{}, fmt.Errorf("structured completion prompt_id must not be empty")
}
sessionID := strings.TrimSpace(req.SessionID)
var execution *promptkit.ExecutionTargetOverride
var targetOverride *promptkit.ExecutionTargetOverride
if c.reasoningEffort != nil {
reasoningEffort := *c.reasoningEffort
execution = &promptkit.ExecutionTargetOverride{
targetOverride = &promptkit.ExecutionTargetOverride{
ReasoningEffort: &reasoningEffort,
}
}
@@ -145,9 +145,15 @@ func (c *PromptKitClient) CompleteStructured(ctx context.Context, req contracts.
SessionID: sessionID,
Inputs: promptKitInputs(req.Inputs),
Vars: promptKitVars(req, sessionID),
Execution: execution,
Execution: targetOverride,
AppendedMessages: appendedMessages,
}
var executionSummary *contracts.LLMExecutionSummary
runReq.Observer = func(event promptkit.ExecutionEvent) {
if event.Kind == promptkit.ExecutionFinished {
executionSummary = promptKitExecutionSummary(event)
}
}
if req.StructuredOutputRepairAttempts != nil {
inspection, err := c.engine.InspectPrompt(ctx, promptID, strings.TrimSpace(req.PromptVersion))
if err != nil {
@@ -172,12 +178,12 @@ func (c *PromptKitClient) CompleteStructured(ctx context.Context, req contracts.
result, err := c.engine.RunPrepared(ctx, prepared)
if err != nil {
if ctxErr := ctx.Err(); ctxErr != nil {
return contracts.StructuredCompletionResponse{}, ctxErr
return promptKitResponseWithExecution(contracts.StructuredCompletionResponse{}, executionSummary), ctxErr
}
if errors.Is(err, promptkit.ErrCapacityExceeded) {
var capacityErr *promptkit.CapacityError
if errors.As(err, &capacityErr) && strings.TrimSpace(capacityErr.BackendID) != "" {
return contracts.StructuredCompletionResponse{}, fmt.Errorf(
return promptKitResponseWithExecution(contracts.StructuredCompletionResponse{}, executionSummary), fmt.Errorf(
"run PromptKit prompt %q on backend %q: %w: %v",
promptID,
strings.TrimSpace(capacityErr.BackendID),
@@ -185,7 +191,7 @@ func (c *PromptKitClient) CompleteStructured(ctx context.Context, req contracts.
redactPromptKitError(err),
)
}
return contracts.StructuredCompletionResponse{}, fmt.Errorf(
return promptKitResponseWithExecution(contracts.StructuredCompletionResponse{}, executionSummary), fmt.Errorf(
"run PromptKit prompt %q: %w: %v",
promptID,
contracts.ErrLLMCapacityExceeded,
@@ -206,14 +212,14 @@ func (c *PromptKitClient) CompleteStructured(ctx context.Context, req contracts.
status = generationErr.StatusCode()
response.Debug.Response = promptKitDebugGenerationError(&preparedDetails, generationErr)
}
return response, contracts.NewLLMGenerationErrorWithMetadata(status, fmt.Sprintf("run PromptKit prompt %q: %v", promptID, redactPromptKitError(err)), metadata)
return promptKitResponseWithExecution(response, executionSummary), contracts.NewLLMGenerationErrorWithMetadata(status, fmt.Sprintf("run PromptKit prompt %q: %v", promptID, redactPromptKitError(err)), metadata)
}
return contracts.StructuredCompletionResponse{}, fmt.Errorf("run PromptKit prompt %q: %v", promptID, redactPromptKitError(err))
return promptKitResponseWithExecution(contracts.StructuredCompletionResponse{}, executionSummary), fmt.Errorf("run PromptKit prompt %q: %v", promptID, redactPromptKitError(err))
}
if result == nil {
return contracts.StructuredCompletionResponse{}, fmt.Errorf("run PromptKit prompt %q: %w: empty result", promptID, contracts.ErrInvalidStructuredOutput)
return promptKitResponseWithExecution(contracts.StructuredCompletionResponse{}, executionSummary), fmt.Errorf("run PromptKit prompt %q: %w: empty result", promptID, contracts.ErrInvalidStructuredOutput)
}
response := c.responseFromResult(result, &preparedDetails)
response := promptKitResponseWithExecution(c.responseFromResult(result, &preparedDetails), executionSummary)
if result.Validation.Status == promptkit.ValidationFailed || !result.Validation.IsValid {
return response, fmt.Errorf("run PromptKit prompt %q: %w: validation failed: %s", promptID, contracts.ErrInvalidStructuredOutput, strings.Join(result.Validation.Errors, "; "))
}
@@ -226,6 +232,47 @@ func (c *PromptKitClient) CompleteStructured(ctx context.Context, req contracts.
return response, nil
}
func promptKitResponseWithExecution(response contracts.StructuredCompletionResponse, execution *contracts.LLMExecutionSummary) contracts.StructuredCompletionResponse {
response.Execution = execution
return response
}
func promptKitExecutionSummary(event promptkit.ExecutionEvent) *contracts.LLMExecutionSummary {
count := func(value promptkit.ObservedTokenCount) contracts.LLMObservedTokenCount {
return contracts.LLMObservedTokenCount{Value: value.Value, Known: value.Known, Complete: value.Complete}
}
return &contracts.LLMExecutionSummary{
Attempts: event.Attempts, TransientRetries: event.TransientRetries, RepairCalls: event.RepairCalls,
CapacityWaitMS: event.CapacityWait.Milliseconds(), GenerationMS: event.GenerationDuration.Milliseconds(), RetryDelayMS: event.RetryDelay.Milliseconds(), DurationMS: event.Duration.Milliseconds(),
Outcome: promptKitExecutionOutcome(event.Outcome), FailureKind: promptKitExecutionFailure(event.Failure), StatusCode: event.StatusCode, RetriesExhausted: event.RetriesExhausted,
Usage: contracts.LLMObservedTokenUsage{PromptTokens: count(event.Usage.PromptTokens), CompletionTokens: count(event.Usage.CompletionTokens), TotalTokens: count(event.Usage.TotalTokens), CachedTokens: count(event.Usage.CachedTokens), CacheWriteTokens: count(event.Usage.CacheWriteTokens)},
}
}
func promptKitExecutionOutcome(value promptkit.ExecutionOutcome) string {
switch value {
case promptkit.ExecutionOutcomeSuccess:
return "success"
case promptkit.ExecutionOutcomeInvalidOutput:
return "invalid_output"
case promptkit.ExecutionOutcomeError:
return "error"
default:
return "other"
}
}
func promptKitExecutionFailure(value promptkit.ExecutionFailureKind) string {
switch value {
case promptkit.ExecutionFailureNone:
return ""
case promptkit.ExecutionFailureCanceled, promptkit.ExecutionFailureDeadline, promptkit.ExecutionFailureTransport, promptkit.ExecutionFailureProviderStatus, promptkit.ExecutionFailureMalformedResponse, promptkit.ExecutionFailureCapacity, promptkit.ExecutionFailureInvalidConfig, promptkit.ExecutionFailureInvalidRequest, promptkit.ExecutionFailurePromptNotFound, promptkit.ExecutionFailureProfileNotFound, promptkit.ExecutionFailurePromptLoad, promptkit.ExecutionFailureProfileLoad, promptkit.ExecutionFailureArtifactLoad, promptkit.ExecutionFailurePromptRender, promptkit.ExecutionFailureValidation, promptkit.ExecutionFailureGeneration:
return string(value)
default:
return "other"
}
}
func promptKitGenerationFailureKind(err error) contracts.LLMGenerationFailureKind {
switch {
case errors.Is(err, promptkit.ErrTransport):