Decode provider reasoning and cache usage details

This commit is contained in:
2026-11-22 18:20:11 +00:00
parent c3257b63f0
commit db1cad6d2d
11 changed files with 108 additions and 9 deletions

View File

@@ -168,6 +168,7 @@ func toDomainTokenUsagePresence(presence *TokenUsagePresence) *domain.TokenUsage
return &domain.TokenUsagePresence{
PromptTokens: presence.PromptTokens,
CompletionTokens: presence.CompletionTokens,
ReasoningTokens: presence.ReasoningTokens,
TotalTokens: presence.TotalTokens,
CachedTokens: presence.CachedTokens,
CacheWriteTokens: presence.CacheWriteTokens,
@@ -356,6 +357,7 @@ func fromDomainTokenUsage(usage domain.TokenUsage) TokenUsage {
return TokenUsage{
PromptTokens: usage.PromptTokens,
CompletionTokens: usage.CompletionTokens,
ReasoningTokens: usage.ReasoningTokens,
TotalTokens: usage.TotalTokens,
CachedTokens: usage.CachedTokens,
CacheWriteTokens: usage.CacheWriteTokens,
@@ -366,6 +368,7 @@ func toDomainTokenUsage(usage TokenUsage) domain.TokenUsage {
return domain.TokenUsage{
PromptTokens: usage.PromptTokens,
CompletionTokens: usage.CompletionTokens,
ReasoningTokens: usage.ReasoningTokens,
TotalTokens: usage.TotalTokens,
CachedTokens: usage.CachedTokens,
CacheWriteTokens: usage.CacheWriteTokens,

View File

@@ -163,7 +163,15 @@ data, a second JSON value, absent choices, missing content, `null` content,
non-string content, and size overflow are malformed responses and return no
partial result.
Each of the five token fields retains whether the provider reported a
For OpenRouter and vLLM, the client also reads reasoning tokens from
`usage.completion_tokens_details.reasoning_tokens`. OpenRouter cache writes
come from `usage.prompt_tokens_details.cache_write_tokens`; vLLM uses
`usage.prompt_tokens_details.created_cache_tokens`. The generic provider keeps
its top-level `usage.cache_write_tokens` mapping. Missing, null, malformed,
fractional, or unrepresentable new details remain unknown without invalidating
otherwise usable content. Existing common count decoding remains strict.
Each common token field retains whether the provider reported a
non-null integer, including zero. Omitted and `null` fields are unknown for
observations. A strictly decoded successful envelope can yield observed usage
even when a later missing choice or content makes the attempt malformed.

View File

@@ -223,6 +223,8 @@ Validation: focused adapter and smoke coordinator tests with controlled HTTP.
## Stage 7 — Reasoning Values And Provider-Specific Token Decoding
**Completed.**
Touch public/domain token declarations, facade conversions, and
`internal/llm/openai_compatible_client.go` plus focused provider files.

View File

@@ -334,6 +334,7 @@ type GenerateResponse struct {
type TokenUsagePresence struct {
PromptTokens bool
CompletionTokens bool
ReasoningTokens bool
TotalTokens bool
CachedTokens bool
CacheWriteTokens bool
@@ -343,6 +344,7 @@ type TokenUsagePresence struct {
type TokenUsage struct {
PromptTokens int
CompletionTokens int
ReasoningTokens int
TotalTokens int
CachedTokens int
CacheWriteTokens int

View File

@@ -56,6 +56,7 @@ type ObservedTokenCount struct {
type ObservedTokenUsage struct {
PromptTokens ObservedTokenCount
CompletionTokens ObservedTokenCount
ReasoningTokens ObservedTokenCount
TotalTokens ObservedTokenCount
CachedTokens ObservedTokenCount
CacheWriteTokens ObservedTokenCount

View File

@@ -88,6 +88,7 @@ func (c *OpenAICompatibleClient) Generate(ctx context.Context, req domain.Genera
// PreparedGeneration holds one encoded request and resolved authentication for
// repeated attempts. Each execution creates a fresh HTTP request and context.
type PreparedGeneration struct {
provider string
endpoint string
payload []byte
apiKey string
@@ -156,7 +157,7 @@ func (c *OpenAICompatibleClient) PrepareGeneration(req domain.GenerateRequest) (
httpClient = &http.Client{Timeout: defaults.LLMRequestTimeoutDefault}
}
return &PreparedGeneration{
endpoint: endpoint, payload: payload, apiKey: apiKey,
endpoint: endpoint, payload: payload, apiKey: apiKey, provider: EffectiveProvider(req.Target.Provider),
timeoutSeconds: req.Target.TimeoutSeconds, httpClient: httpClient,
structuredOutputRequested: wirePayload["response_format"] != nil,
}, nil
@@ -266,16 +267,37 @@ func (p *PreparedGeneration) execute(ctx context.Context, captureEvidence bool)
}
return outcome
}
var writes, reasoning *int
switch p.provider {
case "openrouter", "vllm":
var details map[string]json.RawMessage
if json.Unmarshal(wireResp.Usage.CompletionTokensDetails, &details) == nil {
reasoning = optionalInteger(details["reasoning_tokens"])
}
if p.provider == "openrouter" {
writes = optionalInteger(wireResp.Usage.PromptTokensDetails.CacheWriteTokens)
} else {
writes = optionalInteger(wireResp.Usage.PromptTokensDetails.CreatedCacheTokens)
}
default:
if len(wireResp.Usage.CacheWriteTokens) > 0 && json.Unmarshal(wireResp.Usage.CacheWriteTokens, &writes) != nil {
outcome.Err = fmt.Errorf("%w: failed to decode response", ErrMalformedResponse)
if contextErr := observedContextError(requestContext, ctx, outcome.Err); contextErr != nil {
outcome.Err = contextErr
}
return outcome
}
}
outcome.ResponseDecoded = true
outcome.Usage = domain.TokenUsage{
PromptTokens: tokenCount(wireResp.Usage.PromptTokens), CompletionTokens: tokenCount(wireResp.Usage.CompletionTokens),
TotalTokens: tokenCount(wireResp.Usage.TotalTokens), CachedTokens: tokenCount(wireResp.Usage.PromptTokensDetails.CachedTokens),
CacheWriteTokens: tokenCount(wireResp.Usage.CacheWriteTokens),
CacheWriteTokens: tokenCount(writes), ReasoningTokens: tokenCount(reasoning),
}
outcome.UsagePresence = &domain.TokenUsagePresence{
PromptTokens: wireResp.Usage.PromptTokens != nil, CompletionTokens: wireResp.Usage.CompletionTokens != nil,
TotalTokens: wireResp.Usage.TotalTokens != nil, CachedTokens: wireResp.Usage.PromptTokensDetails.CachedTokens != nil,
CacheWriteTokens: wireResp.Usage.CacheWriteTokens != nil,
CacheWriteTokens: writes != nil, ReasoningTokens: reasoning != nil,
}
if len(wireResp.Choices) == 0 {
@@ -540,12 +562,26 @@ type openAIChatResponse struct {
CompletionTokens *int `json:"completion_tokens"`
TotalTokens *int `json:"total_tokens"`
PromptTokensDetails struct {
CachedTokens *int `json:"cached_tokens"`
CachedTokens *int `json:"cached_tokens"`
CacheWriteTokens json.RawMessage `json:"cache_write_tokens"`
CreatedCacheTokens json.RawMessage `json:"created_cache_tokens"`
} `json:"prompt_tokens_details"`
CacheWriteTokens *int `json:"cache_write_tokens"`
CompletionTokensDetails json.RawMessage `json:"completion_tokens_details"`
CacheWriteTokens json.RawMessage `json:"cache_write_tokens"`
} `json:"usage"`
}
func optionalInteger(raw json.RawMessage) *int {
if len(raw) == 0 {
return nil
}
var value *int
if json.Unmarshal(raw, &value) != nil {
return nil
}
return value
}
type openAIResponseFormat struct {
Type string `json:"type"`
JSONSchema *openAIJSONSchemaEnvelope `json:"json_schema,omitempty"`

View File

@@ -882,6 +882,45 @@ func TestOpenAICompatibleUsagePresence(t *testing.T) {
}
}
func TestProviderSpecificUsageDetails(t *testing.T) {
for _, tc := range []struct {
name, provider, usage string
writes, reasoning int
writesKnown, reasoningKnown bool
}{
{name: "openrouter nested", provider: "openrouter", usage: `"prompt_tokens_details":{"cached_tokens":2,"cache_write_tokens":3,"created_cache_tokens":7},"completion_tokens_details":{"reasoning_tokens":4},"cache_write_tokens":9`, writes: 3, reasoning: 4, writesKnown: true, reasoningKnown: true},
{name: "vllm nested", provider: "vllm", usage: `"prompt_tokens_details":{"cached_tokens":2,"cache_write_tokens":3,"created_cache_tokens":7},"completion_tokens_details":{"reasoning_tokens":0},"cache_write_tokens":9`, writes: 7, writesKnown: true, reasoningKnown: true},
{name: "generic ignores details", provider: "openai-compatible", usage: `"prompt_tokens_details":{"cache_write_tokens":3},"completion_tokens_details":{"reasoning_tokens":4},"cache_write_tokens":9`, writes: 9, writesKnown: true},
{name: "missing selected field", provider: "openrouter", usage: `"prompt_tokens_details":{"created_cache_tokens":7},"cache_write_tokens":9`},
{name: "invalid optional fields", provider: "vllm", usage: `"prompt_tokens_details":{"created_cache_tokens":"7","cached_tokens":2},"completion_tokens_details":{"reasoning_tokens":2.5},"cache_write_tokens":{}`},
{name: "overflow and null", provider: "openrouter", usage: `"prompt_tokens_details":{"cache_write_tokens":999999999999999999999999},"completion_tokens_details":{"reasoning_tokens":null}`},
{name: "vllm disabled details", provider: "vllm", usage: `"prompt_tokens":12`},
} {
t.Run(tc.name, func(t *testing.T) {
provider := newRecordingProvider(t)
provider.respond(http.StatusOK, `{"usage":{`+tc.usage+`},"choices":[{"message":{"content":"ok"}}]}`)
request := provider.generateRequest()
request.Target.Provider = tc.provider
response, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{}).Generate(context.Background(), request)
if err != nil || response.Usage.CacheWriteTokens != tc.writes || response.Usage.ReasoningTokens != tc.reasoning || response.UsagePresence.CacheWriteTokens != tc.writesKnown || response.UsagePresence.ReasoningTokens != tc.reasoningKnown {
t.Fatalf("response=%+v error=%v", response, err)
}
})
}
provider := newRecordingProvider(t)
provider.respond(http.StatusOK, `{"usage":{"completion_tokens_details":{"reasoning_tokens":5}},"choices":[]}`)
request := provider.generateRequest()
request.Target.Provider = "openrouter"
prepared, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{}).PrepareGeneration(request)
if err != nil {
t.Fatal(err)
}
outcome := prepared.Execute(context.Background())
if !errors.Is(outcome.Err, ErrMalformedResponse) || outcome.Usage.ReasoningTokens != 5 || !outcome.UsagePresence.ReasoningTokens {
t.Fatalf("outcome=%+v", outcome)
}
}
func checkOptionalResponseFieldOmission(t *testing.T) {
provider := newRecordingProvider(t)
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{})

View File

@@ -19,9 +19,9 @@ func stringPointerForTest(value string) *string {
func TestGenerateResponseUsagePresenceJSON(t *testing.T) {
value := promptkit.GenerateResponse{
Content: "ok",
Usage: promptkit.TokenUsage{TotalTokens: 5},
Usage: promptkit.TokenUsage{TotalTokens: 5, ReasoningTokens: 2},
UsagePresence: &promptkit.TokenUsagePresence{
PromptTokens: true, CompletionTokens: true, TotalTokens: true,
PromptTokens: true, CompletionTokens: true, TotalTokens: true, ReasoningTokens: true,
CachedTokens: true, CacheWriteTokens: true,
},
}
@@ -30,6 +30,9 @@ func TestGenerateResponseUsagePresenceJSON(t *testing.T) {
t.Fatal(err)
}
object := decodeJSONObject(t, payload)
if string(decodeJSONObject(t, object["usage"])["reasoning_tokens"]) != "2" {
t.Fatalf("reasoning field missing: %s", payload)
}
if _, ok := object["usage_presence"]; !ok {
t.Fatalf("missing usage_presence: %s", payload)
}

View File

@@ -95,6 +95,7 @@ type ObservedTokenCount struct {
type ObservedTokenUsage struct {
PromptTokens ObservedTokenCount
CompletionTokens ObservedTokenCount
ReasoningTokens ObservedTokenCount
TotalTokens ObservedTokenCount
CachedTokens ObservedTokenCount
CacheWriteTokens ObservedTokenCount
@@ -157,7 +158,7 @@ func fromDomainExecutionEvent(value domain.ExecutionEvent) ExecutionEvent {
Outcome: ExecutionOutcome(value.Outcome), Failure: ExecutionFailureKind(value.Failure),
StatusCode: value.StatusCode, RetriesExhausted: value.RetriesExhausted,
Usage: ObservedTokenUsage{PromptTokens: count(value.Usage.PromptTokens), CompletionTokens: count(value.Usage.CompletionTokens),
TotalTokens: count(value.Usage.TotalTokens), CachedTokens: count(value.Usage.CachedTokens), CacheWriteTokens: count(value.Usage.CacheWriteTokens)},
TotalTokens: count(value.Usage.TotalTokens), ReasoningTokens: count(value.Usage.ReasoningTokens), CachedTokens: count(value.Usage.CachedTokens), CacheWriteTokens: count(value.Usage.CacheWriteTokens)},
Attempts: value.Attempts, TransientRetries: value.TransientRetries, RepairCalls: value.RepairCalls,
}
}

View File

@@ -368,6 +368,7 @@ func fromDomainObservedTokenUsage(usage domain.ObservedTokenUsage) ObservedToken
return ObservedTokenUsage{
PromptTokens: fromDomainObservedTokenCount(usage.PromptTokens),
CompletionTokens: fromDomainObservedTokenCount(usage.CompletionTokens),
ReasoningTokens: fromDomainObservedTokenCount(usage.ReasoningTokens),
TotalTokens: fromDomainObservedTokenCount(usage.TotalTokens),
CachedTokens: fromDomainObservedTokenCount(usage.CachedTokens),
CacheWriteTokens: fromDomainObservedTokenCount(usage.CacheWriteTokens),

View File

@@ -568,6 +568,8 @@ type TokenUsage struct {
PromptTokens int `json:"prompt_tokens"`
// CompletionTokens is the reported generated-token count.
CompletionTokens int `json:"completion_tokens"`
// ReasoningTokens is the reported reasoning subset of completion tokens.
ReasoningTokens int `json:"reasoning_tokens"`
// TotalTokens is the reported total-token count.
TotalTokens int `json:"total_tokens"`
// CachedTokens is the reported cached-input-token count.
@@ -697,6 +699,7 @@ type GenerateResponse struct {
type TokenUsagePresence struct {
PromptTokens bool `json:"prompt_tokens"`
CompletionTokens bool `json:"completion_tokens"`
ReasoningTokens bool `json:"reasoning_tokens"`
TotalTokens bool `json:"total_tokens"`
CachedTokens bool `json:"cached_tokens"`
CacheWriteTokens bool `json:"cache_write_tokens"`