Decode provider reasoning and cache usage details
This commit is contained in:
@@ -168,6 +168,7 @@ func toDomainTokenUsagePresence(presence *TokenUsagePresence) *domain.TokenUsage
|
||||
return &domain.TokenUsagePresence{
|
||||
PromptTokens: presence.PromptTokens,
|
||||
CompletionTokens: presence.CompletionTokens,
|
||||
ReasoningTokens: presence.ReasoningTokens,
|
||||
TotalTokens: presence.TotalTokens,
|
||||
CachedTokens: presence.CachedTokens,
|
||||
CacheWriteTokens: presence.CacheWriteTokens,
|
||||
@@ -356,6 +357,7 @@ func fromDomainTokenUsage(usage domain.TokenUsage) TokenUsage {
|
||||
return TokenUsage{
|
||||
PromptTokens: usage.PromptTokens,
|
||||
CompletionTokens: usage.CompletionTokens,
|
||||
ReasoningTokens: usage.ReasoningTokens,
|
||||
TotalTokens: usage.TotalTokens,
|
||||
CachedTokens: usage.CachedTokens,
|
||||
CacheWriteTokens: usage.CacheWriteTokens,
|
||||
@@ -366,6 +368,7 @@ func toDomainTokenUsage(usage TokenUsage) domain.TokenUsage {
|
||||
return domain.TokenUsage{
|
||||
PromptTokens: usage.PromptTokens,
|
||||
CompletionTokens: usage.CompletionTokens,
|
||||
ReasoningTokens: usage.ReasoningTokens,
|
||||
TotalTokens: usage.TotalTokens,
|
||||
CachedTokens: usage.CachedTokens,
|
||||
CacheWriteTokens: usage.CacheWriteTokens,
|
||||
|
||||
@@ -163,7 +163,15 @@ data, a second JSON value, absent choices, missing content, `null` content,
|
||||
non-string content, and size overflow are malformed responses and return no
|
||||
partial result.
|
||||
|
||||
Each of the five token fields retains whether the provider reported a
|
||||
For OpenRouter and vLLM, the client also reads reasoning tokens from
|
||||
`usage.completion_tokens_details.reasoning_tokens`. OpenRouter cache writes
|
||||
come from `usage.prompt_tokens_details.cache_write_tokens`; vLLM uses
|
||||
`usage.prompt_tokens_details.created_cache_tokens`. The generic provider keeps
|
||||
its top-level `usage.cache_write_tokens` mapping. Missing, null, malformed,
|
||||
fractional, or unrepresentable new details remain unknown without invalidating
|
||||
otherwise usable content. Existing common count decoding remains strict.
|
||||
|
||||
Each common token field retains whether the provider reported a
|
||||
non-null integer, including zero. Omitted and `null` fields are unknown for
|
||||
observations. A strictly decoded successful envelope can yield observed usage
|
||||
even when a later missing choice or content makes the attempt malformed.
|
||||
|
||||
@@ -223,6 +223,8 @@ Validation: focused adapter and smoke coordinator tests with controlled HTTP.
|
||||
|
||||
## Stage 7 — Reasoning Values And Provider-Specific Token Decoding
|
||||
|
||||
**Completed.**
|
||||
|
||||
Touch public/domain token declarations, facade conversions, and
|
||||
`internal/llm/openai_compatible_client.go` plus focused provider files.
|
||||
|
||||
|
||||
@@ -334,6 +334,7 @@ type GenerateResponse struct {
|
||||
type TokenUsagePresence struct {
|
||||
PromptTokens bool
|
||||
CompletionTokens bool
|
||||
ReasoningTokens bool
|
||||
TotalTokens bool
|
||||
CachedTokens bool
|
||||
CacheWriteTokens bool
|
||||
@@ -343,6 +344,7 @@ type TokenUsagePresence struct {
|
||||
type TokenUsage struct {
|
||||
PromptTokens int
|
||||
CompletionTokens int
|
||||
ReasoningTokens int
|
||||
TotalTokens int
|
||||
CachedTokens int
|
||||
CacheWriteTokens int
|
||||
|
||||
@@ -56,6 +56,7 @@ type ObservedTokenCount struct {
|
||||
type ObservedTokenUsage struct {
|
||||
PromptTokens ObservedTokenCount
|
||||
CompletionTokens ObservedTokenCount
|
||||
ReasoningTokens ObservedTokenCount
|
||||
TotalTokens ObservedTokenCount
|
||||
CachedTokens ObservedTokenCount
|
||||
CacheWriteTokens ObservedTokenCount
|
||||
|
||||
@@ -88,6 +88,7 @@ func (c *OpenAICompatibleClient) Generate(ctx context.Context, req domain.Genera
|
||||
// PreparedGeneration holds one encoded request and resolved authentication for
|
||||
// repeated attempts. Each execution creates a fresh HTTP request and context.
|
||||
type PreparedGeneration struct {
|
||||
provider string
|
||||
endpoint string
|
||||
payload []byte
|
||||
apiKey string
|
||||
@@ -156,7 +157,7 @@ func (c *OpenAICompatibleClient) PrepareGeneration(req domain.GenerateRequest) (
|
||||
httpClient = &http.Client{Timeout: defaults.LLMRequestTimeoutDefault}
|
||||
}
|
||||
return &PreparedGeneration{
|
||||
endpoint: endpoint, payload: payload, apiKey: apiKey,
|
||||
endpoint: endpoint, payload: payload, apiKey: apiKey, provider: EffectiveProvider(req.Target.Provider),
|
||||
timeoutSeconds: req.Target.TimeoutSeconds, httpClient: httpClient,
|
||||
structuredOutputRequested: wirePayload["response_format"] != nil,
|
||||
}, nil
|
||||
@@ -266,16 +267,37 @@ func (p *PreparedGeneration) execute(ctx context.Context, captureEvidence bool)
|
||||
}
|
||||
return outcome
|
||||
}
|
||||
var writes, reasoning *int
|
||||
switch p.provider {
|
||||
case "openrouter", "vllm":
|
||||
var details map[string]json.RawMessage
|
||||
if json.Unmarshal(wireResp.Usage.CompletionTokensDetails, &details) == nil {
|
||||
reasoning = optionalInteger(details["reasoning_tokens"])
|
||||
}
|
||||
if p.provider == "openrouter" {
|
||||
writes = optionalInteger(wireResp.Usage.PromptTokensDetails.CacheWriteTokens)
|
||||
} else {
|
||||
writes = optionalInteger(wireResp.Usage.PromptTokensDetails.CreatedCacheTokens)
|
||||
}
|
||||
default:
|
||||
if len(wireResp.Usage.CacheWriteTokens) > 0 && json.Unmarshal(wireResp.Usage.CacheWriteTokens, &writes) != nil {
|
||||
outcome.Err = fmt.Errorf("%w: failed to decode response", ErrMalformedResponse)
|
||||
if contextErr := observedContextError(requestContext, ctx, outcome.Err); contextErr != nil {
|
||||
outcome.Err = contextErr
|
||||
}
|
||||
return outcome
|
||||
}
|
||||
}
|
||||
outcome.ResponseDecoded = true
|
||||
outcome.Usage = domain.TokenUsage{
|
||||
PromptTokens: tokenCount(wireResp.Usage.PromptTokens), CompletionTokens: tokenCount(wireResp.Usage.CompletionTokens),
|
||||
TotalTokens: tokenCount(wireResp.Usage.TotalTokens), CachedTokens: tokenCount(wireResp.Usage.PromptTokensDetails.CachedTokens),
|
||||
CacheWriteTokens: tokenCount(wireResp.Usage.CacheWriteTokens),
|
||||
CacheWriteTokens: tokenCount(writes), ReasoningTokens: tokenCount(reasoning),
|
||||
}
|
||||
outcome.UsagePresence = &domain.TokenUsagePresence{
|
||||
PromptTokens: wireResp.Usage.PromptTokens != nil, CompletionTokens: wireResp.Usage.CompletionTokens != nil,
|
||||
TotalTokens: wireResp.Usage.TotalTokens != nil, CachedTokens: wireResp.Usage.PromptTokensDetails.CachedTokens != nil,
|
||||
CacheWriteTokens: wireResp.Usage.CacheWriteTokens != nil,
|
||||
CacheWriteTokens: writes != nil, ReasoningTokens: reasoning != nil,
|
||||
}
|
||||
|
||||
if len(wireResp.Choices) == 0 {
|
||||
@@ -540,12 +562,26 @@ type openAIChatResponse struct {
|
||||
CompletionTokens *int `json:"completion_tokens"`
|
||||
TotalTokens *int `json:"total_tokens"`
|
||||
PromptTokensDetails struct {
|
||||
CachedTokens *int `json:"cached_tokens"`
|
||||
CachedTokens *int `json:"cached_tokens"`
|
||||
CacheWriteTokens json.RawMessage `json:"cache_write_tokens"`
|
||||
CreatedCacheTokens json.RawMessage `json:"created_cache_tokens"`
|
||||
} `json:"prompt_tokens_details"`
|
||||
CacheWriteTokens *int `json:"cache_write_tokens"`
|
||||
CompletionTokensDetails json.RawMessage `json:"completion_tokens_details"`
|
||||
CacheWriteTokens json.RawMessage `json:"cache_write_tokens"`
|
||||
} `json:"usage"`
|
||||
}
|
||||
|
||||
func optionalInteger(raw json.RawMessage) *int {
|
||||
if len(raw) == 0 {
|
||||
return nil
|
||||
}
|
||||
var value *int
|
||||
if json.Unmarshal(raw, &value) != nil {
|
||||
return nil
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
type openAIResponseFormat struct {
|
||||
Type string `json:"type"`
|
||||
JSONSchema *openAIJSONSchemaEnvelope `json:"json_schema,omitempty"`
|
||||
|
||||
@@ -882,6 +882,45 @@ func TestOpenAICompatibleUsagePresence(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestProviderSpecificUsageDetails(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name, provider, usage string
|
||||
writes, reasoning int
|
||||
writesKnown, reasoningKnown bool
|
||||
}{
|
||||
{name: "openrouter nested", provider: "openrouter", usage: `"prompt_tokens_details":{"cached_tokens":2,"cache_write_tokens":3,"created_cache_tokens":7},"completion_tokens_details":{"reasoning_tokens":4},"cache_write_tokens":9`, writes: 3, reasoning: 4, writesKnown: true, reasoningKnown: true},
|
||||
{name: "vllm nested", provider: "vllm", usage: `"prompt_tokens_details":{"cached_tokens":2,"cache_write_tokens":3,"created_cache_tokens":7},"completion_tokens_details":{"reasoning_tokens":0},"cache_write_tokens":9`, writes: 7, writesKnown: true, reasoningKnown: true},
|
||||
{name: "generic ignores details", provider: "openai-compatible", usage: `"prompt_tokens_details":{"cache_write_tokens":3},"completion_tokens_details":{"reasoning_tokens":4},"cache_write_tokens":9`, writes: 9, writesKnown: true},
|
||||
{name: "missing selected field", provider: "openrouter", usage: `"prompt_tokens_details":{"created_cache_tokens":7},"cache_write_tokens":9`},
|
||||
{name: "invalid optional fields", provider: "vllm", usage: `"prompt_tokens_details":{"created_cache_tokens":"7","cached_tokens":2},"completion_tokens_details":{"reasoning_tokens":2.5},"cache_write_tokens":{}`},
|
||||
{name: "overflow and null", provider: "openrouter", usage: `"prompt_tokens_details":{"cache_write_tokens":999999999999999999999999},"completion_tokens_details":{"reasoning_tokens":null}`},
|
||||
{name: "vllm disabled details", provider: "vllm", usage: `"prompt_tokens":12`},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
provider := newRecordingProvider(t)
|
||||
provider.respond(http.StatusOK, `{"usage":{`+tc.usage+`},"choices":[{"message":{"content":"ok"}}]}`)
|
||||
request := provider.generateRequest()
|
||||
request.Target.Provider = tc.provider
|
||||
response, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{}).Generate(context.Background(), request)
|
||||
if err != nil || response.Usage.CacheWriteTokens != tc.writes || response.Usage.ReasoningTokens != tc.reasoning || response.UsagePresence.CacheWriteTokens != tc.writesKnown || response.UsagePresence.ReasoningTokens != tc.reasoningKnown {
|
||||
t.Fatalf("response=%+v error=%v", response, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
provider := newRecordingProvider(t)
|
||||
provider.respond(http.StatusOK, `{"usage":{"completion_tokens_details":{"reasoning_tokens":5}},"choices":[]}`)
|
||||
request := provider.generateRequest()
|
||||
request.Target.Provider = "openrouter"
|
||||
prepared, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{}).PrepareGeneration(request)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
outcome := prepared.Execute(context.Background())
|
||||
if !errors.Is(outcome.Err, ErrMalformedResponse) || outcome.Usage.ReasoningTokens != 5 || !outcome.UsagePresence.ReasoningTokens {
|
||||
t.Fatalf("outcome=%+v", outcome)
|
||||
}
|
||||
}
|
||||
|
||||
func checkOptionalResponseFieldOmission(t *testing.T) {
|
||||
provider := newRecordingProvider(t)
|
||||
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{})
|
||||
|
||||
@@ -19,9 +19,9 @@ func stringPointerForTest(value string) *string {
|
||||
func TestGenerateResponseUsagePresenceJSON(t *testing.T) {
|
||||
value := promptkit.GenerateResponse{
|
||||
Content: "ok",
|
||||
Usage: promptkit.TokenUsage{TotalTokens: 5},
|
||||
Usage: promptkit.TokenUsage{TotalTokens: 5, ReasoningTokens: 2},
|
||||
UsagePresence: &promptkit.TokenUsagePresence{
|
||||
PromptTokens: true, CompletionTokens: true, TotalTokens: true,
|
||||
PromptTokens: true, CompletionTokens: true, TotalTokens: true, ReasoningTokens: true,
|
||||
CachedTokens: true, CacheWriteTokens: true,
|
||||
},
|
||||
}
|
||||
@@ -30,6 +30,9 @@ func TestGenerateResponseUsagePresenceJSON(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
object := decodeJSONObject(t, payload)
|
||||
if string(decodeJSONObject(t, object["usage"])["reasoning_tokens"]) != "2" {
|
||||
t.Fatalf("reasoning field missing: %s", payload)
|
||||
}
|
||||
if _, ok := object["usage_presence"]; !ok {
|
||||
t.Fatalf("missing usage_presence: %s", payload)
|
||||
}
|
||||
|
||||
@@ -95,6 +95,7 @@ type ObservedTokenCount struct {
|
||||
type ObservedTokenUsage struct {
|
||||
PromptTokens ObservedTokenCount
|
||||
CompletionTokens ObservedTokenCount
|
||||
ReasoningTokens ObservedTokenCount
|
||||
TotalTokens ObservedTokenCount
|
||||
CachedTokens ObservedTokenCount
|
||||
CacheWriteTokens ObservedTokenCount
|
||||
@@ -157,7 +158,7 @@ func fromDomainExecutionEvent(value domain.ExecutionEvent) ExecutionEvent {
|
||||
Outcome: ExecutionOutcome(value.Outcome), Failure: ExecutionFailureKind(value.Failure),
|
||||
StatusCode: value.StatusCode, RetriesExhausted: value.RetriesExhausted,
|
||||
Usage: ObservedTokenUsage{PromptTokens: count(value.Usage.PromptTokens), CompletionTokens: count(value.Usage.CompletionTokens),
|
||||
TotalTokens: count(value.Usage.TotalTokens), CachedTokens: count(value.Usage.CachedTokens), CacheWriteTokens: count(value.Usage.CacheWriteTokens)},
|
||||
TotalTokens: count(value.Usage.TotalTokens), ReasoningTokens: count(value.Usage.ReasoningTokens), CachedTokens: count(value.Usage.CachedTokens), CacheWriteTokens: count(value.Usage.CacheWriteTokens)},
|
||||
Attempts: value.Attempts, TransientRetries: value.TransientRetries, RepairCalls: value.RepairCalls,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -368,6 +368,7 @@ func fromDomainObservedTokenUsage(usage domain.ObservedTokenUsage) ObservedToken
|
||||
return ObservedTokenUsage{
|
||||
PromptTokens: fromDomainObservedTokenCount(usage.PromptTokens),
|
||||
CompletionTokens: fromDomainObservedTokenCount(usage.CompletionTokens),
|
||||
ReasoningTokens: fromDomainObservedTokenCount(usage.ReasoningTokens),
|
||||
TotalTokens: fromDomainObservedTokenCount(usage.TotalTokens),
|
||||
CachedTokens: fromDomainObservedTokenCount(usage.CachedTokens),
|
||||
CacheWriteTokens: fromDomainObservedTokenCount(usage.CacheWriteTokens),
|
||||
|
||||
3
types.go
3
types.go
@@ -568,6 +568,8 @@ type TokenUsage struct {
|
||||
PromptTokens int `json:"prompt_tokens"`
|
||||
// CompletionTokens is the reported generated-token count.
|
||||
CompletionTokens int `json:"completion_tokens"`
|
||||
// ReasoningTokens is the reported reasoning subset of completion tokens.
|
||||
ReasoningTokens int `json:"reasoning_tokens"`
|
||||
// TotalTokens is the reported total-token count.
|
||||
TotalTokens int `json:"total_tokens"`
|
||||
// CachedTokens is the reported cached-input-token count.
|
||||
@@ -697,6 +699,7 @@ type GenerateResponse struct {
|
||||
type TokenUsagePresence struct {
|
||||
PromptTokens bool `json:"prompt_tokens"`
|
||||
CompletionTokens bool `json:"completion_tokens"`
|
||||
ReasoningTokens bool `json:"reasoning_tokens"`
|
||||
TotalTokens bool `json:"total_tokens"`
|
||||
CachedTokens bool `json:"cached_tokens"`
|
||||
CacheWriteTokens bool `json:"cache_write_tokens"`
|
||||
|
||||
Reference in New Issue
Block a user