Define observed cost values and aggregation
This commit is contained in:
@@ -134,6 +134,7 @@ func fromDomainRunResult(result *domain.RunResult) *RunResult {
|
||||
EffectiveModelParams: fromDomainExecutionTarget(result.EffectiveModelParams),
|
||||
InputHashes: copyStringMap(result.InputHashes),
|
||||
Usage: fromDomainTokenUsage(result.Usage),
|
||||
CostUSD: fromDomainCost(result.CostUSD),
|
||||
StartTime: result.StartTime,
|
||||
EndTime: result.EndTime,
|
||||
Duration: result.Duration,
|
||||
@@ -158,6 +159,7 @@ func toDomainGenerateResponse(resp *GenerateResponse) *domain.GenerateResponse {
|
||||
Content: resp.Content,
|
||||
Usage: toDomainTokenUsage(resp.Usage),
|
||||
UsagePresence: toDomainTokenUsagePresence(resp.UsagePresence),
|
||||
CostUSD: copyFloat64Ptr(resp.CostUSD),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
30
cost.go
Normal file
30
cost.go
Normal file
@@ -0,0 +1,30 @@
|
||||
package promptkit
|
||||
|
||||
import "go.maximumdirect.net/promptkit/internal/domain"
|
||||
|
||||
// CostSource identifies how an observed USD cost was supplied.
|
||||
type CostSource string
|
||||
|
||||
const (
|
||||
CostSourceNone CostSource = ""
|
||||
CostSourceProvider CostSource = "provider"
|
||||
CostSourceConfiguredZero CostSource = "configured_zero"
|
||||
CostSourceMixed CostSource = "mixed"
|
||||
)
|
||||
|
||||
// ObservedCost reports an available USD subtotal and its coverage and source.
|
||||
// Complete describes coverage of dispatched attempts, not billing verification.
|
||||
type ObservedCost struct {
|
||||
Value float64 `json:"value"`
|
||||
Known bool `json:"known"`
|
||||
Complete bool `json:"complete"`
|
||||
Source CostSource `json:"source"`
|
||||
}
|
||||
|
||||
func fromDomainCost(value domain.ObservedCost) ObservedCost {
|
||||
return ObservedCost{Value: value.Value, Known: value.Known, Complete: value.Complete, Source: CostSource(value.Source)}
|
||||
}
|
||||
|
||||
func toDomainCost(value ObservedCost) domain.ObservedCost {
|
||||
return domain.ObservedCost{Value: value.Value, Known: value.Known, Complete: value.Complete, Source: domain.CostSource(value.Source)}
|
||||
}
|
||||
@@ -276,6 +276,8 @@ Validation: focused token accumulator/group tests and public observation contrac
|
||||
|
||||
## Stage 9 — Public Cost Values And Shared Cost Algebra
|
||||
|
||||
**Completed.**
|
||||
|
||||
Create public cost declarations in a focused root file and matching domain
|
||||
values; add the selected fields to responses, results, events, and summaries.
|
||||
|
||||
|
||||
@@ -29,6 +29,7 @@ type ExecutionGroupSummary struct {
|
||||
TransientRetries uint64 `json:"transient_retries"`
|
||||
RepairCalls uint64 `json:"repair_calls"`
|
||||
Usage ObservedTokenUsage `json:"usage"`
|
||||
CostUSD ObservedCost `json:"cost_usd"`
|
||||
}
|
||||
|
||||
// Observe enrolls an event's run, counting its first terminal event only.
|
||||
@@ -36,7 +37,7 @@ func (g *ExecutionGroup) Observe(event ExecutionEvent) {
|
||||
count := func(c ObservedTokenCount) domain.ObservedTokenCount {
|
||||
return domain.ObservedTokenCount{Value: c.Value, Known: c.Known, Complete: c.Complete}
|
||||
}
|
||||
g.group.Observe(domain.ExecutionEvent{Kind: domain.ExecutionEventKind(event.Kind), RunID: event.RunID, Attempts: event.Attempts, TransientRetries: event.TransientRetries, RepairCalls: event.RepairCalls,
|
||||
g.group.Observe(domain.ExecutionEvent{Kind: domain.ExecutionEventKind(event.Kind), RunID: event.RunID, Attempts: event.Attempts, TransientRetries: event.TransientRetries, RepairCalls: event.RepairCalls, CostUSD: toDomainCost(event.CostUSD),
|
||||
Usage: domain.ObservedTokenUsage{PromptTokens: count(event.Usage.PromptTokens), CompletionTokens: count(event.Usage.CompletionTokens), ReasoningTokens: count(event.Usage.ReasoningTokens), TotalTokens: count(event.Usage.TotalTokens), CachedTokens: count(event.Usage.CachedTokens), CacheWriteTokens: count(event.Usage.CacheWriteTokens)}})
|
||||
}
|
||||
|
||||
@@ -51,6 +52,6 @@ func (g *ExecutionGroup) Snapshot() ExecutionGroupSummary {
|
||||
for i, member := range value.Members {
|
||||
members[i] = ExecutionGroupMember{RunID: member.RunID, TerminalObserved: member.TerminalObserved}
|
||||
}
|
||||
return ExecutionGroupSummary{Members: members, TerminalCoverageComplete: value.TerminalCoverageComplete, Attempts: value.Attempts, TransientRetries: value.TransientRetries, RepairCalls: value.RepairCalls,
|
||||
return ExecutionGroupSummary{Members: members, TerminalCoverageComplete: value.TerminalCoverageComplete, Attempts: value.Attempts, TransientRetries: value.TransientRetries, RepairCalls: value.RepairCalls, CostUSD: fromDomainCost(value.CostUSD),
|
||||
Usage: ObservedTokenUsage{PromptTokens: count(value.Usage.PromptTokens), CompletionTokens: count(value.Usage.CompletionTokens), ReasoningTokens: count(value.Usage.ReasoningTokens), TotalTokens: count(value.Usage.TotalTokens), CachedTokens: count(value.Usage.CachedTokens), CacheWriteTokens: count(value.Usage.CacheWriteTokens)}}
|
||||
}
|
||||
|
||||
17
internal/domain/cost.go
Normal file
17
internal/domain/cost.go
Normal file
@@ -0,0 +1,17 @@
|
||||
package domain
|
||||
|
||||
type CostSource string
|
||||
|
||||
const (
|
||||
CostSourceNone CostSource = ""
|
||||
CostSourceProvider CostSource = "provider"
|
||||
CostSourceConfiguredZero CostSource = "configured_zero"
|
||||
CostSourceMixed CostSource = "mixed"
|
||||
)
|
||||
|
||||
type ObservedCost struct {
|
||||
Value float64
|
||||
Known bool
|
||||
Complete bool
|
||||
Source CostSource
|
||||
}
|
||||
@@ -99,6 +99,7 @@ type RunResult struct {
|
||||
EffectiveModelParams ExecutionTarget
|
||||
InputHashes map[string]string
|
||||
Usage TokenUsage
|
||||
CostUSD ObservedCost
|
||||
StartTime time.Time
|
||||
EndTime time.Time
|
||||
Duration time.Duration
|
||||
@@ -328,6 +329,7 @@ type GenerateResponse struct {
|
||||
Content string
|
||||
Usage TokenUsage
|
||||
UsagePresence *TokenUsagePresence
|
||||
CostUSD *float64
|
||||
}
|
||||
|
||||
// TokenUsagePresence records which token fields a provider reported.
|
||||
|
||||
@@ -76,6 +76,7 @@ type ExecutionGroupSummary struct {
|
||||
TransientRetries uint64
|
||||
RepairCalls uint64
|
||||
Usage ObservedTokenUsage
|
||||
CostUSD ObservedCost
|
||||
}
|
||||
|
||||
// ExecutionEvent contains metadata only; it carries no mutable containers or
|
||||
@@ -101,6 +102,7 @@ type ExecutionEvent struct {
|
||||
StatusCode int
|
||||
RetriesExhausted bool
|
||||
Usage ObservedTokenUsage
|
||||
CostUSD ObservedCost
|
||||
Attempts uint64
|
||||
TransientRetries uint64
|
||||
RepairCalls uint64
|
||||
|
||||
53
internal/observation/cost.go
Normal file
53
internal/observation/cost.go
Normal file
@@ -0,0 +1,53 @@
|
||||
package observation
|
||||
|
||||
import (
|
||||
"math"
|
||||
|
||||
"go.maximumdirect.net/promptkit/internal/domain"
|
||||
)
|
||||
|
||||
// ResolveCost applies reported-cost precedence over a configured-zero fallback.
|
||||
func ResolveCost(reported *float64, zeroCost bool) domain.ObservedCost {
|
||||
if reported != nil && !math.IsNaN(*reported) && !math.IsInf(*reported, 0) {
|
||||
return domain.ObservedCost{Value: *reported, Known: true, Complete: true, Source: domain.CostSourceProvider}
|
||||
}
|
||||
if zeroCost {
|
||||
return domain.ObservedCost{Known: true, Complete: true, Source: domain.CostSourceConfiguredZero}
|
||||
}
|
||||
return domain.ObservedCost{}
|
||||
}
|
||||
|
||||
// CostAccumulator merges dispatched attempts or already aggregated costs.
|
||||
type CostAccumulator struct {
|
||||
value domain.ObservedCost
|
||||
started bool
|
||||
}
|
||||
|
||||
// Add incorporates one contribution.
|
||||
func (c *CostAccumulator) Add(next domain.ObservedCost) {
|
||||
if !c.started {
|
||||
c.started = true
|
||||
c.value = next
|
||||
return
|
||||
}
|
||||
if next.Known {
|
||||
c.value.Value += next.Value
|
||||
if next.Source != domain.CostSourceNone {
|
||||
if c.value.Source == domain.CostSourceNone {
|
||||
c.value.Source = next.Source
|
||||
} else if c.value.Source != next.Source {
|
||||
c.value.Source = domain.CostSourceMixed
|
||||
}
|
||||
}
|
||||
}
|
||||
c.value.Known = c.value.Known || next.Known
|
||||
c.value.Complete = c.value.Complete && next.Complete
|
||||
}
|
||||
|
||||
// Observed returns known complete zero when no contribution has been added.
|
||||
func (c CostAccumulator) Observed() domain.ObservedCost {
|
||||
if !c.started {
|
||||
return domain.ObservedCost{Known: true, Complete: true}
|
||||
}
|
||||
return c.value
|
||||
}
|
||||
63
internal/observation/cost_test.go
Normal file
63
internal/observation/cost_test.go
Normal file
@@ -0,0 +1,63 @@
|
||||
package observation
|
||||
|
||||
import (
|
||||
"math"
|
||||
"testing"
|
||||
|
||||
"go.maximumdirect.net/promptkit/internal/domain"
|
||||
)
|
||||
|
||||
func TestResolveCostPrecedence(t *testing.T) {
|
||||
zero, negative, invalid := 0.0, -1.5, math.Inf(1)
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
reported *float64
|
||||
fallback bool
|
||||
want domain.ObservedCost
|
||||
}{
|
||||
{"unknown", nil, false, domain.ObservedCost{}},
|
||||
{"configured zero", nil, true, domain.ObservedCost{Known: true, Complete: true, Source: domain.CostSourceConfiguredZero}},
|
||||
{"reported zero", &zero, true, domain.ObservedCost{Known: true, Complete: true, Source: domain.CostSourceProvider}},
|
||||
{"negative", &negative, false, domain.ObservedCost{Value: -1.5, Known: true, Complete: true, Source: domain.CostSourceProvider}},
|
||||
{"invalid fallback", &invalid, true, domain.ObservedCost{Known: true, Complete: true, Source: domain.CostSourceConfiguredZero}},
|
||||
{"invalid unknown", &invalid, false, domain.ObservedCost{}},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
if got := ResolveCost(tc.reported, tc.fallback); got != tc.want {
|
||||
t.Fatalf("got %+v want %+v", got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCostAccumulatorMergesSourcesAndCoverage(t *testing.T) {
|
||||
var acc CostAccumulator
|
||||
if got := acc.Observed(); got != (domain.ObservedCost{Known: true, Complete: true}) {
|
||||
t.Fatalf("empty: %+v", got)
|
||||
}
|
||||
acc.Add(domain.ObservedCost{})
|
||||
if got := acc.Observed(); got != (domain.ObservedCost{}) {
|
||||
t.Fatalf("unknown: %+v", got)
|
||||
}
|
||||
a, b := .1, .2
|
||||
acc.Add(ResolveCost(&a, false))
|
||||
acc.Add(ResolveCost(&b, false))
|
||||
acc.Add(ResolveCost(nil, true))
|
||||
got := acc.Observed()
|
||||
if math.Abs(got.Value-.3) > 1e-12 || !got.Known || got.Complete || got.Source != domain.CostSourceMixed {
|
||||
t.Fatalf("merged: %+v", got)
|
||||
}
|
||||
var zeros CostAccumulator
|
||||
zero := 0.0
|
||||
zeros.Add(ResolveCost(&zero, false))
|
||||
zeros.Add(ResolveCost(nil, true))
|
||||
if got := zeros.Observed(); got != (domain.ObservedCost{Known: true, Complete: true, Source: domain.CostSourceMixed}) {
|
||||
t.Fatalf("zeros: %+v", got)
|
||||
}
|
||||
var aggregate CostAccumulator
|
||||
aggregate.Add(domain.ObservedCost{Known: true, Complete: true})
|
||||
aggregate.Add(got)
|
||||
if aggregate.Observed() != got {
|
||||
t.Fatalf("aggregate: %+v", aggregate.Observed())
|
||||
}
|
||||
}
|
||||
@@ -49,6 +49,34 @@ func TestGenerateResponseUsagePresenceJSON(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestCostJSONPreservesUnknownAndReportedZero(t *testing.T) {
|
||||
var old promptkit.RunResult
|
||||
if err := json.Unmarshal([]byte(`{"prompt_id":"old"}`), &old); err != nil || old.CostUSD != (promptkit.ObservedCost{}) {
|
||||
t.Fatalf("old result=%+v err=%v", old, err)
|
||||
}
|
||||
value := promptkit.RunResult{CostUSD: promptkit.ObservedCost{Value: .125, Known: true, Complete: false, Source: promptkit.CostSourceProvider}}
|
||||
data, err := json.Marshal(value)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var decoded promptkit.RunResult
|
||||
if err := json.Unmarshal(data, &decoded); err != nil || decoded.CostUSD != value.CostUSD {
|
||||
t.Fatalf("round trip=%+v err=%v", decoded, err)
|
||||
}
|
||||
zero := 0.0
|
||||
data, err = json.Marshal(promptkit.GenerateResponse{CostUSD: &zero})
|
||||
if err != nil || string(decodeJSONObject(t, data)["cost_usd"]) != "0" {
|
||||
t.Fatalf("reported zero=%s err=%v", data, err)
|
||||
}
|
||||
data, err = json.Marshal(promptkit.GenerateResponse{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, present := decodeJSONObject(t, data)["cost_usd"]; present {
|
||||
t.Fatalf("unknown cost=%s", data)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRemovedPromptVersionIsAbsentFromJSON(t *testing.T) {
|
||||
values := map[string]any{
|
||||
"prepared": promptkit.PreparedRun{PromptID: "prompt"},
|
||||
@@ -176,6 +204,7 @@ func TestRunResultJSONContractRoundTripsAllFields(t *testing.T) {
|
||||
"effective_model_params",
|
||||
"input_hashes",
|
||||
"usage",
|
||||
"cost_usd",
|
||||
"start_time",
|
||||
"end_time",
|
||||
"duration_ms",
|
||||
|
||||
@@ -50,10 +50,12 @@ func TestPublicLLMClientAdapterGivesClientOwnedNestedValues(t *testing.T) {
|
||||
|
||||
func TestPublicLLMClientAdapterCopiesUsagePresence(t *testing.T) {
|
||||
presence := &TokenUsagePresence{PromptTokens: true}
|
||||
cost := .25
|
||||
client := &usagePresenceClient{response: &GenerateResponse{
|
||||
Content: "ok",
|
||||
Usage: TokenUsage{PromptTokens: 0, TotalTokens: 5},
|
||||
UsagePresence: presence,
|
||||
CostUSD: &cost,
|
||||
}}
|
||||
adapter := publicLLMClientAdapter{client: client}
|
||||
got, err := adapter.Generate(context.Background(), domain.GenerateRequest{})
|
||||
@@ -61,9 +63,13 @@ func TestPublicLLMClientAdapterCopiesUsagePresence(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
presence.PromptTokens = false
|
||||
cost = .5
|
||||
if got.UsagePresence == nil || !got.UsagePresence.PromptTokens || got.UsagePresence.TotalTokens || got.Usage.TotalTokens != 5 {
|
||||
t.Fatalf("copied response = %+v", got)
|
||||
}
|
||||
if got.CostUSD == nil || *got.CostUSD != .25 {
|
||||
t.Fatalf("copied cost=%v", got.CostUSD)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPublicLLMClientAdapterPreservesOwnedResponseAlongsideError(t *testing.T) {
|
||||
|
||||
@@ -150,6 +150,7 @@ type ExecutionEvent struct {
|
||||
StatusCode int
|
||||
RetriesExhausted bool
|
||||
Usage ObservedTokenUsage
|
||||
CostUSD ObservedCost
|
||||
Attempts uint64
|
||||
TransientRetries uint64
|
||||
RepairCalls uint64
|
||||
@@ -167,6 +168,7 @@ func fromDomainExecutionEvent(value domain.ExecutionEvent) ExecutionEvent {
|
||||
GenerationDuration: value.GenerationDuration, RetryDelay: value.RetryDelay, Duration: value.Duration,
|
||||
Outcome: ExecutionOutcome(value.Outcome), Failure: ExecutionFailureKind(value.Failure),
|
||||
StatusCode: value.StatusCode, RetriesExhausted: value.RetriesExhausted,
|
||||
CostUSD: fromDomainCost(value.CostUSD),
|
||||
Usage: ObservedTokenUsage{PromptTokens: count(value.Usage.PromptTokens), CompletionTokens: count(value.Usage.CompletionTokens),
|
||||
TotalTokens: count(value.Usage.TotalTokens), ReasoningTokens: count(value.Usage.ReasoningTokens), CachedTokens: count(value.Usage.CachedTokens), CacheWriteTokens: count(value.Usage.CacheWriteTokens)},
|
||||
Attempts: value.Attempts, TransientRetries: value.TransientRetries, RepairCalls: value.RepairCalls,
|
||||
|
||||
4
types.go
4
types.go
@@ -298,6 +298,8 @@ type RunResult struct {
|
||||
InputHashes map[string]string `json:"input_hashes,omitempty"`
|
||||
// Usage is the token accounting reported by the LLM client.
|
||||
Usage TokenUsage `json:"usage"`
|
||||
// CostUSD is the observed charge subtotal across dispatched attempts.
|
||||
CostUSD ObservedCost `json:"cost_usd"`
|
||||
// StartTime is the UTC time immediately before ordinary Run preparation or
|
||||
// after RunPrepared claims its handle.
|
||||
StartTime time.Time `json:"start_time,omitempty"`
|
||||
@@ -692,6 +694,8 @@ type GenerateResponse struct {
|
||||
// UsagePresence states which usage fields were reported, including zeros.
|
||||
// Nil uses the legacy observation fallback described by LLMClient.
|
||||
UsagePresence *TokenUsagePresence `json:"usage_presence,omitempty"`
|
||||
// CostUSD is an optional reported charge; a pointer to zero reports zero.
|
||||
CostUSD *float64 `json:"cost_usd,omitempty"`
|
||||
}
|
||||
|
||||
// TokenUsagePresence identifies provider-reported token counts for observation.
|
||||
|
||||
Reference in New Issue
Block a user