Define observed cost values and aggregation

This commit is contained in:
2026-11-22 18:24:33 +00:00
parent 3d82cad2c9
commit 6ed0e037c4
13 changed files with 215 additions and 2 deletions

View File

@@ -134,6 +134,7 @@ func fromDomainRunResult(result *domain.RunResult) *RunResult {
EffectiveModelParams: fromDomainExecutionTarget(result.EffectiveModelParams),
InputHashes: copyStringMap(result.InputHashes),
Usage: fromDomainTokenUsage(result.Usage),
CostUSD: fromDomainCost(result.CostUSD),
StartTime: result.StartTime,
EndTime: result.EndTime,
Duration: result.Duration,
@@ -158,6 +159,7 @@ func toDomainGenerateResponse(resp *GenerateResponse) *domain.GenerateResponse {
Content: resp.Content,
Usage: toDomainTokenUsage(resp.Usage),
UsagePresence: toDomainTokenUsagePresence(resp.UsagePresence),
CostUSD: copyFloat64Ptr(resp.CostUSD),
}
}

30
cost.go Normal file
View File

@@ -0,0 +1,30 @@
package promptkit
import "go.maximumdirect.net/promptkit/internal/domain"
// CostSource identifies how an observed USD cost was supplied.
type CostSource string
const (
CostSourceNone CostSource = ""
CostSourceProvider CostSource = "provider"
CostSourceConfiguredZero CostSource = "configured_zero"
CostSourceMixed CostSource = "mixed"
)
// ObservedCost reports an available USD subtotal and its coverage and source.
// Complete describes coverage of dispatched attempts, not billing verification.
type ObservedCost struct {
Value float64 `json:"value"`
Known bool `json:"known"`
Complete bool `json:"complete"`
Source CostSource `json:"source"`
}
func fromDomainCost(value domain.ObservedCost) ObservedCost {
return ObservedCost{Value: value.Value, Known: value.Known, Complete: value.Complete, Source: CostSource(value.Source)}
}
func toDomainCost(value ObservedCost) domain.ObservedCost {
return domain.ObservedCost{Value: value.Value, Known: value.Known, Complete: value.Complete, Source: domain.CostSource(value.Source)}
}

View File

@@ -276,6 +276,8 @@ Validation: focused token accumulator/group tests and public observation contrac
## Stage 9 — Public Cost Values And Shared Cost Algebra
**Completed.**
Create public cost declarations in a focused root file and matching domain
values; add the selected fields to responses, results, events, and summaries.

View File

@@ -29,6 +29,7 @@ type ExecutionGroupSummary struct {
TransientRetries uint64 `json:"transient_retries"`
RepairCalls uint64 `json:"repair_calls"`
Usage ObservedTokenUsage `json:"usage"`
CostUSD ObservedCost `json:"cost_usd"`
}
// Observe enrolls an event's run, counting its first terminal event only.
@@ -36,7 +37,7 @@ func (g *ExecutionGroup) Observe(event ExecutionEvent) {
count := func(c ObservedTokenCount) domain.ObservedTokenCount {
return domain.ObservedTokenCount{Value: c.Value, Known: c.Known, Complete: c.Complete}
}
g.group.Observe(domain.ExecutionEvent{Kind: domain.ExecutionEventKind(event.Kind), RunID: event.RunID, Attempts: event.Attempts, TransientRetries: event.TransientRetries, RepairCalls: event.RepairCalls,
g.group.Observe(domain.ExecutionEvent{Kind: domain.ExecutionEventKind(event.Kind), RunID: event.RunID, Attempts: event.Attempts, TransientRetries: event.TransientRetries, RepairCalls: event.RepairCalls, CostUSD: toDomainCost(event.CostUSD),
Usage: domain.ObservedTokenUsage{PromptTokens: count(event.Usage.PromptTokens), CompletionTokens: count(event.Usage.CompletionTokens), ReasoningTokens: count(event.Usage.ReasoningTokens), TotalTokens: count(event.Usage.TotalTokens), CachedTokens: count(event.Usage.CachedTokens), CacheWriteTokens: count(event.Usage.CacheWriteTokens)}})
}
@@ -51,6 +52,6 @@ func (g *ExecutionGroup) Snapshot() ExecutionGroupSummary {
for i, member := range value.Members {
members[i] = ExecutionGroupMember{RunID: member.RunID, TerminalObserved: member.TerminalObserved}
}
return ExecutionGroupSummary{Members: members, TerminalCoverageComplete: value.TerminalCoverageComplete, Attempts: value.Attempts, TransientRetries: value.TransientRetries, RepairCalls: value.RepairCalls,
return ExecutionGroupSummary{Members: members, TerminalCoverageComplete: value.TerminalCoverageComplete, Attempts: value.Attempts, TransientRetries: value.TransientRetries, RepairCalls: value.RepairCalls, CostUSD: fromDomainCost(value.CostUSD),
Usage: ObservedTokenUsage{PromptTokens: count(value.Usage.PromptTokens), CompletionTokens: count(value.Usage.CompletionTokens), ReasoningTokens: count(value.Usage.ReasoningTokens), TotalTokens: count(value.Usage.TotalTokens), CachedTokens: count(value.Usage.CachedTokens), CacheWriteTokens: count(value.Usage.CacheWriteTokens)}}
}

17
internal/domain/cost.go Normal file
View File

@@ -0,0 +1,17 @@
package domain
type CostSource string
const (
CostSourceNone CostSource = ""
CostSourceProvider CostSource = "provider"
CostSourceConfiguredZero CostSource = "configured_zero"
CostSourceMixed CostSource = "mixed"
)
type ObservedCost struct {
Value float64
Known bool
Complete bool
Source CostSource
}

View File

@@ -99,6 +99,7 @@ type RunResult struct {
EffectiveModelParams ExecutionTarget
InputHashes map[string]string
Usage TokenUsage
CostUSD ObservedCost
StartTime time.Time
EndTime time.Time
Duration time.Duration
@@ -328,6 +329,7 @@ type GenerateResponse struct {
Content string
Usage TokenUsage
UsagePresence *TokenUsagePresence
CostUSD *float64
}
// TokenUsagePresence records which token fields a provider reported.

View File

@@ -76,6 +76,7 @@ type ExecutionGroupSummary struct {
TransientRetries uint64
RepairCalls uint64
Usage ObservedTokenUsage
CostUSD ObservedCost
}
// ExecutionEvent contains metadata only; it carries no mutable containers or
@@ -101,6 +102,7 @@ type ExecutionEvent struct {
StatusCode int
RetriesExhausted bool
Usage ObservedTokenUsage
CostUSD ObservedCost
Attempts uint64
TransientRetries uint64
RepairCalls uint64

View File

@@ -0,0 +1,53 @@
package observation
import (
"math"
"go.maximumdirect.net/promptkit/internal/domain"
)
// ResolveCost applies reported-cost precedence over a configured-zero fallback.
func ResolveCost(reported *float64, zeroCost bool) domain.ObservedCost {
if reported != nil && !math.IsNaN(*reported) && !math.IsInf(*reported, 0) {
return domain.ObservedCost{Value: *reported, Known: true, Complete: true, Source: domain.CostSourceProvider}
}
if zeroCost {
return domain.ObservedCost{Known: true, Complete: true, Source: domain.CostSourceConfiguredZero}
}
return domain.ObservedCost{}
}
// CostAccumulator merges dispatched attempts or already aggregated costs.
type CostAccumulator struct {
value domain.ObservedCost
started bool
}
// Add incorporates one contribution.
func (c *CostAccumulator) Add(next domain.ObservedCost) {
if !c.started {
c.started = true
c.value = next
return
}
if next.Known {
c.value.Value += next.Value
if next.Source != domain.CostSourceNone {
if c.value.Source == domain.CostSourceNone {
c.value.Source = next.Source
} else if c.value.Source != next.Source {
c.value.Source = domain.CostSourceMixed
}
}
}
c.value.Known = c.value.Known || next.Known
c.value.Complete = c.value.Complete && next.Complete
}
// Observed returns known complete zero when no contribution has been added.
func (c CostAccumulator) Observed() domain.ObservedCost {
if !c.started {
return domain.ObservedCost{Known: true, Complete: true}
}
return c.value
}

View File

@@ -0,0 +1,63 @@
package observation
import (
"math"
"testing"
"go.maximumdirect.net/promptkit/internal/domain"
)
func TestResolveCostPrecedence(t *testing.T) {
zero, negative, invalid := 0.0, -1.5, math.Inf(1)
for _, tc := range []struct {
name string
reported *float64
fallback bool
want domain.ObservedCost
}{
{"unknown", nil, false, domain.ObservedCost{}},
{"configured zero", nil, true, domain.ObservedCost{Known: true, Complete: true, Source: domain.CostSourceConfiguredZero}},
{"reported zero", &zero, true, domain.ObservedCost{Known: true, Complete: true, Source: domain.CostSourceProvider}},
{"negative", &negative, false, domain.ObservedCost{Value: -1.5, Known: true, Complete: true, Source: domain.CostSourceProvider}},
{"invalid fallback", &invalid, true, domain.ObservedCost{Known: true, Complete: true, Source: domain.CostSourceConfiguredZero}},
{"invalid unknown", &invalid, false, domain.ObservedCost{}},
} {
t.Run(tc.name, func(t *testing.T) {
if got := ResolveCost(tc.reported, tc.fallback); got != tc.want {
t.Fatalf("got %+v want %+v", got, tc.want)
}
})
}
}
func TestCostAccumulatorMergesSourcesAndCoverage(t *testing.T) {
var acc CostAccumulator
if got := acc.Observed(); got != (domain.ObservedCost{Known: true, Complete: true}) {
t.Fatalf("empty: %+v", got)
}
acc.Add(domain.ObservedCost{})
if got := acc.Observed(); got != (domain.ObservedCost{}) {
t.Fatalf("unknown: %+v", got)
}
a, b := .1, .2
acc.Add(ResolveCost(&a, false))
acc.Add(ResolveCost(&b, false))
acc.Add(ResolveCost(nil, true))
got := acc.Observed()
if math.Abs(got.Value-.3) > 1e-12 || !got.Known || got.Complete || got.Source != domain.CostSourceMixed {
t.Fatalf("merged: %+v", got)
}
var zeros CostAccumulator
zero := 0.0
zeros.Add(ResolveCost(&zero, false))
zeros.Add(ResolveCost(nil, true))
if got := zeros.Observed(); got != (domain.ObservedCost{Known: true, Complete: true, Source: domain.CostSourceMixed}) {
t.Fatalf("zeros: %+v", got)
}
var aggregate CostAccumulator
aggregate.Add(domain.ObservedCost{Known: true, Complete: true})
aggregate.Add(got)
if aggregate.Observed() != got {
t.Fatalf("aggregate: %+v", aggregate.Observed())
}
}

View File

@@ -49,6 +49,34 @@ func TestGenerateResponseUsagePresenceJSON(t *testing.T) {
}
}
func TestCostJSONPreservesUnknownAndReportedZero(t *testing.T) {
var old promptkit.RunResult
if err := json.Unmarshal([]byte(`{"prompt_id":"old"}`), &old); err != nil || old.CostUSD != (promptkit.ObservedCost{}) {
t.Fatalf("old result=%+v err=%v", old, err)
}
value := promptkit.RunResult{CostUSD: promptkit.ObservedCost{Value: .125, Known: true, Complete: false, Source: promptkit.CostSourceProvider}}
data, err := json.Marshal(value)
if err != nil {
t.Fatal(err)
}
var decoded promptkit.RunResult
if err := json.Unmarshal(data, &decoded); err != nil || decoded.CostUSD != value.CostUSD {
t.Fatalf("round trip=%+v err=%v", decoded, err)
}
zero := 0.0
data, err = json.Marshal(promptkit.GenerateResponse{CostUSD: &zero})
if err != nil || string(decodeJSONObject(t, data)["cost_usd"]) != "0" {
t.Fatalf("reported zero=%s err=%v", data, err)
}
data, err = json.Marshal(promptkit.GenerateResponse{})
if err != nil {
t.Fatal(err)
}
if _, present := decodeJSONObject(t, data)["cost_usd"]; present {
t.Fatalf("unknown cost=%s", data)
}
}
func TestRemovedPromptVersionIsAbsentFromJSON(t *testing.T) {
values := map[string]any{
"prepared": promptkit.PreparedRun{PromptID: "prompt"},
@@ -176,6 +204,7 @@ func TestRunResultJSONContractRoundTripsAllFields(t *testing.T) {
"effective_model_params",
"input_hashes",
"usage",
"cost_usd",
"start_time",
"end_time",
"duration_ms",

View File

@@ -50,10 +50,12 @@ func TestPublicLLMClientAdapterGivesClientOwnedNestedValues(t *testing.T) {
func TestPublicLLMClientAdapterCopiesUsagePresence(t *testing.T) {
presence := &TokenUsagePresence{PromptTokens: true}
cost := .25
client := &usagePresenceClient{response: &GenerateResponse{
Content: "ok",
Usage: TokenUsage{PromptTokens: 0, TotalTokens: 5},
UsagePresence: presence,
CostUSD: &cost,
}}
adapter := publicLLMClientAdapter{client: client}
got, err := adapter.Generate(context.Background(), domain.GenerateRequest{})
@@ -61,9 +63,13 @@ func TestPublicLLMClientAdapterCopiesUsagePresence(t *testing.T) {
t.Fatal(err)
}
presence.PromptTokens = false
cost = .5
if got.UsagePresence == nil || !got.UsagePresence.PromptTokens || got.UsagePresence.TotalTokens || got.Usage.TotalTokens != 5 {
t.Fatalf("copied response = %+v", got)
}
if got.CostUSD == nil || *got.CostUSD != .25 {
t.Fatalf("copied cost=%v", got.CostUSD)
}
}
func TestPublicLLMClientAdapterPreservesOwnedResponseAlongsideError(t *testing.T) {

View File

@@ -150,6 +150,7 @@ type ExecutionEvent struct {
StatusCode int
RetriesExhausted bool
Usage ObservedTokenUsage
CostUSD ObservedCost
Attempts uint64
TransientRetries uint64
RepairCalls uint64
@@ -167,6 +168,7 @@ func fromDomainExecutionEvent(value domain.ExecutionEvent) ExecutionEvent {
GenerationDuration: value.GenerationDuration, RetryDelay: value.RetryDelay, Duration: value.Duration,
Outcome: ExecutionOutcome(value.Outcome), Failure: ExecutionFailureKind(value.Failure),
StatusCode: value.StatusCode, RetriesExhausted: value.RetriesExhausted,
CostUSD: fromDomainCost(value.CostUSD),
Usage: ObservedTokenUsage{PromptTokens: count(value.Usage.PromptTokens), CompletionTokens: count(value.Usage.CompletionTokens),
TotalTokens: count(value.Usage.TotalTokens), ReasoningTokens: count(value.Usage.ReasoningTokens), CachedTokens: count(value.Usage.CachedTokens), CacheWriteTokens: count(value.Usage.CacheWriteTokens)},
Attempts: value.Attempts, TransientRetries: value.TransientRetries, RepairCalls: value.RepairCalls,

View File

@@ -298,6 +298,8 @@ type RunResult struct {
InputHashes map[string]string `json:"input_hashes,omitempty"`
// Usage is the token accounting reported by the LLM client.
Usage TokenUsage `json:"usage"`
// CostUSD is the observed charge subtotal across dispatched attempts.
CostUSD ObservedCost `json:"cost_usd"`
// StartTime is the UTC time immediately before ordinary Run preparation or
// after RunPrepared claims its handle.
StartTime time.Time `json:"start_time,omitempty"`
@@ -692,6 +694,8 @@ type GenerateResponse struct {
// UsagePresence states which usage fields were reported, including zeros.
// Nil uses the legacy observation fallback described by LLMClient.
UsagePresence *TokenUsagePresence `json:"usage_presence,omitempty"`
// CostUSD is an optional reported charge; a pointer to zero reports zero.
CostUSD *float64 `json:"cost_usd,omitempty"`
}
// TokenUsagePresence identifies provider-reported token counts for observation.