From 6ed0e037c4b050a651063a35b418e41a19d6d963 Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Sun, 22 Nov 2026 18:24:33 +0000 Subject: [PATCH] Define observed cost values and aggregation --- convert.go | 2 + cost.go | 30 +++++++++++++++ docs/roadmap/implementation.md | 2 + execution_group.go | 5 ++- internal/domain/cost.go | 17 +++++++++ internal/domain/domain.go | 2 + internal/domain/observation.go | 2 + internal/observation/cost.go | 53 ++++++++++++++++++++++++++ internal/observation/cost_test.go | 63 +++++++++++++++++++++++++++++++ json_contract_test.go | 29 ++++++++++++++ llm_adapter_internal_test.go | 6 +++ observations.go | 2 + types.go | 4 ++ 13 files changed, 215 insertions(+), 2 deletions(-) create mode 100644 cost.go create mode 100644 internal/domain/cost.go create mode 100644 internal/observation/cost.go create mode 100644 internal/observation/cost_test.go diff --git a/convert.go b/convert.go index 8753254..0278550 100644 --- a/convert.go +++ b/convert.go @@ -134,6 +134,7 @@ func fromDomainRunResult(result *domain.RunResult) *RunResult { EffectiveModelParams: fromDomainExecutionTarget(result.EffectiveModelParams), InputHashes: copyStringMap(result.InputHashes), Usage: fromDomainTokenUsage(result.Usage), + CostUSD: fromDomainCost(result.CostUSD), StartTime: result.StartTime, EndTime: result.EndTime, Duration: result.Duration, @@ -158,6 +159,7 @@ func toDomainGenerateResponse(resp *GenerateResponse) *domain.GenerateResponse { Content: resp.Content, Usage: toDomainTokenUsage(resp.Usage), UsagePresence: toDomainTokenUsagePresence(resp.UsagePresence), + CostUSD: copyFloat64Ptr(resp.CostUSD), } } diff --git a/cost.go b/cost.go new file mode 100644 index 0000000..2449c85 --- /dev/null +++ b/cost.go @@ -0,0 +1,30 @@ +package promptkit + +import "go.maximumdirect.net/promptkit/internal/domain" + +// CostSource identifies how an observed USD cost was supplied. +type CostSource string + +const ( + CostSourceNone CostSource = "" + CostSourceProvider CostSource = "provider" + CostSourceConfiguredZero CostSource = "configured_zero" + CostSourceMixed CostSource = "mixed" +) + +// ObservedCost reports an available USD subtotal and its coverage and source. +// Complete describes coverage of dispatched attempts, not billing verification. +type ObservedCost struct { + Value float64 `json:"value"` + Known bool `json:"known"` + Complete bool `json:"complete"` + Source CostSource `json:"source"` +} + +func fromDomainCost(value domain.ObservedCost) ObservedCost { + return ObservedCost{Value: value.Value, Known: value.Known, Complete: value.Complete, Source: CostSource(value.Source)} +} + +func toDomainCost(value ObservedCost) domain.ObservedCost { + return domain.ObservedCost{Value: value.Value, Known: value.Known, Complete: value.Complete, Source: domain.CostSource(value.Source)} +} diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index 3db1644..95ddf1a 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -276,6 +276,8 @@ Validation: focused token accumulator/group tests and public observation contrac ## Stage 9 — Public Cost Values And Shared Cost Algebra +**Completed.** + Create public cost declarations in a focused root file and matching domain values; add the selected fields to responses, results, events, and summaries. diff --git a/execution_group.go b/execution_group.go index 88bd121..ad7b0b2 100644 --- a/execution_group.go +++ b/execution_group.go @@ -29,6 +29,7 @@ type ExecutionGroupSummary struct { TransientRetries uint64 `json:"transient_retries"` RepairCalls uint64 `json:"repair_calls"` Usage ObservedTokenUsage `json:"usage"` + CostUSD ObservedCost `json:"cost_usd"` } // Observe enrolls an event's run, counting its first terminal event only. @@ -36,7 +37,7 @@ func (g *ExecutionGroup) Observe(event ExecutionEvent) { count := func(c ObservedTokenCount) domain.ObservedTokenCount { return domain.ObservedTokenCount{Value: c.Value, Known: c.Known, Complete: c.Complete} } - g.group.Observe(domain.ExecutionEvent{Kind: domain.ExecutionEventKind(event.Kind), RunID: event.RunID, Attempts: event.Attempts, TransientRetries: event.TransientRetries, RepairCalls: event.RepairCalls, + g.group.Observe(domain.ExecutionEvent{Kind: domain.ExecutionEventKind(event.Kind), RunID: event.RunID, Attempts: event.Attempts, TransientRetries: event.TransientRetries, RepairCalls: event.RepairCalls, CostUSD: toDomainCost(event.CostUSD), Usage: domain.ObservedTokenUsage{PromptTokens: count(event.Usage.PromptTokens), CompletionTokens: count(event.Usage.CompletionTokens), ReasoningTokens: count(event.Usage.ReasoningTokens), TotalTokens: count(event.Usage.TotalTokens), CachedTokens: count(event.Usage.CachedTokens), CacheWriteTokens: count(event.Usage.CacheWriteTokens)}}) } @@ -51,6 +52,6 @@ func (g *ExecutionGroup) Snapshot() ExecutionGroupSummary { for i, member := range value.Members { members[i] = ExecutionGroupMember{RunID: member.RunID, TerminalObserved: member.TerminalObserved} } - return ExecutionGroupSummary{Members: members, TerminalCoverageComplete: value.TerminalCoverageComplete, Attempts: value.Attempts, TransientRetries: value.TransientRetries, RepairCalls: value.RepairCalls, + return ExecutionGroupSummary{Members: members, TerminalCoverageComplete: value.TerminalCoverageComplete, Attempts: value.Attempts, TransientRetries: value.TransientRetries, RepairCalls: value.RepairCalls, CostUSD: fromDomainCost(value.CostUSD), Usage: ObservedTokenUsage{PromptTokens: count(value.Usage.PromptTokens), CompletionTokens: count(value.Usage.CompletionTokens), ReasoningTokens: count(value.Usage.ReasoningTokens), TotalTokens: count(value.Usage.TotalTokens), CachedTokens: count(value.Usage.CachedTokens), CacheWriteTokens: count(value.Usage.CacheWriteTokens)}} } diff --git a/internal/domain/cost.go b/internal/domain/cost.go new file mode 100644 index 0000000..af3b39e --- /dev/null +++ b/internal/domain/cost.go @@ -0,0 +1,17 @@ +package domain + +type CostSource string + +const ( + CostSourceNone CostSource = "" + CostSourceProvider CostSource = "provider" + CostSourceConfiguredZero CostSource = "configured_zero" + CostSourceMixed CostSource = "mixed" +) + +type ObservedCost struct { + Value float64 + Known bool + Complete bool + Source CostSource +} diff --git a/internal/domain/domain.go b/internal/domain/domain.go index 2c6cc4a..ae8f0f3 100644 --- a/internal/domain/domain.go +++ b/internal/domain/domain.go @@ -99,6 +99,7 @@ type RunResult struct { EffectiveModelParams ExecutionTarget InputHashes map[string]string Usage TokenUsage + CostUSD ObservedCost StartTime time.Time EndTime time.Time Duration time.Duration @@ -328,6 +329,7 @@ type GenerateResponse struct { Content string Usage TokenUsage UsagePresence *TokenUsagePresence + CostUSD *float64 } // TokenUsagePresence records which token fields a provider reported. diff --git a/internal/domain/observation.go b/internal/domain/observation.go index 2836c63..348c68f 100644 --- a/internal/domain/observation.go +++ b/internal/domain/observation.go @@ -76,6 +76,7 @@ type ExecutionGroupSummary struct { TransientRetries uint64 RepairCalls uint64 Usage ObservedTokenUsage + CostUSD ObservedCost } // ExecutionEvent contains metadata only; it carries no mutable containers or @@ -101,6 +102,7 @@ type ExecutionEvent struct { StatusCode int RetriesExhausted bool Usage ObservedTokenUsage + CostUSD ObservedCost Attempts uint64 TransientRetries uint64 RepairCalls uint64 diff --git a/internal/observation/cost.go b/internal/observation/cost.go new file mode 100644 index 0000000..60ee947 --- /dev/null +++ b/internal/observation/cost.go @@ -0,0 +1,53 @@ +package observation + +import ( + "math" + + "go.maximumdirect.net/promptkit/internal/domain" +) + +// ResolveCost applies reported-cost precedence over a configured-zero fallback. +func ResolveCost(reported *float64, zeroCost bool) domain.ObservedCost { + if reported != nil && !math.IsNaN(*reported) && !math.IsInf(*reported, 0) { + return domain.ObservedCost{Value: *reported, Known: true, Complete: true, Source: domain.CostSourceProvider} + } + if zeroCost { + return domain.ObservedCost{Known: true, Complete: true, Source: domain.CostSourceConfiguredZero} + } + return domain.ObservedCost{} +} + +// CostAccumulator merges dispatched attempts or already aggregated costs. +type CostAccumulator struct { + value domain.ObservedCost + started bool +} + +// Add incorporates one contribution. +func (c *CostAccumulator) Add(next domain.ObservedCost) { + if !c.started { + c.started = true + c.value = next + return + } + if next.Known { + c.value.Value += next.Value + if next.Source != domain.CostSourceNone { + if c.value.Source == domain.CostSourceNone { + c.value.Source = next.Source + } else if c.value.Source != next.Source { + c.value.Source = domain.CostSourceMixed + } + } + } + c.value.Known = c.value.Known || next.Known + c.value.Complete = c.value.Complete && next.Complete +} + +// Observed returns known complete zero when no contribution has been added. +func (c CostAccumulator) Observed() domain.ObservedCost { + if !c.started { + return domain.ObservedCost{Known: true, Complete: true} + } + return c.value +} diff --git a/internal/observation/cost_test.go b/internal/observation/cost_test.go new file mode 100644 index 0000000..52edc1a --- /dev/null +++ b/internal/observation/cost_test.go @@ -0,0 +1,63 @@ +package observation + +import ( + "math" + "testing" + + "go.maximumdirect.net/promptkit/internal/domain" +) + +func TestResolveCostPrecedence(t *testing.T) { + zero, negative, invalid := 0.0, -1.5, math.Inf(1) + for _, tc := range []struct { + name string + reported *float64 + fallback bool + want domain.ObservedCost + }{ + {"unknown", nil, false, domain.ObservedCost{}}, + {"configured zero", nil, true, domain.ObservedCost{Known: true, Complete: true, Source: domain.CostSourceConfiguredZero}}, + {"reported zero", &zero, true, domain.ObservedCost{Known: true, Complete: true, Source: domain.CostSourceProvider}}, + {"negative", &negative, false, domain.ObservedCost{Value: -1.5, Known: true, Complete: true, Source: domain.CostSourceProvider}}, + {"invalid fallback", &invalid, true, domain.ObservedCost{Known: true, Complete: true, Source: domain.CostSourceConfiguredZero}}, + {"invalid unknown", &invalid, false, domain.ObservedCost{}}, + } { + t.Run(tc.name, func(t *testing.T) { + if got := ResolveCost(tc.reported, tc.fallback); got != tc.want { + t.Fatalf("got %+v want %+v", got, tc.want) + } + }) + } +} + +func TestCostAccumulatorMergesSourcesAndCoverage(t *testing.T) { + var acc CostAccumulator + if got := acc.Observed(); got != (domain.ObservedCost{Known: true, Complete: true}) { + t.Fatalf("empty: %+v", got) + } + acc.Add(domain.ObservedCost{}) + if got := acc.Observed(); got != (domain.ObservedCost{}) { + t.Fatalf("unknown: %+v", got) + } + a, b := .1, .2 + acc.Add(ResolveCost(&a, false)) + acc.Add(ResolveCost(&b, false)) + acc.Add(ResolveCost(nil, true)) + got := acc.Observed() + if math.Abs(got.Value-.3) > 1e-12 || !got.Known || got.Complete || got.Source != domain.CostSourceMixed { + t.Fatalf("merged: %+v", got) + } + var zeros CostAccumulator + zero := 0.0 + zeros.Add(ResolveCost(&zero, false)) + zeros.Add(ResolveCost(nil, true)) + if got := zeros.Observed(); got != (domain.ObservedCost{Known: true, Complete: true, Source: domain.CostSourceMixed}) { + t.Fatalf("zeros: %+v", got) + } + var aggregate CostAccumulator + aggregate.Add(domain.ObservedCost{Known: true, Complete: true}) + aggregate.Add(got) + if aggregate.Observed() != got { + t.Fatalf("aggregate: %+v", aggregate.Observed()) + } +} diff --git a/json_contract_test.go b/json_contract_test.go index 2e9f8d6..c0bca6e 100644 --- a/json_contract_test.go +++ b/json_contract_test.go @@ -49,6 +49,34 @@ func TestGenerateResponseUsagePresenceJSON(t *testing.T) { } } +func TestCostJSONPreservesUnknownAndReportedZero(t *testing.T) { + var old promptkit.RunResult + if err := json.Unmarshal([]byte(`{"prompt_id":"old"}`), &old); err != nil || old.CostUSD != (promptkit.ObservedCost{}) { + t.Fatalf("old result=%+v err=%v", old, err) + } + value := promptkit.RunResult{CostUSD: promptkit.ObservedCost{Value: .125, Known: true, Complete: false, Source: promptkit.CostSourceProvider}} + data, err := json.Marshal(value) + if err != nil { + t.Fatal(err) + } + var decoded promptkit.RunResult + if err := json.Unmarshal(data, &decoded); err != nil || decoded.CostUSD != value.CostUSD { + t.Fatalf("round trip=%+v err=%v", decoded, err) + } + zero := 0.0 + data, err = json.Marshal(promptkit.GenerateResponse{CostUSD: &zero}) + if err != nil || string(decodeJSONObject(t, data)["cost_usd"]) != "0" { + t.Fatalf("reported zero=%s err=%v", data, err) + } + data, err = json.Marshal(promptkit.GenerateResponse{}) + if err != nil { + t.Fatal(err) + } + if _, present := decodeJSONObject(t, data)["cost_usd"]; present { + t.Fatalf("unknown cost=%s", data) + } +} + func TestRemovedPromptVersionIsAbsentFromJSON(t *testing.T) { values := map[string]any{ "prepared": promptkit.PreparedRun{PromptID: "prompt"}, @@ -176,6 +204,7 @@ func TestRunResultJSONContractRoundTripsAllFields(t *testing.T) { "effective_model_params", "input_hashes", "usage", + "cost_usd", "start_time", "end_time", "duration_ms", diff --git a/llm_adapter_internal_test.go b/llm_adapter_internal_test.go index 2c701cc..9fc2ec0 100644 --- a/llm_adapter_internal_test.go +++ b/llm_adapter_internal_test.go @@ -50,10 +50,12 @@ func TestPublicLLMClientAdapterGivesClientOwnedNestedValues(t *testing.T) { func TestPublicLLMClientAdapterCopiesUsagePresence(t *testing.T) { presence := &TokenUsagePresence{PromptTokens: true} + cost := .25 client := &usagePresenceClient{response: &GenerateResponse{ Content: "ok", Usage: TokenUsage{PromptTokens: 0, TotalTokens: 5}, UsagePresence: presence, + CostUSD: &cost, }} adapter := publicLLMClientAdapter{client: client} got, err := adapter.Generate(context.Background(), domain.GenerateRequest{}) @@ -61,9 +63,13 @@ func TestPublicLLMClientAdapterCopiesUsagePresence(t *testing.T) { t.Fatal(err) } presence.PromptTokens = false + cost = .5 if got.UsagePresence == nil || !got.UsagePresence.PromptTokens || got.UsagePresence.TotalTokens || got.Usage.TotalTokens != 5 { t.Fatalf("copied response = %+v", got) } + if got.CostUSD == nil || *got.CostUSD != .25 { + t.Fatalf("copied cost=%v", got.CostUSD) + } } func TestPublicLLMClientAdapterPreservesOwnedResponseAlongsideError(t *testing.T) { diff --git a/observations.go b/observations.go index 1347ddb..e9d101a 100644 --- a/observations.go +++ b/observations.go @@ -150,6 +150,7 @@ type ExecutionEvent struct { StatusCode int RetriesExhausted bool Usage ObservedTokenUsage + CostUSD ObservedCost Attempts uint64 TransientRetries uint64 RepairCalls uint64 @@ -167,6 +168,7 @@ func fromDomainExecutionEvent(value domain.ExecutionEvent) ExecutionEvent { GenerationDuration: value.GenerationDuration, RetryDelay: value.RetryDelay, Duration: value.Duration, Outcome: ExecutionOutcome(value.Outcome), Failure: ExecutionFailureKind(value.Failure), StatusCode: value.StatusCode, RetriesExhausted: value.RetriesExhausted, + CostUSD: fromDomainCost(value.CostUSD), Usage: ObservedTokenUsage{PromptTokens: count(value.Usage.PromptTokens), CompletionTokens: count(value.Usage.CompletionTokens), TotalTokens: count(value.Usage.TotalTokens), ReasoningTokens: count(value.Usage.ReasoningTokens), CachedTokens: count(value.Usage.CachedTokens), CacheWriteTokens: count(value.Usage.CacheWriteTokens)}, Attempts: value.Attempts, TransientRetries: value.TransientRetries, RepairCalls: value.RepairCalls, diff --git a/types.go b/types.go index 1ff2aeb..7e9c66c 100644 --- a/types.go +++ b/types.go @@ -298,6 +298,8 @@ type RunResult struct { InputHashes map[string]string `json:"input_hashes,omitempty"` // Usage is the token accounting reported by the LLM client. Usage TokenUsage `json:"usage"` + // CostUSD is the observed charge subtotal across dispatched attempts. + CostUSD ObservedCost `json:"cost_usd"` // StartTime is the UTC time immediately before ordinary Run preparation or // after RunPrepared claims its handle. StartTime time.Time `json:"start_time,omitempty"` @@ -692,6 +694,8 @@ type GenerateResponse struct { // UsagePresence states which usage fields were reported, including zeros. // Nil uses the legacy observation fallback described by LLMClient. UsagePresence *TokenUsagePresence `json:"usage_presence,omitempty"` + // CostUSD is an optional reported charge; a pointer to zero reports zero. + CostUSD *float64 `json:"cost_usd,omitempty"` } // TokenUsagePresence identifies provider-reported token counts for observation.