Report costs for ordinary runs

This commit is contained in:
2026-11-22 18:28:39 +00:00
parent 81a2a7591d
commit d77e3e2f83
5 changed files with 142 additions and 5 deletions

124
cost_run_contract_test.go Normal file
View File

@@ -0,0 +1,124 @@
package promptkit_test
import (
"context"
"errors"
"io"
"net/http"
"strings"
"testing"
"go.maximumdirect.net/configkit"
"go.maximumdirect.net/promptkit"
)
type costRoundTrip func(*http.Request) (*http.Response, error)
func (f costRoundTrip) RoundTrip(req *http.Request) (*http.Response, error) { return f(req) }
func costEngine(t *testing.T, zero bool, client *http.Client, options ...promptkit.Option) *promptkit.Engine {
t.Helper()
input := configkit.Input{Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("cost", map[string]any{
"backends": []any{map[string]any{"id": "backend", "endpoint": "https://example.invalid/v1", "provider": "openrouter", "zero_cost": zero}},
"profiles": []any{map[string]any{"id": "profile", "backend": "backend", "model": "model"}},
"prompts": []any{map[string]any{"id": "prompt", "default_profile": "profile", "messages": []any{map[string]any{"role": "user", "content": "hello"}}, "output": map[string]any{"format": "text", "validation_mode": "none"}}},
})}
options = append(options, promptkit.WithBundledOpenRouter(false))
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &input}, HTTPClient: client}, options...)
if err != nil {
t.Fatal(err)
}
return engine
}
func TestOrdinaryRunCostMatchesTerminalWithOrWithoutObserver(t *testing.T) {
for _, observed := range []bool{false, true} {
t.Run(map[bool]string{false: "without observer", true: "with observer"}[observed], func(t *testing.T) {
calls := 0
transport := costRoundTrip(func(*http.Request) (*http.Response, error) {
calls++
if calls == 1 {
return &http.Response{StatusCode: 503, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(`{"error":{}}`))}, nil
}
return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(`{"usage":{"cost":0.02},"choices":[{"message":{"content":"ok"}}]}`))}, nil
})
engine := costEngine(t, false, &http.Client{Transport: transport})
var events []promptkit.ExecutionEvent
req := promptkit.RunRequest{PromptID: "prompt"}
if observed {
req.Observer = func(e promptkit.ExecutionEvent) { events = append(events, e) }
}
result, err := engine.Run(context.Background(), req)
want := promptkit.ObservedCost{Value: .02, Known: true, Complete: false, Source: promptkit.CostSourceProvider}
if err != nil || result == nil || result.CostUSD != want || calls != 2 {
t.Fatalf("result=%+v calls=%d err=%v", result, calls, err)
}
if observed && (len(events) != 5 || events[len(events)-1].CostUSD != want) {
t.Fatalf("events=%+v", events)
}
})
}
}
func TestOrdinaryRunCostFailureAndFallback(t *testing.T) {
charge := .125
for _, tc := range []struct {
name string
response *promptkit.GenerateResponse
err error
want promptkit.ObservedCost
}{
{"reported takes precedence", &promptkit.GenerateResponse{Content: "ok", CostUSD: &charge}, nil, promptkit.ObservedCost{Value: .125, Known: true, Complete: true, Source: promptkit.CostSourceProvider}},
{"failed dispatch fallback", nil, errors.New("failure"), promptkit.ObservedCost{Known: true, Complete: true, Source: promptkit.CostSourceConfiguredZero}},
} {
t.Run(tc.name, func(t *testing.T) {
engine := costEngine(t, true, nil, promptkit.WithLLMClient(observationClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
return tc.response, tc.err
})))
var terminal promptkit.ExecutionEvent
result, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "prompt", Observer: func(e promptkit.ExecutionEvent) {
if e.Kind == promptkit.ExecutionFinished {
terminal = e
}
}})
if (tc.err != nil) != (err != nil) || terminal.CostUSD != tc.want || (result != nil && result.CostUSD != tc.want) {
t.Fatalf("result=%+v terminal=%+v err=%v", result, terminal, err)
}
})
}
}
func TestOrdinaryRunCombinesRepairCostSources(t *testing.T) {
charge := .1
calls := 0
engine := costEngine(t, true, nil, promptkit.WithLLMClient(observationClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
calls++
if calls == 1 {
return &promptkit.GenerateResponse{Content: "invalid"}, nil
}
return &promptkit.GenerateResponse{Content: `{"valid":true}`, CostUSD: &charge}, nil
})))
var terminal promptkit.ExecutionEvent
result, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "prompt", Validation: &promptkit.OutputContractInput{Format: promptkit.FormatJSON, ValidationMode: promptkit.ValidationJSON, RepairAttempts: new(1)}, Observer: func(e promptkit.ExecutionEvent) {
if e.Kind == promptkit.ExecutionFinished {
terminal = e
}
}})
want := promptkit.ObservedCost{Value: .1, Known: true, Complete: true, Source: promptkit.CostSourceMixed}
if err != nil || result == nil || calls != 2 || result.CostUSD != want || terminal.CostUSD != want {
t.Fatalf("result=%+v terminal=%+v calls=%d err=%v", result, terminal, calls, err)
}
}
func TestOrdinaryRunBeforeDispatchHasEmptyCost(t *testing.T) {
engine := costEngine(t, true, nil)
var terminal promptkit.ExecutionEvent
result, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "missing", Observer: func(e promptkit.ExecutionEvent) {
if e.Kind == promptkit.ExecutionFinished {
terminal = e
}
}})
if result != nil || err == nil || terminal.CostUSD != (promptkit.ObservedCost{Known: true, Complete: true}) {
t.Fatalf("result=%+v terminal=%+v err=%v", result, terminal, err)
}
}

View File

@@ -56,10 +56,11 @@ attempts above the first in a generation, and repair generations with at least
one dispatch. A terminal event is an aggregate and must not be added to its
attempt events.
Ordinary runs install invocation-local cost accounting even without an observer.
The generation coordinator resolves cost once per dispatched attempt and adds
it to invocation-local accounting independently of event delivery. A recorder
can read that same accumulator at termination; groups merge first terminal
costs independently of terminal coverage.
it to that state independently of event delivery. The result and recorder read
the same accumulator; groups merge first terminal costs independently of
terminal coverage.
Reasoning counts use the same per-field rules and are a subset of completion
tokens. The public cache-hit helper derives a ratio from complete, consistent

View File

@@ -353,6 +353,8 @@ Validation: generation, observation recorder/group, and diagnostic-boundary test
## Stage 12 — Ordinary Run Cost Lifetime And Result Accounting
**Completed.**
- Create/install a new private accumulator at the public ordinary Run boundary,
before optional recorder creation. Pass that same instance to the recorder.
Ensure direct internal runner execution creates a local accumulator if one

View File

@@ -602,6 +602,8 @@ func (e *Engine) PrepareExecution(ctx context.Context, req RunRequest) (*Prepare
// result on error.
func (e *Engine) Run(ctx context.Context, req RunRequest) (*RunResult, error) {
ctx = observation.WithoutExecutionState(ctx)
cost := &observation.CostAccumulator{}
ctx = observation.WithCostAccumulator(ctx, cost)
scope := usecase.NewOperationScope(ctx, req.OperationTimeout)
defer scope.Close()
ctx = scope.Context
@@ -617,6 +619,7 @@ func (e *Engine) Run(ctx context.Context, req RunRequest) (*RunResult, error) {
return result, err
}
recorder, idErr := observation.New(observer)
recorder.SetCostAccumulator(cost)
if req.ContinueFrom != nil && req.ContinueFrom.internal != nil {
recorder.SetParentRunID(req.ContinueFrom.internal.RunID())
}

View File

@@ -137,6 +137,9 @@ func NewRunnerWithRepairer(
}
func (r *Runner) Run(ctx context.Context, req domain.RunRequest) (*domain.RunResult, error) {
if observation.CostFromContext(ctx) == nil {
ctx = observation.WithCostAccumulator(ctx, &observation.CostAccumulator{})
}
recorder := observation.FromContext(ctx)
runID := ""
if recorder != nil {
@@ -274,7 +277,7 @@ func (r *Runner) executePreparedRun(
exchange = domain.CaptureExchange(parent, runID, prepared.Messages[parent.MessageCount():], genResp.Content)
}
return &domain.RunResult{
result := &domain.RunResult{
Exchange: exchange,
ParentRunID: prepared.ParentRunID,
RunID: runID,
@@ -295,7 +298,11 @@ func (r *Runner) executePreparedRun(
StartTime: start,
EndTime: end,
Duration: end.Sub(start),
}, nil
}
if cost := observation.CostFromContext(ctx); cost != nil {
result.CostUSD = cost.Observed()
}
return result, nil
}
func wrapGenerationError(err error) error {