From ba4de9d11e18ada8c3048844b88cf95f8b808a67 Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Sun, 22 Nov 2026 18:30:02 +0000 Subject: [PATCH] Isolate cost accounting for prepared executions --- cost_prepared_contract_test.go | 131 +++++++++++++++++++++++++ docs/internal/observation.md | 3 +- docs/roadmap/implementation.md | 2 + internal/usecase/prepared_execution.go | 3 + 4 files changed, 138 insertions(+), 1 deletion(-) create mode 100644 cost_prepared_contract_test.go diff --git a/cost_prepared_contract_test.go b/cost_prepared_contract_test.go new file mode 100644 index 0000000..accdc0b --- /dev/null +++ b/cost_prepared_contract_test.go @@ -0,0 +1,131 @@ +package promptkit_test + +import ( + "context" + "errors" + "math" + "testing" + + "go.maximumdirect.net/promptkit" +) + +func TestPreparedCostAndRejectedClaims(t *testing.T) { + calls := 0 + engine := costEngine(t, true, nil, promptkit.WithLLMClient(observationClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { + calls++ + if calls == 1 { + return &promptkit.GenerateResponse{Content: "bad"}, nil + } + return nil, errors.New("repair failed") + }))) + ctx := context.Background() + var events []promptkit.ExecutionEvent + ctx = promptkit.WithExecutionObserver(ctx, func(e promptkit.ExecutionEvent) { events = append(events, e) }) + handle, err := engine.PrepareExecution(ctx, promptkit.RunRequest{PromptID: "prompt", Validation: &promptkit.OutputContractInput{Format: promptkit.FormatJSON, ValidationMode: promptkit.ValidationJSON, RepairAttempts: new(1)}}) + if err != nil || len(events) != 0 { + t.Fatalf("prepare error=%v events=%+v", err, events) + } + result, err := engine.RunPrepared(ctx, handle) + want := promptkit.ObservedCost{Known: true, Complete: true, Source: promptkit.CostSourceConfiguredZero} + if result != nil || err == nil || calls != 2 || len(events) != 4 || events[len(events)-1].CostUSD != want { + t.Fatalf("result=%+v events=%+v calls=%d err=%v", result, events, calls, err) + } + events = nil + if _, err := engine.RunPrepared(ctx, handle); !errors.Is(err, promptkit.ErrInvalidRequest) || len(events) != 0 { + t.Fatalf("reused err=%v events=%+v", err, events) + } +} + +func TestNestedOrdinaryAndPreparedRunsIsolateCosts(t *testing.T) { + childCalls := 0 + child := costEngine(t, false, nil, promptkit.WithLLMClient(observationClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { + childCalls++ + charge := .2 + if childCalls == 2 { + charge = .3 + } + return &promptkit.GenerateResponse{Content: "child", CostUSD: &charge}, nil + }))) + handle, err := child.PrepareExecution(context.Background(), promptkit.RunRequest{PromptID: "prompt"}) + if err != nil { + t.Fatal(err) + } + var childRun, childPrepared *promptkit.RunResult + outer := costEngine(t, false, nil, promptkit.WithLLMClient(observationClientFunc(func(ctx context.Context, _ promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { + var err error + childRun, err = child.Run(ctx, promptkit.RunRequest{PromptID: "prompt"}) + if err != nil { + return nil, err + } + childPrepared, err = child.RunPrepared(ctx, handle) + if err != nil { + return nil, err + } + charge := 1.0 + return &promptkit.GenerateResponse{Content: "parent", CostUSD: &charge}, nil + }))) + var group promptkit.ExecutionGroup + ctx := promptkit.WithExecutionObserver(context.Background(), group.Observe) + result, err := outer.Run(ctx, promptkit.RunRequest{PromptID: "prompt"}) + if err != nil { + t.Fatal(err) + } + if result.CostUSD.Value != 1 || childRun.CostUSD.Value != .2 || childPrepared.CostUSD.Value != .3 { + t.Fatalf("parent=%+v child=%+v prepared=%+v", result.CostUSD, childRun.CostUSD, childPrepared.CostUSD) + } + summary := group.Snapshot() + if len(summary.Members) != 3 || !summary.TerminalCoverageComplete || !summary.CostUSD.Complete || summary.CostUSD.Source != promptkit.CostSourceProvider || math.Abs(summary.CostUSD.Value-1.5) > 1e-12 { + t.Fatalf("summary=%+v", summary) + } +} + +func TestNestedDiagnosticDoesNotContributeCostOrEvents(t *testing.T) { + diagnostic := costEngine(t, false, nil, promptkit.WithLLMClient(observationClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { + charge := 2.0 + return &promptkit.GenerateResponse{Content: "OK", CostUSD: &charge}, nil + }))) + outer := costEngine(t, false, nil, promptkit.WithLLMClient(observationClientFunc(func(ctx context.Context, _ promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { + if _, err := diagnostic.SmokeTest(ctx, promptkit.SmokeTestRequest{Kind: promptkit.SmokeTestTextInference, ProfileID: "profile"}); err != nil { + return nil, err + } + charge := 1.0 + return &promptkit.GenerateResponse{Content: "parent", CostUSD: &charge}, nil + }))) + var group promptkit.ExecutionGroup + result, err := outer.Run(promptkit.WithExecutionObserver(context.Background(), group.Observe), promptkit.RunRequest{PromptID: "prompt"}) + if err != nil { + t.Fatal(err) + } + summary := group.Snapshot() + if result.CostUSD.Value != 1 || summary.CostUSD.Value != 1 || len(summary.Members) != 1 { + t.Fatalf("result=%+v group=%+v", result.CostUSD, summary) + } +} + +func TestPreparedCallbackChildHasIndependentCost(t *testing.T) { + child := costEngine(t, false, nil, promptkit.WithLLMClient(observationClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { + charge := .5 + return &promptkit.GenerateResponse{Content: "child", CostUSD: &charge}, nil + }))) + parent := costEngine(t, false, nil, promptkit.WithLLMClient(observationClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { + charge := 1.0 + return &promptkit.GenerateResponse{Content: "parent", CostUSD: &charge}, nil + }))) + var childResult *promptkit.RunResult + var childErr error + handle, err := parent.PrepareExecution(context.Background(), promptkit.RunRequest{PromptID: "prompt", OnPrepared: func(ctx context.Context, _ promptkit.PreparedRun) { + childResult, childErr = child.Run(ctx, promptkit.RunRequest{PromptID: "prompt"}) + }}) + if err != nil { + t.Fatal(err) + } + var group promptkit.ExecutionGroup + result, err := parent.RunPrepared(promptkit.WithExecutionObserver(context.Background(), group.Observe), handle) + if err != nil || childErr != nil || result.CostUSD.Value != 1 || childResult == nil || childResult.CostUSD.Value != .5 { + t.Fatalf("parent=%+v child=%+v errors=%v,%v", result, childResult, err, childErr) + } + summary := group.Snapshot() + if len(summary.Members) != 2 || math.Abs(summary.CostUSD.Value-1.5) > 1e-12 { + t.Fatalf("summary=%+v", summary) + } +} diff --git a/docs/internal/observation.md b/docs/internal/observation.md index 9bca0cf..1980f2c 100644 --- a/docs/internal/observation.md +++ b/docs/internal/observation.md @@ -56,7 +56,8 @@ attempts above the first in a generation, and repair generations with at least one dispatch. A terminal event is an aggregate and must not be added to its attempt events. -Ordinary runs install invocation-local cost accounting even without an observer. +Ordinary runs and claimed prepared executions install invocation-local cost +accounting even without an observer. The generation coordinator resolves cost once per dispatched attempt and adds it to that state independently of event delivery. The result and recorder read the same accumulator; groups merge first terminal costs independently of diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index 92b7e5f..f7815d6 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -373,6 +373,8 @@ Validation: assembled ordinary run/reliable execution contract tests. ## Stage 13 — Prepared Cost Lifetime And Nested Isolation +**Completed.** + - In internal RunPrepared, after successful claim create/install its own cost accumulator before recorder creation and generation. Use the same result and terminal accounting path as ordinary Run. Do not store an accumulator or diff --git a/internal/usecase/prepared_execution.go b/internal/usecase/prepared_execution.go index 86317f7..929abab 100644 --- a/internal/usecase/prepared_execution.go +++ b/internal/usecase/prepared_execution.go @@ -110,6 +110,8 @@ func (r *Runner) RunPrepared(ctx context.Context, prepared *PreparedExecution) ( return nil, err } defer payload.clear() + cost := &observation.CostAccumulator{} + ctx = observation.WithCostAccumulator(ctx, cost) scope := NewOperationScope(ctx, payload.operationTimeout) defer scope.Close() defer func() { err = scope.Finalize(err) }() @@ -121,6 +123,7 @@ func (r *Runner) RunPrepared(ctx context.Context, prepared *PreparedExecution) ( if observer != nil { var idErr error recorder, idErr = observation.New(observer) + recorder.SetCostAccumulator(cost) recorder.SetParentRunID(payload.parent.RunID()) observation.RegisterClaim(ctx, recorder) ctx = observation.WithRecorder(ctx, recorder)