From d4509b163edbac22b2fb528128ff0b17976e292e Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Sun, 22 Nov 2026 18:32:29 +0000 Subject: [PATCH] Cover concurrent multi-provider invocation accounting --- docs/roadmap/implementation.md | 2 + invocation_workflow_contract_test.go | 127 +++++++++++++++++++++++++++ 2 files changed, 129 insertions(+) create mode 100644 invocation_workflow_contract_test.go diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index f7815d6..67dcd60 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -394,6 +394,8 @@ Validation: prepared and nested-boundary contracts, including race-enabled focus ## Stage 14 — End-To-End Consumer Regression Coverage +**Completed.** + - Add one representative offline invocation workflow using two engines and concurrent runs under one group. Use controlled transport/injected clients for OpenRouter reported cost, vLLM details/configured zero, a retry, and a diff --git a/invocation_workflow_contract_test.go b/invocation_workflow_contract_test.go new file mode 100644 index 0000000..27a4cf8 --- /dev/null +++ b/invocation_workflow_contract_test.go @@ -0,0 +1,127 @@ +package promptkit_test + +import ( + "context" + "encoding/json" + "io" + "math" + "net/http" + "strings" + "sync" + "sync/atomic" + "testing" + + "go.maximumdirect.net/configkit" + "go.maximumdirect.net/promptkit" +) + +func TestConcurrentInvocationObservationAcrossProviders(t *testing.T) { + pendingStarted, release := make(chan struct{}), make(chan struct{}) + releasePending := sync.OnceFunc(func() { close(release) }) + defer releasePending() + var retryCalls atomic.Int32 + transport := costRoundTrip(func(req *http.Request) (*http.Response, error) { + var body struct { + Model string `json:"model"` + } + if err := json.NewDecoder(req.Body).Decode(&body); err != nil { + return nil, err + } + status, payload := 200, "" + switch body.Model { + case "router": + if retryCalls.Add(1) == 1 { + status, payload = 503, `{"error":{}}` + } else { + payload = `{"usage":{"prompt_tokens":100,"completion_tokens":20,"total_tokens":120,"prompt_tokens_details":{"cached_tokens":40,"cache_write_tokens":5},"completion_tokens_details":{"reasoning_tokens":3},"cost":0.02},"choices":[{"message":{"content":"ok"}}]}` + } + case "failure": + payload = `{"usage":{"prompt_tokens":30,"completion_tokens":6,"total_tokens":36,"prompt_tokens_details":{"cached_tokens":15,"cache_write_tokens":1},"completion_tokens_details":{"reasoning_tokens":1},"cost":0.01},"choices":[]}` + case "vllm": + payload = `{"usage":{"prompt_tokens":20,"completion_tokens":8,"total_tokens":28,"prompt_tokens_details":{"cached_tokens":10,"created_cache_tokens":4},"completion_tokens_details":{"reasoning_tokens":2}},"choices":[{"message":{"content":"ok"}}]}` + case "pending": + close(pendingStarted) + <-release + payload = `{"usage":{"prompt_tokens":10,"completion_tokens":2,"total_tokens":12,"prompt_tokens_details":{"cached_tokens":0,"created_cache_tokens":0},"completion_tokens_details":{"reasoning_tokens":0}},"choices":[{"message":{"content":"ok"}}]}` + default: + t.Errorf("unexpected model %q", body.Model) + } + return &http.Response{StatusCode: status, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(payload))}, nil + }) + engine := func(provider, model string) *promptkit.Engine { + t.Helper() + input := configkit.Input{Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("workflow", map[string]any{ + "backends": []any{map[string]any{"id": "backend", "endpoint": "https://example.invalid/v1", "provider": provider, "zero_cost": true}}, + "profiles": []any{map[string]any{"id": "profile", "backend": "backend", "model": model}}, + "prompts": []any{map[string]any{"id": "prompt", "default_profile": "profile", "messages": []any{map[string]any{"role": "user", "content": "hello"}}, "output": map[string]any{"format": "text", "validation_mode": "none"}}}, + })} + result, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &input}, HTTPClient: &http.Client{Transport: transport}}, promptkit.WithBundledOpenRouter(false)) + if err != nil { + t.Fatal(err) + } + return result + } + router, vllm := engine("openrouter", "router"), engine("vllm", "vllm") + var group promptkit.ExecutionGroup + if empty := group.Snapshot(); len(empty.Members) != 0 || empty.CostUSD != (promptkit.ObservedCost{Known: true, Complete: true}) { + t.Fatalf("empty=%+v", empty) + } + ctx := promptkit.WithExecutionObserver(context.Background(), group.Observe) + results := make(chan error, 4) + var completed sync.WaitGroup + var pendingDone sync.WaitGroup + start := func(e *promptkit.Engine, model string, fail bool) { + completed.Add(1) + go func() { + defer completed.Done() + req := promptkit.RunRequest{PromptID: "prompt"} + if model != "" { + req.Execution = &promptkit.ExecutionTargetOverride{Model: model} + } + _, err := e.Run(ctx, req) + if fail && err != nil { + err = nil + } else if fail { + err = io.ErrUnexpectedEOF + } + results <- err + }() + } + pendingDone.Add(1) + go func() { + defer pendingDone.Done() + _, err := vllm.Run(ctx, promptkit.RunRequest{PromptID: "prompt", Execution: &promptkit.ExecutionTargetOverride{Model: "pending"}}) + results <- err + }() + <-pendingStarted + if pending := group.Snapshot(); len(pending.Members) != 1 || pending.CostUSD != (promptkit.ObservedCost{}) || pending.TerminalCoverageComplete { + t.Fatalf("pending=%+v", pending) + } + start(router, "", false) + start(router, "failure", true) + start(vllm, "", false) + completed.Wait() + for range 3 { + if err := <-results; err != nil { + t.Fatal(err) + } + } + snapshot := group.Snapshot() + if len(snapshot.Members) != 4 || snapshot.TerminalCoverageComplete || snapshot.Attempts != 4 || !snapshot.CostUSD.Known || !snapshot.CostUSD.Complete || snapshot.CostUSD.Source != promptkit.CostSourceMixed || math.Abs(snapshot.CostUSD.Value-.03) > 1e-12 { + t.Fatalf("snapshot=%+v", snapshot) + } + if snapshot.Usage.PromptTokens.Value != 150 || snapshot.Usage.ReasoningTokens.Value != 6 || snapshot.Usage.CachedTokens.Value != 65 || snapshot.Usage.CacheWriteTokens.Value != 10 || snapshot.Usage.PromptTokens.Complete { + t.Fatalf("usage=%+v", snapshot.Usage) + } + if _, ok := snapshot.Usage.CacheHitRatio(); ok { + t.Fatal("ratio available despite an unreported retry") + } + releasePending() + if err := <-results; err != nil { + t.Fatal(err) + } + pendingDone.Wait() + if final := group.Snapshot(); !final.TerminalCoverageComplete || len(final.Members) != 4 || final.Attempts != 5 || math.Abs(final.CostUSD.Value-.03) > 1e-12 { + t.Fatalf("final=%+v", final) + } +}