49 lines
1.5 KiB
Go
49 lines
1.5 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"net/http"
|
|
"testing"
|
|
|
|
pk "gitea.maximumdirect.net/eric/promptkit"
|
|
)
|
|
|
|
func TestPartialUsageOnFailedRun(t *testing.T) {
|
|
calls := 0
|
|
r := testRuntime(t, roundTripFunc(func(req *http.Request) (*http.Response, error) {
|
|
calls++
|
|
if calls == 1 {
|
|
return providerResponse(200, "bad", map[string]any{"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0}), nil
|
|
}
|
|
return providerResponse(503, "", nil), nil
|
|
}), intOverride("transient_retries", 1), intOverride("repair_attempts", 1))
|
|
recorder := &Recorder{}
|
|
result, err := r.call(context.Background(), "proposal", roleVars(), recorder.observer("proposal"))
|
|
if result != nil || !errors.Is(err, pk.ErrRetriesExhausted) {
|
|
t.Fatalf("result %v error %v", result, err)
|
|
}
|
|
a, rt, rp := recorder.totals()
|
|
if a != 3 || rt != 1 || rp != 1 {
|
|
t.Fatalf("totals %d/%d/%d", a, rt, rp)
|
|
}
|
|
knownZero, unknown, terminal := false, false, false
|
|
for _, o := range recorder.snapshot() {
|
|
if o.Kind == string(pk.ExecutionAttempt) && o.StatusCode == 200 {
|
|
knownZero = o.Usage.Total.Known && o.Usage.Total.Complete && o.Usage.Total.Value == 0
|
|
}
|
|
if o.Kind == string(pk.ExecutionAttempt) && o.StatusCode == 503 {
|
|
unknown = !o.Usage.Total.Known
|
|
}
|
|
if o.Kind == string(pk.ExecutionFinished) {
|
|
terminal = true
|
|
if !o.Usage.Total.Known || o.Usage.Total.Complete || o.Usage.Total.Value != 0 || !o.RetriesExhausted {
|
|
t.Fatalf("partial total %+v", o)
|
|
}
|
|
}
|
|
}
|
|
if !knownZero || !unknown || !terminal {
|
|
t.Fatal("usage evidence missing")
|
|
}
|
|
}
|