Prototype ConfigKit and PromptKit adoption and record the findings

examples/library-adoption/observations_test.go

@@ -0,0 +1,48 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"net/http"
+	"testing"
+
+	pk "gitea.maximumdirect.net/eric/promptkit"
+)
+
+func TestPartialUsageOnFailedRun(t *testing.T) {
+	calls := 0
+	r := testRuntime(t, roundTripFunc(func(req *http.Request) (*http.Response, error) {
+		calls++
+		if calls == 1 {
+			return providerResponse(200, "bad", map[string]any{"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0}), nil
+		}
+		return providerResponse(503, "", nil), nil
+	}), intOverride("transient_retries", 1), intOverride("repair_attempts", 1))
+	recorder := &Recorder{}
+	result, err := r.call(context.Background(), "proposal", roleVars(), recorder.observer("proposal"))
+	if result != nil || !errors.Is(err, pk.ErrRetriesExhausted) {
+		t.Fatalf("result %v error %v", result, err)
+	}
+	a, rt, rp := recorder.totals()
+	if a != 3 || rt != 1 || rp != 1 {
+		t.Fatalf("totals %d/%d/%d", a, rt, rp)
+	}
+	knownZero, unknown, terminal := false, false, false
+	for _, o := range recorder.snapshot() {
+		if o.Kind == string(pk.ExecutionAttempt) && o.StatusCode == 200 {
+			knownZero = o.Usage.Total.Known && o.Usage.Total.Complete && o.Usage.Total.Value == 0
+		}
+		if o.Kind == string(pk.ExecutionAttempt) && o.StatusCode == 503 {
+			unknown = !o.Usage.Total.Known
+		}
+		if o.Kind == string(pk.ExecutionFinished) {
+			terminal = true
+			if !o.Usage.Total.Known || o.Usage.Total.Complete || o.Usage.Total.Value != 0 || !o.RetriesExhausted {
+				t.Fatalf("partial total %+v", o)
+			}
+		}
+	}
+	if !knownZero || !unknown || !terminal {
+		t.Fatal("usage evidence missing")
+	}
+}