eric/audita
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 6 Wiki Activity

Classify model failures and capture execution counts

Browse Source
This commit is contained in:
Eric Rakestraw
2026-09-24 17:07:01 +00:00
parent 7dc1be5a08
commit 51169ff9e9
4 changed files with 243 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
internal/framework/llm/adapter_capacity_test.go
View File

@@ -15,6 +15,7 @@ import (
ck "gitea.maximumdirect.net/eric/ConfigKit"
"gitea.maximumdirect.net/eric/audita/internal/core/config"
"gitea.maximumdirect.net/eric/audita/internal/framework/outcomes"
"gitea.maximumdirect.net/eric/audita/internal/framework/responseschema"
"gitea.maximumdirect.net/eric/audita/internal/framework/structuredoutput"
"gitea.maximumdirect.net/eric/audita/internal/prompts"
@@ -75,10 +76,14 @@ func TestRolesShareBackendCapacityAndReleaseAfterCancellation(t *testing.T) {
t.Fatal("first call did not enter provider")
}
var output map[string]any
_, err = validation.CompleteStructured(context.Background(), request, &output)
capacityResponse, err := validation.CompleteStructured(context.Background(), request, &output)
if !errors.Is(err, pk.ErrCapacityExceeded) {
t.Fatalf("other role did not share capacity: %v", err)
}
var capacityFailure *structuredoutput.Failure
if !errors.As(err, &capacityFailure) || capacityFailure.Class != structuredoutput.FailureRecoverable || capacityFailure.Reason != outcomes.ReasonCapacityExceeded || !capacityResponse.Execution.Available || capacityResponse.Execution.ProviderAttempts != 0 {
t.Fatalf("capacity outcome: failure=%#v counts=%+v", capacityFailure, capacityResponse.Execution)
}
select {
case <-entered:
t.Fatal("second role reached provider despite full backend")
@@ -131,6 +136,9 @@ func TestRolesShareBackendCapacityAndReleaseAfterCancellation(t *testing.T) {
if !errors.Is(err, context.DeadlineExceeded) {
t.Fatalf("queued call did not honor deadline: %v", err)
}
if !errors.As(err, &capacityFailure) || capacityFailure.Class != structuredoutput.FailureCanceled || capacityFailure.Reason != outcomes.ReasonCanceled {
t.Fatalf("queued parent deadline classification: %#v", capacityFailure)
}
close(release)
if err := <-firstDone; err != nil {
t.Fatal(err)