eric/audita
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 6 Wiki Activity

Classify model failures and capture execution counts

Browse Source
This commit is contained in:
Eric Rakestraw
2026-09-24 17:07:01 +00:00
parent 7dc1be5a08
commit 51169ff9e9
4 changed files with 243 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files

121
internal/framework/llm/adapter_test.go
View File

@@ -13,9 +13,11 @@ import (
"sync"
"sync/atomic"
"testing"
"time"
ck "gitea.maximumdirect.net/eric/ConfigKit"
"gitea.maximumdirect.net/eric/audita/internal/core/config"
"gitea.maximumdirect.net/eric/audita/internal/framework/outcomes"
"gitea.maximumdirect.net/eric/audita/internal/framework/responseschema"
"gitea.maximumdirect.net/eric/audita/internal/framework/structuredoutput"
"gitea.maximumdirect.net/eric/audita/internal/prompts"
@@ -102,18 +104,19 @@ func TestPreparedAdapterRepairsAndClassifies(t *testing.T) {
replies []reply
retries, repairs, attempts, repairCalls, transientRetries int
malformed bool
reason outcomes.ReasonCode
cause error
}{
{"success", []reply{{200, valid}}, 0, 0, 1, 0, 0, false, nil},
{"repair succeeds", []reply{{200, `{"wrong":true}`}, {200, valid}}, 0, 1, 2, 1, 0, false, nil},
{"repair exhausted", []reply{{200, `{"wrong":true}`}, {200, `{"wrong":true}`}}, 0, 1, 2, 1, 0, true, nil},
{"429 recovery", []reply{{429, ""}, {200, valid}}, 1, 0, 2, 0, 1, false, nil},
{"5xx recovery", []reply{{503, ""}, {200, valid}}, 1, 0, 2, 0, 1, false, nil},
{"zero retry", []reply{{503, ""}}, 0, 0, 1, 0, 0, false, pk.ErrLLMGenerate},
{"positive retry exhausted", []reply{{503, ""}, {503, ""}}, 1, 0, 2, 0, 1, false, pk.ErrRetriesExhausted},
{"transport failure", []reply{{0, ""}}, 0, 0, 1, 0, 0, false, pk.ErrTransport},
{"malformed envelope", []reply{{200, "MALFORMED_ENVELOPE"}}, 0, 0, 1, 0, 0, false, pk.ErrMalformedResponse},
{"retry and repair", []reply{{503, ""}, {200, `{"wrong":true}`}, {503, ""}, {200, valid}}, 1, 1, 4, 1, 2, false, nil},
{"success", []reply{{200, valid}}, 0, 0, 1, 0, 0, false, "", nil},
{"repair succeeds", []reply{{200, `{"wrong":true}`}, {200, valid}}, 0, 1, 2, 1, 0, false, "", nil},
{"repair exhausted", []reply{{200, `{"wrong":true}`}, {200, `{"wrong":true}`}}, 0, 1, 2, 1, 0, true, outcomes.ReasonInvalidOutputExhausted, nil},
{"429 recovery", []reply{{429, ""}, {200, valid}}, 1, 0, 2, 0, 1, false, "", nil},
{"5xx recovery", []reply{{503, ""}, {200, valid}}, 1, 0, 2, 0, 1, false, "", nil},
{"zero retry", []reply{{503, ""}}, 0, 0, 1, 0, 0, false, outcomes.ReasonProviderFailed, pk.ErrLLMGenerate},
{"positive retry exhausted", []reply{{503, ""}, {503, ""}}, 1, 0, 2, 0, 1, false, outcomes.ReasonProviderFailed, pk.ErrRetriesExhausted},
{"transport failure", []reply{{0, ""}}, 0, 0, 1, 0, 0, false, outcomes.ReasonTransportFailed, pk.ErrTransport},
{"malformed envelope", []reply{{200, "MALFORMED_ENVELOPE"}}, 0, 0, 1, 0, 0, false, outcomes.ReasonProviderResponseInvalid, pk.ErrMalformedResponse},
{"retry and repair", []reply{{503, ""}, {200, `{"wrong":true}`}, {503, ""}, {200, valid}}, 1, 1, 4, 1, 2, false, "", nil},
} {
t.Run(string(schema)+"/"+tc.name, func(t *testing.T) {
runtime, count, _ := adapterFixture(t, tc.replies, tc.retries, tc.repairs, nil)
@@ -141,6 +144,24 @@ func TestPreparedAdapterRepairsAndClassifies(t *testing.T) {
if tc.cause == nil && !tc.malformed && err != nil {
t.Fatal(err)
}
var failure *structuredoutput.Failure
if tc.reason != "" {
if !errors.As(err, &failure) || failure.Class != structuredoutput.FailureRecoverable || failure.Reason != tc.reason || !structuredoutput.IsRecoverable(err) {
t.Fatalf("failure classification: %#v, %v", failure, err)
}
if strings.Contains(err.Error(), "provider error") || strings.Contains(err.Error(), "adapter-secret") {
t.Fatalf("unsafe model error: %v", err)
}
}
if !response.Execution.Available || response.Execution.ProviderAttempts != tc.attempts || response.Execution.StructuralRepairs != tc.repairCalls || response.Execution.TransientRetries != tc.transientRetries {
t.Fatalf("returned execution counts: %+v", response.Execution)
}
if response.Execution.RetriesExhausted != (tc.name == "positive retry exhausted") {
t.Fatalf("retry exhaustion: %+v", response.Execution)
}
if tc.reason == outcomes.ReasonProviderFailed && (response.Execution.HTTPStatus == nil || *response.Execution.HTTPStatus != http.StatusServiceUnavailable) {
t.Fatalf("provider status missing: %+v", response.Execution)
}
if tc.name == "zero retry" && errors.Is(err, pk.ErrRetriesExhausted) {
t.Fatal("zero retry reported exhaustion")
}
@@ -256,3 +277,83 @@ func TestPreparedAdapterCancellationStopsRetryWait(t *testing.T) {
t.Fatalf("cancellation: %v, HTTP calls=%d", err, count.Load())
}
}
func TestPreparedAdapterDistinguishesOperationTimeoutFromParentCancellation(t *testing.T) {
blocking := func(*http.Request) { time.Sleep(500 * time.Millisecond) }
runtime, _, _ := adapterFixture(t, []reply{{200, `{"corrections":[]}`}}, 0, 0, blocking)
runtime.Config.LLM.OperationTimeout = 200 * time.Millisecond
client, err := runtime.NewRoleClient("proposal", CallHooks{})
if err != nil {
t.Fatal(err)
}
var output map[string]any
response, err := client.CompleteStructured(context.Background(), adapterRequest(responseschema.CorrectionSetKey), &output)
var failure *structuredoutput.Failure
if !errors.As(err, &failure) || failure.Class != structuredoutput.FailureRecoverable || failure.Reason != outcomes.ReasonOperationTimeout || !response.Execution.Available {
t.Fatalf("local timeout: failure=%#v counts=%+v", failure, response.Execution)
}
parent, cancel := context.WithCancel(context.Background())
cancel()
response, err = client.CompleteStructured(parent, adapterRequest(responseschema.CorrectionSetKey), &output)
if !errors.As(err, &failure) || failure.Class != structuredoutput.FailureCanceled || failure.Reason != outcomes.ReasonCanceled || !errors.Is(err, context.Canceled) || response.Execution.Available {
t.Fatalf("parent cancellation: failure=%#v counts=%+v", failure, response.Execution)
}
}
func TestPreparedAdapterClassifiesSetupAndDomainDecodeFailures(t *testing.T) {
runtime, _, _ := adapterFixture(t, []reply{{200, `{"validations":[{"correction_index":9999999999999999999999999,"approved":true,"confidence":0.8,"reason":"ok"}]}`}}, 0, 0, nil)
client, err := runtime.NewRoleClient("validation", CallHooks{})
if err != nil {
t.Fatal(err)
}
var output struct {
Validations []struct {
CorrectionIndex int `json:"correction_index"`
} `json:"validations"`
}
response, err := client.CompleteStructured(context.Background(), adapterRequest(responseschema.ValidatorDecisionSetKey), &output)
var failure *structuredoutput.Failure
if !errors.As(err, &failure) || failure.Class != structuredoutput.FailureRecoverable || failure.Reason != outcomes.ReasonInvalidOutput || !response.Execution.Available {
t.Fatalf("decode overflow: failure=%#v counts=%+v", failure, response.Execution)
}
_, err = client.CompleteStructured(context.Background(), adapterRequest(responseschema.CorrectionSetKey), nil)
if !errors.As(err, &failure) || failure.Class != structuredoutput.FailureFatal || failure.Reason != outcomes.ReasonSetupFailure {
t.Fatalf("invalid target: %#v", failure)
}
req := adapterRequest("missing_schema")
_, err = client.CompleteStructured(context.Background(), req, &output)
if !errors.As(err, &failure) || failure.Class != structuredoutput.FailureFatal || failure.Reason != outcomes.ReasonSetupFailure {
t.Fatalf("unknown schema: %#v", failure)
}
req = adapterRequest(responseschema.ValidatorDecisionSetKey)
req.PromptID = "missing.prompt"
_, err = client.CompleteStructured(context.Background(), req, &output)
if !errors.As(err, &failure) || failure.Class != structuredoutput.FailureFatal || failure.Reason != outcomes.ReasonSetupFailure {
t.Fatalf("missing prompt: %#v", failure)
}
}
func TestPreparedAdapterNonRetryableProviderStatus(t *testing.T) {
runtime, count, _ := adapterFixture(t, []reply{{http.StatusBadRequest, ""}}, 2, 0, nil)
client, err := runtime.NewRoleClient("proposal", CallHooks{})
if err != nil {
t.Fatal(err)
}
var output map[string]any
response, err := client.CompleteStructured(context.Background(), adapterRequest(responseschema.CorrectionSetKey), &output)
var failure *structuredoutput.Failure
if !errors.As(err, &failure) || failure.Class != structuredoutput.FailureRecoverable || failure.Reason != outcomes.ReasonProviderFailed || count.Load() != 1 || response.Execution.HTTPStatus == nil || *response.Execution.HTTPStatus != http.StatusBadRequest {
t.Fatalf("non-retryable provider failure: failure=%#v calls=%d counts=%+v", failure, count.Load(), response.Execution)
}
}
func TestPreparedAdapterUnknownValidationFailureRemainsFatal(t *testing.T) {
parent, cancel := context.WithCancel(context.Background())
defer cancel()
cause := fmt.Errorf("%w: embedded contract defect", pk.ErrValidation)
failure := classifiedFailure(parent, parent, cause, nil)
if failure.Class != structuredoutput.FailureFatal || failure.Reason != outcomes.ReasonSetupFailure || !errors.Is(failure, pk.ErrValidation) {
t.Fatalf("unknown validation failure: %#v", failure)
}
}