eric/audita
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 6 Wiki Activity

Classify model failures and capture execution counts

Browse Source
This commit is contained in:
Eric Rakestraw
2026-09-24 17:07:01 +00:00
parent 7dc1be5a08
commit 51169ff9e9
4 changed files with 243 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
docs/roadmap/implementation.md
View File

@@ -436,6 +436,15 @@ partial validation, and recovery detail without any filesystem/report dependency
## Stage 2 — Classify Model Failures and Return Execution Facts
**Completed 2026-09-24.** The PromptKit adapter now classifies parent
cancellation, setup defects, local deadlines, capacity, transport, provider
envelopes/statuses, exhausted invalid output, and domain decode overflow with
typed safe failures. It returns terminal attempt/retry/repair/status counts on
successful and failed calls, while preserving observation hooks and typed causes.
`go test ./internal/framework/llm -count=1`, its race suite, and
`git diff --check` passed. Whole-repository compilation remains deferred while
the warning consumers are reconnected in later stages.
**Work:** Implement C in `llm/adapter.go` using the pinned library's public types.
Capture terminal counts locally for each call and return them even on failure;
forward existing observation hooks without deriving recovery classifications in

123
internal/framework/llm/adapter.go
View File

@@ -3,10 +3,13 @@ package llm
import (
"context"
"encoding/json"
"errors"
"fmt"
"strings"
"sync"
"time"
"gitea.maximumdirect.net/eric/audita/internal/framework/outcomes"
"gitea.maximumdirect.net/eric/audita/internal/framework/responseschema"
"gitea.maximumdirect.net/eric/audita/internal/framework/structuredoutput"
"gitea.maximumdirect.net/eric/audita/internal/prompts"
@@ -39,6 +42,26 @@ func (r *Runtime) NewRoleClient(role string, hooks CallHooks) (*RoleClient, erro
func (c *RoleClient) CompleteStructured(parent context.Context, req structuredoutput.Request, out any) (response structuredoutput.Response, err error) {
ctx, cancel := context.WithTimeout(parent, c.runtime.Config.LLM.OperationTimeout)
defer cancel()
var terminalMu sync.Mutex
var terminal pk.ExecutionEvent
terminalAvailable := false
defer func() {
terminalMu.Lock()
if terminalAvailable {
response.Execution = outcomes.ExecutionCounts{
Available: true,
ProviderAttempts: int(terminal.Attempts),
TransientRetries: int(terminal.TransientRetries),
StructuralRepairs: int(terminal.RepairCalls),
RetriesExhausted: terminal.RetriesExhausted,
}
if terminal.StatusCode != 0 {
status := terminal.StatusCode
response.Execution.HTTPStatus = &status
}
}
terminalMu.Unlock()
}()
record := structuredoutput.CallRecord{CallID: c.runtime.nextCallID.Add(1), Context: copyContext(req.Context), Started: time.Now().UTC()}
record.Context.Role = c.role
profile := c.runtime.Profiles[c.role]
@@ -49,7 +72,8 @@ func (c *RoleClient) CompleteStructured(parent context.Context, req structuredou
defer func() {
record.Finished = time.Now().UTC()
if err != nil {
if structuredoutput.IsMalformedError(err) {
var failure *structuredoutput.Failure
if errors.As(err, &failure) && failure.Reason == outcomes.ReasonInvalidOutputExhausted {
record.Failure = structuredoutput.FailureMalformedOutput
} else {
record.Failure = structuredoutput.FailureRuntime
@@ -59,12 +83,15 @@ func (c *RoleClient) CompleteStructured(parent context.Context, req structuredou
c.hooks.OnCall(record)
}
}()
if parent.Err() != nil {
return response, classifiedFailure(parent, ctx, parent.Err(), nil)
}
if err = validateOutputTarget(out); err != nil {
return response, &structuredoutput.Failure{Kind: structuredoutput.FailureRuntime, Message: "invalid structured output target", Cause: err}
return response, &structuredoutput.Failure{Class: structuredoutput.FailureFatal, Reason: outcomes.ReasonSetupFailure, Message: "invalid structured output target", Cause: err}
}
schemaPath, ok := responseschema.AssetPath(req.Schema)
if !ok {
return response, &structuredoutput.Failure{Kind: structuredoutput.FailureRuntime, Message: "unknown structured response schema", Cause: fmt.Errorf("schema %q", req.Schema)}
return response, &structuredoutput.Failure{Class: structuredoutput.FailureFatal, Reason: outcomes.ReasonSetupFailure, Message: "unknown structured response schema", Cause: fmt.Errorf("schema %q", req.Schema)}
}
backend := c.runtime.Config.Backends[profile.EffectiveModelParams.BackendID]
vars := make(map[string]string, len(req.Variables)+1)
@@ -81,6 +108,12 @@ func (c *RoleClient) CompleteStructured(parent context.Context, req structuredou
ProfileID: profile.ProfileID, APIKey: apiKey, Vars: vars,
Validation: &pk.OutputContract{Format: pk.FormatJSON, ValidationMode: pk.ValidationJSONSchema, SchemaPath: schemaPath, RepairAttempts: c.runtime.Config.LLM.RepairAttempts},
Observer: func(event pk.ExecutionEvent) {
if event.Kind == pk.ExecutionFinished {
terminalMu.Lock()
terminal = event
terminalAvailable = true
terminalMu.Unlock()
}
if c.hooks.OnEvent != nil {
c.hooks.OnEvent(event)
}
@@ -90,7 +123,7 @@ func (c *RoleClient) CompleteStructured(parent context.Context, req structuredou
},
})
if prepErr != nil {
return response, &structuredoutput.Failure{Kind: structuredoutput.FailureRuntime, Message: "could not prepare structured model call", Cause: prepErr}
return response, classifiedFailure(parent, ctx, prepErr, nil)
}
defer handle.Discard()
details := handle.Details()
@@ -102,7 +135,7 @@ func (c *RoleClient) CompleteStructured(parent context.Context, req structuredou
selected, exists := c.runtime.Config.Backends[details.SelectedBackendID]
endpoint := func(s string) string { return strings.TrimRight(strings.TrimSpace(s), "/") }
if !exists || details.SelectedProfileID != profile.ProfileID || details.SelectedBackendID != profile.EffectiveModelParams.BackendID || endpoint(details.EffectiveModelParams.Endpoint) != endpoint(selected.Endpoint) || details.EffectiveModelParams.APIKeyEnv != "" {
return response, &structuredoutput.Failure{Kind: structuredoutput.FailureRuntime, Message: "structured model target changed after configuration inspection", Cause: fmt.Errorf("prepared target does not match registered role backend")}
return response, &structuredoutput.Failure{Class: structuredoutput.FailureFatal, Reason: outcomes.ReasonSetupFailure, Message: "structured model target changed after configuration inspection", Cause: fmt.Errorf("prepared target does not match registered role backend")}
}
result, runErr := c.runtime.Engine.RunPrepared(ctx, handle)
if result != nil {
@@ -114,22 +147,94 @@ func (c *RoleClient) CompleteStructured(parent context.Context, req structuredou
response.TotalTokens = result.Usage.TotalTokens
}
if runErr != nil {
return response, &structuredoutput.Failure{Kind: structuredoutput.FailureRuntime, Message: "structured model call failed", Cause: runErr}
return response, classifiedFailure(parent, ctx, runErr, result)
}
if result == nil {
return response, &structuredoutput.Failure{Kind: structuredoutput.FailureRuntime, Message: "structured model call returned no result"}
return response, &structuredoutput.Failure{Class: structuredoutput.FailureFatal, Reason: outcomes.ReasonInternalFailure, Message: "structured model call returned no result"}
}
if !result.Validation.IsValid {
return response, &structuredoutput.Failure{Kind: structuredoutput.FailureMalformedOutput, Message: "malformed structured output"}
return response, classifiedFailure(parent, ctx, nil, result)
}
if decodeErr := json.Unmarshal([]byte(result.RawOutput), out); decodeErr != nil {
return response, &structuredoutput.Failure{Kind: structuredoutput.FailureRuntime, Message: "validated structured output could not be decoded", Cause: decodeErr}
var typeError *json.UnmarshalTypeError
if errors.As(decodeErr, &typeError) {
return response, &structuredoutput.Failure{Class: structuredoutput.FailureRecoverable, Reason: outcomes.ReasonInvalidOutput, Message: "structured output contains an unusable value", Cause: decodeErr}
}
return response, &structuredoutput.Failure{Class: structuredoutput.FailureFatal, Reason: outcomes.ReasonInternalFailure, Message: "validated structured output could not be decoded", Cause: decodeErr}
}
response.Content = json.RawMessage(result.RawOutput)
response.Provider = "openai-compatible"
return response, nil
}
// classifiedFailure deliberately checks setup defects before broad PromptKit
// validation/generation wrappers. Provider text is kept only in the typed cause.
func classifiedFailure(parent, local context.Context, cause error, result *pk.RunResult) *structuredoutput.Failure {
failure := &structuredoutput.Failure{Class: structuredoutput.FailureFatal, Reason: outcomes.ReasonInternalFailure, Message: "structured model call failed", Cause: cause}
if parent.Err() != nil {
failure.Class = structuredoutput.FailureCanceled
failure.Reason = outcomes.ReasonCanceled
failure.Message = "processing canceled"
failure.Cause = parent.Err()
return failure
}
if isSetupFailure(cause) {
failure.Reason = outcomes.ReasonSetupFailure
failure.Message = "structured model setup failed"
return failure
}
if errors.Is(local.Err(), context.DeadlineExceeded) || errors.Is(cause, context.DeadlineExceeded) {
failure.Class = structuredoutput.FailureRecoverable
failure.Reason = outcomes.ReasonOperationTimeout
failure.Message = "model operation timed out"
return failure
}
if errors.Is(cause, pk.ErrCapacityExceeded) {
failure.Class = structuredoutput.FailureRecoverable
failure.Reason = outcomes.ReasonCapacityExceeded
failure.Message = "model capacity was unavailable"
return failure
}
if errors.Is(cause, pk.ErrTransport) {
failure.Class = structuredoutput.FailureRecoverable
failure.Reason = outcomes.ReasonTransportFailed
failure.Message = "model transport failed"
return failure
}
if errors.Is(cause, pk.ErrMalformedResponse) {
failure.Class = structuredoutput.FailureRecoverable
failure.Reason = outcomes.ReasonProviderResponseInvalid
failure.Message = "model provider response was invalid"
return failure
}
var provider *pk.GenerationError
if errors.As(cause, &provider) && provider.StatusCode() != 0 {
failure.Class = structuredoutput.FailureRecoverable
failure.Reason = outcomes.ReasonProviderFailed
failure.Message = "model provider failed"
return failure
}
if result != nil && !result.Validation.IsValid {
failure.Class = structuredoutput.FailureRecoverable
failure.Kind = structuredoutput.FailureMalformedOutput
failure.Reason = outcomes.ReasonInvalidOutputExhausted
failure.Message = "structured output remained invalid"
return failure
}
if errors.Is(cause, pk.ErrValidation) {
failure.Reason = outcomes.ReasonSetupFailure
failure.Message = "structured validation contract failed"
}
return failure
}
func isSetupFailure(err error) bool {
return errors.Is(err, pk.ErrInvalidConfig) || errors.Is(err, pk.ErrInvalidRequest) ||
errors.Is(err, pk.ErrPromptNotFound) || errors.Is(err, pk.ErrProfileNotFound) ||
errors.Is(err, pk.ErrPromptLoad) || errors.Is(err, pk.ErrProfileLoad) ||
errors.Is(err, pk.ErrArtifactLoad) || errors.Is(err, pk.ErrPromptRender)
}
func promptMetadata(details pk.PreparedRun) structuredoutput.PromptMetadata {
return structuredoutput.PromptMetadata{
PromptID: details.PromptID, PromptVersion: details.PromptVersion,

10
internal/framework/llm/adapter_capacity_test.go
View File

@@ -15,6 +15,7 @@ import (
ck "gitea.maximumdirect.net/eric/ConfigKit"
"gitea.maximumdirect.net/eric/audita/internal/core/config"
"gitea.maximumdirect.net/eric/audita/internal/framework/outcomes"
"gitea.maximumdirect.net/eric/audita/internal/framework/responseschema"
"gitea.maximumdirect.net/eric/audita/internal/framework/structuredoutput"
"gitea.maximumdirect.net/eric/audita/internal/prompts"
@@ -75,10 +76,14 @@ func TestRolesShareBackendCapacityAndReleaseAfterCancellation(t *testing.T) {
t.Fatal("first call did not enter provider")
}
var output map[string]any
_, err = validation.CompleteStructured(context.Background(), request, &output)
capacityResponse, err := validation.CompleteStructured(context.Background(), request, &output)
if !errors.Is(err, pk.ErrCapacityExceeded) {
t.Fatalf("other role did not share capacity: %v", err)
}
var capacityFailure *structuredoutput.Failure
if !errors.As(err, &capacityFailure) || capacityFailure.Class != structuredoutput.FailureRecoverable || capacityFailure.Reason != outcomes.ReasonCapacityExceeded || !capacityResponse.Execution.Available || capacityResponse.Execution.ProviderAttempts != 0 {
t.Fatalf("capacity outcome: failure=%#v counts=%+v", capacityFailure, capacityResponse.Execution)
}
select {
case <-entered:
t.Fatal("second role reached provider despite full backend")
@@ -131,6 +136,9 @@ func TestRolesShareBackendCapacityAndReleaseAfterCancellation(t *testing.T) {
if !errors.Is(err, context.DeadlineExceeded) {
t.Fatalf("queued call did not honor deadline: %v", err)
}
if !errors.As(err, &capacityFailure) || capacityFailure.Class != structuredoutput.FailureCanceled || capacityFailure.Reason != outcomes.ReasonCanceled {
t.Fatalf("queued parent deadline classification: %#v", capacityFailure)
}
close(release)
if err := <-firstDone; err != nil {
t.Fatal(err)

121
internal/framework/llm/adapter_test.go
View File

@@ -13,9 +13,11 @@ import (
"sync"
"sync/atomic"
"testing"
"time"
ck "gitea.maximumdirect.net/eric/ConfigKit"
"gitea.maximumdirect.net/eric/audita/internal/core/config"
"gitea.maximumdirect.net/eric/audita/internal/framework/outcomes"
"gitea.maximumdirect.net/eric/audita/internal/framework/responseschema"
"gitea.maximumdirect.net/eric/audita/internal/framework/structuredoutput"
"gitea.maximumdirect.net/eric/audita/internal/prompts"
@@ -102,18 +104,19 @@ func TestPreparedAdapterRepairsAndClassifies(t *testing.T) {
replies []reply
retries, repairs, attempts, repairCalls, transientRetries int
malformed bool
reason outcomes.ReasonCode
cause error
}{
{"success", []reply{{200, valid}}, 0, 0, 1, 0, 0, false, nil},
{"repair succeeds", []reply{{200, `{"wrong":true}`}, {200, valid}}, 0, 1, 2, 1, 0, false, nil},
{"repair exhausted", []reply{{200, `{"wrong":true}`}, {200, `{"wrong":true}`}}, 0, 1, 2, 1, 0, true, nil},
{"429 recovery", []reply{{429, ""}, {200, valid}}, 1, 0, 2, 0, 1, false, nil},
{"5xx recovery", []reply{{503, ""}, {200, valid}}, 1, 0, 2, 0, 1, false, nil},
{"zero retry", []reply{{503, ""}}, 0, 0, 1, 0, 0, false, pk.ErrLLMGenerate},
{"positive retry exhausted", []reply{{503, ""}, {503, ""}}, 1, 0, 2, 0, 1, false, pk.ErrRetriesExhausted},
{"transport failure", []reply{{0, ""}}, 0, 0, 1, 0, 0, false, pk.ErrTransport},
{"malformed envelope", []reply{{200, "MALFORMED_ENVELOPE"}}, 0, 0, 1, 0, 0, false, pk.ErrMalformedResponse},
{"retry and repair", []reply{{503, ""}, {200, `{"wrong":true}`}, {503, ""}, {200, valid}}, 1, 1, 4, 1, 2, false, nil},
{"success", []reply{{200, valid}}, 0, 0, 1, 0, 0, false, "", nil},
{"repair succeeds", []reply{{200, `{"wrong":true}`}, {200, valid}}, 0, 1, 2, 1, 0, false, "", nil},
{"repair exhausted", []reply{{200, `{"wrong":true}`}, {200, `{"wrong":true}`}}, 0, 1, 2, 1, 0, true, outcomes.ReasonInvalidOutputExhausted, nil},
{"429 recovery", []reply{{429, ""}, {200, valid}}, 1, 0, 2, 0, 1, false, "", nil},
{"5xx recovery", []reply{{503, ""}, {200, valid}}, 1, 0, 2, 0, 1, false, "", nil},
{"zero retry", []reply{{503, ""}}, 0, 0, 1, 0, 0, false, outcomes.ReasonProviderFailed, pk.ErrLLMGenerate},
{"positive retry exhausted", []reply{{503, ""}, {503, ""}}, 1, 0, 2, 0, 1, false, outcomes.ReasonProviderFailed, pk.ErrRetriesExhausted},
{"transport failure", []reply{{0, ""}}, 0, 0, 1, 0, 0, false, outcomes.ReasonTransportFailed, pk.ErrTransport},
{"malformed envelope", []reply{{200, "MALFORMED_ENVELOPE"}}, 0, 0, 1, 0, 0, false, outcomes.ReasonProviderResponseInvalid, pk.ErrMalformedResponse},
{"retry and repair", []reply{{503, ""}, {200, `{"wrong":true}`}, {503, ""}, {200, valid}}, 1, 1, 4, 1, 2, false, "", nil},
} {
t.Run(string(schema)+"/"+tc.name, func(t *testing.T) {
runtime, count, _ := adapterFixture(t, tc.replies, tc.retries, tc.repairs, nil)
@@ -141,6 +144,24 @@ func TestPreparedAdapterRepairsAndClassifies(t *testing.T) {
if tc.cause == nil && !tc.malformed && err != nil {
t.Fatal(err)
}
var failure *structuredoutput.Failure
if tc.reason != "" {
if !errors.As(err, &failure) || failure.Class != structuredoutput.FailureRecoverable || failure.Reason != tc.reason || !structuredoutput.IsRecoverable(err) {
t.Fatalf("failure classification: %#v, %v", failure, err)
}
if strings.Contains(err.Error(), "provider error") || strings.Contains(err.Error(), "adapter-secret") {
t.Fatalf("unsafe model error: %v", err)
}
}
if !response.Execution.Available || response.Execution.ProviderAttempts != tc.attempts || response.Execution.StructuralRepairs != tc.repairCalls || response.Execution.TransientRetries != tc.transientRetries {
t.Fatalf("returned execution counts: %+v", response.Execution)
}
if response.Execution.RetriesExhausted != (tc.name == "positive retry exhausted") {
t.Fatalf("retry exhaustion: %+v", response.Execution)
}
if tc.reason == outcomes.ReasonProviderFailed && (response.Execution.HTTPStatus == nil || *response.Execution.HTTPStatus != http.StatusServiceUnavailable) {
t.Fatalf("provider status missing: %+v", response.Execution)
}
if tc.name == "zero retry" && errors.Is(err, pk.ErrRetriesExhausted) {
t.Fatal("zero retry reported exhaustion")
}
@@ -256,3 +277,83 @@ func TestPreparedAdapterCancellationStopsRetryWait(t *testing.T) {
t.Fatalf("cancellation: %v, HTTP calls=%d", err, count.Load())
}
}
func TestPreparedAdapterDistinguishesOperationTimeoutFromParentCancellation(t *testing.T) {
blocking := func(*http.Request) { time.Sleep(500 * time.Millisecond) }
runtime, _, _ := adapterFixture(t, []reply{{200, `{"corrections":[]}`}}, 0, 0, blocking)
runtime.Config.LLM.OperationTimeout = 200 * time.Millisecond
client, err := runtime.NewRoleClient("proposal", CallHooks{})
if err != nil {
t.Fatal(err)
}
var output map[string]any
response, err := client.CompleteStructured(context.Background(), adapterRequest(responseschema.CorrectionSetKey), &output)
var failure *structuredoutput.Failure
if !errors.As(err, &failure) || failure.Class != structuredoutput.FailureRecoverable || failure.Reason != outcomes.ReasonOperationTimeout || !response.Execution.Available {
t.Fatalf("local timeout: failure=%#v counts=%+v", failure, response.Execution)
}
parent, cancel := context.WithCancel(context.Background())
cancel()
response, err = client.CompleteStructured(parent, adapterRequest(responseschema.CorrectionSetKey), &output)
if !errors.As(err, &failure) || failure.Class != structuredoutput.FailureCanceled || failure.Reason != outcomes.ReasonCanceled || !errors.Is(err, context.Canceled) || response.Execution.Available {
t.Fatalf("parent cancellation: failure=%#v counts=%+v", failure, response.Execution)
}
}
func TestPreparedAdapterClassifiesSetupAndDomainDecodeFailures(t *testing.T) {
runtime, _, _ := adapterFixture(t, []reply{{200, `{"validations":[{"correction_index":9999999999999999999999999,"approved":true,"confidence":0.8,"reason":"ok"}]}`}}, 0, 0, nil)
client, err := runtime.NewRoleClient("validation", CallHooks{})
if err != nil {
t.Fatal(err)
}
var output struct {
Validations []struct {
CorrectionIndex int `json:"correction_index"`
} `json:"validations"`
}
response, err := client.CompleteStructured(context.Background(), adapterRequest(responseschema.ValidatorDecisionSetKey), &output)
var failure *structuredoutput.Failure
if !errors.As(err, &failure) || failure.Class != structuredoutput.FailureRecoverable || failure.Reason != outcomes.ReasonInvalidOutput || !response.Execution.Available {
t.Fatalf("decode overflow: failure=%#v counts=%+v", failure, response.Execution)
}
_, err = client.CompleteStructured(context.Background(), adapterRequest(responseschema.CorrectionSetKey), nil)
if !errors.As(err, &failure) || failure.Class != structuredoutput.FailureFatal || failure.Reason != outcomes.ReasonSetupFailure {
t.Fatalf("invalid target: %#v", failure)
}
req := adapterRequest("missing_schema")
_, err = client.CompleteStructured(context.Background(), req, &output)
if !errors.As(err, &failure) || failure.Class != structuredoutput.FailureFatal || failure.Reason != outcomes.ReasonSetupFailure {
t.Fatalf("unknown schema: %#v", failure)
}
req = adapterRequest(responseschema.ValidatorDecisionSetKey)
req.PromptID = "missing.prompt"
_, err = client.CompleteStructured(context.Background(), req, &output)
if !errors.As(err, &failure) || failure.Class != structuredoutput.FailureFatal || failure.Reason != outcomes.ReasonSetupFailure {
t.Fatalf("missing prompt: %#v", failure)
}
}
func TestPreparedAdapterNonRetryableProviderStatus(t *testing.T) {
runtime, count, _ := adapterFixture(t, []reply{{http.StatusBadRequest, ""}}, 2, 0, nil)
client, err := runtime.NewRoleClient("proposal", CallHooks{})
if err != nil {
t.Fatal(err)
}
var output map[string]any
response, err := client.CompleteStructured(context.Background(), adapterRequest(responseschema.CorrectionSetKey), &output)
var failure *structuredoutput.Failure
if !errors.As(err, &failure) || failure.Class != structuredoutput.FailureRecoverable || failure.Reason != outcomes.ReasonProviderFailed || count.Load() != 1 || response.Execution.HTTPStatus == nil || *response.Execution.HTTPStatus != http.StatusBadRequest {
t.Fatalf("non-retryable provider failure: failure=%#v calls=%d counts=%+v", failure, count.Load(), response.Execution)
}
}
func TestPreparedAdapterUnknownValidationFailureRemainsFatal(t *testing.T) {
parent, cancel := context.WithCancel(context.Background())
defer cancel()
cause := fmt.Errorf("%w: embedded contract defect", pk.ErrValidation)
failure := classifiedFailure(parent, parent, cause, nil)
if failure.Class != structuredoutput.FailureFatal || failure.Reason != outcomes.ReasonSetupFailure || !errors.Is(failure, pk.ErrValidation) {
t.Fatalf("unknown validation failure: %#v", failure)
}
}