diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index 3c9a98e..2f61b7d 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -436,6 +436,15 @@ partial validation, and recovery detail without any filesystem/report dependency ## Stage 2 — Classify Model Failures and Return Execution Facts +**Completed 2026-09-24.** The PromptKit adapter now classifies parent +cancellation, setup defects, local deadlines, capacity, transport, provider +envelopes/statuses, exhausted invalid output, and domain decode overflow with +typed safe failures. It returns terminal attempt/retry/repair/status counts on +successful and failed calls, while preserving observation hooks and typed causes. +`go test ./internal/framework/llm -count=1`, its race suite, and +`git diff --check` passed. Whole-repository compilation remains deferred while +the warning consumers are reconnected in later stages. + **Work:** Implement C in `llm/adapter.go` using the pinned library's public types. Capture terminal counts locally for each call and return them even on failure; forward existing observation hooks without deriving recovery classifications in diff --git a/internal/framework/llm/adapter.go b/internal/framework/llm/adapter.go index ddd0a51..31aa027 100644 --- a/internal/framework/llm/adapter.go +++ b/internal/framework/llm/adapter.go @@ -3,10 +3,13 @@ package llm import ( "context" "encoding/json" + "errors" "fmt" "strings" + "sync" "time" + "gitea.maximumdirect.net/eric/audita/internal/framework/outcomes" "gitea.maximumdirect.net/eric/audita/internal/framework/responseschema" "gitea.maximumdirect.net/eric/audita/internal/framework/structuredoutput" "gitea.maximumdirect.net/eric/audita/internal/prompts" @@ -39,6 +42,26 @@ func (r *Runtime) NewRoleClient(role string, hooks CallHooks) (*RoleClient, erro func (c *RoleClient) CompleteStructured(parent context.Context, req structuredoutput.Request, out any) (response structuredoutput.Response, err error) { ctx, cancel := context.WithTimeout(parent, c.runtime.Config.LLM.OperationTimeout) defer cancel() + var terminalMu sync.Mutex + var terminal pk.ExecutionEvent + terminalAvailable := false + defer func() { + terminalMu.Lock() + if terminalAvailable { + response.Execution = outcomes.ExecutionCounts{ + Available: true, + ProviderAttempts: int(terminal.Attempts), + TransientRetries: int(terminal.TransientRetries), + StructuralRepairs: int(terminal.RepairCalls), + RetriesExhausted: terminal.RetriesExhausted, + } + if terminal.StatusCode != 0 { + status := terminal.StatusCode + response.Execution.HTTPStatus = &status + } + } + terminalMu.Unlock() + }() record := structuredoutput.CallRecord{CallID: c.runtime.nextCallID.Add(1), Context: copyContext(req.Context), Started: time.Now().UTC()} record.Context.Role = c.role profile := c.runtime.Profiles[c.role] @@ -49,7 +72,8 @@ func (c *RoleClient) CompleteStructured(parent context.Context, req structuredou defer func() { record.Finished = time.Now().UTC() if err != nil { - if structuredoutput.IsMalformedError(err) { + var failure *structuredoutput.Failure + if errors.As(err, &failure) && failure.Reason == outcomes.ReasonInvalidOutputExhausted { record.Failure = structuredoutput.FailureMalformedOutput } else { record.Failure = structuredoutput.FailureRuntime @@ -59,12 +83,15 @@ func (c *RoleClient) CompleteStructured(parent context.Context, req structuredou c.hooks.OnCall(record) } }() + if parent.Err() != nil { + return response, classifiedFailure(parent, ctx, parent.Err(), nil) + } if err = validateOutputTarget(out); err != nil { - return response, &structuredoutput.Failure{Kind: structuredoutput.FailureRuntime, Message: "invalid structured output target", Cause: err} + return response, &structuredoutput.Failure{Class: structuredoutput.FailureFatal, Reason: outcomes.ReasonSetupFailure, Message: "invalid structured output target", Cause: err} } schemaPath, ok := responseschema.AssetPath(req.Schema) if !ok { - return response, &structuredoutput.Failure{Kind: structuredoutput.FailureRuntime, Message: "unknown structured response schema", Cause: fmt.Errorf("schema %q", req.Schema)} + return response, &structuredoutput.Failure{Class: structuredoutput.FailureFatal, Reason: outcomes.ReasonSetupFailure, Message: "unknown structured response schema", Cause: fmt.Errorf("schema %q", req.Schema)} } backend := c.runtime.Config.Backends[profile.EffectiveModelParams.BackendID] vars := make(map[string]string, len(req.Variables)+1) @@ -81,6 +108,12 @@ func (c *RoleClient) CompleteStructured(parent context.Context, req structuredou ProfileID: profile.ProfileID, APIKey: apiKey, Vars: vars, Validation: &pk.OutputContract{Format: pk.FormatJSON, ValidationMode: pk.ValidationJSONSchema, SchemaPath: schemaPath, RepairAttempts: c.runtime.Config.LLM.RepairAttempts}, Observer: func(event pk.ExecutionEvent) { + if event.Kind == pk.ExecutionFinished { + terminalMu.Lock() + terminal = event + terminalAvailable = true + terminalMu.Unlock() + } if c.hooks.OnEvent != nil { c.hooks.OnEvent(event) } @@ -90,7 +123,7 @@ func (c *RoleClient) CompleteStructured(parent context.Context, req structuredou }, }) if prepErr != nil { - return response, &structuredoutput.Failure{Kind: structuredoutput.FailureRuntime, Message: "could not prepare structured model call", Cause: prepErr} + return response, classifiedFailure(parent, ctx, prepErr, nil) } defer handle.Discard() details := handle.Details() @@ -102,7 +135,7 @@ func (c *RoleClient) CompleteStructured(parent context.Context, req structuredou selected, exists := c.runtime.Config.Backends[details.SelectedBackendID] endpoint := func(s string) string { return strings.TrimRight(strings.TrimSpace(s), "/") } if !exists || details.SelectedProfileID != profile.ProfileID || details.SelectedBackendID != profile.EffectiveModelParams.BackendID || endpoint(details.EffectiveModelParams.Endpoint) != endpoint(selected.Endpoint) || details.EffectiveModelParams.APIKeyEnv != "" { - return response, &structuredoutput.Failure{Kind: structuredoutput.FailureRuntime, Message: "structured model target changed after configuration inspection", Cause: fmt.Errorf("prepared target does not match registered role backend")} + return response, &structuredoutput.Failure{Class: structuredoutput.FailureFatal, Reason: outcomes.ReasonSetupFailure, Message: "structured model target changed after configuration inspection", Cause: fmt.Errorf("prepared target does not match registered role backend")} } result, runErr := c.runtime.Engine.RunPrepared(ctx, handle) if result != nil { @@ -114,22 +147,94 @@ func (c *RoleClient) CompleteStructured(parent context.Context, req structuredou response.TotalTokens = result.Usage.TotalTokens } if runErr != nil { - return response, &structuredoutput.Failure{Kind: structuredoutput.FailureRuntime, Message: "structured model call failed", Cause: runErr} + return response, classifiedFailure(parent, ctx, runErr, result) } if result == nil { - return response, &structuredoutput.Failure{Kind: structuredoutput.FailureRuntime, Message: "structured model call returned no result"} + return response, &structuredoutput.Failure{Class: structuredoutput.FailureFatal, Reason: outcomes.ReasonInternalFailure, Message: "structured model call returned no result"} } if !result.Validation.IsValid { - return response, &structuredoutput.Failure{Kind: structuredoutput.FailureMalformedOutput, Message: "malformed structured output"} + return response, classifiedFailure(parent, ctx, nil, result) } if decodeErr := json.Unmarshal([]byte(result.RawOutput), out); decodeErr != nil { - return response, &structuredoutput.Failure{Kind: structuredoutput.FailureRuntime, Message: "validated structured output could not be decoded", Cause: decodeErr} + var typeError *json.UnmarshalTypeError + if errors.As(decodeErr, &typeError) { + return response, &structuredoutput.Failure{Class: structuredoutput.FailureRecoverable, Reason: outcomes.ReasonInvalidOutput, Message: "structured output contains an unusable value", Cause: decodeErr} + } + return response, &structuredoutput.Failure{Class: structuredoutput.FailureFatal, Reason: outcomes.ReasonInternalFailure, Message: "validated structured output could not be decoded", Cause: decodeErr} } response.Content = json.RawMessage(result.RawOutput) response.Provider = "openai-compatible" return response, nil } +// classifiedFailure deliberately checks setup defects before broad PromptKit +// validation/generation wrappers. Provider text is kept only in the typed cause. +func classifiedFailure(parent, local context.Context, cause error, result *pk.RunResult) *structuredoutput.Failure { + failure := &structuredoutput.Failure{Class: structuredoutput.FailureFatal, Reason: outcomes.ReasonInternalFailure, Message: "structured model call failed", Cause: cause} + if parent.Err() != nil { + failure.Class = structuredoutput.FailureCanceled + failure.Reason = outcomes.ReasonCanceled + failure.Message = "processing canceled" + failure.Cause = parent.Err() + return failure + } + if isSetupFailure(cause) { + failure.Reason = outcomes.ReasonSetupFailure + failure.Message = "structured model setup failed" + return failure + } + if errors.Is(local.Err(), context.DeadlineExceeded) || errors.Is(cause, context.DeadlineExceeded) { + failure.Class = structuredoutput.FailureRecoverable + failure.Reason = outcomes.ReasonOperationTimeout + failure.Message = "model operation timed out" + return failure + } + if errors.Is(cause, pk.ErrCapacityExceeded) { + failure.Class = structuredoutput.FailureRecoverable + failure.Reason = outcomes.ReasonCapacityExceeded + failure.Message = "model capacity was unavailable" + return failure + } + if errors.Is(cause, pk.ErrTransport) { + failure.Class = structuredoutput.FailureRecoverable + failure.Reason = outcomes.ReasonTransportFailed + failure.Message = "model transport failed" + return failure + } + if errors.Is(cause, pk.ErrMalformedResponse) { + failure.Class = structuredoutput.FailureRecoverable + failure.Reason = outcomes.ReasonProviderResponseInvalid + failure.Message = "model provider response was invalid" + return failure + } + var provider *pk.GenerationError + if errors.As(cause, &provider) && provider.StatusCode() != 0 { + failure.Class = structuredoutput.FailureRecoverable + failure.Reason = outcomes.ReasonProviderFailed + failure.Message = "model provider failed" + return failure + } + if result != nil && !result.Validation.IsValid { + failure.Class = structuredoutput.FailureRecoverable + failure.Kind = structuredoutput.FailureMalformedOutput + failure.Reason = outcomes.ReasonInvalidOutputExhausted + failure.Message = "structured output remained invalid" + return failure + } + if errors.Is(cause, pk.ErrValidation) { + failure.Reason = outcomes.ReasonSetupFailure + failure.Message = "structured validation contract failed" + } + return failure +} + +func isSetupFailure(err error) bool { + return errors.Is(err, pk.ErrInvalidConfig) || errors.Is(err, pk.ErrInvalidRequest) || + errors.Is(err, pk.ErrPromptNotFound) || errors.Is(err, pk.ErrProfileNotFound) || + errors.Is(err, pk.ErrPromptLoad) || errors.Is(err, pk.ErrProfileLoad) || + errors.Is(err, pk.ErrArtifactLoad) || errors.Is(err, pk.ErrPromptRender) +} + func promptMetadata(details pk.PreparedRun) structuredoutput.PromptMetadata { return structuredoutput.PromptMetadata{ PromptID: details.PromptID, PromptVersion: details.PromptVersion, diff --git a/internal/framework/llm/adapter_capacity_test.go b/internal/framework/llm/adapter_capacity_test.go index 4518da0..6ac5bc2 100644 --- a/internal/framework/llm/adapter_capacity_test.go +++ b/internal/framework/llm/adapter_capacity_test.go @@ -15,6 +15,7 @@ import ( ck "gitea.maximumdirect.net/eric/ConfigKit" "gitea.maximumdirect.net/eric/audita/internal/core/config" + "gitea.maximumdirect.net/eric/audita/internal/framework/outcomes" "gitea.maximumdirect.net/eric/audita/internal/framework/responseschema" "gitea.maximumdirect.net/eric/audita/internal/framework/structuredoutput" "gitea.maximumdirect.net/eric/audita/internal/prompts" @@ -75,10 +76,14 @@ func TestRolesShareBackendCapacityAndReleaseAfterCancellation(t *testing.T) { t.Fatal("first call did not enter provider") } var output map[string]any - _, err = validation.CompleteStructured(context.Background(), request, &output) + capacityResponse, err := validation.CompleteStructured(context.Background(), request, &output) if !errors.Is(err, pk.ErrCapacityExceeded) { t.Fatalf("other role did not share capacity: %v", err) } + var capacityFailure *structuredoutput.Failure + if !errors.As(err, &capacityFailure) || capacityFailure.Class != structuredoutput.FailureRecoverable || capacityFailure.Reason != outcomes.ReasonCapacityExceeded || !capacityResponse.Execution.Available || capacityResponse.Execution.ProviderAttempts != 0 { + t.Fatalf("capacity outcome: failure=%#v counts=%+v", capacityFailure, capacityResponse.Execution) + } select { case <-entered: t.Fatal("second role reached provider despite full backend") @@ -131,6 +136,9 @@ func TestRolesShareBackendCapacityAndReleaseAfterCancellation(t *testing.T) { if !errors.Is(err, context.DeadlineExceeded) { t.Fatalf("queued call did not honor deadline: %v", err) } + if !errors.As(err, &capacityFailure) || capacityFailure.Class != structuredoutput.FailureCanceled || capacityFailure.Reason != outcomes.ReasonCanceled { + t.Fatalf("queued parent deadline classification: %#v", capacityFailure) + } close(release) if err := <-firstDone; err != nil { t.Fatal(err) diff --git a/internal/framework/llm/adapter_test.go b/internal/framework/llm/adapter_test.go index 9140546..9029171 100644 --- a/internal/framework/llm/adapter_test.go +++ b/internal/framework/llm/adapter_test.go @@ -13,9 +13,11 @@ import ( "sync" "sync/atomic" "testing" + "time" ck "gitea.maximumdirect.net/eric/ConfigKit" "gitea.maximumdirect.net/eric/audita/internal/core/config" + "gitea.maximumdirect.net/eric/audita/internal/framework/outcomes" "gitea.maximumdirect.net/eric/audita/internal/framework/responseschema" "gitea.maximumdirect.net/eric/audita/internal/framework/structuredoutput" "gitea.maximumdirect.net/eric/audita/internal/prompts" @@ -102,18 +104,19 @@ func TestPreparedAdapterRepairsAndClassifies(t *testing.T) { replies []reply retries, repairs, attempts, repairCalls, transientRetries int malformed bool + reason outcomes.ReasonCode cause error }{ - {"success", []reply{{200, valid}}, 0, 0, 1, 0, 0, false, nil}, - {"repair succeeds", []reply{{200, `{"wrong":true}`}, {200, valid}}, 0, 1, 2, 1, 0, false, nil}, - {"repair exhausted", []reply{{200, `{"wrong":true}`}, {200, `{"wrong":true}`}}, 0, 1, 2, 1, 0, true, nil}, - {"429 recovery", []reply{{429, ""}, {200, valid}}, 1, 0, 2, 0, 1, false, nil}, - {"5xx recovery", []reply{{503, ""}, {200, valid}}, 1, 0, 2, 0, 1, false, nil}, - {"zero retry", []reply{{503, ""}}, 0, 0, 1, 0, 0, false, pk.ErrLLMGenerate}, - {"positive retry exhausted", []reply{{503, ""}, {503, ""}}, 1, 0, 2, 0, 1, false, pk.ErrRetriesExhausted}, - {"transport failure", []reply{{0, ""}}, 0, 0, 1, 0, 0, false, pk.ErrTransport}, - {"malformed envelope", []reply{{200, "MALFORMED_ENVELOPE"}}, 0, 0, 1, 0, 0, false, pk.ErrMalformedResponse}, - {"retry and repair", []reply{{503, ""}, {200, `{"wrong":true}`}, {503, ""}, {200, valid}}, 1, 1, 4, 1, 2, false, nil}, + {"success", []reply{{200, valid}}, 0, 0, 1, 0, 0, false, "", nil}, + {"repair succeeds", []reply{{200, `{"wrong":true}`}, {200, valid}}, 0, 1, 2, 1, 0, false, "", nil}, + {"repair exhausted", []reply{{200, `{"wrong":true}`}, {200, `{"wrong":true}`}}, 0, 1, 2, 1, 0, true, outcomes.ReasonInvalidOutputExhausted, nil}, + {"429 recovery", []reply{{429, ""}, {200, valid}}, 1, 0, 2, 0, 1, false, "", nil}, + {"5xx recovery", []reply{{503, ""}, {200, valid}}, 1, 0, 2, 0, 1, false, "", nil}, + {"zero retry", []reply{{503, ""}}, 0, 0, 1, 0, 0, false, outcomes.ReasonProviderFailed, pk.ErrLLMGenerate}, + {"positive retry exhausted", []reply{{503, ""}, {503, ""}}, 1, 0, 2, 0, 1, false, outcomes.ReasonProviderFailed, pk.ErrRetriesExhausted}, + {"transport failure", []reply{{0, ""}}, 0, 0, 1, 0, 0, false, outcomes.ReasonTransportFailed, pk.ErrTransport}, + {"malformed envelope", []reply{{200, "MALFORMED_ENVELOPE"}}, 0, 0, 1, 0, 0, false, outcomes.ReasonProviderResponseInvalid, pk.ErrMalformedResponse}, + {"retry and repair", []reply{{503, ""}, {200, `{"wrong":true}`}, {503, ""}, {200, valid}}, 1, 1, 4, 1, 2, false, "", nil}, } { t.Run(string(schema)+"/"+tc.name, func(t *testing.T) { runtime, count, _ := adapterFixture(t, tc.replies, tc.retries, tc.repairs, nil) @@ -141,6 +144,24 @@ func TestPreparedAdapterRepairsAndClassifies(t *testing.T) { if tc.cause == nil && !tc.malformed && err != nil { t.Fatal(err) } + var failure *structuredoutput.Failure + if tc.reason != "" { + if !errors.As(err, &failure) || failure.Class != structuredoutput.FailureRecoverable || failure.Reason != tc.reason || !structuredoutput.IsRecoverable(err) { + t.Fatalf("failure classification: %#v, %v", failure, err) + } + if strings.Contains(err.Error(), "provider error") || strings.Contains(err.Error(), "adapter-secret") { + t.Fatalf("unsafe model error: %v", err) + } + } + if !response.Execution.Available || response.Execution.ProviderAttempts != tc.attempts || response.Execution.StructuralRepairs != tc.repairCalls || response.Execution.TransientRetries != tc.transientRetries { + t.Fatalf("returned execution counts: %+v", response.Execution) + } + if response.Execution.RetriesExhausted != (tc.name == "positive retry exhausted") { + t.Fatalf("retry exhaustion: %+v", response.Execution) + } + if tc.reason == outcomes.ReasonProviderFailed && (response.Execution.HTTPStatus == nil || *response.Execution.HTTPStatus != http.StatusServiceUnavailable) { + t.Fatalf("provider status missing: %+v", response.Execution) + } if tc.name == "zero retry" && errors.Is(err, pk.ErrRetriesExhausted) { t.Fatal("zero retry reported exhaustion") } @@ -256,3 +277,83 @@ func TestPreparedAdapterCancellationStopsRetryWait(t *testing.T) { t.Fatalf("cancellation: %v, HTTP calls=%d", err, count.Load()) } } + +func TestPreparedAdapterDistinguishesOperationTimeoutFromParentCancellation(t *testing.T) { + blocking := func(*http.Request) { time.Sleep(500 * time.Millisecond) } + runtime, _, _ := adapterFixture(t, []reply{{200, `{"corrections":[]}`}}, 0, 0, blocking) + runtime.Config.LLM.OperationTimeout = 200 * time.Millisecond + client, err := runtime.NewRoleClient("proposal", CallHooks{}) + if err != nil { + t.Fatal(err) + } + var output map[string]any + response, err := client.CompleteStructured(context.Background(), adapterRequest(responseschema.CorrectionSetKey), &output) + var failure *structuredoutput.Failure + if !errors.As(err, &failure) || failure.Class != structuredoutput.FailureRecoverable || failure.Reason != outcomes.ReasonOperationTimeout || !response.Execution.Available { + t.Fatalf("local timeout: failure=%#v counts=%+v", failure, response.Execution) + } + + parent, cancel := context.WithCancel(context.Background()) + cancel() + response, err = client.CompleteStructured(parent, adapterRequest(responseschema.CorrectionSetKey), &output) + if !errors.As(err, &failure) || failure.Class != structuredoutput.FailureCanceled || failure.Reason != outcomes.ReasonCanceled || !errors.Is(err, context.Canceled) || response.Execution.Available { + t.Fatalf("parent cancellation: failure=%#v counts=%+v", failure, response.Execution) + } +} + +func TestPreparedAdapterClassifiesSetupAndDomainDecodeFailures(t *testing.T) { + runtime, _, _ := adapterFixture(t, []reply{{200, `{"validations":[{"correction_index":9999999999999999999999999,"approved":true,"confidence":0.8,"reason":"ok"}]}`}}, 0, 0, nil) + client, err := runtime.NewRoleClient("validation", CallHooks{}) + if err != nil { + t.Fatal(err) + } + var output struct { + Validations []struct { + CorrectionIndex int `json:"correction_index"` + } `json:"validations"` + } + response, err := client.CompleteStructured(context.Background(), adapterRequest(responseschema.ValidatorDecisionSetKey), &output) + var failure *structuredoutput.Failure + if !errors.As(err, &failure) || failure.Class != structuredoutput.FailureRecoverable || failure.Reason != outcomes.ReasonInvalidOutput || !response.Execution.Available { + t.Fatalf("decode overflow: failure=%#v counts=%+v", failure, response.Execution) + } + _, err = client.CompleteStructured(context.Background(), adapterRequest(responseschema.CorrectionSetKey), nil) + if !errors.As(err, &failure) || failure.Class != structuredoutput.FailureFatal || failure.Reason != outcomes.ReasonSetupFailure { + t.Fatalf("invalid target: %#v", failure) + } + req := adapterRequest("missing_schema") + _, err = client.CompleteStructured(context.Background(), req, &output) + if !errors.As(err, &failure) || failure.Class != structuredoutput.FailureFatal || failure.Reason != outcomes.ReasonSetupFailure { + t.Fatalf("unknown schema: %#v", failure) + } + req = adapterRequest(responseschema.ValidatorDecisionSetKey) + req.PromptID = "missing.prompt" + _, err = client.CompleteStructured(context.Background(), req, &output) + if !errors.As(err, &failure) || failure.Class != structuredoutput.FailureFatal || failure.Reason != outcomes.ReasonSetupFailure { + t.Fatalf("missing prompt: %#v", failure) + } +} + +func TestPreparedAdapterNonRetryableProviderStatus(t *testing.T) { + runtime, count, _ := adapterFixture(t, []reply{{http.StatusBadRequest, ""}}, 2, 0, nil) + client, err := runtime.NewRoleClient("proposal", CallHooks{}) + if err != nil { + t.Fatal(err) + } + var output map[string]any + response, err := client.CompleteStructured(context.Background(), adapterRequest(responseschema.CorrectionSetKey), &output) + var failure *structuredoutput.Failure + if !errors.As(err, &failure) || failure.Class != structuredoutput.FailureRecoverable || failure.Reason != outcomes.ReasonProviderFailed || count.Load() != 1 || response.Execution.HTTPStatus == nil || *response.Execution.HTTPStatus != http.StatusBadRequest { + t.Fatalf("non-retryable provider failure: failure=%#v calls=%d counts=%+v", failure, count.Load(), response.Execution) + } +} + +func TestPreparedAdapterUnknownValidationFailureRemainsFatal(t *testing.T) { + parent, cancel := context.WithCancel(context.Background()) + defer cancel() + cause := fmt.Errorf("%w: embedded contract defect", pk.ErrValidation) + failure := classifiedFailure(parent, parent, cause, nil) + if failure.Class != structuredoutput.FailureFatal || failure.Reason != outcomes.ReasonSetupFailure || !errors.Is(failure, pk.ErrValidation) { + t.Fatalf("unknown validation failure: %#v", failure) + } +}