diff --git a/docs/consumers/pkg-promptkit.md b/docs/consumers/pkg-promptkit.md index e723395..5b57d96 100644 --- a/docs/consumers/pkg-promptkit.md +++ b/docs/consumers/pkg-promptkit.md @@ -558,6 +558,14 @@ disclosure policy. Promptkit does not assign retry or presentation behavior. The [`GenerationError` GoDoc](../../generation_error.go) owns the exact typed error contract. +Generation failures also match `ErrTransport` when the built-in HTTP request +fails, or `ErrMalformedResponse` when a successful provider response cannot be +decoded or an injected client returns a nil successful response. Both retain +`ErrLLMGenerate`; transport failures retain their original causes for +`errors.Is` and `errors.As`. Caller cancellation and deadlines retain their +standard context identities. Arbitrary injected-client errors retain their own +identities without Promptkit guessing a failure category from their text. + ## Application Boundary Promptkit is an importable library. It does not own a command, inbound HTTP diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index 61f095b..434e177 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -216,6 +216,8 @@ admission slot. No asynchronous delivery or durable history. ## Stage 1 — Expose Missing Generation Failure Categories +**Completed:** 2026-09-23. + Read `errors.go`, `generation_error.go`, `llm_adapter.go`, the LLM error types, and public error-boundary tests. Add public transport/malformed sentinels and map current built-in errors while preserving cause identity and redacted text. diff --git a/engine.go b/engine.go index 7b470b4..04a28dd 100644 --- a/engine.go +++ b/engine.go @@ -61,6 +61,15 @@ var ( // [GenerationError]. Errors returned by an injected LLMClient remain // available through errors.Is. ErrLLMGenerate = errors.New("failed to generate output") + // ErrTransport identifies a failed HTTP request by the built-in model + // client. It also matches ErrLLMGenerate and retains the transport cause + // for errors.Is and errors.As. It does not imply that the provider received + // no work. + ErrTransport = errors.New("model transport failed") + // ErrMalformedResponse identifies an invalid successful model response or + // a nil successful response from an injected client. It also matches + // ErrLLMGenerate. + ErrMalformedResponse = errors.New("malformed model response") // ErrValidation identifies an operational failure to load or compile a // schema or validate output. A completed validation whose Status is // ValidationFailed is returned in RunResult without this error. @@ -483,7 +492,10 @@ func (e *Engine) PrepareExecution(ctx context.Context, req RunRequest) (*Prepare // no partial result. // // Run can return every error category documented by [Engine.Prepare], plus -// ErrCapacityExceeded and ErrLLMGenerate. An engine admission rejection is +// ErrCapacityExceeded and ErrLLMGenerate. Built-in HTTP execution failures +// additionally match ErrTransport; malformed successful responses and nil +// successful injected responses additionally match ErrMalformedResponse. +// An engine admission rejection is // discoverable as [CapacityError] and still matches ErrCapacityExceeded. It // occurs before artifacts, schemas, rendering, or model generation because the // selected backend's admission capacity is full; it does not match @@ -532,7 +544,9 @@ func (e *Engine) Run(ctx context.Context, req RunRequest) (*RunResult, error) { // preserving documented collaborator and context identities. An engine // admission rejection is discoverable as [CapacityError] and still matches // ErrCapacityExceeded. A built-in OpenAI-compatible non-2xx response is -// discoverable as [GenerationError]. A completed content-validation rejection, +// discoverable as [GenerationError]. Built-in HTTP execution failures match +// ErrTransport; malformed successful responses and nil successful injected +// responses match ErrMalformedResponse. A completed content-validation rejection, // including repair exhaustion, is returned in RunResult, not as an operational // error. An operational error returns no partial RunResult. func (e *Engine) RunPrepared(ctx context.Context, prepared *PreparedExecution) (*RunResult, error) { diff --git a/errors.go b/errors.go index 612d0dc..014bd31 100644 --- a/errors.go +++ b/errors.go @@ -38,7 +38,14 @@ func mapPublicError(err error) error { if publicErr == nil { return err } - return fmt.Errorf("%w: %w", publicErr, err) + switch { + case errors.Is(err, llm.ErrTransport): + return fmt.Errorf("%w: %w: %w", publicErr, ErrTransport, err) + case errors.Is(err, llm.ErrMalformedResponse): + return fmt.Errorf("%w: %w: %w", publicErr, ErrMalformedResponse, err) + default: + return fmt.Errorf("%w: %w", publicErr, err) + } } func publicErrorFor(err error) error { diff --git a/errors_internal_test.go b/errors_internal_test.go index 82d066d..6657949 100644 --- a/errors_internal_test.go +++ b/errors_internal_test.go @@ -73,3 +73,10 @@ func TestMapPublicErrorPreservesValidationAroundGenerationError(t *testing.T) { t.Fatalf("mapped error exposes internal ProviderHTTPError: %v", err) } } + +func TestMapPublicErrorDoesNotTreatRequestEncodingAsTransport(t *testing.T) { + err := mapPublicError(fmt.Errorf("%w: %w", usecase.ErrLLMGenerate, llm.ErrRequestFailed)) + if !errors.Is(err, ErrLLMGenerate) || errors.Is(err, ErrTransport) { + t.Fatalf("request encoding category = %v", err) + } +} diff --git a/generation_error_contract_test.go b/generation_error_contract_test.go index d1252a7..20ace5f 100644 --- a/generation_error_contract_test.go +++ b/generation_error_contract_test.go @@ -94,6 +94,89 @@ func TestBuiltInRepairGenerationError(t *testing.T) { assertGenerationError(t, err, http.StatusUnprocessableEntity, codeMarker, typeMarker, messageMarker) } +func TestBuiltInGenerationFailureCategories(t *testing.T) { + transportCause := errors.New("private transport diagnostic") + config := contractConfig(frameworkSchemaDir) + config.HTTPClient = &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) { + return nil, transportCause + })} + engine, err := promptkit.NewEngine(config) + if err != nil { + t.Fatal(err) + } + result, err := engine.Run(context.Background(), generationErrorRunRequest()) + if result != nil || !errors.Is(err, promptkit.ErrLLMGenerate) || !errors.Is(err, promptkit.ErrTransport) || !errors.Is(err, transportCause) { + t.Fatalf("transport result=%#v, error=%v", result, err) + } + if errors.Is(err, promptkit.ErrMalformedResponse) || strings.Contains(fmt.Sprintf("%#v", err), transportCause.Error()) { + t.Fatalf("transport category or formatting = %v", err) + } + + for _, tc := range []struct { + name string + body string + }{ + {name: "invalid framing", body: `{"choices":[]}{}`}, + {name: "missing choices", body: `{"choices":[]}`}, + {name: "missing content", body: `{"choices":[{"message":{}}]}`}, + } { + t.Run(tc.name, func(t *testing.T) { + engine := newBuiltInGenerationErrorEngine(t, http.StatusOK, tc.body) + result, err := engine.Run(context.Background(), generationErrorRunRequest()) + if result != nil || !errors.Is(err, promptkit.ErrLLMGenerate) || !errors.Is(err, promptkit.ErrMalformedResponse) || errors.Is(err, promptkit.ErrTransport) { + t.Fatalf("malformed result=%#v, error=%v", result, err) + } + }) + } +} + +func TestBuiltInGenerationPreservesContextIdentity(t *testing.T) { + for _, tc := range []struct { + name string + cause error + }{ + {name: "canceled", cause: context.Canceled}, + {name: "deadline", cause: context.DeadlineExceeded}, + } { + t.Run(tc.name, func(t *testing.T) { + config := contractConfig(frameworkSchemaDir) + config.HTTPClient = &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) { + return nil, tc.cause + })} + engine, err := promptkit.NewEngine(config) + if err != nil { + t.Fatal(err) + } + result, err := engine.Run(context.Background(), generationErrorRunRequest()) + if result != nil || !errors.Is(err, tc.cause) || !errors.Is(err, promptkit.ErrLLMGenerate) || !errors.Is(err, promptkit.ErrTransport) { + t.Fatalf("context result=%#v, error=%v", result, err) + } + }) + } +} + +func TestInjectedGenerationFailureCategories(t *testing.T) { + for _, tc := range []struct { + name string + client *fakeLLMClient + malformed bool + }{ + {name: "nil response", client: &fakeLLMClient{}, malformed: true}, + {name: "arbitrary error", client: &fakeLLMClient{err: errors.New("injected failure")}}, + } { + t.Run(tc.name, func(t *testing.T) { + engine := newContractEngineWithOptions(t, frameworkSchemaDir, promptkit.WithLLMClient(tc.client)) + result, err := engine.Run(context.Background(), generationErrorRunRequest()) + if result != nil || !errors.Is(err, promptkit.ErrLLMGenerate) || errors.Is(err, promptkit.ErrTransport) || errors.Is(err, promptkit.ErrMalformedResponse) != tc.malformed { + t.Fatalf("injected result=%#v, error=%v", result, err) + } + if tc.client.err != nil && !errors.Is(err, tc.client.err) { + t.Fatalf("injected cause lost: %v", err) + } + }) + } +} + func assertGenerationError(t *testing.T, err error, statusCode int, code, providerType, message string) { t.Helper() diff --git a/internal/llm/openai_compatible_client.go b/internal/llm/openai_compatible_client.go index ba0c760..1892e48 100644 --- a/internal/llm/openai_compatible_client.go +++ b/internal/llm/openai_compatible_client.go @@ -21,6 +21,7 @@ var ( ErrInvalidConfig = errors.New("invalid llm client configuration") ErrInvalidRequest = errors.New("invalid generate request") ErrRequestFailed = errors.New("llm request failed") + ErrTransport = errors.New("llm transport failed") ErrUnexpectedStatus = errors.New("llm returned non-success status") ErrMalformedResponse = errors.New("malformed llm response") ) @@ -36,7 +37,7 @@ func (e *requestFailedError) Error() string { } func (e *requestFailedError) Unwrap() []error { - return []error{ErrRequestFailed, e.cause} + return []error{ErrRequestFailed, ErrTransport, e.cause} } type OpenAICompatibleConfig struct { diff --git a/llm_adapter.go b/llm_adapter.go index 9efbbcb..d4a2c46 100644 --- a/llm_adapter.go +++ b/llm_adapter.go @@ -17,7 +17,7 @@ func (a publicLLMClientAdapter) Generate(ctx context.Context, req domain.Generat return nil, err } if resp == nil { - return nil, fmt.Errorf("%w: llm client returned nil response", ErrLLMGenerate) + return nil, fmt.Errorf("%w: %w: llm client returned nil response", ErrLLMGenerate, ErrMalformedResponse) } return toDomainGenerateResponse(resp), nil }