Expose transport and malformed generation errors
This commit is contained in:
@@ -558,6 +558,14 @@ disclosure policy. Promptkit does not assign retry or presentation behavior.
|
||||
The [`GenerationError` GoDoc](../../generation_error.go) owns the exact typed
|
||||
error contract.
|
||||
|
||||
Generation failures also match `ErrTransport` when the built-in HTTP request
|
||||
fails, or `ErrMalformedResponse` when a successful provider response cannot be
|
||||
decoded or an injected client returns a nil successful response. Both retain
|
||||
`ErrLLMGenerate`; transport failures retain their original causes for
|
||||
`errors.Is` and `errors.As`. Caller cancellation and deadlines retain their
|
||||
standard context identities. Arbitrary injected-client errors retain their own
|
||||
identities without Promptkit guessing a failure category from their text.
|
||||
|
||||
## Application Boundary
|
||||
|
||||
Promptkit is an importable library. It does not own a command, inbound HTTP
|
||||
|
||||
@@ -216,6 +216,8 @@ admission slot. No asynchronous delivery or durable history.
|
||||
|
||||
## Stage 1 — Expose Missing Generation Failure Categories
|
||||
|
||||
**Completed:** 2026-09-23.
|
||||
|
||||
Read `errors.go`, `generation_error.go`, `llm_adapter.go`, the LLM error types,
|
||||
and public error-boundary tests. Add public transport/malformed sentinels and
|
||||
map current built-in errors while preserving cause identity and redacted text.
|
||||
|
||||
18
engine.go
18
engine.go
@@ -61,6 +61,15 @@ var (
|
||||
// [GenerationError]. Errors returned by an injected LLMClient remain
|
||||
// available through errors.Is.
|
||||
ErrLLMGenerate = errors.New("failed to generate output")
|
||||
// ErrTransport identifies a failed HTTP request by the built-in model
|
||||
// client. It also matches ErrLLMGenerate and retains the transport cause
|
||||
// for errors.Is and errors.As. It does not imply that the provider received
|
||||
// no work.
|
||||
ErrTransport = errors.New("model transport failed")
|
||||
// ErrMalformedResponse identifies an invalid successful model response or
|
||||
// a nil successful response from an injected client. It also matches
|
||||
// ErrLLMGenerate.
|
||||
ErrMalformedResponse = errors.New("malformed model response")
|
||||
// ErrValidation identifies an operational failure to load or compile a
|
||||
// schema or validate output. A completed validation whose Status is
|
||||
// ValidationFailed is returned in RunResult without this error.
|
||||
@@ -483,7 +492,10 @@ func (e *Engine) PrepareExecution(ctx context.Context, req RunRequest) (*Prepare
|
||||
// no partial result.
|
||||
//
|
||||
// Run can return every error category documented by [Engine.Prepare], plus
|
||||
// ErrCapacityExceeded and ErrLLMGenerate. An engine admission rejection is
|
||||
// ErrCapacityExceeded and ErrLLMGenerate. Built-in HTTP execution failures
|
||||
// additionally match ErrTransport; malformed successful responses and nil
|
||||
// successful injected responses additionally match ErrMalformedResponse.
|
||||
// An engine admission rejection is
|
||||
// discoverable as [CapacityError] and still matches ErrCapacityExceeded. It
|
||||
// occurs before artifacts, schemas, rendering, or model generation because the
|
||||
// selected backend's admission capacity is full; it does not match
|
||||
@@ -532,7 +544,9 @@ func (e *Engine) Run(ctx context.Context, req RunRequest) (*RunResult, error) {
|
||||
// preserving documented collaborator and context identities. An engine
|
||||
// admission rejection is discoverable as [CapacityError] and still matches
|
||||
// ErrCapacityExceeded. A built-in OpenAI-compatible non-2xx response is
|
||||
// discoverable as [GenerationError]. A completed content-validation rejection,
|
||||
// discoverable as [GenerationError]. Built-in HTTP execution failures match
|
||||
// ErrTransport; malformed successful responses and nil successful injected
|
||||
// responses match ErrMalformedResponse. A completed content-validation rejection,
|
||||
// including repair exhaustion, is returned in RunResult, not as an operational
|
||||
// error. An operational error returns no partial RunResult.
|
||||
func (e *Engine) RunPrepared(ctx context.Context, prepared *PreparedExecution) (*RunResult, error) {
|
||||
|
||||
@@ -38,8 +38,15 @@ func mapPublicError(err error) error {
|
||||
if publicErr == nil {
|
||||
return err
|
||||
}
|
||||
switch {
|
||||
case errors.Is(err, llm.ErrTransport):
|
||||
return fmt.Errorf("%w: %w: %w", publicErr, ErrTransport, err)
|
||||
case errors.Is(err, llm.ErrMalformedResponse):
|
||||
return fmt.Errorf("%w: %w: %w", publicErr, ErrMalformedResponse, err)
|
||||
default:
|
||||
return fmt.Errorf("%w: %w", publicErr, err)
|
||||
}
|
||||
}
|
||||
|
||||
func publicErrorFor(err error) error {
|
||||
switch {
|
||||
|
||||
@@ -73,3 +73,10 @@ func TestMapPublicErrorPreservesValidationAroundGenerationError(t *testing.T) {
|
||||
t.Fatalf("mapped error exposes internal ProviderHTTPError: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMapPublicErrorDoesNotTreatRequestEncodingAsTransport(t *testing.T) {
|
||||
err := mapPublicError(fmt.Errorf("%w: %w", usecase.ErrLLMGenerate, llm.ErrRequestFailed))
|
||||
if !errors.Is(err, ErrLLMGenerate) || errors.Is(err, ErrTransport) {
|
||||
t.Fatalf("request encoding category = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -94,6 +94,89 @@ func TestBuiltInRepairGenerationError(t *testing.T) {
|
||||
assertGenerationError(t, err, http.StatusUnprocessableEntity, codeMarker, typeMarker, messageMarker)
|
||||
}
|
||||
|
||||
func TestBuiltInGenerationFailureCategories(t *testing.T) {
|
||||
transportCause := errors.New("private transport diagnostic")
|
||||
config := contractConfig(frameworkSchemaDir)
|
||||
config.HTTPClient = &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
|
||||
return nil, transportCause
|
||||
})}
|
||||
engine, err := promptkit.NewEngine(config)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
result, err := engine.Run(context.Background(), generationErrorRunRequest())
|
||||
if result != nil || !errors.Is(err, promptkit.ErrLLMGenerate) || !errors.Is(err, promptkit.ErrTransport) || !errors.Is(err, transportCause) {
|
||||
t.Fatalf("transport result=%#v, error=%v", result, err)
|
||||
}
|
||||
if errors.Is(err, promptkit.ErrMalformedResponse) || strings.Contains(fmt.Sprintf("%#v", err), transportCause.Error()) {
|
||||
t.Fatalf("transport category or formatting = %v", err)
|
||||
}
|
||||
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
body string
|
||||
}{
|
||||
{name: "invalid framing", body: `{"choices":[]}{}`},
|
||||
{name: "missing choices", body: `{"choices":[]}`},
|
||||
{name: "missing content", body: `{"choices":[{"message":{}}]}`},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
engine := newBuiltInGenerationErrorEngine(t, http.StatusOK, tc.body)
|
||||
result, err := engine.Run(context.Background(), generationErrorRunRequest())
|
||||
if result != nil || !errors.Is(err, promptkit.ErrLLMGenerate) || !errors.Is(err, promptkit.ErrMalformedResponse) || errors.Is(err, promptkit.ErrTransport) {
|
||||
t.Fatalf("malformed result=%#v, error=%v", result, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuiltInGenerationPreservesContextIdentity(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
cause error
|
||||
}{
|
||||
{name: "canceled", cause: context.Canceled},
|
||||
{name: "deadline", cause: context.DeadlineExceeded},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
config := contractConfig(frameworkSchemaDir)
|
||||
config.HTTPClient = &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
|
||||
return nil, tc.cause
|
||||
})}
|
||||
engine, err := promptkit.NewEngine(config)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
result, err := engine.Run(context.Background(), generationErrorRunRequest())
|
||||
if result != nil || !errors.Is(err, tc.cause) || !errors.Is(err, promptkit.ErrLLMGenerate) || !errors.Is(err, promptkit.ErrTransport) {
|
||||
t.Fatalf("context result=%#v, error=%v", result, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestInjectedGenerationFailureCategories(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
client *fakeLLMClient
|
||||
malformed bool
|
||||
}{
|
||||
{name: "nil response", client: &fakeLLMClient{}, malformed: true},
|
||||
{name: "arbitrary error", client: &fakeLLMClient{err: errors.New("injected failure")}},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
engine := newContractEngineWithOptions(t, frameworkSchemaDir, promptkit.WithLLMClient(tc.client))
|
||||
result, err := engine.Run(context.Background(), generationErrorRunRequest())
|
||||
if result != nil || !errors.Is(err, promptkit.ErrLLMGenerate) || errors.Is(err, promptkit.ErrTransport) || errors.Is(err, promptkit.ErrMalformedResponse) != tc.malformed {
|
||||
t.Fatalf("injected result=%#v, error=%v", result, err)
|
||||
}
|
||||
if tc.client.err != nil && !errors.Is(err, tc.client.err) {
|
||||
t.Fatalf("injected cause lost: %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func assertGenerationError(t *testing.T, err error, statusCode int, code, providerType, message string) {
|
||||
t.Helper()
|
||||
|
||||
|
||||
@@ -21,6 +21,7 @@ var (
|
||||
ErrInvalidConfig = errors.New("invalid llm client configuration")
|
||||
ErrInvalidRequest = errors.New("invalid generate request")
|
||||
ErrRequestFailed = errors.New("llm request failed")
|
||||
ErrTransport = errors.New("llm transport failed")
|
||||
ErrUnexpectedStatus = errors.New("llm returned non-success status")
|
||||
ErrMalformedResponse = errors.New("malformed llm response")
|
||||
)
|
||||
@@ -36,7 +37,7 @@ func (e *requestFailedError) Error() string {
|
||||
}
|
||||
|
||||
func (e *requestFailedError) Unwrap() []error {
|
||||
return []error{ErrRequestFailed, e.cause}
|
||||
return []error{ErrRequestFailed, ErrTransport, e.cause}
|
||||
}
|
||||
|
||||
type OpenAICompatibleConfig struct {
|
||||
|
||||
@@ -17,7 +17,7 @@ func (a publicLLMClientAdapter) Generate(ctx context.Context, req domain.Generat
|
||||
return nil, err
|
||||
}
|
||||
if resp == nil {
|
||||
return nil, fmt.Errorf("%w: llm client returned nil response", ErrLLMGenerate)
|
||||
return nil, fmt.Errorf("%w: %w: llm client returned nil response", ErrLLMGenerate, ErrMalformedResponse)
|
||||
}
|
||||
return toDomainGenerateResponse(resp), nil
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user