From 159b02116f4e93ebe17370f239ccd1c919afa6fb Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Sun, 23 Aug 2026 19:01:38 +0000 Subject: [PATCH] Expose structured generation errors to consumers --- doc.go | 16 ++++-- docs/roadmap/implementation.md | 2 + engine.go | 24 ++++---- engine_test.go | 11 +++- errors.go | 10 ++++ generation_error.go | 87 ++++++++++++++++++++++++++++ generation_error_contract_test.go | 95 +++++++++++++++++++++++++++++++ generation_error_internal_test.go | 32 +++++++++++ types.go | 8 +-- 9 files changed, 263 insertions(+), 22 deletions(-) create mode 100644 generation_error.go create mode 100644 generation_error_contract_test.go create mode 100644 generation_error_internal_test.go diff --git a/doc.go b/doc.go index 0fbd334..899f56f 100644 --- a/doc.go +++ b/doc.go @@ -23,8 +23,9 @@ // InspectProfile return copied inspection values. Returned values and values // passed to extension interfaces are likewise isolated from engine state. // Callers own those copies and may mutate them after the call that supplied or -// returned them. Returned structured errors are likewise caller-owned and may -// be mutated without affecting engine state or another error. +// returned them. [CapacityError] values are caller-owned and may be mutated +// without affecting engine state or another error. Immutable [GenerationError] +// values are also caller-owned and do not retain shared engine state. // // # Security and sensitive data // @@ -53,9 +54,14 @@ // Construction, inspection, handle, and error values, including [Config], // [Backend], [RunRequest], [ArtifactRef], [ExecutionTargetOverride], [Profile], // [OpenAICompatibleProfileConfig], [ProfileInspection], -// [PromptInputDefinition], [PromptInspection], [PreparedExecution], and -// [CapacityError], do not have stable JSON representations. Direct API keys -// are nevertheless excluded from JSON for every public value. +// [PromptInputDefinition], [PromptInspection], [PreparedExecution], +// [CapacityError], and [GenerationError], do not have stable JSON +// representations. Direct API keys are nevertheless excluded from JSON for +// every public value. + +// Provider-derived [GenerationError] accessor values are untrusted and can +// contain sensitive request or schema fragments. Applications must apply their +// own disclosure policy before logging, displaying, or returning them. // // JSON timestamps use time.Time's RFC 3339 encoding and are omitted when zero. // PreparedRun and RunResult durations are encoded as integer milliseconds in diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index ff31253..0989925 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -359,6 +359,8 @@ go test ./... Stage 4 is complete when consumers can inspect built-in non-2xx details through the stable root contract and injected errors remain untouched. +**Status:** Complete. + ## Stage 5: Update Canonical Documentation and Run Full Validation ### Objective diff --git a/engine.go b/engine.go index b1fb3b5..0ff88d5 100644 --- a/engine.go +++ b/engine.go @@ -69,8 +69,9 @@ var ( // request, an LLM or provider rate-limit response, or ErrLLMGenerate. ErrCapacityExceeded = errors.New("backend capacity exceeded") // ErrLLMGenerate identifies a model-client failure or a nil successful - // response. Errors returned by an injected LLMClient remain available - // through errors.Is. + // response. A built-in OpenAI-compatible non-2xx response is available as a + // [GenerationError]. Errors returned by an injected LLMClient remain + // available through errors.Is. ErrLLMGenerate = errors.New("failed to generate output") // ErrValidation identifies an operational failure to load or compile a // schema or validate output. A completed validation whose Status is @@ -646,11 +647,13 @@ func (e *Engine) PrepareExecution(ctx context.Context, req RunRequest) (*Prepare // discoverable as [CapacityError] and still matches ErrCapacityExceeded. It // occurs before artifacts, schemas, rendering, or model generation because the // selected backend's admission capacity is full; it does not match -// ErrInvalidRequest or ErrLLMGenerate. Errors from injected clients remain -// available through errors.Is. Cancellation while waiting for model-generation -// capacity matches both ErrLLMGenerate and the context error. Cancellation -// otherwise follows the active collaborator's documented behavior. A nil -// Engine returns ErrInvalidConfig. Run returns no partial result on error. +// ErrInvalidRequest or ErrLLMGenerate. A built-in OpenAI-compatible non-2xx +// response is discoverable as [GenerationError]. Errors from injected clients +// remain available through errors.Is. Cancellation while waiting for +// model-generation capacity matches both ErrLLMGenerate and the context error. +// Cancellation otherwise follows the active collaborator's documented +// behavior. A nil Engine returns ErrInvalidConfig. Run returns no partial +// result on error. func (e *Engine) Run(ctx context.Context, req RunRequest) (*RunResult, error) { if e == nil || e.runner == nil { return nil, fmt.Errorf("%w: engine is nil", ErrInvalidConfig) @@ -688,9 +691,10 @@ func (e *Engine) Run(ctx context.Context, req RunRequest) (*RunResult, error) { // ErrCapacityExceeded, ErrLLMGenerate, or ErrValidation as applicable while // preserving documented collaborator and context identities. An engine // admission rejection is discoverable as [CapacityError] and still matches -// ErrCapacityExceeded. A completed content-validation rejection is returned -// in RunResult, not as an operational error. An operational error returns no -// partial RunResult. +// ErrCapacityExceeded. A built-in OpenAI-compatible non-2xx response is +// discoverable as [GenerationError]. A completed content-validation rejection +// is returned in RunResult, not as an operational error. An operational error +// returns no partial RunResult. func (e *Engine) RunPrepared(ctx context.Context, prepared *PreparedExecution) (*RunResult, error) { if e == nil || e.runner == nil { return nil, fmt.Errorf("%w: engine is nil", ErrInvalidConfig) diff --git a/engine_test.go b/engine_test.go index 35e28f4..10b4a57 100644 --- a/engine_test.go +++ b/engine_test.go @@ -1164,8 +1164,9 @@ func TestArtifactReaderFailuresPreserveArtifactLoadErrors(t *testing.T) { } func TestRunAddsLLMGenerateToCollaboratorPublicError(t *testing.T) { + injectedErr := errors.New("injected model client failure") engine := newContractEngineWithOptions(t, frameworkSchemaDir, - promptkit.WithLLMClient(&fakeLLMClient{err: promptkit.ErrArtifactLoad}), + promptkit.WithLLMClient(&fakeLLMClient{err: injectedErr}), ) _, err := engine.Run(context.Background(), promptkit.RunRequest{ @@ -1178,8 +1179,12 @@ func TestRunAddsLLMGenerateToCollaboratorPublicError(t *testing.T) { if !errors.Is(err, promptkit.ErrLLMGenerate) { t.Fatalf("expected ErrLLMGenerate, got %v", err) } - if !errors.Is(err, promptkit.ErrArtifactLoad) { - t.Fatalf("expected preserved ErrArtifactLoad, got %v", err) + if !errors.Is(err, injectedErr) { + t.Fatalf("expected preserved injected error, got %v", err) + } + var generationErr *promptkit.GenerationError + if errors.As(err, &generationErr) { + t.Fatalf("injected error became GenerationError: %v", err) } } diff --git a/errors.go b/errors.go index 0844431..4cee2b2 100644 --- a/errors.go +++ b/errors.go @@ -6,6 +6,7 @@ import ( "strings" "gitea.maximumdirect.net/eric/promptkit/internal/capacity" + "gitea.maximumdirect.net/eric/promptkit/internal/llm" "gitea.maximumdirect.net/eric/promptkit/internal/profile" "gitea.maximumdirect.net/eric/promptkit/internal/promptdef" "gitea.maximumdirect.net/eric/promptkit/internal/usecase" @@ -20,6 +21,15 @@ func mapPublicError(err error) error { strings.TrimSpace(internalCapacityError.BackendID) != "" { return &CapacityError{BackendID: internalCapacityError.BackendID} } + var providerHTTPError *llm.ProviderHTTPError + if errors.As(err, &providerHTTPError) && providerHTTPError != nil { + return newGenerationError( + providerHTTPError.StatusCode(), + providerHTTPError.ProviderCode(), + providerHTTPError.ProviderType(), + providerHTTPError.ProviderMessage(), + ) + } publicErr := publicErrorFor(err) if publicErr == nil { return err diff --git a/generation_error.go b/generation_error.go new file mode 100644 index 0000000..4120949 --- /dev/null +++ b/generation_error.go @@ -0,0 +1,87 @@ +package promptkit + +import "fmt" + +// GenerationError reports a non-2xx response from Promptkit's built-in +// OpenAI-compatible client during [Engine.Run] or [Engine.RunPrepared]. +// +// Engine-produced values are immutable, caller-owned values. Use errors.Is to +// match [ErrLLMGenerate] and errors.As with a *GenerationError target to obtain +// this type. The four provider accessors expose untrusted provider-controlled +// values that can contain sensitive request or schema fragments. Applications +// must apply their own disclosure policy before logging, displaying, or +// returning them to another caller. +// +// Accessors, Error, GoString, and Unwrap are safe on a nil receiver and a zero +// value. Default and Go-syntax formatting deliberately redact provider details. +// GenerationError has no stable JSON representation. +type GenerationError struct { + statusCode int + providerCode string + providerType string + providerMessage string +} + +func newGenerationError(statusCode int, providerCode, providerType, providerMessage string) *GenerationError { + return &GenerationError{ + statusCode: statusCode, + providerCode: providerCode, + providerType: providerType, + providerMessage: providerMessage, + } +} + +// StatusCode returns the received provider HTTP status code, or zero for a nil +// receiver or zero value. +func (e *GenerationError) StatusCode() int { + if e == nil { + return 0 + } + return e.statusCode +} + +// ProviderCode returns the normalized provider error code, if present. Its +// value is untrusted and may contain sensitive data. +func (e *GenerationError) ProviderCode() string { + if e == nil { + return "" + } + return e.providerCode +} + +// ProviderType returns the normalized provider error type, if present. Its +// value is untrusted and may contain sensitive data. +func (e *GenerationError) ProviderType() string { + if e == nil { + return "" + } + return e.providerType +} + +// ProviderMessage returns the bounded normalized provider diagnostic, if +// present. Its value is untrusted and may contain sensitive data. +func (e *GenerationError) ProviderMessage() string { + if e == nil { + return "" + } + return e.providerMessage +} + +// Error returns a redacted diagnostic that is not a parsing contract. +func (e *GenerationError) Error() string { + if e == nil || e.statusCode == 0 { + return ErrLLMGenerate.Error() + } + return fmt.Sprintf("%s: provider returned HTTP status %d", ErrLLMGenerate, e.statusCode) +} + +// GoString returns the same redacted diagnostic as Error. +func (e *GenerationError) GoString() string { + return e.Error() +} + +// Unwrap returns ErrLLMGenerate. It is safe to call on a nil receiver or zero +// value. +func (e *GenerationError) Unwrap() error { + return ErrLLMGenerate +} diff --git a/generation_error_contract_test.go b/generation_error_contract_test.go new file mode 100644 index 0000000..b6beaac --- /dev/null +++ b/generation_error_contract_test.go @@ -0,0 +1,95 @@ +package promptkit_test + +import ( + "context" + "errors" + "fmt" + "io" + "net/http" + "strings" + "testing" + + "gitea.maximumdirect.net/eric/promptkit" +) + +func TestBuiltInGenerationError(t *testing.T) { + const ( + codeMarker = "provider-code-marker" + typeMarker = "provider-type-marker" + messageMarker = "provider-message-marker" + ) + engine := newBuiltInGenerationErrorEngine(t, http.StatusUnprocessableEntity, + `{"error":{"code":"`+codeMarker+`","type":"`+typeMarker+`","message":"`+messageMarker+`"}}`) + + result, err := engine.Run(context.Background(), generationErrorRunRequest()) + if result != nil { + t.Fatalf("Run result = %#v, want nil", result) + } + assertGenerationError(t, err, http.StatusUnprocessableEntity, codeMarker, typeMarker, messageMarker) + + preparedEngine := newBuiltInGenerationErrorEngine(t, http.StatusServiceUnavailable, `{"error":{}}`) + prepared, err := preparedEngine.PrepareExecution(context.Background(), generationErrorRunRequest()) + if err != nil { + t.Fatalf("PrepareExecution: %v", err) + } + result, err = preparedEngine.RunPrepared(context.Background(), prepared) + if result != nil { + t.Fatalf("RunPrepared result = %#v, want nil", result) + } + assertGenerationError(t, err, http.StatusServiceUnavailable, "", "", "") +} + +func assertGenerationError(t *testing.T, err error, statusCode int, code, providerType, message string) { + t.Helper() + + if !errors.Is(err, promptkit.ErrLLMGenerate) { + t.Fatalf("errors.Is(%v, ErrLLMGenerate) = false", err) + } + var generationErr *promptkit.GenerationError + if !errors.As(err, &generationErr) || generationErr == nil { + t.Fatalf("error = %T, want *GenerationError", err) + } + if generationErr.StatusCode() != statusCode || generationErr.ProviderCode() != code || generationErr.ProviderType() != providerType || generationErr.ProviderMessage() != message { + t.Fatalf("GenerationError = %#v", generationErr) + } + + wantFormatted := fmt.Sprintf("failed to generate output: provider returned HTTP status %d", statusCode) + for _, rendered := range []string{fmt.Sprintf("%v", generationErr), fmt.Sprintf("%+v", generationErr), fmt.Sprintf("%#v", generationErr)} { + if rendered != wantFormatted { + t.Fatalf("formatted error = %q, want %q", rendered, wantFormatted) + } + for _, marker := range []string{code, providerType, message} { + if marker != "" && strings.Contains(rendered, marker) { + t.Fatalf("formatted error exposed provider marker %q: %q", marker, rendered) + } + } + } +} + +func newBuiltInGenerationErrorEngine(t *testing.T, statusCode int, body string) *promptkit.Engine { + t.Helper() + + config := contractConfig(frameworkSchemaDir) + config.HTTPClient = &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) { + return &http.Response{ + StatusCode: statusCode, + ContentLength: int64(len(body)), + Body: io.NopCloser(strings.NewReader(body)), + }, nil + })} + engine, err := promptkit.NewEngine(config) + if err != nil { + t.Fatalf("NewEngine: %v", err) + } + return engine +} + +func generationErrorRunRequest() promptkit.RunRequest { + return promptkit.RunRequest{ + PromptID: frameworkMarkdownSummaryPromptID, + Inputs: map[string]promptkit.ArtifactRef{ + "transcript": promptkit.Inline("Rin opens the gate."), + "glossary": promptkit.Inline("gate: A guarded passage."), + }, + } +} diff --git a/generation_error_internal_test.go b/generation_error_internal_test.go new file mode 100644 index 0000000..cb7eb97 --- /dev/null +++ b/generation_error_internal_test.go @@ -0,0 +1,32 @@ +package promptkit + +import ( + "errors" + "fmt" + "testing" +) + +func TestGenerationErrorNilAndZeroValue(t *testing.T) { + var nilError *GenerationError + zeroError := &GenerationError{} + + for name, err := range map[string]*GenerationError{ + "nil": nilError, + "zero": zeroError, + } { + t.Run(name, func(t *testing.T) { + if err.StatusCode() != 0 || err.ProviderCode() != "" || err.ProviderType() != "" || err.ProviderMessage() != "" { + t.Fatalf("accessors returned provider details: %#v", err) + } + if err.Error() != "failed to generate output" || err.GoString() != "failed to generate output" { + t.Fatalf("redacted formatting = (%q, %q)", err.Error(), err.GoString()) + } + if fmt.Sprintf("%v", err) != "failed to generate output" || fmt.Sprintf("%#v", err) != "failed to generate output" { + t.Fatalf("formatted error = (%q, %q)", fmt.Sprintf("%v", err), fmt.Sprintf("%#v", err)) + } + if !errors.Is(err, ErrLLMGenerate) { + t.Fatalf("errors.Is(%v, ErrLLMGenerate) = false", err) + } + }) + } +} diff --git a/types.go b/types.go index 87f17e5..68545c7 100644 --- a/types.go +++ b/types.go @@ -665,10 +665,10 @@ type StructuredOutputJSONSpec struct { // and retained copies. It is responsible for the cancellation behavior of any // work it starts and for synchronizing access to retained or shared data. // -// A returned error makes Run or RunPrepared return ErrLLMGenerate while -// preserving the client error through errors.Is. A nil response with a nil -// error also produces ErrLLMGenerate. Promptkit copies the non-nil response -// before returning from either method. +// An arbitrary returned error makes Run or RunPrepared return ErrLLMGenerate +// while preserving the client error through errors.Is rather than translating +// it. A nil response with a nil error also produces ErrLLMGenerate. Promptkit +// copies the non-nil response before returning from either method. type LLMClient interface { Generate(context.Context, GenerateRequest) (*GenerateResponse, error) }