Expose structured generation errors to consumers

doc.go

@@ -23,8 +23,9 @@
 // InspectProfile return copied inspection values. Returned values and values
 // passed to extension interfaces are likewise isolated from engine state.
 // Callers own those copies and may mutate them after the call that supplied or
-// returned them. Returned structured errors are likewise caller-owned and may
-// be mutated without affecting engine state or another error.
+// returned them. [CapacityError] values are caller-owned and may be mutated
+// without affecting engine state or another error. Immutable [GenerationError]
+// values are also caller-owned and do not retain shared engine state.
 //
 // # Security and sensitive data
 //
@@ -53,9 +54,14 @@
 // Construction, inspection, handle, and error values, including [Config],
 // [Backend], [RunRequest], [ArtifactRef], [ExecutionTargetOverride], [Profile],
 // [OpenAICompatibleProfileConfig], [ProfileInspection],
-// [PromptInputDefinition], [PromptInspection], [PreparedExecution], and
-// [CapacityError], do not have stable JSON representations. Direct API keys
-// are nevertheless excluded from JSON for every public value.
+// [PromptInputDefinition], [PromptInspection], [PreparedExecution],
+// [CapacityError], and [GenerationError], do not have stable JSON
+// representations. Direct API keys are nevertheless excluded from JSON for
+// every public value.
+
+// Provider-derived [GenerationError] accessor values are untrusted and can
+// contain sensitive request or schema fragments. Applications must apply their
+// own disclosure policy before logging, displaying, or returning them.
 //
 // JSON timestamps use time.Time's RFC 3339 encoding and are omitted when zero.
 // PreparedRun and RunResult durations are encoded as integer milliseconds in

docs/roadmap/implementation.md

@@ -359,6 +359,8 @@ go test ./...
 Stage 4 is complete when consumers can inspect built-in non-2xx details through
 the stable root contract and injected errors remain untouched.
 
+**Status:** Complete.
+
 ## Stage 5: Update Canonical Documentation and Run Full Validation
 
 ### Objective

engine.go

@@ -69,8 +69,9 @@ var (
 	// request, an LLM or provider rate-limit response, or ErrLLMGenerate.
 	ErrCapacityExceeded = errors.New("backend capacity exceeded")
 	// ErrLLMGenerate identifies a model-client failure or a nil successful
-	// response. Errors returned by an injected LLMClient remain available
-	// through errors.Is.
+	// response. A built-in OpenAI-compatible non-2xx response is available as a
+	// [GenerationError]. Errors returned by an injected LLMClient remain
+	// available through errors.Is.
 	ErrLLMGenerate = errors.New("failed to generate output")
 	// ErrValidation identifies an operational failure to load or compile a
 	// schema or validate output. A completed validation whose Status is
@@ -646,11 +647,13 @@ func (e *Engine) PrepareExecution(ctx context.Context, req RunRequest) (*Prepare
 // discoverable as [CapacityError] and still matches ErrCapacityExceeded. It
 // occurs before artifacts, schemas, rendering, or model generation because the
 // selected backend's admission capacity is full; it does not match
-// ErrInvalidRequest or ErrLLMGenerate. Errors from injected clients remain
-// available through errors.Is. Cancellation while waiting for model-generation
-// capacity matches both ErrLLMGenerate and the context error. Cancellation
-// otherwise follows the active collaborator's documented behavior. A nil
-// Engine returns ErrInvalidConfig. Run returns no partial result on error.
+// ErrInvalidRequest or ErrLLMGenerate. A built-in OpenAI-compatible non-2xx
+// response is discoverable as [GenerationError]. Errors from injected clients
+// remain available through errors.Is. Cancellation while waiting for
+// model-generation capacity matches both ErrLLMGenerate and the context error.
+// Cancellation otherwise follows the active collaborator's documented
+// behavior. A nil Engine returns ErrInvalidConfig. Run returns no partial
+// result on error.
 func (e *Engine) Run(ctx context.Context, req RunRequest) (*RunResult, error) {
 	if e == nil || e.runner == nil {
 		return nil, fmt.Errorf("%w: engine is nil", ErrInvalidConfig)
@@ -688,9 +691,10 @@ func (e *Engine) Run(ctx context.Context, req RunRequest) (*RunResult, error) {
 // ErrCapacityExceeded, ErrLLMGenerate, or ErrValidation as applicable while
 // preserving documented collaborator and context identities. An engine
 // admission rejection is discoverable as [CapacityError] and still matches
-// ErrCapacityExceeded. A completed content-validation rejection is returned
-// in RunResult, not as an operational error. An operational error returns no
-// partial RunResult.
+// ErrCapacityExceeded. A built-in OpenAI-compatible non-2xx response is
+// discoverable as [GenerationError]. A completed content-validation rejection
+// is returned in RunResult, not as an operational error. An operational error
+// returns no partial RunResult.
 func (e *Engine) RunPrepared(ctx context.Context, prepared *PreparedExecution) (*RunResult, error) {
 	if e == nil || e.runner == nil {
 		return nil, fmt.Errorf("%w: engine is nil", ErrInvalidConfig)

engine_test.go

@@ -1164,8 +1164,9 @@ func TestArtifactReaderFailuresPreserveArtifactLoadErrors(t *testing.T) {
 }
 
 func TestRunAddsLLMGenerateToCollaboratorPublicError(t *testing.T) {
+	injectedErr := errors.New("injected model client failure")
 	engine := newContractEngineWithOptions(t, frameworkSchemaDir,
-		promptkit.WithLLMClient(&fakeLLMClient{err: promptkit.ErrArtifactLoad}),
+		promptkit.WithLLMClient(&fakeLLMClient{err: injectedErr}),
 	)
 
 	_, err := engine.Run(context.Background(), promptkit.RunRequest{
@@ -1178,8 +1179,12 @@ func TestRunAddsLLMGenerateToCollaboratorPublicError(t *testing.T) {
 	if !errors.Is(err, promptkit.ErrLLMGenerate) {
 		t.Fatalf("expected ErrLLMGenerate, got %v", err)
 	}
-	if !errors.Is(err, promptkit.ErrArtifactLoad) {
-		t.Fatalf("expected preserved ErrArtifactLoad, got %v", err)
+	if !errors.Is(err, injectedErr) {
+		t.Fatalf("expected preserved injected error, got %v", err)
+	}
+	var generationErr *promptkit.GenerationError
+	if errors.As(err, &generationErr) {
+		t.Fatalf("injected error became GenerationError: %v", err)
 	}
 }
 

errors.go

@@ -6,6 +6,7 @@ import (
 	"strings"
 
 	"gitea.maximumdirect.net/eric/promptkit/internal/capacity"
+	"gitea.maximumdirect.net/eric/promptkit/internal/llm"
 	"gitea.maximumdirect.net/eric/promptkit/internal/profile"
 	"gitea.maximumdirect.net/eric/promptkit/internal/promptdef"
 	"gitea.maximumdirect.net/eric/promptkit/internal/usecase"
@@ -20,6 +21,15 @@ func mapPublicError(err error) error {
 		strings.TrimSpace(internalCapacityError.BackendID) != "" {
 		return &CapacityError{BackendID: internalCapacityError.BackendID}
 	}
+	var providerHTTPError *llm.ProviderHTTPError
+	if errors.As(err, &providerHTTPError) && providerHTTPError != nil {
+		return newGenerationError(
+			providerHTTPError.StatusCode(),
+			providerHTTPError.ProviderCode(),
+			providerHTTPError.ProviderType(),
+			providerHTTPError.ProviderMessage(),
+		)
+	}
 	publicErr := publicErrorFor(err)
 	if publicErr == nil {
 		return err

generation_error.go

@@ -0,0 +1,87 @@
+package promptkit
+
+import "fmt"
+
+// GenerationError reports a non-2xx response from Promptkit's built-in
+// OpenAI-compatible client during [Engine.Run] or [Engine.RunPrepared].
+//
+// Engine-produced values are immutable, caller-owned values. Use errors.Is to
+// match [ErrLLMGenerate] and errors.As with a *GenerationError target to obtain
+// this type. The four provider accessors expose untrusted provider-controlled
+// values that can contain sensitive request or schema fragments. Applications
+// must apply their own disclosure policy before logging, displaying, or
+// returning them to another caller.
+//
+// Accessors, Error, GoString, and Unwrap are safe on a nil receiver and a zero
+// value. Default and Go-syntax formatting deliberately redact provider details.
+// GenerationError has no stable JSON representation.
+type GenerationError struct {
+	statusCode      int
+	providerCode    string
+	providerType    string
+	providerMessage string
+}
+
+func newGenerationError(statusCode int, providerCode, providerType, providerMessage string) *GenerationError {
+	return &GenerationError{
+		statusCode:      statusCode,
+		providerCode:    providerCode,
+		providerType:    providerType,
+		providerMessage: providerMessage,
+	}
+}
+
+// StatusCode returns the received provider HTTP status code, or zero for a nil
+// receiver or zero value.
+func (e *GenerationError) StatusCode() int {
+	if e == nil {
+		return 0
+	}
+	return e.statusCode
+}
+
+// ProviderCode returns the normalized provider error code, if present. Its
+// value is untrusted and may contain sensitive data.
+func (e *GenerationError) ProviderCode() string {
+	if e == nil {
+		return ""
+	}
+	return e.providerCode
+}
+
+// ProviderType returns the normalized provider error type, if present. Its
+// value is untrusted and may contain sensitive data.
+func (e *GenerationError) ProviderType() string {
+	if e == nil {
+		return ""
+	}
+	return e.providerType
+}
+
+// ProviderMessage returns the bounded normalized provider diagnostic, if
+// present. Its value is untrusted and may contain sensitive data.
+func (e *GenerationError) ProviderMessage() string {
+	if e == nil {
+		return ""
+	}
+	return e.providerMessage
+}
+
+// Error returns a redacted diagnostic that is not a parsing contract.
+func (e *GenerationError) Error() string {
+	if e == nil || e.statusCode == 0 {
+		return ErrLLMGenerate.Error()
+	}
+	return fmt.Sprintf("%s: provider returned HTTP status %d", ErrLLMGenerate, e.statusCode)
+}
+
+// GoString returns the same redacted diagnostic as Error.
+func (e *GenerationError) GoString() string {
+	return e.Error()
+}
+
+// Unwrap returns ErrLLMGenerate. It is safe to call on a nil receiver or zero
+// value.
+func (e *GenerationError) Unwrap() error {
+	return ErrLLMGenerate
+}

generation_error_contract_test.go

@@ -0,0 +1,95 @@
+package promptkit_test
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"strings"
+	"testing"
+
+	"gitea.maximumdirect.net/eric/promptkit"
+)
+
+func TestBuiltInGenerationError(t *testing.T) {
+	const (
+		codeMarker    = "provider-code-marker"
+		typeMarker    = "provider-type-marker"
+		messageMarker = "provider-message-marker"
+	)
+	engine := newBuiltInGenerationErrorEngine(t, http.StatusUnprocessableEntity,
+		`{"error":{"code":"`+codeMarker+`","type":"`+typeMarker+`","message":"`+messageMarker+`"}}`)
+
+	result, err := engine.Run(context.Background(), generationErrorRunRequest())
+	if result != nil {
+		t.Fatalf("Run result = %#v, want nil", result)
+	}
+	assertGenerationError(t, err, http.StatusUnprocessableEntity, codeMarker, typeMarker, messageMarker)
+
+	preparedEngine := newBuiltInGenerationErrorEngine(t, http.StatusServiceUnavailable, `{"error":{}}`)
+	prepared, err := preparedEngine.PrepareExecution(context.Background(), generationErrorRunRequest())
+	if err != nil {
+		t.Fatalf("PrepareExecution: %v", err)
+	}
+	result, err = preparedEngine.RunPrepared(context.Background(), prepared)
+	if result != nil {
+		t.Fatalf("RunPrepared result = %#v, want nil", result)
+	}
+	assertGenerationError(t, err, http.StatusServiceUnavailable, "", "", "")
+}
+
+func assertGenerationError(t *testing.T, err error, statusCode int, code, providerType, message string) {
+	t.Helper()
+
+	if !errors.Is(err, promptkit.ErrLLMGenerate) {
+		t.Fatalf("errors.Is(%v, ErrLLMGenerate) = false", err)
+	}
+	var generationErr *promptkit.GenerationError
+	if !errors.As(err, &generationErr) || generationErr == nil {
+		t.Fatalf("error = %T, want *GenerationError", err)
+	}
+	if generationErr.StatusCode() != statusCode || generationErr.ProviderCode() != code || generationErr.ProviderType() != providerType || generationErr.ProviderMessage() != message {
+		t.Fatalf("GenerationError = %#v", generationErr)
+	}
+
+	wantFormatted := fmt.Sprintf("failed to generate output: provider returned HTTP status %d", statusCode)
+	for _, rendered := range []string{fmt.Sprintf("%v", generationErr), fmt.Sprintf("%+v", generationErr), fmt.Sprintf("%#v", generationErr)} {
+		if rendered != wantFormatted {
+			t.Fatalf("formatted error = %q, want %q", rendered, wantFormatted)
+		}
+		for _, marker := range []string{code, providerType, message} {
+			if marker != "" && strings.Contains(rendered, marker) {
+				t.Fatalf("formatted error exposed provider marker %q: %q", marker, rendered)
+			}
+		}
+	}
+}
+
+func newBuiltInGenerationErrorEngine(t *testing.T, statusCode int, body string) *promptkit.Engine {
+	t.Helper()
+
+	config := contractConfig(frameworkSchemaDir)
+	config.HTTPClient = &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
+		return &http.Response{
+			StatusCode:    statusCode,
+			ContentLength: int64(len(body)),
+			Body:          io.NopCloser(strings.NewReader(body)),
+		}, nil
+	})}
+	engine, err := promptkit.NewEngine(config)
+	if err != nil {
+		t.Fatalf("NewEngine: %v", err)
+	}
+	return engine
+}
+
+func generationErrorRunRequest() promptkit.RunRequest {
+	return promptkit.RunRequest{
+		PromptID: frameworkMarkdownSummaryPromptID,
+		Inputs: map[string]promptkit.ArtifactRef{
+			"transcript": promptkit.Inline("Rin opens the gate."),
+			"glossary":   promptkit.Inline("gate: A guarded passage."),
+		},
+	}
+}

generation_error_internal_test.go

@@ -0,0 +1,32 @@
+package promptkit
+
+import (
+	"errors"
+	"fmt"
+	"testing"
+)
+
+func TestGenerationErrorNilAndZeroValue(t *testing.T) {
+	var nilError *GenerationError
+	zeroError := &GenerationError{}
+
+	for name, err := range map[string]*GenerationError{
+		"nil":  nilError,
+		"zero": zeroError,
+	} {
+		t.Run(name, func(t *testing.T) {
+			if err.StatusCode() != 0 || err.ProviderCode() != "" || err.ProviderType() != "" || err.ProviderMessage() != "" {
+				t.Fatalf("accessors returned provider details: %#v", err)
+			}
+			if err.Error() != "failed to generate output" || err.GoString() != "failed to generate output" {
+				t.Fatalf("redacted formatting = (%q, %q)", err.Error(), err.GoString())
+			}
+			if fmt.Sprintf("%v", err) != "failed to generate output" || fmt.Sprintf("%#v", err) != "failed to generate output" {
+				t.Fatalf("formatted error = (%q, %q)", fmt.Sprintf("%v", err), fmt.Sprintf("%#v", err))
+			}
+			if !errors.Is(err, ErrLLMGenerate) {
+				t.Fatalf("errors.Is(%v, ErrLLMGenerate) = false", err)
+			}
+		})
+	}
+}

types.go

@@ -665,10 +665,10 @@ type StructuredOutputJSONSpec struct {
 // and retained copies. It is responsible for the cancellation behavior of any
 // work it starts and for synchronizing access to retained or shared data.
 //
-// A returned error makes Run or RunPrepared return ErrLLMGenerate while
-// preserving the client error through errors.Is. A nil response with a nil
-// error also produces ErrLLMGenerate. Promptkit copies the non-nil response
-// before returning from either method.
+// An arbitrary returned error makes Run or RunPrepared return ErrLLMGenerate
+// while preserving the client error through errors.Is rather than translating
+// it. A nil response with a nil error also produces ErrLLMGenerate. Promptkit
+// copies the non-nil response before returning from either method.
 type LLMClient interface {
 	Generate(context.Context, GenerateRequest) (*GenerateResponse, error)
 }