diff --git a/docs/integrations/pkg-promptkit.md b/docs/integrations/pkg-promptkit.md index d3fe5ae3..90751f92 100644 --- a/docs/integrations/pkg-promptkit.md +++ b/docs/integrations/pkg-promptkit.md @@ -75,6 +75,12 @@ error type. PromptKit's built-in client retries eligible transient HTTP failures with its v0.11.0 default of two additional attempts per generation; Notarius keeps pipeline-stage retries outside that completion boundary. +The adapter projects built-in generation failures as the application-owned +`LLMGenerationError`, retaining a safe failure kind, HTTP status when present, +and retry-exhaustion attempt count. Transport, malformed successful envelopes, +provider statuses, and generic generation failures remain distinct without +exposing PromptKit error values to Notarius callers. + ## Profile Sources And Compatibility Notarius gives PromptKit the configured operator profile source, registered diff --git a/docs/internal/llm.md b/docs/internal/llm.md index 7473898e..a3318ce6 100644 --- a/docs/internal/llm.md +++ b/docs/internal/llm.md @@ -216,6 +216,13 @@ than output-validation failures. Apart from documented context, capacity, and invalid-output categories, provider error values and types do not cross the adapter error chain; callers receive only a credential-redacted diagnostic. +`LLMGenerationError` preserves the provider-neutral failure kind (`transport`, +`provider_status`, `malformed_response`, or generic `generation`) and any HTTP +status. When PromptKit exhausts an eligible transient-status retry budget, it +also records that exhaustion and the attempts in that generation. Those fields +do not include repair generations, and zero attempts means unavailable rather +than proof that no request was sent. + When PromptKit rejects backend admission before generation, the adapter maps `promptkit.ErrCapacityExceeded` to `contracts.ErrLLMCapacityExceeded`, retaining prompt context and a redacted diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index 5132a570..b5be18c2 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -174,6 +174,8 @@ and `git diff --check`. ## Stage 4 — Preserve Typed Failure And Exhaustion Metadata +**Status:** Completed 2026-09-25. + **Goal:** Distinguish provider failures without coupling callers to PromptKit. One prompt; defer execution observations to the next stage. diff --git a/internal/framework/contracts/errors.go b/internal/framework/contracts/errors.go index a9c1d421..166b7505 100644 --- a/internal/framework/contracts/errors.go +++ b/internal/framework/contracts/errors.go @@ -13,16 +13,42 @@ var ErrLLMCapacityExceeded = errors.New("LLM capacity exceeded") // ErrLLMGeneration identifies a provider generation failure. var ErrLLMGeneration = errors.New("LLM generation failed") +type LLMGenerationFailureKind string + +const ( + LLMGenerationFailureGeneration LLMGenerationFailureKind = "generation" + LLMGenerationFailureTransport LLMGenerationFailureKind = "transport" + LLMGenerationFailureProviderStatus LLMGenerationFailureKind = "provider_status" + LLMGenerationFailureMalformedResponse LLMGenerationFailureKind = "malformed_response" +) + +type LLMGenerationMetadata struct { + FailureKind LLMGenerationFailureKind + RetriesExhausted bool + Attempts uint64 +} + type LLMGenerationError struct { - status int - diagnostic string + status int + diagnostic string + failureKind LLMGenerationFailureKind + exhausted bool + attempts uint64 } func NewLLMGenerationError(status int, diagnostic string) *LLMGenerationError { + return NewLLMGenerationErrorWithMetadata(status, diagnostic, LLMGenerationMetadata{}) +} + +func NewLLMGenerationErrorWithMetadata(status int, diagnostic string, metadata LLMGenerationMetadata) *LLMGenerationError { if status < 0 { status = 0 } - return &LLMGenerationError{status: status, diagnostic: diagnostic} + kind := metadata.FailureKind + if kind == "" { + kind = LLMGenerationFailureGeneration + } + return &LLMGenerationError{status: status, diagnostic: diagnostic, failureKind: kind, exhausted: metadata.RetriesExhausted, attempts: metadata.Attempts} } func (e *LLMGenerationError) Error() string { if e == nil || e.diagnostic == "" { @@ -37,3 +63,21 @@ func (e *LLMGenerationError) StatusCode() int { } return e.status } + +func (e *LLMGenerationError) FailureKind() LLMGenerationFailureKind { + if e == nil || e.failureKind == "" { + return LLMGenerationFailureGeneration + } + return e.failureKind +} + +func (e *LLMGenerationError) RetriesExhausted() bool { + return e != nil && e.exhausted +} + +func (e *LLMGenerationError) Attempts() uint64 { + if e == nil { + return 0 + } + return e.attempts +} diff --git a/internal/framework/contracts/errors_test.go b/internal/framework/contracts/errors_test.go new file mode 100644 index 00000000..24b656db --- /dev/null +++ b/internal/framework/contracts/errors_test.go @@ -0,0 +1,22 @@ +package contracts + +import ( + "errors" + "testing" +) + +func TestLLMGenerationErrorPreservesApplicationMetadata(t *testing.T) { + legacy := NewLLMGenerationError(503, "provider failure") + if !errors.Is(legacy, ErrLLMGeneration) || legacy.StatusCode() != 503 || legacy.FailureKind() != LLMGenerationFailureGeneration || legacy.RetriesExhausted() || legacy.Attempts() != 0 { + t.Fatalf("legacy generation error = %#v", legacy) + } + + err := NewLLMGenerationErrorWithMetadata(429, "provider failure", LLMGenerationMetadata{ + FailureKind: LLMGenerationFailureProviderStatus, + RetriesExhausted: true, + Attempts: 3, + }) + if !errors.Is(err, ErrLLMGeneration) || err.StatusCode() != 429 || err.FailureKind() != LLMGenerationFailureProviderStatus || !err.RetriesExhausted() || err.Attempts() != 3 { + t.Fatalf("generation error = %#v", err) + } +} diff --git a/internal/framework/llm/promptkit_client.go b/internal/framework/llm/promptkit_client.go index d91aab18..3c5e39bd 100644 --- a/internal/framework/llm/promptkit_client.go +++ b/internal/framework/llm/promptkit_client.go @@ -194,13 +194,19 @@ func (c *PromptKitClient) CompleteStructured(ctx context.Context, req contracts. } if errors.Is(err, promptkit.ErrLLMGenerate) { var generationErr *promptkit.GenerationError + var exhaustedErr *promptkit.RetryExhaustedError status := 0 + metadata := contracts.LLMGenerationMetadata{FailureKind: promptKitGenerationFailureKind(err)} + if errors.As(err, &exhaustedErr) { + metadata.RetriesExhausted = true + metadata.Attempts = exhaustedErr.Attempts() + } response := contracts.StructuredCompletionResponse{Debug: &contracts.LLMDebugMaterial{Prompt: promptKitDebugPrompt(&preparedDetails)}} if errors.As(err, &generationErr) { status = generationErr.StatusCode() response.Debug.Response = promptKitDebugGenerationError(&preparedDetails, generationErr) } - return response, contracts.NewLLMGenerationError(status, fmt.Sprintf("run PromptKit prompt %q: %v", promptID, redactPromptKitError(err))) + return response, contracts.NewLLMGenerationErrorWithMetadata(status, fmt.Sprintf("run PromptKit prompt %q: %v", promptID, redactPromptKitError(err)), metadata) } return contracts.StructuredCompletionResponse{}, fmt.Errorf("run PromptKit prompt %q: %v", promptID, redactPromptKitError(err)) } @@ -220,6 +226,20 @@ func (c *PromptKitClient) CompleteStructured(ctx context.Context, req contracts. return response, nil } +func promptKitGenerationFailureKind(err error) contracts.LLMGenerationFailureKind { + switch { + case errors.Is(err, promptkit.ErrTransport): + return contracts.LLMGenerationFailureTransport + case errors.Is(err, promptkit.ErrMalformedResponse): + return contracts.LLMGenerationFailureMalformedResponse + } + var generationErr *promptkit.GenerationError + if errors.As(err, &generationErr) && generationErr.StatusCode() != 0 { + return contracts.LLMGenerationFailureProviderStatus + } + return contracts.LLMGenerationFailureGeneration +} + func promptKitCorrectionMessages(correction *contracts.SemanticCorrection) ([]promptkit.RenderedMessage, error) { owned, err := contracts.CloneSemanticCorrection(correction) if err != nil { diff --git a/internal/framework/llm/promptkit_client_test.go b/internal/framework/llm/promptkit_client_test.go index 48deb48a..2cb7f5a5 100644 --- a/internal/framework/llm/promptkit_client_test.go +++ b/internal/framework/llm/promptkit_client_test.go @@ -1028,6 +1028,19 @@ func TestPromptKitClientAppliesConfiguredBuiltInTransientRetries(t *testing.T) { if (err != nil) != test.wantErr { t.Fatalf("CompleteStructured() error = %v, want error=%t", err, test.wantErr) } + if test.wantErr { + var generation *contracts.LLMGenerationError + if !errors.As(err, &generation) || generation.FailureKind() != contracts.LLMGenerationFailureProviderStatus { + t.Fatalf("generation error = %#v, want provider-status classification", generation) + } + wantExhausted := test.name == "exhausted" + if generation.RetriesExhausted() != wantExhausted { + t.Fatalf("retries exhausted = %t, want %t", generation.RetriesExhausted(), wantExhausted) + } + if wantExhausted && (generation.Attempts() != 2 || generation.StatusCode() != http.StatusServiceUnavailable) { + t.Fatalf("exhausted generation = %#v", generation) + } + } if got := calls.Load(); got != test.wantCalls { t.Fatalf("HTTP calls = %d, want %d", got, test.wantCalls) } @@ -1373,6 +1386,51 @@ func TestPromptKitClientForwardsConfiguredTransportTimeout(t *testing.T) { } } +func TestPromptKitClientClassifiesBuiltInTransportAndMalformedResponses(t *testing.T) { + for _, test := range []struct { + name string + kind contracts.LLMGenerationFailureKind + roundTrip func(*http.Request) (*http.Response, error) + }{ + { + name: "transport", + kind: contracts.LLMGenerationFailureTransport, + roundTrip: func(*http.Request) (*http.Response, error) { + return nil, errors.New("transport unavailable") + }, + }, + { + name: "malformed response", + kind: contracts.LLMGenerationFailureMalformedResponse, + roundTrip: func(request *http.Request) (*http.Response, error) { + return &http.Response{StatusCode: http.StatusOK, Header: make(http.Header), Body: io.NopCloser(strings.NewReader("not-json")), Request: request}, nil + }, + }, + } { + t.Run(test.name, func(t *testing.T) { + client, err := NewPromptKitClient(PromptKitClientConfig{ + Assets: newTestPromptKitAssets(t), + HTTPClient: &http.Client{Transport: roundTripFunc(test.roundTrip)}, + EngineOptions: []promptkit.Option{promptkit.WithProfiles(promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{ + ID: "default-profile", Endpoint: "http://promptkit.test/v1", Model: "default-model", + }))}, + }) + if err != nil { + t.Fatal(err) + } + var out map[string]any + _, err = client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{ + PromptID: "adapter.test", SessionID: "failure-kind-test", + Inputs: contracts.LLMInputSet{"transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", "")}, + }, &out) + var generation *contracts.LLMGenerationError + if !errors.As(err, &generation) || generation.FailureKind() != test.kind || generation.RetriesExhausted() || generation.Attempts() != 0 { + t.Fatalf("generation error = %#v, want kind %q", generation, test.kind) + } + }) + } +} + func TestPromptKitClientClassifiesEmptyStructuredCompletion(t *testing.T) { client := newTestPromptKitClient(t, &fakePromptKitLLM{allowEmpty: true})