eric/notarius
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Preserve LLM generation failure metadata

Browse Source
This commit is contained in:
Eric Rakestraw
2026-09-25 21:59:15 +00:00
parent 10ac29d149
commit bd83255a8e
7 changed files with 163 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

50
internal/framework/contracts/errors.go
View File

@@ -13,16 +13,42 @@ var ErrLLMCapacityExceeded = errors.New("LLM capacity exceeded")
// ErrLLMGeneration identifies a provider generation failure.
var ErrLLMGeneration = errors.New("LLM generation failed")
type LLMGenerationFailureKind string
const (
LLMGenerationFailureGeneration LLMGenerationFailureKind = "generation"
LLMGenerationFailureTransport LLMGenerationFailureKind = "transport"
LLMGenerationFailureProviderStatus LLMGenerationFailureKind = "provider_status"
LLMGenerationFailureMalformedResponse LLMGenerationFailureKind = "malformed_response"
)
type LLMGenerationMetadata struct {
FailureKind LLMGenerationFailureKind
RetriesExhausted bool
Attempts uint64
}
type LLMGenerationError struct {
status int
diagnostic string
status int
diagnostic string
failureKind LLMGenerationFailureKind
exhausted bool
attempts uint64
}
func NewLLMGenerationError(status int, diagnostic string) *LLMGenerationError {
return NewLLMGenerationErrorWithMetadata(status, diagnostic, LLMGenerationMetadata{})
}
func NewLLMGenerationErrorWithMetadata(status int, diagnostic string, metadata LLMGenerationMetadata) *LLMGenerationError {
if status < 0 {
status = 0
}
return &LLMGenerationError{status: status, diagnostic: diagnostic}
kind := metadata.FailureKind
if kind == "" {
kind = LLMGenerationFailureGeneration
}
return &LLMGenerationError{status: status, diagnostic: diagnostic, failureKind: kind, exhausted: metadata.RetriesExhausted, attempts: metadata.Attempts}
}
func (e *LLMGenerationError) Error() string {
if e == nil || e.diagnostic == "" {
@@ -37,3 +63,21 @@ func (e *LLMGenerationError) StatusCode() int {
}
return e.status
}
func (e *LLMGenerationError) FailureKind() LLMGenerationFailureKind {
if e == nil || e.failureKind == "" {
return LLMGenerationFailureGeneration
}
return e.failureKind
}
func (e *LLMGenerationError) RetriesExhausted() bool {
return e != nil && e.exhausted
}
func (e *LLMGenerationError) Attempts() uint64 {
if e == nil {
return 0
}
return e.attempts
}

22
internal/framework/contracts/errors_test.go Normal file
View File

@@ -0,0 +1,22 @@
package contracts
import (
"errors"
"testing"
)
func TestLLMGenerationErrorPreservesApplicationMetadata(t *testing.T) {
legacy := NewLLMGenerationError(503, "provider failure")
if !errors.Is(legacy, ErrLLMGeneration) || legacy.StatusCode() != 503 || legacy.FailureKind() != LLMGenerationFailureGeneration || legacy.RetriesExhausted() || legacy.Attempts() != 0 {
t.Fatalf("legacy generation error = %#v", legacy)
}
err := NewLLMGenerationErrorWithMetadata(429, "provider failure", LLMGenerationMetadata{
FailureKind: LLMGenerationFailureProviderStatus,
RetriesExhausted: true,
Attempts: 3,
})
if !errors.Is(err, ErrLLMGeneration) || err.StatusCode() != 429 || err.FailureKind() != LLMGenerationFailureProviderStatus || !err.RetriesExhausted() || err.Attempts() != 3 {
t.Fatalf("generation error = %#v", err)
}
}

22
internal/framework/llm/promptkit_client.go
View File

@@ -194,13 +194,19 @@ func (c *PromptKitClient) CompleteStructured(ctx context.Context, req contracts.
}
if errors.Is(err, promptkit.ErrLLMGenerate) {
var generationErr *promptkit.GenerationError
var exhaustedErr *promptkit.RetryExhaustedError
status := 0
metadata := contracts.LLMGenerationMetadata{FailureKind: promptKitGenerationFailureKind(err)}
if errors.As(err, &exhaustedErr) {
metadata.RetriesExhausted = true
metadata.Attempts = exhaustedErr.Attempts()
}
response := contracts.StructuredCompletionResponse{Debug: &contracts.LLMDebugMaterial{Prompt: promptKitDebugPrompt(&preparedDetails)}}
if errors.As(err, &generationErr) {
status = generationErr.StatusCode()
response.Debug.Response = promptKitDebugGenerationError(&preparedDetails, generationErr)
}
return response, contracts.NewLLMGenerationError(status, fmt.Sprintf("run PromptKit prompt %q: %v", promptID, redactPromptKitError(err)))
return response, contracts.NewLLMGenerationErrorWithMetadata(status, fmt.Sprintf("run PromptKit prompt %q: %v", promptID, redactPromptKitError(err)), metadata)
}
return contracts.StructuredCompletionResponse{}, fmt.Errorf("run PromptKit prompt %q: %v", promptID, redactPromptKitError(err))
}
@@ -220,6 +226,20 @@ func (c *PromptKitClient) CompleteStructured(ctx context.Context, req contracts.
return response, nil
}
func promptKitGenerationFailureKind(err error) contracts.LLMGenerationFailureKind {
switch {
case errors.Is(err, promptkit.ErrTransport):
return contracts.LLMGenerationFailureTransport
case errors.Is(err, promptkit.ErrMalformedResponse):
return contracts.LLMGenerationFailureMalformedResponse
}
var generationErr *promptkit.GenerationError
if errors.As(err, &generationErr) && generationErr.StatusCode() != 0 {
return contracts.LLMGenerationFailureProviderStatus
}
return contracts.LLMGenerationFailureGeneration
}
func promptKitCorrectionMessages(correction *contracts.SemanticCorrection) ([]promptkit.RenderedMessage, error) {
owned, err := contracts.CloneSemanticCorrection(correction)
if err != nil {

58
internal/framework/llm/promptkit_client_test.go
View File

@@ -1028,6 +1028,19 @@ func TestPromptKitClientAppliesConfiguredBuiltInTransientRetries(t *testing.T) {
if (err != nil) != test.wantErr {
t.Fatalf("CompleteStructured() error = %v, want error=%t", err, test.wantErr)
}
if test.wantErr {
var generation *contracts.LLMGenerationError
if !errors.As(err, &generation) || generation.FailureKind() != contracts.LLMGenerationFailureProviderStatus {
t.Fatalf("generation error = %#v, want provider-status classification", generation)
}
wantExhausted := test.name == "exhausted"
if generation.RetriesExhausted() != wantExhausted {
t.Fatalf("retries exhausted = %t, want %t", generation.RetriesExhausted(), wantExhausted)
}
if wantExhausted && (generation.Attempts() != 2 || generation.StatusCode() != http.StatusServiceUnavailable) {
t.Fatalf("exhausted generation = %#v", generation)
}
}
if got := calls.Load(); got != test.wantCalls {
t.Fatalf("HTTP calls = %d, want %d", got, test.wantCalls)
}
@@ -1373,6 +1386,51 @@ func TestPromptKitClientForwardsConfiguredTransportTimeout(t *testing.T) {
}
}
func TestPromptKitClientClassifiesBuiltInTransportAndMalformedResponses(t *testing.T) {
for _, test := range []struct {
name string
kind contracts.LLMGenerationFailureKind
roundTrip func(*http.Request) (*http.Response, error)
}{
{
name: "transport",
kind: contracts.LLMGenerationFailureTransport,
roundTrip: func(*http.Request) (*http.Response, error) {
return nil, errors.New("transport unavailable")
},
},
{
name: "malformed response",
kind: contracts.LLMGenerationFailureMalformedResponse,
roundTrip: func(request *http.Request) (*http.Response, error) {
return &http.Response{StatusCode: http.StatusOK, Header: make(http.Header), Body: io.NopCloser(strings.NewReader("not-json")), Request: request}, nil
},
},
} {
t.Run(test.name, func(t *testing.T) {
client, err := NewPromptKitClient(PromptKitClientConfig{
Assets: newTestPromptKitAssets(t),
HTTPClient: &http.Client{Transport: roundTripFunc(test.roundTrip)},
EngineOptions: []promptkit.Option{promptkit.WithProfiles(promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{
ID: "default-profile", Endpoint: "http://promptkit.test/v1", Model: "default-model",
}))},
})
if err != nil {
t.Fatal(err)
}
var out map[string]any
_, err = client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
PromptID: "adapter.test", SessionID: "failure-kind-test",
Inputs: contracts.LLMInputSet{"transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", "")},
}, &out)
var generation *contracts.LLMGenerationError
if !errors.As(err, &generation) || generation.FailureKind() != test.kind || generation.RetriesExhausted() || generation.Attempts() != 0 {
t.Fatalf("generation error = %#v, want kind %q", generation, test.kind)
}
})
}
}
func TestPromptKitClientClassifiesEmptyStructuredCompletion(t *testing.T) {
client := newTestPromptKitClient(t, &fakePromptKitLLM{allowEmpty: true})