eric/notarius
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Preserve LLM generation failure metadata

Browse Source
This commit is contained in:
Eric Rakestraw
2026-09-25 21:59:15 +00:00
parent 10ac29d149
commit bd83255a8e
7 changed files with 163 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
internal/framework/llm/promptkit_client.go
View File

@@ -194,13 +194,19 @@ func (c *PromptKitClient) CompleteStructured(ctx context.Context, req contracts.
}
if errors.Is(err, promptkit.ErrLLMGenerate) {
var generationErr *promptkit.GenerationError
var exhaustedErr *promptkit.RetryExhaustedError
status := 0
metadata := contracts.LLMGenerationMetadata{FailureKind: promptKitGenerationFailureKind(err)}
if errors.As(err, &exhaustedErr) {
metadata.RetriesExhausted = true
metadata.Attempts = exhaustedErr.Attempts()
}
response := contracts.StructuredCompletionResponse{Debug: &contracts.LLMDebugMaterial{Prompt: promptKitDebugPrompt(&preparedDetails)}}
if errors.As(err, &generationErr) {
status = generationErr.StatusCode()
response.Debug.Response = promptKitDebugGenerationError(&preparedDetails, generationErr)
}
return response, contracts.NewLLMGenerationError(status, fmt.Sprintf("run PromptKit prompt %q: %v", promptID, redactPromptKitError(err)))
return response, contracts.NewLLMGenerationErrorWithMetadata(status, fmt.Sprintf("run PromptKit prompt %q: %v", promptID, redactPromptKitError(err)), metadata)
}
return contracts.StructuredCompletionResponse{}, fmt.Errorf("run PromptKit prompt %q: %v", promptID, redactPromptKitError(err))
}
@@ -220,6 +226,20 @@ func (c *PromptKitClient) CompleteStructured(ctx context.Context, req contracts.
return response, nil
}
func promptKitGenerationFailureKind(err error) contracts.LLMGenerationFailureKind {
switch {
case errors.Is(err, promptkit.ErrTransport):
return contracts.LLMGenerationFailureTransport
case errors.Is(err, promptkit.ErrMalformedResponse):
return contracts.LLMGenerationFailureMalformedResponse
}
var generationErr *promptkit.GenerationError
if errors.As(err, &generationErr) && generationErr.StatusCode() != 0 {
return contracts.LLMGenerationFailureProviderStatus
}
return contracts.LLMGenerationFailureGeneration
}
func promptKitCorrectionMessages(correction *contracts.SemanticCorrection) ([]promptkit.RenderedMessage, error) {
owned, err := contracts.CloneSemanticCorrection(correction)
if err != nil {

58
internal/framework/llm/promptkit_client_test.go
View File

@@ -1028,6 +1028,19 @@ func TestPromptKitClientAppliesConfiguredBuiltInTransientRetries(t *testing.T) {
if (err != nil) != test.wantErr {
t.Fatalf("CompleteStructured() error = %v, want error=%t", err, test.wantErr)
}
if test.wantErr {
var generation *contracts.LLMGenerationError
if !errors.As(err, &generation) || generation.FailureKind() != contracts.LLMGenerationFailureProviderStatus {
t.Fatalf("generation error = %#v, want provider-status classification", generation)
}
wantExhausted := test.name == "exhausted"
if generation.RetriesExhausted() != wantExhausted {
t.Fatalf("retries exhausted = %t, want %t", generation.RetriesExhausted(), wantExhausted)
}
if wantExhausted && (generation.Attempts() != 2 || generation.StatusCode() != http.StatusServiceUnavailable) {
t.Fatalf("exhausted generation = %#v", generation)
}
}
if got := calls.Load(); got != test.wantCalls {
t.Fatalf("HTTP calls = %d, want %d", got, test.wantCalls)
}
@@ -1373,6 +1386,51 @@ func TestPromptKitClientForwardsConfiguredTransportTimeout(t *testing.T) {
}
}
func TestPromptKitClientClassifiesBuiltInTransportAndMalformedResponses(t *testing.T) {
for _, test := range []struct {
name string
kind contracts.LLMGenerationFailureKind
roundTrip func(*http.Request) (*http.Response, error)
}{
{
name: "transport",
kind: contracts.LLMGenerationFailureTransport,
roundTrip: func(*http.Request) (*http.Response, error) {
return nil, errors.New("transport unavailable")
},
},
{
name: "malformed response",
kind: contracts.LLMGenerationFailureMalformedResponse,
roundTrip: func(request *http.Request) (*http.Response, error) {
return &http.Response{StatusCode: http.StatusOK, Header: make(http.Header), Body: io.NopCloser(strings.NewReader("not-json")), Request: request}, nil
},
},
} {
t.Run(test.name, func(t *testing.T) {
client, err := NewPromptKitClient(PromptKitClientConfig{
Assets: newTestPromptKitAssets(t),
HTTPClient: &http.Client{Transport: roundTripFunc(test.roundTrip)},
EngineOptions: []promptkit.Option{promptkit.WithProfiles(promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{
ID: "default-profile", Endpoint: "http://promptkit.test/v1", Model: "default-model",
}))},
})
if err != nil {
t.Fatal(err)
}
var out map[string]any
_, err = client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
PromptID: "adapter.test", SessionID: "failure-kind-test",
Inputs: contracts.LLMInputSet{"transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", "")},
}, &out)
var generation *contracts.LLMGenerationError
if !errors.As(err, &generation) || generation.FailureKind() != test.kind || generation.RetriesExhausted() || generation.Attempts() != 0 {
t.Fatalf("generation error = %#v, want kind %q", generation, test.kind)
}
})
}
}
func TestPromptKitClientClassifiesEmptyStructuredCompletion(t *testing.T) {
client := newTestPromptKitClient(t, &fakePromptKitLLM{allowEmpty: true})