eric/notarius
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Classify PromptKit generation errors safely

Browse Source
This commit is contained in:
Eric Rakestraw
2026-08-25 19:34:27 +00:00
parent 75a3f51cee
commit 3bd3c7ebf7
7 changed files with 131 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files

46
internal/framework/llm/promptkit_client_test.go
View File

@@ -629,6 +629,44 @@ func TestPromptKitClientAllowsMissingOptionalFilesystemCredential(t *testing.T)
}
}
func TestPromptKitClientMapsHTTPGenerationErrorToApplicationBoundary(t *testing.T) {
const credential = "selected-test-credential"
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusServiceUnavailable)
_, _ = w.Write([]byte(`{"error":{"code":"temporary","type":"provider_error","message":"marker ` + credential + ` Bearer bearer-secret"}}`))
}))
defer server.Close()
t.Setenv("NOTARIUS_GENERATION_TEST_KEY", credential)
profilePath := filepath.Join(t.TempDir(), "profile.yaml")
if err := os.WriteFile(profilePath, []byte("id: generation-profile\nendpoint: "+server.URL+"/v1\nmodel: test\napi_key_env: NOTARIUS_GENERATION_TEST_KEY\n"), 0o600); err != nil {
t.Fatal(err)
}
client, err := NewPromptKitClient(PromptKitClientConfig{Assets: newTestPromptKitAssets(t), ProfileFile: profilePath})
if err != nil {
t.Fatal(err)
}
var out map[string]any
response, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{PromptID: "adapter.test", ProfileID: "generation-profile", SessionID: "generation-error-test", Inputs: contracts.LLMInputSet{"transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", "")}}, &out)
if !errors.Is(err, contracts.ErrLLMGeneration) {
t.Fatalf("error = %v", err)
}
var generation *contracts.LLMGenerationError
if !errors.As(err, &generation) || generation.StatusCode() != http.StatusServiceUnavailable {
t.Fatalf("generation error = %#v", generation)
}
if strings.Contains(err.Error(), credential) || strings.Contains(err.Error(), "marker") {
t.Fatalf("ordinary error leaked provider detail: %v", err)
}
if response.Debug == nil || response.Debug.Response == nil || response.Debug.Response.ProviderError == nil {
t.Fatalf("debug = %#v", response.Debug)
}
debug := response.Debug.Response.ProviderError
if debug.StatusCode != http.StatusServiceUnavailable || debug.Code != "temporary" || strings.Contains(debug.Message, credential) || strings.Contains(debug.Message, "bearer-secret") || !strings.Contains(debug.Message, "marker") {
t.Fatalf("debug provider error = %#v", debug)
}
}
func TestPromptKitClientCheckpointFingerprintTracksFallbackProfileAssets(t *testing.T) {
fingerprintFor := func(content string) CheckpointFingerprint {
t.Helper()
@@ -945,8 +983,8 @@ func TestPromptKitClientProviderFailureIncludesContextAndRedactsBearerToken(t *t
t.Fatalf("error chain exposes credential: %v", err)
}
}
if errors.Unwrap(err) != nil {
t.Fatalf("provider failure must not expose a wrapped diagnostic: %v", err)
if !errors.Is(err, contracts.ErrLLMGeneration) {
t.Fatalf("provider failure = %v, want application generation classification", err)
}
var recoveredProviderErr *credentialBearingProviderError
if errors.As(err, &recoveredProviderErr) {
@@ -955,8 +993,8 @@ func TestPromptKitClientProviderFailureIncludesContextAndRedactsBearerToken(t *t
if errors.Is(err, promptkit.ErrLLMGenerate) {
t.Fatalf("provider failure exposes PromptKit generation sentinel: %v", err)
}
if resp.Debug != nil {
t.Fatalf("debug material = %#v, want none for provider failure without result", resp.Debug)
if resp.Debug == nil || resp.Debug.Prompt == nil || resp.Debug.Response != nil {
t.Fatalf("debug material = %#v, want prepared prompt without provider details", resp.Debug)
}
}