eric/notarius
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Rewrite the debug path to provide raw LLM prompt and response artifacts

Browse Source
This commit is contained in:
Eric Rakestraw
2026-07-08 08:37:26 -05:00
parent 451f6c0bb9
commit 68ec69f2e4
9 changed files with 389 additions and 71 deletions
Download Patch File Download Diff File Expand all files Collapse all files

33
internal/framework/llm/scriptorium_client_test.go
View File

@@ -45,6 +45,25 @@ func TestScriptoriumClientMapsPromptRequestAndUnmarshalsOutput(t *testing.T) {
if resp.PromptTokens != 11 || resp.CompletionTokens != 7 || resp.TotalTokens != 18 {
t.Fatalf("usage = %#v, want mapped token counts", resp)
}
if resp.Debug == nil || resp.Debug.Prompt == nil {
t.Fatalf("debug prompt = nil, want prepared prompt material")
}
if resp.Debug.Prompt.PromptID != "adapter.test" || resp.Debug.Prompt.SelectedProfileID != "explicit-profile" {
t.Fatalf("debug prompt metadata = %#v, want prompt/profile", resp.Debug.Prompt)
}
if len(resp.Debug.Prompt.Messages) != 1 || !strings.Contains(resp.Debug.Prompt.Messages[0].Content, `{"source":true}`) {
t.Fatalf("debug prompt messages = %#v, want rendered input content", resp.Debug.Prompt.Messages)
}
if resp.Debug.Response == nil || resp.Debug.Response.Content != `{"ok":true}` {
t.Fatalf("debug response = %#v, want raw response content", resp.Debug.Response)
}
debugJSON, err := json.Marshal(resp.Debug)
if err != nil {
t.Fatalf("marshal debug material: %v", err)
}
if strings.Contains(string(debugJSON), "secret-token") || strings.Contains(string(debugJSON), "sk-") {
t.Fatalf("debug material contains secret material: %s", debugJSON)
}
gotReq := fake.lastRequest()
if gotReq.Prompt.SessionID != "session-123" {
t.Fatalf("session id = %q, want session-123", gotReq.Prompt.SessionID)
@@ -100,6 +119,12 @@ func TestScriptoriumClientValidationFailureReturnsError(t *testing.T) {
if got := string(resp.Content); got != `{"bad":true}` {
t.Fatalf("response content = %q, want raw failed output", got)
}
if resp.Debug == nil || resp.Debug.Response == nil || resp.Debug.Response.Content != `{"bad":true}` {
t.Fatalf("debug response = %#v, want raw failed output", resp.Debug)
}
if resp.Debug.Prompt == nil || len(resp.Debug.Prompt.Messages) == 0 {
t.Fatalf("debug prompt = %#v, want prepared prompt material", resp.Debug.Prompt)
}
}
func TestScriptoriumClientDecodeFailureReturnsRawResponse(t *testing.T) {
@@ -119,13 +144,16 @@ func TestScriptoriumClientDecodeFailureReturnsRawResponse(t *testing.T) {
if got := string(resp.Content); got != `{"ok":true}` {
t.Fatalf("response content = %q, want raw decode-failed output", got)
}
if resp.Debug == nil || resp.Debug.Response == nil || resp.Debug.Response.Content != `{"ok":true}` {
t.Fatalf("debug response = %#v, want raw decode-failed output", resp.Debug)
}
}
func TestScriptoriumClientProviderFailureIncludesContextAndRedactsBearerToken(t *testing.T) {
client := newTestScriptoriumClient(t, &fakeScriptoriumLLM{err: errors.New("provider failed with Bearer secret-token")})
var out map[string]any
_, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
resp, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
PromptID: "adapter.test",
SessionID: "session-123",
Inputs: contracts.LLMInputSet{
@@ -141,6 +169,9 @@ func TestScriptoriumClientProviderFailureIncludesContextAndRedactsBearerToken(t
if strings.Contains(err.Error(), "secret-token") || !strings.Contains(err.Error(), "Bearer [REDACTED]") {
t.Fatalf("error = %q, want redacted bearer token", err.Error())
}
if resp.Debug != nil {
t.Fatalf("debug material = %#v, want none for provider failure without result", resp.Debug)
}
}
func TestScriptoriumClientContextCancellationIsRespected(t *testing.T) {