eric/notarius
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Rewrite the debug path to provide raw LLM prompt and response artifacts

Browse Source
This commit is contained in:
Eric Rakestraw
2026-07-08 08:37:26 -05:00
parent 451f6c0bb9
commit 68ec69f2e4
9 changed files with 389 additions and 71 deletions
Download Patch File Download Diff File Expand all files Collapse all files

47
internal/framework/pipeline/runner_test.go
View File

@@ -1310,22 +1310,36 @@ func TestRunDebugFailedChunkAttemptReferencesScopedLLMOutput(t *testing.T) {
t.Fatalf("llm_calls = %#v, want one scoped call", attempt.LLMCalls)
}
call := attempt.LLMCalls[0]
if call.CallID != "call-0001" || call.CanonicalPath != "llm/call-0001.json" || call.ScopedPath != "chunk/attempt-01/llm-call-0001.json" {
t.Fatalf("llm call reference = %#v, want canonical and scoped paths", call)
if call.CallID != "0001" || call.PromptPath != "chunk/attempt-01/prompt-0001.json" || call.ResponsePath != "chunk/attempt-01/response-0001.json" {
t.Fatalf("llm call reference = %#v, want prompt and response paths", call)
}
if call.PromptID != "runner.chunk" || call.ProfileID != "debug-profile" || call.Error {
if call.PromptID != "runner.chunk" || call.ProfileID != "debug-profile" || call.Model != "debug-model" || call.Error {
t.Fatalf("llm call metadata = %#v, want prompt/profile and no call error", call)
}
scoped := recorder.envelope(t, call.ScopedPath)
scopedPayload, ok := scoped.Payload.(debugStructuredLLMCall)
prompt := recorder.envelope(t, call.PromptPath)
promptPayload, ok := prompt.Payload.(debugLLMPromptArtifact)
if !ok {
t.Fatalf("scoped payload type = %T, want debugStructuredLLMCall", scoped.Payload)
t.Fatalf("prompt payload type = %T, want debugLLMPromptArtifact", prompt.Payload)
}
if scopedPayload.Response.Content != `{"raw":true}` {
t.Fatalf("scoped response content = %q, want raw LLM response", scopedPayload.Response.Content)
if promptPayload.Prompt == nil || len(promptPayload.Prompt.Messages) != 1 || promptPayload.Prompt.Messages[0].Content != "raw prompt text" {
t.Fatalf("prompt payload = %#v, want raw prompt message", promptPayload)
}
response := recorder.envelope(t, call.ResponsePath)
responsePayload, ok := response.Payload.(debugLLMResponseArtifact)
if !ok {
t.Fatalf("response payload type = %T, want debugLLMResponseArtifact", response.Payload)
}
if responsePayload.Response == nil || responsePayload.Response.Content != `{"raw":true}` {
t.Fatalf("response payload = %#v, want raw LLM response", responsePayload)
}
if _, ok := recorder.payloads["llm/call-0001.json"]; ok {
t.Fatalf("old canonical LLM debug artifact was written")
}
if _, ok := recorder.payloads["chunk/attempt-01/llm-call-0001.json"]; ok {
t.Fatalf("old scoped LLM debug artifact was written")
}
_ = recorder.envelope(t, call.CanonicalPath)
}
func TestRunStopsRetryAfterConfiguredAttemptsAndRecordsAttemptCount(t *testing.T) {
@@ -2332,7 +2346,22 @@ func (client debugResponseLLMClient) CompleteStructured(ctx context.Context, req
}
return contracts.StructuredCompletionResponse{
Content: append([]byte(nil), client.content...),
Model: "debug-model",
ProfileID: profileID,
Debug: &contracts.LLMDebugMaterial{
Prompt: &contracts.LLMDebugPrompt{
PromptID: req.PromptID,
SelectedProfileID: profileID,
Messages: []contracts.LLMDebugMessage{
{Role: "user", Content: "raw prompt text"},
},
},
Response: &contracts.LLMDebugResponse{
Content: string(client.content),
SelectedProfileID: profileID,
ModelName: "debug-model",
},
},
}, client.err
}