eric/notarius
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Rewrite the debug path to provide raw LLM prompt and response artifacts

Browse Source
This commit is contained in:
Eric Rakestraw
2026-07-08 08:37:26 -05:00
parent 451f6c0bb9
commit 68ec69f2e4
9 changed files with 389 additions and 71 deletions
Download Patch File Download Diff File Expand all files Collapse all files

55
internal/cli/run_test.go
View File

@@ -2244,8 +2244,9 @@ func TestRunPipelineWritesDebugWhenWorkspaceDebugEnabled(t *testing.T) {
"chunk/input.json",
"chunk/output.json",
"extract/spells/input.json",
"extract/spells/chunk-000001-attempt-01.json",
"extract/spells/chunk-000001-attempt-01/llm-call-0001.json",
"extract/spells/chunk-000001/attempt-01.json",
"extract/spells/chunk-000001/attempt-01/prompt-0001.json",
"extract/spells/chunk-000001/attempt-01/response-0001.json",
"extract/spells/output.json",
"merge/spells/input.json",
"merge/spells/output.json",
@@ -2253,15 +2254,23 @@ func TestRunPipelineWritesDebugWhenWorkspaceDebugEnabled(t *testing.T) {
"normalize/spells/output.json",
"output/input.json",
"output/output.json",
"llm/call-0001.json",
} {
if _, err := os.Stat(filepath.Join(debugDir, name)); err != nil {
t.Fatalf("expected debug artifact %q: %v", name, err)
}
}
attemptDebug := string(readFile(t, filepath.Join(debugDir, "extract/spells/chunk-000001-attempt-01.json")))
if !strings.Contains(attemptDebug, `"llm_calls"`) || !strings.Contains(attemptDebug, `"scoped_path"`) {
t.Fatalf("extract attempt debug = %s, want scoped llm_calls", attemptDebug)
attemptDebug := string(readFile(t, filepath.Join(debugDir, "extract/spells/chunk-000001/attempt-01.json")))
if !strings.Contains(attemptDebug, `"llm_calls"`) || !strings.Contains(attemptDebug, `"prompt_path"`) || !strings.Contains(attemptDebug, `"response_path"`) {
t.Fatalf("extract attempt debug = %s, want prompt/response llm_calls", attemptDebug)
}
responseDebug := string(readFile(t, filepath.Join(debugDir, "extract/spells/chunk-000001/attempt-01/response-0001.json")))
if !strings.Contains(responseDebug, `"content"`) || !strings.Contains(responseDebug, `spell_casts`) {
t.Fatalf("response debug = %s, want raw response content", responseDebug)
}
assertPathNotExist(t, filepath.Join(debugDir, "llm/call-0001.json"))
assertPathNotExist(t, filepath.Join(debugDir, "extract/spells/chunk-000001-attempt-01/llm-call-0001.json"))
if _, err := os.Stat(filepath.Join(debugDir, "extract/spells/chunk-000001-attempt-01.json")); !os.IsNotExist(err) {
t.Fatalf("old extract attempt path still exists: %v", err)
}
assertPathNotExist(t, filepath.Join(workspaceDir, "checkpoints"))
}
@@ -3367,7 +3376,7 @@ func (client *fakeRunLLMClient) CompleteStructured(ctx context.Context, req cont
if err := json.Unmarshal(encoded, out); err != nil {
return contracts.StructuredCompletionResponse{}, err
}
return contracts.StructuredCompletionResponse{Content: encoded}, nil
return fakeRunStructuredResponse(req, encoded), nil
}
startUnitID := 1
if client.invalidSourceRef {
@@ -3400,7 +3409,37 @@ func (client *fakeRunLLMClient) CompleteStructured(ctx context.Context, req cont
if err := json.Unmarshal(encoded, out); err != nil {
return contracts.StructuredCompletionResponse{}, err
}
return contracts.StructuredCompletionResponse{Content: encoded}, nil
return fakeRunStructuredResponse(req, encoded), nil
}
func fakeRunStructuredResponse(req contracts.StructuredCompletionRequest, content []byte) contracts.StructuredCompletionResponse {
profileID := req.ProfileID
if profileID == "" {
profileID = "fake-profile"
}
return contracts.StructuredCompletionResponse{
Content: content,
Model: "fake-model",
ProfileID: profileID,
Debug: &contracts.LLMDebugMaterial{
Prompt: &contracts.LLMDebugPrompt{
PromptID: req.PromptID,
PromptVersion: req.PromptVersion,
SelectedProfileID: profileID,
SessionID: req.SessionID,
Messages: []contracts.LLMDebugMessage{
{Role: "user", Content: "fake rendered prompt for " + req.PromptID},
},
},
Response: &contracts.LLMDebugResponse{
Content: string(content),
PromptID: req.PromptID,
PromptVersion: req.PromptVersion,
SelectedProfileID: profileID,
ModelName: "fake-model",
},
},
}
}
func fakeLLMFactory(client contracts.StructuredLLMClient, err error) LLMClientFactory {