eric/notarius
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Enhance debug output to include response content files and update related metadata handling

Browse Source
This commit is contained in:
Eric Rakestraw
2026-07-08 08:52:19 -05:00
parent 68ec69f2e4
commit 610bdb4fea
6 changed files with 178 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
internal/cli/run_test.go
View File

@@ -2247,6 +2247,7 @@ func TestRunPipelineWritesDebugWhenWorkspaceDebugEnabled(t *testing.T) {
"extract/spells/chunk-000001/attempt-01.json",
"extract/spells/chunk-000001/attempt-01/prompt-0001.json",
"extract/spells/chunk-000001/attempt-01/response-0001.json",
"extract/spells/chunk-000001/attempt-01/response-content-0001.json",
"extract/spells/output.json",
"merge/spells/input.json",
"merge/spells/output.json",
@@ -2260,12 +2261,16 @@ func TestRunPipelineWritesDebugWhenWorkspaceDebugEnabled(t *testing.T) {
}
}
attemptDebug := string(readFile(t, filepath.Join(debugDir, "extract/spells/chunk-000001/attempt-01.json")))
if !strings.Contains(attemptDebug, `"llm_calls"`) || !strings.Contains(attemptDebug, `"prompt_path"`) || !strings.Contains(attemptDebug, `"response_path"`) {
if !strings.Contains(attemptDebug, `"llm_calls"`) || !strings.Contains(attemptDebug, `"prompt_path"`) || !strings.Contains(attemptDebug, `"response_path"`) || !strings.Contains(attemptDebug, `"response_content_path"`) {
t.Fatalf("extract attempt debug = %s, want prompt/response llm_calls", attemptDebug)
}
responseDebug := string(readFile(t, filepath.Join(debugDir, "extract/spells/chunk-000001/attempt-01/response-0001.json")))
if !strings.Contains(responseDebug, `"content"`) || !strings.Contains(responseDebug, `spell_casts`) {
t.Fatalf("response debug = %s, want raw response content", responseDebug)
if !strings.Contains(responseDebug, `"content_path"`) || strings.Contains(responseDebug, `spell_casts`) {
t.Fatalf("response debug = %s, want metadata with content path and no inline response", responseDebug)
}
responseContent := string(readFile(t, filepath.Join(debugDir, "extract/spells/chunk-000001/attempt-01/response-content-0001.json")))
if !strings.Contains(responseContent, `"spell_casts"`) || !strings.Contains(responseContent, "\n ") {
t.Fatalf("response content = %s, want pretty JSON response body", responseContent)
}
assertPathNotExist(t, filepath.Join(debugDir, "llm/call-0001.json"))
assertPathNotExist(t, filepath.Join(debugDir, "extract/spells/chunk-000001-attempt-01/llm-call-0001.json"))
@@ -3751,6 +3756,9 @@ func assertDebugTreeDoesNotContain(t *testing.T, root string, forbidden ...strin
if entry.IsDir() {
return nil
}
if strings.HasPrefix(filepath.Base(path), "response-content-") {
return nil
}
data, err := os.ReadFile(path)
if err != nil {
return err

7
internal/framework/debug/recorder.go
View File

@@ -32,3 +32,10 @@ func (r *WorkspaceRecorder) WriteJSON(name string, payload any) error {
}
return coreworkspace.WriteJSON(r.root, name, payload)
}
func (r *WorkspaceRecorder) WriteBytes(name string, data []byte) error {
if !r.Enabled() {
return nil
}
return coreworkspace.WriteBytes(r.root, name, data)
}

114
internal/framework/pipeline/debug.go
View File

@@ -1,10 +1,12 @@
package pipeline
import (
"bytes"
"context"
"crypto/sha256"
"encoding/base64"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"path"
@@ -22,14 +24,16 @@ import (
type DebugRecorder interface {
Enabled() bool
WriteJSON(name string, payload any) error
WriteBytes(name string, data []byte) error
}
type noopDebugRecorder struct{}
func NoopDebugRecorder() DebugRecorder { return noopDebugRecorder{} }
func (noopDebugRecorder) Enabled() bool { return false }
func (noopDebugRecorder) WriteJSON(string, any) error { return nil }
func (noopDebugRecorder) Enabled() bool { return false }
func (noopDebugRecorder) WriteJSON(string, any) error { return nil }
func (noopDebugRecorder) WriteBytes(string, []byte) error { return nil }
func debugPathComponent(value string) string {
value = strings.TrimSpace(value)
if value == "" {
@@ -171,20 +175,22 @@ type debugLLMPromptArtifact struct {
}
type debugLLMResponseArtifact struct {
CallID string `json:"call_id"`
Response *contracts.LLMDebugResponse `json:"response,omitempty"`
Fallback *debugStructuredCompletionResponse `json:"fallback,omitempty"`
Error string `json:"error,omitempty"`
CallID string `json:"call_id"`
Response *contracts.LLMDebugResponse `json:"response,omitempty"`
Fallback *debugStructuredCompletionResponse `json:"fallback,omitempty"`
ContentPath string `json:"content_path,omitempty"`
Error string `json:"error,omitempty"`
}
type debugLLMCallReference struct {
CallID string `json:"call_id"`
PromptPath string `json:"prompt_path,omitempty"`
ResponsePath string `json:"response_path"`
PromptID string `json:"prompt_id,omitempty"`
ProfileID string `json:"profile_id,omitempty"`
Model string `json:"model,omitempty"`
Error bool `json:"error,omitempty"`
CallID string `json:"call_id"`
PromptPath string `json:"prompt_path,omitempty"`
ResponsePath string `json:"response_path"`
ResponseContentPath string `json:"response_content_path,omitempty"`
PromptID string `json:"prompt_id,omitempty"`
ProfileID string `json:"profile_id,omitempty"`
Model string `json:"model,omitempty"`
Error bool `json:"error,omitempty"`
}
type debugValidationRequest struct {
@@ -274,6 +280,10 @@ func (client *debugLLMClient) CompleteStructured(ctx context.Context, req contra
}))
}
responsePath := path.Join(scopePrefix, "response-"+callID+".json")
responseMaterial := debugResponseMaterial(response)
fallbackMaterial := debugCompletionFallback(response)
responseContentPath, responseForArtifact, fallbackForArtifact, contentErr := writeDebugResponseContent(client.recorder, scopePrefix, callID, responseMaterial, fallbackMaterial)
writeErr = errors.Join(writeErr, contentErr)
writeErr = errors.Join(writeErr, writeDebugTimed(client.recorder, responsePath, debugTimedEnvelope{
Stage: req.StageName,
ModuleKey: req.StageName,
@@ -281,21 +291,23 @@ func (client *debugLLMClient) CompleteStructured(ctx context.Context, req contra
CompletedAt: completed,
DurationMS: completed.Sub(started).Milliseconds(),
Payload: debugLLMResponseArtifact{
CallID: callID,
Response: debugResponseMaterial(response),
Fallback: debugCompletionFallback(response),
Error: errorText,
CallID: callID,
Response: responseForArtifact,
Fallback: fallbackForArtifact,
ContentPath: responseContentPath,
Error: errorText,
},
Error: errorText,
}))
callRef := debugLLMCallReference{
CallID: callID,
PromptPath: promptPath,
ResponsePath: responsePath,
PromptID: req.PromptID,
ProfileID: debugFirstNonEmptyString(response.ProfileID, req.ProfileID),
Model: debugFirstNonEmptyString(response.Model, debugResponseModel(response)),
Error: err != nil,
CallID: callID,
PromptPath: promptPath,
ResponsePath: responsePath,
ResponseContentPath: responseContentPath,
PromptID: req.PromptID,
ProfileID: debugFirstNonEmptyString(response.ProfileID, req.ProfileID),
Model: debugFirstNonEmptyString(response.Model, debugResponseModel(response)),
Error: err != nil,
}
if scope := debugLLMScopeFromContext(ctx); scope != nil {
scope.record(callRef)
@@ -593,6 +605,60 @@ func debugCompletionFallback(response contracts.StructuredCompletionResponse) *d
return &fallback
}
func writeDebugResponseContent(recorder DebugRecorder, scopePrefix string, callID string, response *contracts.LLMDebugResponse, fallback *debugStructuredCompletionResponse) (string, *contracts.LLMDebugResponse, *debugStructuredCompletionResponse, error) {
responseCopy := cloneDebugResponseWithoutContent(response)
fallbackCopy := cloneDebugFallbackWithoutContent(fallback)
content := ""
if response != nil {
content = response.Content
}
if content == "" && fallback != nil {
content = fallback.Content
}
if content == "" {
return "", responseCopy, fallbackCopy, nil
}
contentPath, data := debugResponseContentFile(scopePrefix, callID, content)
if recorder == nil || !recorder.Enabled() {
return contentPath, responseCopy, fallbackCopy, nil
}
if err := recorder.WriteBytes(contentPath, data); err != nil {
return contentPath, responseCopy, fallbackCopy, err
}
return contentPath, responseCopy, fallbackCopy, nil
}
func cloneDebugResponseWithoutContent(response *contracts.LLMDebugResponse) *contracts.LLMDebugResponse {
if response == nil {
return nil
}
clone := *response
clone.Content = ""
return &clone
}
func cloneDebugFallbackWithoutContent(fallback *debugStructuredCompletionResponse) *debugStructuredCompletionResponse {
if fallback == nil {
return nil
}
clone := *fallback
clone.Content = ""
return &clone
}
func debugResponseContentFile(scopePrefix string, callID string, content string) (string, []byte) {
raw := []byte(content)
if json.Valid(raw) {
var formatted bytes.Buffer
if err := json.Indent(&formatted, raw, "", " "); err == nil {
formatted.WriteByte('\n')
return path.Join(scopePrefix, "response-content-"+callID+".json"), formatted.Bytes()
}
}
return path.Join(scopePrefix, "response-content-"+callID+".txt"), raw
}
func debugResponseMaterial(response contracts.StructuredCompletionResponse) *contracts.LLMDebugResponse {
if response.Debug == nil {
return nil

60
internal/framework/pipeline/runner_test.go
View File

@@ -1310,7 +1310,7 @@ func TestRunDebugFailedChunkAttemptReferencesScopedLLMOutput(t *testing.T) {
t.Fatalf("llm_calls = %#v, want one scoped call", attempt.LLMCalls)
}
call := attempt.LLMCalls[0]
if call.CallID != "0001" || call.PromptPath != "chunk/attempt-01/prompt-0001.json" || call.ResponsePath != "chunk/attempt-01/response-0001.json" {
if call.CallID != "0001" || call.PromptPath != "chunk/attempt-01/prompt-0001.json" || call.ResponsePath != "chunk/attempt-01/response-0001.json" || call.ResponseContentPath != "chunk/attempt-01/response-content-0001.json" {
t.Fatalf("llm call reference = %#v, want prompt and response paths", call)
}
if call.PromptID != "runner.chunk" || call.ProfileID != "debug-profile" || call.Model != "debug-model" || call.Error {
@@ -1331,8 +1331,14 @@ func TestRunDebugFailedChunkAttemptReferencesScopedLLMOutput(t *testing.T) {
if !ok {
t.Fatalf("response payload type = %T, want debugLLMResponseArtifact", response.Payload)
}
if responsePayload.Response == nil || responsePayload.Response.Content != `{"raw":true}` {
t.Fatalf("response payload = %#v, want raw LLM response", responsePayload)
if responsePayload.ContentPath != call.ResponseContentPath {
t.Fatalf("response content path = %q, want %q", responsePayload.ContentPath, call.ResponseContentPath)
}
if responsePayload.Response == nil || responsePayload.Response.Content != "" {
t.Fatalf("response payload = %#v, want metadata without inline content", responsePayload)
}
if got := string(recorder.bytes[call.ResponseContentPath]); got != "{\n \"raw\": true\n}\n" {
t.Fatalf("response content file = %q, want pretty JSON", got)
}
if _, ok := recorder.payloads["llm/call-0001.json"]; ok {
t.Fatalf("old canonical LLM debug artifact was written")
@@ -1342,6 +1348,43 @@ func TestRunDebugFailedChunkAttemptReferencesScopedLLMOutput(t *testing.T) {
}
}
func TestRunDebugWritesNonJSONLLMResponseContentAsText(t *testing.T) {
modules := defaultRunnerModules()
modules.chunker.callLLM = true
modules.chunker.llmPromptID = "runner.chunk"
modules.chunker.err = errors.New("malformed structured output")
recorder := newMemoryDebugRecorder()
_, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{
Pipeline: resolvedPipeline(),
LLMClient: debugResponseLLMClient{content: []byte("plain text response"), profileID: "debug-profile"},
Debug: recorder,
})
if err == nil || !strings.Contains(err.Error(), "malformed structured output") {
t.Fatalf("Run() error = %v, want chunk failure", err)
}
attempt := recorder.envelope(t, "chunk/attempt-01.json")
if len(attempt.LLMCalls) != 1 {
t.Fatalf("llm_calls = %#v, want one scoped call", attempt.LLMCalls)
}
call := attempt.LLMCalls[0]
if call.ResponseContentPath != "chunk/attempt-01/response-content-0001.txt" {
t.Fatalf("response content path = %q, want .txt file", call.ResponseContentPath)
}
if got := string(recorder.bytes[call.ResponseContentPath]); got != "plain text response" {
t.Fatalf("response text content = %q, want raw text", got)
}
response := recorder.envelope(t, call.ResponsePath)
responsePayload, ok := response.Payload.(debugLLMResponseArtifact)
if !ok {
t.Fatalf("response payload type = %T, want debugLLMResponseArtifact", response.Payload)
}
if responsePayload.Response == nil || responsePayload.Response.Content != "" {
t.Fatalf("response payload = %#v, want metadata without inline content", responsePayload)
}
}
func TestRunStopsRetryAfterConfiguredAttemptsAndRecordsAttemptCount(t *testing.T) {
modules := defaultRunnerModules()
validator := &runnerChainValidator{name: "chain-extract", approved: []bool{false}, reason: "bad_extract", message: "extract rejected"}
@@ -2367,10 +2410,14 @@ func (client debugResponseLLMClient) CompleteStructured(ctx context.Context, req
type memoryDebugRecorder struct {
payloads map[string]any
bytes map[string][]byte
}
func newMemoryDebugRecorder() *memoryDebugRecorder {
return &memoryDebugRecorder{payloads: map[string]any{}}
return &memoryDebugRecorder{
payloads: map[string]any{},
bytes: map[string][]byte{},
}
}
func (recorder *memoryDebugRecorder) Enabled() bool { return true }
@@ -2380,6 +2427,11 @@ func (recorder *memoryDebugRecorder) WriteJSON(name string, payload any) error {
return nil
}
func (recorder *memoryDebugRecorder) WriteBytes(name string, data []byte) error {
recorder.bytes[name] = append([]byte(nil), data...)
return nil
}
func (recorder *memoryDebugRecorder) envelope(t *testing.T, name string) debugTimedEnvelope {
t.Helper()
payload, ok := recorder.payloads[name]