eric/notarius
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Enhance debug output to include response content files and update related metadata handling

Browse Source
This commit is contained in:
Eric Rakestraw
2026-07-08 08:52:19 -05:00
parent 68ec69f2e4
commit 610bdb4fea
6 changed files with 178 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files

60
internal/framework/pipeline/runner_test.go
View File

@@ -1310,7 +1310,7 @@ func TestRunDebugFailedChunkAttemptReferencesScopedLLMOutput(t *testing.T) {
t.Fatalf("llm_calls = %#v, want one scoped call", attempt.LLMCalls)
}
call := attempt.LLMCalls[0]
if call.CallID != "0001" || call.PromptPath != "chunk/attempt-01/prompt-0001.json" || call.ResponsePath != "chunk/attempt-01/response-0001.json" {
if call.CallID != "0001" || call.PromptPath != "chunk/attempt-01/prompt-0001.json" || call.ResponsePath != "chunk/attempt-01/response-0001.json" || call.ResponseContentPath != "chunk/attempt-01/response-content-0001.json" {
t.Fatalf("llm call reference = %#v, want prompt and response paths", call)
}
if call.PromptID != "runner.chunk" || call.ProfileID != "debug-profile" || call.Model != "debug-model" || call.Error {
@@ -1331,8 +1331,14 @@ func TestRunDebugFailedChunkAttemptReferencesScopedLLMOutput(t *testing.T) {
if !ok {
t.Fatalf("response payload type = %T, want debugLLMResponseArtifact", response.Payload)
}
if responsePayload.Response == nil || responsePayload.Response.Content != `{"raw":true}` {
t.Fatalf("response payload = %#v, want raw LLM response", responsePayload)
if responsePayload.ContentPath != call.ResponseContentPath {
t.Fatalf("response content path = %q, want %q", responsePayload.ContentPath, call.ResponseContentPath)
}
if responsePayload.Response == nil || responsePayload.Response.Content != "" {
t.Fatalf("response payload = %#v, want metadata without inline content", responsePayload)
}
if got := string(recorder.bytes[call.ResponseContentPath]); got != "{\n \"raw\": true\n}\n" {
t.Fatalf("response content file = %q, want pretty JSON", got)
}
if _, ok := recorder.payloads["llm/call-0001.json"]; ok {
t.Fatalf("old canonical LLM debug artifact was written")
@@ -1342,6 +1348,43 @@ func TestRunDebugFailedChunkAttemptReferencesScopedLLMOutput(t *testing.T) {
}
}
func TestRunDebugWritesNonJSONLLMResponseContentAsText(t *testing.T) {
modules := defaultRunnerModules()
modules.chunker.callLLM = true
modules.chunker.llmPromptID = "runner.chunk"
modules.chunker.err = errors.New("malformed structured output")
recorder := newMemoryDebugRecorder()
_, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{
Pipeline: resolvedPipeline(),
LLMClient: debugResponseLLMClient{content: []byte("plain text response"), profileID: "debug-profile"},
Debug: recorder,
})
if err == nil || !strings.Contains(err.Error(), "malformed structured output") {
t.Fatalf("Run() error = %v, want chunk failure", err)
}
attempt := recorder.envelope(t, "chunk/attempt-01.json")
if len(attempt.LLMCalls) != 1 {
t.Fatalf("llm_calls = %#v, want one scoped call", attempt.LLMCalls)
}
call := attempt.LLMCalls[0]
if call.ResponseContentPath != "chunk/attempt-01/response-content-0001.txt" {
t.Fatalf("response content path = %q, want .txt file", call.ResponseContentPath)
}
if got := string(recorder.bytes[call.ResponseContentPath]); got != "plain text response" {
t.Fatalf("response text content = %q, want raw text", got)
}
response := recorder.envelope(t, call.ResponsePath)
responsePayload, ok := response.Payload.(debugLLMResponseArtifact)
if !ok {
t.Fatalf("response payload type = %T, want debugLLMResponseArtifact", response.Payload)
}
if responsePayload.Response == nil || responsePayload.Response.Content != "" {
t.Fatalf("response payload = %#v, want metadata without inline content", responsePayload)
}
}
func TestRunStopsRetryAfterConfiguredAttemptsAndRecordsAttemptCount(t *testing.T) {
modules := defaultRunnerModules()
validator := &runnerChainValidator{name: "chain-extract", approved: []bool{false}, reason: "bad_extract", message: "extract rejected"}
@@ -2367,10 +2410,14 @@ func (client debugResponseLLMClient) CompleteStructured(ctx context.Context, req
type memoryDebugRecorder struct {
payloads map[string]any
bytes map[string][]byte
}
func newMemoryDebugRecorder() *memoryDebugRecorder {
return &memoryDebugRecorder{payloads: map[string]any{}}
return &memoryDebugRecorder{
payloads: map[string]any{},
bytes: map[string][]byte{},
}
}
func (recorder *memoryDebugRecorder) Enabled() bool { return true }
@@ -2380,6 +2427,11 @@ func (recorder *memoryDebugRecorder) WriteJSON(name string, payload any) error {
return nil
}
func (recorder *memoryDebugRecorder) WriteBytes(name string, data []byte) error {
recorder.bytes[name] = append([]byte(nil), data...)
return nil
}
func (recorder *memoryDebugRecorder) envelope(t *testing.T, name string) debugTimedEnvelope {
t.Helper()
payload, ok := recorder.payloads[name]