eric/notarius
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Stop base64 encoding LLM responses in debug output

Browse Source
This commit is contained in:
Eric Rakestraw
2026-07-07 23:14:17 -05:00
parent 3011dd91ca
commit 451f6c0bb9
4 changed files with 11 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
docs/operations.md
View File

@@ -141,10 +141,12 @@ response data from Notarius contracts, validator requests and results, timing,
and retry attempt metadata. Canonical LLM call artifacts are written under and retry attempt metadata. Canonical LLM call artifacts are written under
`llm/call-000N.json`; LLM calls made inside a retry or validator attempt are `llm/call-000N.json`; LLM calls made inside a retry or validator attempt are
also copied under that attempt directory and linked from the attempt also copied under that attempt directory and linked from the attempt
`llm_calls` array. Debug artifacts may contain source material, reference `llm_calls` array. LLM response content in those artifacts is written as raw
text for inspection. Debug artifacts may contain source material, reference
material, prompt inputs, model outputs, and other sensitive data. Obvious material, prompt inputs, model outputs, and other sensitive data. Obvious
credential-shaped values and sensitive map keys are redacted, but debug credential-shaped values and sensitive map keys are redacted in framework
directories should still be protected as sensitive local state. envelopes, but debug directories should still be protected as sensitive local
state.
## Retention ## Retention

5
internal/cli/run_test.go
View File

@@ -3700,7 +3700,8 @@ func assertDistinctRoots(t *testing.T, roots ...string) {
} }
} }
var debugBase64FieldPattern = regexp.MustCompile(`"(?:content_base64|content)"\s*:\s*"([^"]*)"`) var debugBase64FieldPattern = regexp.MustCompile(`"content_base64"\s*:\s*"([^"]*)"`)
var debugRawLLMResponseContentPattern = regexp.MustCompile(`"content"\s*:\s*"(?:\\.|[^"\\])*"`)
func assertDebugTreeDoesNotContain(t *testing.T, root string, forbidden ...string) { func assertDebugTreeDoesNotContain(t *testing.T, root string, forbidden ...string) {
t.Helper() t.Helper()
@@ -3715,7 +3716,7 @@ func assertDebugTreeDoesNotContain(t *testing.T, root string, forbidden ...strin
if err != nil { if err != nil {
return err return err
} }
text := string(data) text := debugRawLLMResponseContentPattern.ReplaceAllString(string(data), `"content":"[RAW_LLM_RESPONSE]"`)
for _, value := range forbidden { for _, value := range forbidden {
if strings.Contains(text, value) { if strings.Contains(text, value) {
t.Fatalf("debug artifact %q contains forbidden value %q", path, value) t.Fatalf("debug artifact %q contains forbidden value %q", path, value)

2
internal/framework/pipeline/debug.go
View File

@@ -538,7 +538,7 @@ func debugCompletionRequest(req contracts.StructuredCompletionRequest) debugStru
func debugCompletionResponse(response contracts.StructuredCompletionResponse) debugStructuredCompletionResponse { func debugCompletionResponse(response contracts.StructuredCompletionResponse) debugStructuredCompletionResponse {
return debugStructuredCompletionResponse{ return debugStructuredCompletionResponse{
Content: base64.StdEncoding.EncodeToString(redactSecretBytes(response.Content)), Content: string(response.Content),
Provider: response.Provider, Provider: response.Provider,
Model: response.Model, Model: response.Model,
ProfileID: response.ProfileID, ProfileID: response.ProfileID,

6
internal/framework/pipeline/runner_test.go
View File

@@ -2,7 +2,6 @@ package pipeline
import ( import (
"context" "context"
"encoding/base64"
"errors" "errors"
"reflect" "reflect"
"strings" "strings"
@@ -1323,9 +1322,8 @@ func TestRunDebugFailedChunkAttemptReferencesScopedLLMOutput(t *testing.T) {
if !ok { if !ok {
t.Fatalf("scoped payload type = %T, want debugStructuredLLMCall", scoped.Payload) t.Fatalf("scoped payload type = %T, want debugStructuredLLMCall", scoped.Payload)
} }
wantContent := base64.StdEncoding.EncodeToString([]byte(`{"raw":true}`)) if scopedPayload.Response.Content != `{"raw":true}` {
if scopedPayload.Response.Content != wantContent { t.Fatalf("scoped response content = %q, want raw LLM response", scopedPayload.Response.Content)
t.Fatalf("scoped response content = %q, want %q", scopedPayload.Response.Content, wantContent)
} }
_ = recorder.envelope(t, call.CanonicalPath) _ = recorder.envelope(t, call.CanonicalPath)
} }