diff --git a/docs/operations.md b/docs/operations.md index 193107f..3d9021a 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -141,10 +141,12 @@ response data from Notarius contracts, validator requests and results, timing, and retry attempt metadata. Canonical LLM call artifacts are written under `llm/call-000N.json`; LLM calls made inside a retry or validator attempt are also copied under that attempt directory and linked from the attempt -`llm_calls` array. Debug artifacts may contain source material, reference +`llm_calls` array. LLM response content in those artifacts is written as raw +text for inspection. Debug artifacts may contain source material, reference material, prompt inputs, model outputs, and other sensitive data. Obvious -credential-shaped values and sensitive map keys are redacted, but debug -directories should still be protected as sensitive local state. +credential-shaped values and sensitive map keys are redacted in framework +envelopes, but debug directories should still be protected as sensitive local +state. ## Retention diff --git a/internal/cli/run_test.go b/internal/cli/run_test.go index b552652..efdc19c 100644 --- a/internal/cli/run_test.go +++ b/internal/cli/run_test.go @@ -3700,7 +3700,8 @@ func assertDistinctRoots(t *testing.T, roots ...string) { } } -var debugBase64FieldPattern = regexp.MustCompile(`"(?:content_base64|content)"\s*:\s*"([^"]*)"`) +var debugBase64FieldPattern = regexp.MustCompile(`"content_base64"\s*:\s*"([^"]*)"`) +var debugRawLLMResponseContentPattern = regexp.MustCompile(`"content"\s*:\s*"(?:\\.|[^"\\])*"`) func assertDebugTreeDoesNotContain(t *testing.T, root string, forbidden ...string) { t.Helper() @@ -3715,7 +3716,7 @@ func assertDebugTreeDoesNotContain(t *testing.T, root string, forbidden ...strin if err != nil { return err } - text := string(data) + text := debugRawLLMResponseContentPattern.ReplaceAllString(string(data), `"content":"[RAW_LLM_RESPONSE]"`) for _, value := range forbidden { if strings.Contains(text, value) { t.Fatalf("debug artifact %q contains forbidden value %q", path, value) diff --git a/internal/framework/pipeline/debug.go b/internal/framework/pipeline/debug.go index 8ece1f2..94868e6 100644 --- a/internal/framework/pipeline/debug.go +++ b/internal/framework/pipeline/debug.go @@ -538,7 +538,7 @@ func debugCompletionRequest(req contracts.StructuredCompletionRequest) debugStru func debugCompletionResponse(response contracts.StructuredCompletionResponse) debugStructuredCompletionResponse { return debugStructuredCompletionResponse{ - Content: base64.StdEncoding.EncodeToString(redactSecretBytes(response.Content)), + Content: string(response.Content), Provider: response.Provider, Model: response.Model, ProfileID: response.ProfileID, diff --git a/internal/framework/pipeline/runner_test.go b/internal/framework/pipeline/runner_test.go index 7ae09ae..a657987 100644 --- a/internal/framework/pipeline/runner_test.go +++ b/internal/framework/pipeline/runner_test.go @@ -2,7 +2,6 @@ package pipeline import ( "context" - "encoding/base64" "errors" "reflect" "strings" @@ -1323,9 +1322,8 @@ func TestRunDebugFailedChunkAttemptReferencesScopedLLMOutput(t *testing.T) { if !ok { t.Fatalf("scoped payload type = %T, want debugStructuredLLMCall", scoped.Payload) } - wantContent := base64.StdEncoding.EncodeToString([]byte(`{"raw":true}`)) - if scopedPayload.Response.Content != wantContent { - t.Fatalf("scoped response content = %q, want %q", scopedPayload.Response.Content, wantContent) + if scopedPayload.Response.Content != `{"raw":true}` { + t.Fatalf("scoped response content = %q, want raw LLM response", scopedPayload.Response.Content) } _ = recorder.envelope(t, call.CanonicalPath) }