eric/weatherreporter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

Harden prompt debug redaction

Browse Source
This commit is contained in:
Eric Rakestraw
2026-08-13 01:19:35 +00:00
parent 27849813db
commit a38d291f63
6 changed files with 148 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files

75
internal/promptdebug/debug_writer_test.go
View File

@@ -42,7 +42,7 @@ func TestPromptDebugWriterWritesIsolatedArtifacts(t *testing.T) {
t.Fatalf("WriteExecution() directory = %q, want %q", executionDir, preparationDir)
}
preparationData := readPromptDebugFile(t, filepath.Join(preparationDir, "preparation.json"))
for _, want := range []string{"weatherreporter.prompt_preparation_debug.v2", "Use the supplied weather facts.", `"type": "object"`, "https://llm.example.test/v1/chat?api_key=%5Bredacted%5D", `"temperature": 0.2`, `"api_key": "[redacted]"`} {
for _, want := range []string{"weatherreporter.prompt_preparation_debug.v2", "Use the supplied weather facts.", `"type": "object"`, "https://llm.example.test", `"temperature": 0.2`} {
if !strings.Contains(string(preparationData), want) {
t.Fatalf("preparation debug artifact missing %q:\n%s", want, preparationData)
}
@@ -66,6 +66,79 @@ func TestPromptDebugWriterWritesIsolatedArtifacts(t *testing.T) {
}
}
func TestPromptDebugWriterProjectsProviderConfigurationSafely(t *testing.T) {
const marker = "private-debug-marker"
tests := []struct {
name string
endpoint string
}{
{
name: "user info signed path and access key query",
endpoint: "https://operator:" + marker + "@llm.example.test/signed/" + marker + "?access-key=" + marker + "&Signature=" + marker,
},
{
name: "case and separator query aliases",
endpoint: "https://llm.example.test/" + marker + "?X-Amz-Signature=" + marker + "&AUTH=" + marker + "&session_cookie=" + marker,
},
}
parameters := []byte(`{
"temperature": 0.2,
"max_tokens": 400,
"top_p": 0.9,
"timeout_seconds": 30,
"service_tier": "flex",
"reasoning_effort": "high",
"access-key": "private-debug-marker",
"signature": "private-debug-marker",
"auth": "private-debug-marker",
"cookie": "private-debug-marker",
"nested": {"X Api Key": "private-debug-marker"},
"extra_params": {"authorization": "private-debug-marker"}
}`)
for index, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
writer, err := NewPromptDebugWriter(filepath.Join(t.TempDir(), "debug"))
if err != nil {
t.Fatalf("NewPromptDebugWriter() error = %v", err)
}
ref := promptDebugRef()
ref.RunID = fmt.Sprintf("run-%d", index)
directory, err := writer.WritePreparation(ref, promptDebugPreparationFixture(), &promptexec.PreparationDebug{
Endpoint: tt.endpoint,
ParametersJSON: parameters,
})
if err != nil {
t.Fatalf("WritePreparation() error = %v", err)
}
data := readPromptDebugFile(t, filepath.Join(directory, "preparation.json"))
text := string(data)
if strings.Contains(text, marker) || strings.Contains(text, "signed") || strings.Contains(text, "access-key") || strings.Contains(text, "signature") || strings.Contains(text, "cookie") {
t.Fatalf("preparation debug artifact leaked provider configuration:\n%s", data)
}
for _, want := range []string{"https://llm.example.test", `"temperature": 0.2`, `"max_tokens": 400`, `"top_p": 0.9`, `"timeout_seconds": 30`, `"service_tier": "flex"`, `"reasoning_effort": "high"`} {
if !strings.Contains(text, want) {
t.Fatalf("preparation debug artifact missing safe value %q:\n%s", want, data)
}
}
})
}
}
func TestPromptDebugWriterDoesNotProjectParameterValuesInErrors(t *testing.T) {
const marker = "private-debug-marker"
writer, err := NewPromptDebugWriter(filepath.Join(t.TempDir(), "debug"))
if err != nil {
t.Fatalf("NewPromptDebugWriter() error = %v", err)
}
_, err = writer.WritePreparation(promptDebugRef(), promptDebugPreparationFixture(), &promptexec.PreparationDebug{
ParametersJSON: []byte(`{"access-key":"` + marker),
})
if err == nil || strings.Contains(err.Error(), marker) {
t.Fatalf("WritePreparation() error = %v, want safe parameter projection error", err)
}
}
func TestPromptDebugWriterAtomicallyReplacesArtifacts(t *testing.T) {
writer, err := NewPromptDebugWriter(filepath.Join(t.TempDir(), "debug"))
if err != nil {