eric/weatherreporter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

Harden prompt debug redaction

Browse Source
This commit is contained in:
Eric Rakestraw
2026-08-13 01:19:35 +00:00
parent 27849813db
commit a38d291f63
6 changed files with 148 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
internal/adapters/promptkit/adapter.go
View File

@@ -247,13 +247,12 @@ func copyInputHashes(values map[string]string) map[string]string {
func marshalDebugParameters(value promptkit.ExecutionTarget) []byte {
parameters := struct {
Temperature float64 `json:"temperature"`
MaxTokens int `json:"max_tokens"`
TopP float64 `json:"top_p"`
TimeoutSeconds int `json:"timeout_seconds"`
ServiceTier string `json:"service_tier"`
ReasoningEffort string `json:"reasoning_effort"`
ExtraParams map[string]any `json:"extra_params"`
Temperature float64 `json:"temperature"`
MaxTokens int `json:"max_tokens"`
TopP float64 `json:"top_p"`
TimeoutSeconds int `json:"timeout_seconds"`
ServiceTier string `json:"service_tier"`
ReasoningEffort string `json:"reasoning_effort"`
}{
Temperature: value.Temperature,
MaxTokens: value.MaxTokens,
@@ -261,7 +260,6 @@ func marshalDebugParameters(value promptkit.ExecutionTarget) []byte {
TimeoutSeconds: value.TimeoutSeconds,
ServiceTier: value.ServiceTier,
ReasoningEffort: value.ReasoningEffort,
ExtraParams: value.ExtraParams,
}
data, _ := json.Marshal(parameters)
return data

24
internal/adapters/promptkit/adapter_test.go
View File

@@ -316,6 +316,30 @@ func TestExecuteCapturesSensitiveDebugOnlyWhenRequested(t *testing.T) {
}
}
func TestMarshalDebugParametersOmitsProviderExtras(t *testing.T) {
const marker = "private-debug-marker"
parameters := string(marshalDebugParameters(promptkit.ExecutionTarget{
Temperature: 0.2,
MaxTokens: 400,
TopP: 0.9,
TimeoutSeconds: 30,
ServiceTier: "flex",
ReasoningEffort: "high",
ExtraParams: map[string]any{
"access-key": marker,
"signature": marker,
},
}))
if strings.Contains(parameters, marker) || strings.Contains(parameters, "extra_params") {
t.Fatalf("debug parameters leaked provider extras: %s", parameters)
}
for _, want := range []string{`"temperature":0.2`, `"max_tokens":400`, `"top_p":0.9`, `"timeout_seconds":30`, `"service_tier":"flex"`, `"reasoning_effort":"high"`} {
if !strings.Contains(parameters, want) {
t.Fatalf("debug parameters missing safe value %q: %s", want, parameters)
}
}
}
func TestExecuteCallbackFailurePreventsGeneration(t *testing.T) {
client := &fakeClient{response: validResponse()}
adapter := newTestAdapter(t, client)

72
internal/promptdebug/debug_writer.go
View File

@@ -171,7 +171,7 @@ func (w *PromptDebugWriter) WritePreparation(ref PromptDebugRef, preparation pro
artifact.RenderedMessages = promptDebugMessages(debug.RenderedMessages)
artifact.StructuredSchema = copyRawJSON(debug.StructuredSchema)
artifact.Endpoint = safePromptDebugEndpoint(debug.Endpoint)
parameters, err := redactPromptDebugParameters(debug.ParametersJSON)
parameters, err := safePromptDebugParameters(debug.ParametersJSON)
if err != nil {
return "", err
}
@@ -404,55 +404,55 @@ func copyRawJSON(value []byte) json.RawMessage {
func safePromptDebugEndpoint(value string) string {
parsed, err := url.Parse(value)
if err != nil {
if err != nil || parsed.Scheme == "" || parsed.Host == "" {
return ""
}
parsed.User = nil
parameters := parsed.Query()
for key := range parameters {
if isPromptDebugSecretKey(key) {
parameters[key] = []string{"[redacted]"}
}
}
parsed.RawQuery = parameters.Encode()
parsed.Fragment = ""
return parsed.String()
return (&url.URL{Scheme: parsed.Scheme, Host: parsed.Host}).String()
}
func redactPromptDebugParameters(value []byte) (json.RawMessage, error) {
func safePromptDebugParameters(value []byte) (json.RawMessage, error) {
if len(value) == 0 {
return nil, nil
}
var decoded any
var decoded map[string]json.RawMessage
if err := json.Unmarshal(value, &decoded); err != nil {
return nil, fmt.Errorf("decode prompt debug parameters: %w", err)
return nil, fmt.Errorf("prompt debug parameters must be a JSON object")
}
redactPromptDebugValue(decoded)
encoded, err := json.Marshal(decoded)
parameters := make(map[string]json.RawMessage)
for _, field := range []struct {
name string
value func(json.RawMessage) bool
}{
{name: "temperature", value: promptDebugJSONNumber},
{name: "max_tokens", value: promptDebugJSONInteger},
{name: "top_p", value: promptDebugJSONNumber},
{name: "timeout_seconds", value: promptDebugJSONInteger},
{name: "service_tier", value: promptDebugJSONString},
{name: "reasoning_effort", value: promptDebugJSONString},
} {
value, ok := decoded[field.name]
if ok && field.value(value) {
parameters[field.name] = append(json.RawMessage(nil), value...)
}
}
encoded, err := json.Marshal(parameters)
if err != nil {
return nil, fmt.Errorf("encode prompt debug parameters: %w", err)
return nil, fmt.Errorf("encode safe prompt debug parameters: %w", err)
}
return encoded, nil
}
func redactPromptDebugValue(value any) {
switch typed := value.(type) {
case map[string]any:
for key, item := range typed {
if isPromptDebugSecretKey(key) {
typed[key] = "[redacted]"
continue
}
redactPromptDebugValue(item)
}
case []any:
for _, item := range typed {
redactPromptDebugValue(item)
}
}
func promptDebugJSONNumber(value json.RawMessage) bool {
var decoded float64
return json.Unmarshal(value, &decoded) == nil
}
func isPromptDebugSecretKey(key string) bool {
normalized := strings.NewReplacer("_", "", "-", "", " ", "").Replace(strings.ToLower(key))
return strings.Contains(normalized, "credential") || strings.Contains(normalized, "secret") || strings.Contains(normalized, "password") || strings.Contains(normalized, "token") || strings.Contains(normalized, "apikey") || strings.Contains(normalized, "authorization")
func promptDebugJSONInteger(value json.RawMessage) bool {
var decoded int
return json.Unmarshal(value, &decoded) == nil
}
func promptDebugJSONString(value json.RawMessage) bool {
var decoded string
return json.Unmarshal(value, &decoded) == nil
}

75
internal/promptdebug/debug_writer_test.go
View File

@@ -42,7 +42,7 @@ func TestPromptDebugWriterWritesIsolatedArtifacts(t *testing.T) {
t.Fatalf("WriteExecution() directory = %q, want %q", executionDir, preparationDir)
}
preparationData := readPromptDebugFile(t, filepath.Join(preparationDir, "preparation.json"))
for _, want := range []string{"weatherreporter.prompt_preparation_debug.v2", "Use the supplied weather facts.", `"type": "object"`, "https://llm.example.test/v1/chat?api_key=%5Bredacted%5D", `"temperature": 0.2`, `"api_key": "[redacted]"`} {
for _, want := range []string{"weatherreporter.prompt_preparation_debug.v2", "Use the supplied weather facts.", `"type": "object"`, "https://llm.example.test", `"temperature": 0.2`} {
if !strings.Contains(string(preparationData), want) {
t.Fatalf("preparation debug artifact missing %q:\n%s", want, preparationData)
}
@@ -66,6 +66,79 @@ func TestPromptDebugWriterWritesIsolatedArtifacts(t *testing.T) {
}
}
func TestPromptDebugWriterProjectsProviderConfigurationSafely(t *testing.T) {
const marker = "private-debug-marker"
tests := []struct {
name string
endpoint string
}{
{
name: "user info signed path and access key query",
endpoint: "https://operator:" + marker + "@llm.example.test/signed/" + marker + "?access-key=" + marker + "&Signature=" + marker,
},
{
name: "case and separator query aliases",
endpoint: "https://llm.example.test/" + marker + "?X-Amz-Signature=" + marker + "&AUTH=" + marker + "&session_cookie=" + marker,
},
}
parameters := []byte(`{
"temperature": 0.2,
"max_tokens": 400,
"top_p": 0.9,
"timeout_seconds": 30,
"service_tier": "flex",
"reasoning_effort": "high",
"access-key": "private-debug-marker",
"signature": "private-debug-marker",
"auth": "private-debug-marker",
"cookie": "private-debug-marker",
"nested": {"X Api Key": "private-debug-marker"},
"extra_params": {"authorization": "private-debug-marker"}
}`)
for index, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
writer, err := NewPromptDebugWriter(filepath.Join(t.TempDir(), "debug"))
if err != nil {
t.Fatalf("NewPromptDebugWriter() error = %v", err)
}
ref := promptDebugRef()
ref.RunID = fmt.Sprintf("run-%d", index)
directory, err := writer.WritePreparation(ref, promptDebugPreparationFixture(), &promptexec.PreparationDebug{
Endpoint: tt.endpoint,
ParametersJSON: parameters,
})
if err != nil {
t.Fatalf("WritePreparation() error = %v", err)
}
data := readPromptDebugFile(t, filepath.Join(directory, "preparation.json"))
text := string(data)
if strings.Contains(text, marker) || strings.Contains(text, "signed") || strings.Contains(text, "access-key") || strings.Contains(text, "signature") || strings.Contains(text, "cookie") {
t.Fatalf("preparation debug artifact leaked provider configuration:\n%s", data)
}
for _, want := range []string{"https://llm.example.test", `"temperature": 0.2`, `"max_tokens": 400`, `"top_p": 0.9`, `"timeout_seconds": 30`, `"service_tier": "flex"`, `"reasoning_effort": "high"`} {
if !strings.Contains(text, want) {
t.Fatalf("preparation debug artifact missing safe value %q:\n%s", want, data)
}
}
})
}
}
func TestPromptDebugWriterDoesNotProjectParameterValuesInErrors(t *testing.T) {
const marker = "private-debug-marker"
writer, err := NewPromptDebugWriter(filepath.Join(t.TempDir(), "debug"))
if err != nil {
t.Fatalf("NewPromptDebugWriter() error = %v", err)
}
_, err = writer.WritePreparation(promptDebugRef(), promptDebugPreparationFixture(), &promptexec.PreparationDebug{
ParametersJSON: []byte(`{"access-key":"` + marker),
})
if err == nil || strings.Contains(err.Error(), marker) {
t.Fatalf("WritePreparation() error = %v, want safe parameter projection error", err)
}
}
func TestPromptDebugWriterAtomicallyReplacesArtifacts(t *testing.T) {
writer, err := NewPromptDebugWriter(filepath.Join(t.TempDir(), "debug"))
if err != nil {