From a38d291f63037b3ab61da5264c6318c83445ae52 Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Thu, 13 Aug 2026 01:19:35 +0000 Subject: [PATCH] Harden prompt debug redaction --- docs/integrations/promptkit.md | 4 ++ docs/operations.md | 4 ++ internal/adapters/promptkit/adapter.go | 14 ++-- internal/adapters/promptkit/adapter_test.go | 24 +++++++ internal/promptdebug/debug_writer.go | 72 ++++++++++---------- internal/promptdebug/debug_writer_test.go | 75 ++++++++++++++++++++- 6 files changed, 148 insertions(+), 45 deletions(-) diff --git a/docs/integrations/promptkit.md b/docs/integrations/promptkit.md index 7889309..068cff3 100644 --- a/docs/integrations/promptkit.md +++ b/docs/integrations/promptkit.md @@ -29,6 +29,10 @@ Profiles that require a direct API key are unsupported; a profile that reports ` Promptkit receives the YAML data package as an inline input and returns structured JSON that Weatherreporter validates before rendering its own Markdown template. The package contains only reviewed prompt-facing warning summaries, never source transport or provenance details. Safe active provenance remains in memory. Content-rich diagnostics are opt-in through `--llm-debug-dir`; see [operations](../operations.md) for retention and permissions. +When capture is enabled, its preparation artifact projects a provider endpoint +to its scheme and host and retains only reviewed execution settings. Provider +extras and URL user information, paths, queries, and fragments are omitted. + ## Comparison Execution For `compare`, Weatherreporter validates one exact prompt and every explicitly diff --git a/docs/operations.md b/docs/operations.md index 10889fe..83e4aa5 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -143,6 +143,10 @@ remain distinct. Normal output, summaries, and routine logs omit that sensitive content. Debug capture is never created for an ordinary command without `--llm-debug-dir`. +Preparation captures retain only the provider endpoint origin and reviewed +execution settings. URL user information, paths, queries, fragments, and +unrecognized provider parameters are omitted. + If capture creation or writing fails, the affected run fails rather than silently continuing without the requested diagnostics. diff --git a/internal/adapters/promptkit/adapter.go b/internal/adapters/promptkit/adapter.go index 4f2ca4d..0b1ff99 100644 --- a/internal/adapters/promptkit/adapter.go +++ b/internal/adapters/promptkit/adapter.go @@ -247,13 +247,12 @@ func copyInputHashes(values map[string]string) map[string]string { func marshalDebugParameters(value promptkit.ExecutionTarget) []byte { parameters := struct { - Temperature float64 `json:"temperature"` - MaxTokens int `json:"max_tokens"` - TopP float64 `json:"top_p"` - TimeoutSeconds int `json:"timeout_seconds"` - ServiceTier string `json:"service_tier"` - ReasoningEffort string `json:"reasoning_effort"` - ExtraParams map[string]any `json:"extra_params"` + Temperature float64 `json:"temperature"` + MaxTokens int `json:"max_tokens"` + TopP float64 `json:"top_p"` + TimeoutSeconds int `json:"timeout_seconds"` + ServiceTier string `json:"service_tier"` + ReasoningEffort string `json:"reasoning_effort"` }{ Temperature: value.Temperature, MaxTokens: value.MaxTokens, @@ -261,7 +260,6 @@ func marshalDebugParameters(value promptkit.ExecutionTarget) []byte { TimeoutSeconds: value.TimeoutSeconds, ServiceTier: value.ServiceTier, ReasoningEffort: value.ReasoningEffort, - ExtraParams: value.ExtraParams, } data, _ := json.Marshal(parameters) return data diff --git a/internal/adapters/promptkit/adapter_test.go b/internal/adapters/promptkit/adapter_test.go index f60dc17..546972c 100644 --- a/internal/adapters/promptkit/adapter_test.go +++ b/internal/adapters/promptkit/adapter_test.go @@ -316,6 +316,30 @@ func TestExecuteCapturesSensitiveDebugOnlyWhenRequested(t *testing.T) { } } +func TestMarshalDebugParametersOmitsProviderExtras(t *testing.T) { + const marker = "private-debug-marker" + parameters := string(marshalDebugParameters(promptkit.ExecutionTarget{ + Temperature: 0.2, + MaxTokens: 400, + TopP: 0.9, + TimeoutSeconds: 30, + ServiceTier: "flex", + ReasoningEffort: "high", + ExtraParams: map[string]any{ + "access-key": marker, + "signature": marker, + }, + })) + if strings.Contains(parameters, marker) || strings.Contains(parameters, "extra_params") { + t.Fatalf("debug parameters leaked provider extras: %s", parameters) + } + for _, want := range []string{`"temperature":0.2`, `"max_tokens":400`, `"top_p":0.9`, `"timeout_seconds":30`, `"service_tier":"flex"`, `"reasoning_effort":"high"`} { + if !strings.Contains(parameters, want) { + t.Fatalf("debug parameters missing safe value %q: %s", want, parameters) + } + } +} + func TestExecuteCallbackFailurePreventsGeneration(t *testing.T) { client := &fakeClient{response: validResponse()} adapter := newTestAdapter(t, client) diff --git a/internal/promptdebug/debug_writer.go b/internal/promptdebug/debug_writer.go index 2bd5b9c..f84cd04 100644 --- a/internal/promptdebug/debug_writer.go +++ b/internal/promptdebug/debug_writer.go @@ -171,7 +171,7 @@ func (w *PromptDebugWriter) WritePreparation(ref PromptDebugRef, preparation pro artifact.RenderedMessages = promptDebugMessages(debug.RenderedMessages) artifact.StructuredSchema = copyRawJSON(debug.StructuredSchema) artifact.Endpoint = safePromptDebugEndpoint(debug.Endpoint) - parameters, err := redactPromptDebugParameters(debug.ParametersJSON) + parameters, err := safePromptDebugParameters(debug.ParametersJSON) if err != nil { return "", err } @@ -404,55 +404,55 @@ func copyRawJSON(value []byte) json.RawMessage { func safePromptDebugEndpoint(value string) string { parsed, err := url.Parse(value) - if err != nil { + if err != nil || parsed.Scheme == "" || parsed.Host == "" { return "" } - parsed.User = nil - parameters := parsed.Query() - for key := range parameters { - if isPromptDebugSecretKey(key) { - parameters[key] = []string{"[redacted]"} - } - } - parsed.RawQuery = parameters.Encode() - parsed.Fragment = "" - return parsed.String() + return (&url.URL{Scheme: parsed.Scheme, Host: parsed.Host}).String() } -func redactPromptDebugParameters(value []byte) (json.RawMessage, error) { +func safePromptDebugParameters(value []byte) (json.RawMessage, error) { if len(value) == 0 { return nil, nil } - var decoded any + var decoded map[string]json.RawMessage if err := json.Unmarshal(value, &decoded); err != nil { - return nil, fmt.Errorf("decode prompt debug parameters: %w", err) + return nil, fmt.Errorf("prompt debug parameters must be a JSON object") } - redactPromptDebugValue(decoded) - encoded, err := json.Marshal(decoded) + parameters := make(map[string]json.RawMessage) + for _, field := range []struct { + name string + value func(json.RawMessage) bool + }{ + {name: "temperature", value: promptDebugJSONNumber}, + {name: "max_tokens", value: promptDebugJSONInteger}, + {name: "top_p", value: promptDebugJSONNumber}, + {name: "timeout_seconds", value: promptDebugJSONInteger}, + {name: "service_tier", value: promptDebugJSONString}, + {name: "reasoning_effort", value: promptDebugJSONString}, + } { + value, ok := decoded[field.name] + if ok && field.value(value) { + parameters[field.name] = append(json.RawMessage(nil), value...) + } + } + encoded, err := json.Marshal(parameters) if err != nil { - return nil, fmt.Errorf("encode prompt debug parameters: %w", err) + return nil, fmt.Errorf("encode safe prompt debug parameters: %w", err) } return encoded, nil } -func redactPromptDebugValue(value any) { - switch typed := value.(type) { - case map[string]any: - for key, item := range typed { - if isPromptDebugSecretKey(key) { - typed[key] = "[redacted]" - continue - } - redactPromptDebugValue(item) - } - case []any: - for _, item := range typed { - redactPromptDebugValue(item) - } - } +func promptDebugJSONNumber(value json.RawMessage) bool { + var decoded float64 + return json.Unmarshal(value, &decoded) == nil } -func isPromptDebugSecretKey(key string) bool { - normalized := strings.NewReplacer("_", "", "-", "", " ", "").Replace(strings.ToLower(key)) - return strings.Contains(normalized, "credential") || strings.Contains(normalized, "secret") || strings.Contains(normalized, "password") || strings.Contains(normalized, "token") || strings.Contains(normalized, "apikey") || strings.Contains(normalized, "authorization") +func promptDebugJSONInteger(value json.RawMessage) bool { + var decoded int + return json.Unmarshal(value, &decoded) == nil +} + +func promptDebugJSONString(value json.RawMessage) bool { + var decoded string + return json.Unmarshal(value, &decoded) == nil } diff --git a/internal/promptdebug/debug_writer_test.go b/internal/promptdebug/debug_writer_test.go index ee76c51..8590593 100644 --- a/internal/promptdebug/debug_writer_test.go +++ b/internal/promptdebug/debug_writer_test.go @@ -42,7 +42,7 @@ func TestPromptDebugWriterWritesIsolatedArtifacts(t *testing.T) { t.Fatalf("WriteExecution() directory = %q, want %q", executionDir, preparationDir) } preparationData := readPromptDebugFile(t, filepath.Join(preparationDir, "preparation.json")) - for _, want := range []string{"weatherreporter.prompt_preparation_debug.v2", "Use the supplied weather facts.", `"type": "object"`, "https://llm.example.test/v1/chat?api_key=%5Bredacted%5D", `"temperature": 0.2`, `"api_key": "[redacted]"`} { + for _, want := range []string{"weatherreporter.prompt_preparation_debug.v2", "Use the supplied weather facts.", `"type": "object"`, "https://llm.example.test", `"temperature": 0.2`} { if !strings.Contains(string(preparationData), want) { t.Fatalf("preparation debug artifact missing %q:\n%s", want, preparationData) } @@ -66,6 +66,79 @@ func TestPromptDebugWriterWritesIsolatedArtifacts(t *testing.T) { } } +func TestPromptDebugWriterProjectsProviderConfigurationSafely(t *testing.T) { + const marker = "private-debug-marker" + tests := []struct { + name string + endpoint string + }{ + { + name: "user info signed path and access key query", + endpoint: "https://operator:" + marker + "@llm.example.test/signed/" + marker + "?access-key=" + marker + "&Signature=" + marker, + }, + { + name: "case and separator query aliases", + endpoint: "https://llm.example.test/" + marker + "?X-Amz-Signature=" + marker + "&AUTH=" + marker + "&session_cookie=" + marker, + }, + } + parameters := []byte(`{ + "temperature": 0.2, + "max_tokens": 400, + "top_p": 0.9, + "timeout_seconds": 30, + "service_tier": "flex", + "reasoning_effort": "high", + "access-key": "private-debug-marker", + "signature": "private-debug-marker", + "auth": "private-debug-marker", + "cookie": "private-debug-marker", + "nested": {"X Api Key": "private-debug-marker"}, + "extra_params": {"authorization": "private-debug-marker"} +}`) + + for index, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + writer, err := NewPromptDebugWriter(filepath.Join(t.TempDir(), "debug")) + if err != nil { + t.Fatalf("NewPromptDebugWriter() error = %v", err) + } + ref := promptDebugRef() + ref.RunID = fmt.Sprintf("run-%d", index) + directory, err := writer.WritePreparation(ref, promptDebugPreparationFixture(), &promptexec.PreparationDebug{ + Endpoint: tt.endpoint, + ParametersJSON: parameters, + }) + if err != nil { + t.Fatalf("WritePreparation() error = %v", err) + } + data := readPromptDebugFile(t, filepath.Join(directory, "preparation.json")) + text := string(data) + if strings.Contains(text, marker) || strings.Contains(text, "signed") || strings.Contains(text, "access-key") || strings.Contains(text, "signature") || strings.Contains(text, "cookie") { + t.Fatalf("preparation debug artifact leaked provider configuration:\n%s", data) + } + for _, want := range []string{"https://llm.example.test", `"temperature": 0.2`, `"max_tokens": 400`, `"top_p": 0.9`, `"timeout_seconds": 30`, `"service_tier": "flex"`, `"reasoning_effort": "high"`} { + if !strings.Contains(text, want) { + t.Fatalf("preparation debug artifact missing safe value %q:\n%s", want, data) + } + } + }) + } +} + +func TestPromptDebugWriterDoesNotProjectParameterValuesInErrors(t *testing.T) { + const marker = "private-debug-marker" + writer, err := NewPromptDebugWriter(filepath.Join(t.TempDir(), "debug")) + if err != nil { + t.Fatalf("NewPromptDebugWriter() error = %v", err) + } + _, err = writer.WritePreparation(promptDebugRef(), promptDebugPreparationFixture(), &promptexec.PreparationDebug{ + ParametersJSON: []byte(`{"access-key":"` + marker), + }) + if err == nil || strings.Contains(err.Error(), marker) { + t.Fatalf("WritePreparation() error = %v, want safe parameter projection error", err) + } +} + func TestPromptDebugWriterAtomicallyReplacesArtifacts(t *testing.T) { writer, err := NewPromptDebugWriter(filepath.Join(t.TempDir(), "debug")) if err != nil {