eric/weatherreporter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

Harden prompt debug redaction

Browse Source
This commit is contained in:
Eric Rakestraw
2026-08-13 01:19:35 +00:00
parent 27849813db
commit a38d291f63
6 changed files with 148 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
docs/integrations/promptkit.md
View File

@@ -29,6 +29,10 @@ Profiles that require a direct API key are unsupported; a profile that reports `
Promptkit receives the YAML data package as an inline input and returns structured JSON that Weatherreporter validates before rendering its own Markdown template. The package contains only reviewed prompt-facing warning summaries, never source transport or provenance details. Safe active provenance remains in memory. Content-rich diagnostics are opt-in through `--llm-debug-dir`; see [operations](../operations.md) for retention and permissions.
When capture is enabled, its preparation artifact projects a provider endpoint
to its scheme and host and retains only reviewed execution settings. Provider
extras and URL user information, paths, queries, and fragments are omitted.
## Comparison Execution
For `compare`, Weatherreporter validates one exact prompt and every explicitly

4
docs/operations.md
View File

@@ -143,6 +143,10 @@ remain distinct. Normal output, summaries, and routine logs omit that sensitive
content. Debug capture is never created for an ordinary command without
`--llm-debug-dir`.
Preparation captures retain only the provider endpoint origin and reviewed
execution settings. URL user information, paths, queries, fragments, and
unrecognized provider parameters are omitted.
If capture creation or writing fails, the affected run fails rather than
silently continuing without the requested diagnostics.

14
internal/adapters/promptkit/adapter.go
View File

@@ -247,13 +247,12 @@ func copyInputHashes(values map[string]string) map[string]string {
func marshalDebugParameters(value promptkit.ExecutionTarget) []byte {
parameters := struct {
Temperature float64 `json:"temperature"`
MaxTokens int `json:"max_tokens"`
TopP float64 `json:"top_p"`
TimeoutSeconds int `json:"timeout_seconds"`
ServiceTier string `json:"service_tier"`
ReasoningEffort string `json:"reasoning_effort"`
ExtraParams map[string]any `json:"extra_params"`
Temperature float64 `json:"temperature"`
MaxTokens int `json:"max_tokens"`
TopP float64 `json:"top_p"`
TimeoutSeconds int `json:"timeout_seconds"`
ServiceTier string `json:"service_tier"`
ReasoningEffort string `json:"reasoning_effort"`
}{
Temperature: value.Temperature,
MaxTokens: value.MaxTokens,
@@ -261,7 +260,6 @@ func marshalDebugParameters(value promptkit.ExecutionTarget) []byte {
TimeoutSeconds: value.TimeoutSeconds,
ServiceTier: value.ServiceTier,
ReasoningEffort: value.ReasoningEffort,
ExtraParams: value.ExtraParams,
}
data, _ := json.Marshal(parameters)
return data

24
internal/adapters/promptkit/adapter_test.go
View File

@@ -316,6 +316,30 @@ func TestExecuteCapturesSensitiveDebugOnlyWhenRequested(t *testing.T) {
}
}
func TestMarshalDebugParametersOmitsProviderExtras(t *testing.T) {
const marker = "private-debug-marker"
parameters := string(marshalDebugParameters(promptkit.ExecutionTarget{
Temperature: 0.2,
MaxTokens: 400,
TopP: 0.9,
TimeoutSeconds: 30,
ServiceTier: "flex",
ReasoningEffort: "high",
ExtraParams: map[string]any{
"access-key": marker,
"signature": marker,
},
}))
if strings.Contains(parameters, marker) || strings.Contains(parameters, "extra_params") {
t.Fatalf("debug parameters leaked provider extras: %s", parameters)
}
for _, want := range []string{`"temperature":0.2`, `"max_tokens":400`, `"top_p":0.9`, `"timeout_seconds":30`, `"service_tier":"flex"`, `"reasoning_effort":"high"`} {
if !strings.Contains(parameters, want) {
t.Fatalf("debug parameters missing safe value %q: %s", want, parameters)
}
}
}
func TestExecuteCallbackFailurePreventsGeneration(t *testing.T) {
client := &fakeClient{response: validResponse()}
adapter := newTestAdapter(t, client)

72
internal/promptdebug/debug_writer.go
View File

@@ -171,7 +171,7 @@ func (w *PromptDebugWriter) WritePreparation(ref PromptDebugRef, preparation pro
artifact.RenderedMessages = promptDebugMessages(debug.RenderedMessages)
artifact.StructuredSchema = copyRawJSON(debug.StructuredSchema)
artifact.Endpoint = safePromptDebugEndpoint(debug.Endpoint)
parameters, err := redactPromptDebugParameters(debug.ParametersJSON)
parameters, err := safePromptDebugParameters(debug.ParametersJSON)
if err != nil {
return "", err
}
@@ -404,55 +404,55 @@ func copyRawJSON(value []byte) json.RawMessage {
func safePromptDebugEndpoint(value string) string {
parsed, err := url.Parse(value)
if err != nil {
if err != nil || parsed.Scheme == "" || parsed.Host == "" {
return ""
}
parsed.User = nil
parameters := parsed.Query()
for key := range parameters {
if isPromptDebugSecretKey(key) {
parameters[key] = []string{"[redacted]"}
}
}
parsed.RawQuery = parameters.Encode()
parsed.Fragment = ""
return parsed.String()
return (&url.URL{Scheme: parsed.Scheme, Host: parsed.Host}).String()
}
func redactPromptDebugParameters(value []byte) (json.RawMessage, error) {
func safePromptDebugParameters(value []byte) (json.RawMessage, error) {
if len(value) == 0 {
return nil, nil
}
var decoded any
var decoded map[string]json.RawMessage
if err := json.Unmarshal(value, &decoded); err != nil {
return nil, fmt.Errorf("decode prompt debug parameters: %w", err)
return nil, fmt.Errorf("prompt debug parameters must be a JSON object")
}
redactPromptDebugValue(decoded)
encoded, err := json.Marshal(decoded)
parameters := make(map[string]json.RawMessage)
for _, field := range []struct {
name string
value func(json.RawMessage) bool
}{
{name: "temperature", value: promptDebugJSONNumber},
{name: "max_tokens", value: promptDebugJSONInteger},
{name: "top_p", value: promptDebugJSONNumber},
{name: "timeout_seconds", value: promptDebugJSONInteger},
{name: "service_tier", value: promptDebugJSONString},
{name: "reasoning_effort", value: promptDebugJSONString},
} {
value, ok := decoded[field.name]
if ok && field.value(value) {
parameters[field.name] = append(json.RawMessage(nil), value...)
}
}
encoded, err := json.Marshal(parameters)
if err != nil {
return nil, fmt.Errorf("encode prompt debug parameters: %w", err)
return nil, fmt.Errorf("encode safe prompt debug parameters: %w", err)
}
return encoded, nil
}
func redactPromptDebugValue(value any) {
switch typed := value.(type) {
case map[string]any:
for key, item := range typed {
if isPromptDebugSecretKey(key) {
typed[key] = "[redacted]"
continue
}
redactPromptDebugValue(item)
}
case []any:
for _, item := range typed {
redactPromptDebugValue(item)
}
}
func promptDebugJSONNumber(value json.RawMessage) bool {
var decoded float64
return json.Unmarshal(value, &decoded) == nil
}
func isPromptDebugSecretKey(key string) bool {
normalized := strings.NewReplacer("_", "", "-", "", " ", "").Replace(strings.ToLower(key))
return strings.Contains(normalized, "credential") || strings.Contains(normalized, "secret") || strings.Contains(normalized, "password") || strings.Contains(normalized, "token") || strings.Contains(normalized, "apikey") || strings.Contains(normalized, "authorization")
func promptDebugJSONInteger(value json.RawMessage) bool {
var decoded int
return json.Unmarshal(value, &decoded) == nil
}
func promptDebugJSONString(value json.RawMessage) bool {
var decoded string
return json.Unmarshal(value, &decoded) == nil
}

75
internal/promptdebug/debug_writer_test.go
View File

@@ -42,7 +42,7 @@ func TestPromptDebugWriterWritesIsolatedArtifacts(t *testing.T) {
t.Fatalf("WriteExecution() directory = %q, want %q", executionDir, preparationDir)
}
preparationData := readPromptDebugFile(t, filepath.Join(preparationDir, "preparation.json"))
for _, want := range []string{"weatherreporter.prompt_preparation_debug.v2", "Use the supplied weather facts.", `"type": "object"`, "https://llm.example.test/v1/chat?api_key=%5Bredacted%5D", `"temperature": 0.2`, `"api_key": "[redacted]"`} {
for _, want := range []string{"weatherreporter.prompt_preparation_debug.v2", "Use the supplied weather facts.", `"type": "object"`, "https://llm.example.test", `"temperature": 0.2`} {
if !strings.Contains(string(preparationData), want) {
t.Fatalf("preparation debug artifact missing %q:\n%s", want, preparationData)
}
@@ -66,6 +66,79 @@ func TestPromptDebugWriterWritesIsolatedArtifacts(t *testing.T) {
}
}
func TestPromptDebugWriterProjectsProviderConfigurationSafely(t *testing.T) {
const marker = "private-debug-marker"
tests := []struct {
name string
endpoint string
}{
{
name: "user info signed path and access key query",
endpoint: "https://operator:" + marker + "@llm.example.test/signed/" + marker + "?access-key=" + marker + "&Signature=" + marker,
},
{
name: "case and separator query aliases",
endpoint: "https://llm.example.test/" + marker + "?X-Amz-Signature=" + marker + "&AUTH=" + marker + "&session_cookie=" + marker,
},
}
parameters := []byte(`{
"temperature": 0.2,
"max_tokens": 400,
"top_p": 0.9,
"timeout_seconds": 30,
"service_tier": "flex",
"reasoning_effort": "high",
"access-key": "private-debug-marker",
"signature": "private-debug-marker",
"auth": "private-debug-marker",
"cookie": "private-debug-marker",
"nested": {"X Api Key": "private-debug-marker"},
"extra_params": {"authorization": "private-debug-marker"}
}`)
for index, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
writer, err := NewPromptDebugWriter(filepath.Join(t.TempDir(), "debug"))
if err != nil {
t.Fatalf("NewPromptDebugWriter() error = %v", err)
}
ref := promptDebugRef()
ref.RunID = fmt.Sprintf("run-%d", index)
directory, err := writer.WritePreparation(ref, promptDebugPreparationFixture(), &promptexec.PreparationDebug{
Endpoint: tt.endpoint,
ParametersJSON: parameters,
})
if err != nil {
t.Fatalf("WritePreparation() error = %v", err)
}
data := readPromptDebugFile(t, filepath.Join(directory, "preparation.json"))
text := string(data)
if strings.Contains(text, marker) || strings.Contains(text, "signed") || strings.Contains(text, "access-key") || strings.Contains(text, "signature") || strings.Contains(text, "cookie") {
t.Fatalf("preparation debug artifact leaked provider configuration:\n%s", data)
}
for _, want := range []string{"https://llm.example.test", `"temperature": 0.2`, `"max_tokens": 400`, `"top_p": 0.9`, `"timeout_seconds": 30`, `"service_tier": "flex"`, `"reasoning_effort": "high"`} {
if !strings.Contains(text, want) {
t.Fatalf("preparation debug artifact missing safe value %q:\n%s", want, data)
}
}
})
}
}
func TestPromptDebugWriterDoesNotProjectParameterValuesInErrors(t *testing.T) {
const marker = "private-debug-marker"
writer, err := NewPromptDebugWriter(filepath.Join(t.TempDir(), "debug"))
if err != nil {
t.Fatalf("NewPromptDebugWriter() error = %v", err)
}
_, err = writer.WritePreparation(promptDebugRef(), promptDebugPreparationFixture(), &promptexec.PreparationDebug{
ParametersJSON: []byte(`{"access-key":"` + marker),
})
if err == nil || strings.Contains(err.Error(), marker) {
t.Fatalf("WritePreparation() error = %v, want safe parameter projection error", err)
}
}
func TestPromptDebugWriterAtomicallyReplacesArtifacts(t *testing.T) {
writer, err := NewPromptDebugWriter(filepath.Join(t.TempDir(), "debug"))
if err != nil {