377 lines
14 KiB
Go
377 lines
14 KiB
Go
//go:build unix
|
|
|
|
package promptdebug
|
|
|
|
import (
|
|
"fmt"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec"
|
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/report"
|
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/testutil"
|
|
)
|
|
|
|
func TestPromptDebugWriterWritesIsolatedArtifacts(t *testing.T) {
|
|
root := filepath.Join(t.TempDir(), "operator-debug")
|
|
writer, err := NewPromptDebugWriter(root)
|
|
if err != nil {
|
|
t.Fatalf("NewPromptDebugWriter() error = %v", err)
|
|
}
|
|
ref := promptDebugRef()
|
|
preparationDir, err := writer.WritePreparation(ref, promptDebugPreparationFixture(), &promptexec.PreparationDebug{
|
|
RenderedMessages: []promptexec.RenderedMessage{{Role: "system", Content: "Use the supplied weather facts."}},
|
|
StructuredSchema: []byte(`{"type":"object","required":["summary"]}`),
|
|
Endpoint: "https://direct:resolved-secret-value@llm.example.test/v1/chat?api_key=resolved-secret-value",
|
|
ParametersJSON: []byte(`{"temperature":0.2,"api_key":"resolved-secret-value"}`),
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("WritePreparation() error = %v", err)
|
|
}
|
|
wantDirectory := filepath.Join(root, "daily", "2026-05-29", "run-123")
|
|
if preparationDir != wantDirectory {
|
|
t.Fatalf("WritePreparation() directory = %q, want %q", preparationDir, wantDirectory)
|
|
}
|
|
executionDir, err := writer.WriteExecution(ref, promptDebugExecutionFixture())
|
|
if err != nil {
|
|
t.Fatalf("WriteExecution() error = %v", err)
|
|
}
|
|
if executionDir != preparationDir {
|
|
t.Fatalf("WriteExecution() directory = %q, want %q", executionDir, preparationDir)
|
|
}
|
|
preparationData := readPromptDebugFile(t, filepath.Join(preparationDir, "preparation.json"))
|
|
for _, want := range []string{"weatherreporter.prompt_preparation_debug.v2", "Use the supplied weather facts.", `"type": "object"`, "https://llm.example.test", `"temperature": 0.2`} {
|
|
if !strings.Contains(string(preparationData), want) {
|
|
t.Fatalf("preparation debug artifact missing %q:\n%s", want, preparationData)
|
|
}
|
|
}
|
|
executionData := readPromptDebugFile(t, filepath.Join(executionDir, "execution.json"))
|
|
for _, want := range []string{"weatherreporter.prompt_execution_debug.v2", "Generated forecast prose.", "validation details", `"status": "passed"`} {
|
|
if !strings.Contains(string(executionData), want) {
|
|
t.Fatalf("execution debug artifact missing %q:\n%s", want, executionData)
|
|
}
|
|
}
|
|
for _, data := range [][]byte{preparationData, executionData} {
|
|
if strings.Contains(string(data), "credential") || strings.Contains(string(data), "resolved-secret-value") || strings.Contains(string(data), "dataPackagePath") {
|
|
t.Fatalf("debug artifact contains credentials:\n%s", data)
|
|
}
|
|
}
|
|
assertPromptDebugMode(t, root, debugDirectoryMode)
|
|
assertPromptDebugMode(t, preparationDir, debugDirectoryMode)
|
|
assertPromptDebugMode(t, filepath.Join(preparationDir, "preparation.json"), debugFileMode)
|
|
assertPromptDebugMode(t, filepath.Join(executionDir, "execution.json"), debugFileMode)
|
|
}
|
|
|
|
func TestPromptDebugWriterProjectsProviderConfigurationSafely(t *testing.T) {
|
|
const marker = "private-debug-marker"
|
|
tests := []struct {
|
|
name string
|
|
endpoint string
|
|
}{
|
|
{
|
|
name: "user info signed path and access key query",
|
|
endpoint: "https://operator:" + marker + "@llm.example.test/signed/" + marker + "?access-key=" + marker + "&Signature=" + marker,
|
|
},
|
|
{
|
|
name: "case and separator query aliases",
|
|
endpoint: "https://llm.example.test/" + marker + "?X-Amz-Signature=" + marker + "&AUTH=" + marker + "&session_cookie=" + marker,
|
|
},
|
|
}
|
|
parameters := []byte(`{
|
|
"temperature": 0.2,
|
|
"max_tokens": 400,
|
|
"top_p": 0.9,
|
|
"timeout_seconds": 30,
|
|
"service_tier": "flex",
|
|
"reasoning_effort": "high",
|
|
"access-key": "private-debug-marker",
|
|
"signature": "private-debug-marker",
|
|
"auth": "private-debug-marker",
|
|
"cookie": "private-debug-marker",
|
|
"nested": {"X Api Key": "private-debug-marker"},
|
|
"extra_params": {"authorization": "private-debug-marker"}
|
|
}`)
|
|
|
|
for index, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
writer, err := NewPromptDebugWriter(filepath.Join(t.TempDir(), "debug"))
|
|
if err != nil {
|
|
t.Fatalf("NewPromptDebugWriter() error = %v", err)
|
|
}
|
|
ref := promptDebugRef()
|
|
ref.RunID = fmt.Sprintf("run-%d", index)
|
|
directory, err := writer.WritePreparation(ref, promptDebugPreparationFixture(), &promptexec.PreparationDebug{
|
|
Endpoint: tt.endpoint,
|
|
ParametersJSON: parameters,
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("WritePreparation() error = %v", err)
|
|
}
|
|
data := readPromptDebugFile(t, filepath.Join(directory, "preparation.json"))
|
|
text := string(data)
|
|
if strings.Contains(text, marker) || strings.Contains(text, "signed") || strings.Contains(text, "access-key") || strings.Contains(text, "signature") || strings.Contains(text, "cookie") {
|
|
t.Fatalf("preparation debug artifact leaked provider configuration:\n%s", data)
|
|
}
|
|
for _, want := range []string{"https://llm.example.test", `"temperature": 0.2`, `"max_tokens": 400`, `"top_p": 0.9`, `"timeout_seconds": 30`, `"service_tier": "flex"`, `"reasoning_effort": "high"`} {
|
|
if !strings.Contains(text, want) {
|
|
t.Fatalf("preparation debug artifact missing safe value %q:\n%s", want, data)
|
|
}
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestPromptDebugWriterDoesNotProjectParameterValuesInErrors(t *testing.T) {
|
|
const marker = "private-debug-marker"
|
|
writer, err := NewPromptDebugWriter(filepath.Join(t.TempDir(), "debug"))
|
|
if err != nil {
|
|
t.Fatalf("NewPromptDebugWriter() error = %v", err)
|
|
}
|
|
_, err = writer.WritePreparation(promptDebugRef(), promptDebugPreparationFixture(), &promptexec.PreparationDebug{
|
|
ParametersJSON: []byte(`{"access-key":"` + marker),
|
|
})
|
|
if err == nil || strings.Contains(err.Error(), marker) {
|
|
t.Fatalf("WritePreparation() error = %v, want safe parameter projection error", err)
|
|
}
|
|
}
|
|
|
|
func TestPromptDebugWriterAtomicallyReplacesArtifacts(t *testing.T) {
|
|
writer, err := NewPromptDebugWriter(filepath.Join(t.TempDir(), "debug"))
|
|
if err != nil {
|
|
t.Fatalf("NewPromptDebugWriter() error = %v", err)
|
|
}
|
|
ref := promptDebugRef()
|
|
first := promptDebugPreparationFixture()
|
|
first.PromptHash = "old-hash"
|
|
if _, err := writer.WritePreparation(ref, first, nil); err != nil {
|
|
t.Fatalf("first WritePreparation() error = %v", err)
|
|
}
|
|
second := promptDebugPreparationFixture()
|
|
second.PromptHash = "new-hash"
|
|
directory, err := writer.WritePreparation(ref, second, nil)
|
|
if err != nil {
|
|
t.Fatalf("second WritePreparation() error = %v", err)
|
|
}
|
|
data := readPromptDebugFile(t, filepath.Join(directory, "preparation.json"))
|
|
if strings.Contains(string(data), "old-hash") || !strings.Contains(string(data), "new-hash") {
|
|
t.Fatalf("replacement artifact = %s", data)
|
|
}
|
|
temporary, err := filepath.Glob(filepath.Join(directory, ".preparation.json.*.tmp"))
|
|
if err != nil || len(temporary) != 0 {
|
|
t.Fatalf("temporary files = %v, %v", temporary, err)
|
|
}
|
|
}
|
|
|
|
func TestPromptDebugWriterCreatesSharedMissingAncestorsConcurrently(t *testing.T) {
|
|
root := filepath.Join(t.TempDir(), "debug")
|
|
writer, err := NewPromptDebugWriter(root)
|
|
if err != nil {
|
|
t.Fatalf("NewPromptDebugWriter() error = %v", err)
|
|
}
|
|
|
|
const writerCount = 8
|
|
start := make(chan struct{})
|
|
type writeResult struct {
|
|
directory string
|
|
err error
|
|
}
|
|
results := make(chan writeResult, writerCount)
|
|
var writers sync.WaitGroup
|
|
for index := 0; index < writerCount; index++ {
|
|
writers.Add(1)
|
|
go func(index int) {
|
|
defer writers.Done()
|
|
<-start
|
|
directory, err := writer.WritePreparation(PromptDebugRef{
|
|
ReportID: report.Daily, ValidDate: "2026-05-29", RunID: fmt.Sprintf("run-%02d", index),
|
|
}, promptDebugPreparationFixture(), nil)
|
|
results <- writeResult{directory: directory, err: err}
|
|
}(index)
|
|
}
|
|
close(start)
|
|
|
|
finished := make(chan struct{})
|
|
go func() {
|
|
writers.Wait()
|
|
close(finished)
|
|
}()
|
|
select {
|
|
case <-finished:
|
|
case <-time.After(5 * time.Second):
|
|
t.Fatal("concurrent prompt debug writes did not finish")
|
|
}
|
|
close(results)
|
|
|
|
directories := map[string]struct{}{}
|
|
for result := range results {
|
|
if result.err != nil {
|
|
t.Fatalf("WritePreparation() error = %v", result.err)
|
|
}
|
|
if _, duplicate := directories[result.directory]; duplicate {
|
|
t.Fatalf("duplicate debug directory %q", result.directory)
|
|
}
|
|
directories[result.directory] = struct{}{}
|
|
if _, err := os.Stat(filepath.Join(result.directory, "preparation.json")); err != nil {
|
|
t.Fatalf("preparation artifact %q: %v", result.directory, err)
|
|
}
|
|
}
|
|
if len(directories) != writerCount {
|
|
t.Fatalf("debug directories = %#v", directories)
|
|
}
|
|
assertPromptDebugMode(t, root, debugDirectoryMode)
|
|
assertPromptDebugMode(t, filepath.Join(root, "daily"), debugDirectoryMode)
|
|
assertPromptDebugMode(t, filepath.Join(root, "daily", "2026-05-29"), debugDirectoryMode)
|
|
for directory := range directories {
|
|
assertPromptDebugMode(t, directory, debugDirectoryMode)
|
|
assertPromptDebugMode(t, filepath.Join(directory, "preparation.json"), debugFileMode)
|
|
}
|
|
}
|
|
|
|
func TestPromptDebugWriterKeepsWritesAnchoredToOpenedRoot(t *testing.T) {
|
|
parent := t.TempDir()
|
|
root := filepath.Join(parent, "debug")
|
|
writer, err := NewPromptDebugWriter(root)
|
|
if err != nil {
|
|
t.Fatalf("NewPromptDebugWriter() error = %v", err)
|
|
}
|
|
defer writer.Close()
|
|
|
|
anchoredRoot := filepath.Join(parent, "anchored-debug")
|
|
outside := filepath.Join(parent, "outside")
|
|
if err := os.Mkdir(outside, debugDirectoryMode); err != nil {
|
|
t.Fatalf("create outside directory: %v", err)
|
|
}
|
|
if err := os.Rename(root, anchoredRoot); err != nil {
|
|
t.Fatalf("replace opened root: %v", err)
|
|
}
|
|
testutil.RequireSymlink(t, outside, root)
|
|
|
|
ref := promptDebugRef()
|
|
ref.RunID = "run-anchored"
|
|
if _, err := writer.WritePreparation(ref, promptDebugPreparationFixture(), nil); err != nil {
|
|
t.Fatalf("WritePreparation() error = %v", err)
|
|
}
|
|
anchoredArtifact := filepath.Join(anchoredRoot, "daily", "2026-05-29", "run-anchored", "preparation.json")
|
|
if _, err := os.Stat(anchoredArtifact); err != nil {
|
|
t.Fatalf("anchored preparation artifact: %v", err)
|
|
}
|
|
entries, err := os.ReadDir(outside)
|
|
if err != nil {
|
|
t.Fatalf("read outside directory: %v", err)
|
|
}
|
|
if len(entries) != 0 {
|
|
t.Fatalf("outside directory received debug artifacts: %#v", entries)
|
|
}
|
|
}
|
|
|
|
func TestPromptDebugWriterDisabledDoesNotAccessFilesystem(t *testing.T) {
|
|
writer, err := NewPromptDebugWriter("")
|
|
if err != nil {
|
|
t.Fatalf("NewPromptDebugWriter() error = %v", err)
|
|
}
|
|
if writer.Enabled() {
|
|
t.Fatal("disabled writer reports enabled")
|
|
}
|
|
directory, err := writer.WritePreparation(PromptDebugRef{ReportID: report.ID("../unsafe")}, promptDebugPreparationFixture(), nil)
|
|
if err != nil || directory != "" {
|
|
t.Fatalf("disabled WritePreparation() = %q, %v", directory, err)
|
|
}
|
|
}
|
|
|
|
func TestPromptDebugWriterRejectsUnsafeRootsAndReferences(t *testing.T) {
|
|
root := t.TempDir()
|
|
if _, err := NewPromptDebugWriter("relative-debug"); err == nil {
|
|
t.Fatal("NewPromptDebugWriter(relative) error = nil")
|
|
}
|
|
nonDirectory := filepath.Join(root, "not-a-directory")
|
|
if err := os.WriteFile(nonDirectory, []byte("x"), 0o600); err != nil {
|
|
t.Fatalf("write non-directory root: %v", err)
|
|
}
|
|
if _, err := NewPromptDebugWriter(nonDirectory); err == nil {
|
|
t.Fatal("NewPromptDebugWriter(file) error = nil")
|
|
}
|
|
target := filepath.Join(root, "target")
|
|
link := filepath.Join(root, "root-link")
|
|
if err := os.Mkdir(target, debugDirectoryMode); err != nil {
|
|
t.Fatalf("create root symlink target: %v", err)
|
|
}
|
|
testutil.RequireSymlink(t, target, link)
|
|
if _, err := NewPromptDebugWriter(link); err == nil {
|
|
t.Fatal("NewPromptDebugWriter(symlink) error = nil")
|
|
}
|
|
|
|
writer, err := NewPromptDebugWriter(filepath.Join(root, "debug"))
|
|
if err != nil {
|
|
t.Fatalf("NewPromptDebugWriter() error = %v", err)
|
|
}
|
|
for _, ref := range []PromptDebugRef{
|
|
{ReportID: report.ID("../daily"), ValidDate: "2026-05-29", RunID: "run-123"},
|
|
{ReportID: report.Daily, ValidDate: "2026/05/29", RunID: "run-123"},
|
|
{ReportID: report.Daily, ValidDate: "2026-05-29", RunID: "/run-123"},
|
|
{ReportID: report.Daily, ValidDate: "not-a-date", RunID: "run-123"},
|
|
} {
|
|
if _, err := writer.WritePreparation(ref, promptDebugPreparationFixture(), nil); err == nil {
|
|
t.Fatalf("WritePreparation(%#v) error = nil", ref)
|
|
}
|
|
}
|
|
outside := filepath.Join(root, "outside")
|
|
if err := os.Mkdir(outside, debugDirectoryMode); err != nil {
|
|
t.Fatalf("create symlink component target: %v", err)
|
|
}
|
|
testutil.RequireSymlink(t, outside, filepath.Join(root, "debug", "daily"))
|
|
if _, err := writer.WritePreparation(promptDebugRef(), promptDebugPreparationFixture(), nil); err == nil {
|
|
t.Fatal("WritePreparation(symlink component) error = nil")
|
|
}
|
|
}
|
|
|
|
func promptDebugRef() PromptDebugRef {
|
|
return PromptDebugRef{ReportID: report.Daily, ValidDate: "2026-05-29", RunID: "run-123"}
|
|
}
|
|
|
|
func promptDebugPreparationFixture() promptexec.Preparation {
|
|
startedAt := time.Date(2026, 5, 29, 12, 0, 0, 0, time.UTC)
|
|
return promptexec.Preparation{
|
|
PromptID: "weather.daily", PromptVersion: "v1", PromptHash: "prompt-hash", RenderedPromptHash: "rendered-hash",
|
|
InputHashes: map[string]string{"data_package": "input-hash"}, ProfileID: "local", BackendID: "local", ModelName: "weather-model",
|
|
Output: promptexec.OutputContract{Format: "json_schema", ValidationMode: "strict", SchemaPath: "schemas/daily.json"},
|
|
StartedAt: startedAt, EndedAt: startedAt.Add(time.Second), Duration: time.Second,
|
|
}
|
|
}
|
|
|
|
func promptDebugExecutionFixture() promptexec.Execution {
|
|
startedAt := time.Date(2026, 5, 29, 12, 0, 0, 0, time.UTC)
|
|
return promptexec.Execution{
|
|
RunID: "run-123", PromptID: "weather.daily", PromptVersion: "v1", PromptHash: "prompt-hash", RenderedPromptHash: "rendered-hash",
|
|
InputHashes: map[string]string{"data_package": "input-hash"}, ProfileID: "local", BackendID: "local", ModelName: "weather-model",
|
|
GeneratedHash: "generated-hash", Usage: promptexec.TokenUsage{PromptTokens: 10, CompletionTokens: 5, TotalTokens: 15},
|
|
StartedAt: startedAt, EndedAt: startedAt.Add(time.Second), Duration: time.Second,
|
|
Validation: promptexec.NewValidation(promptexec.ValidationPassed, "strict", "schemas/daily.json", []string{"validation details"}),
|
|
RawOutput: []byte("Generated forecast prose."),
|
|
Debug: &promptexec.ExecutionDebug{ValidationDiagnostics: []string{"validation details"}},
|
|
}
|
|
}
|
|
|
|
func readPromptDebugFile(t *testing.T, path string) []byte {
|
|
t.Helper()
|
|
data, err := os.ReadFile(path)
|
|
if err != nil {
|
|
t.Fatalf("read %q: %v", path, err)
|
|
}
|
|
return data
|
|
}
|
|
|
|
func assertPromptDebugMode(t *testing.T, path string, want os.FileMode) {
|
|
t.Helper()
|
|
info, err := os.Stat(path)
|
|
if err != nil {
|
|
t.Fatalf("stat %q: %v", path, err)
|
|
}
|
|
if got := info.Mode().Perm(); got != want {
|
|
t.Fatalf("mode for %q = %#o, want %#o", path, got, want)
|
|
}
|
|
}
|