//go:build unix package promptdebug import ( "fmt" "os" "path/filepath" "strings" "sync" "testing" "time" "gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec" "gitea.maximumdirect.net/eric/weatherreporter/internal/report" "gitea.maximumdirect.net/eric/weatherreporter/internal/testutil" ) func TestPromptDebugWriterWritesIsolatedArtifacts(t *testing.T) { root := filepath.Join(t.TempDir(), "operator-debug") writer, err := NewPromptDebugWriter(root) if err != nil { t.Fatalf("NewPromptDebugWriter() error = %v", err) } ref := promptDebugRef() preparationDir, err := writer.WritePreparation(ref, promptDebugPreparationFixture(), &promptexec.PreparationDebug{ RenderedMessages: []promptexec.RenderedMessage{{Role: "system", Content: "Use the supplied weather facts."}}, StructuredSchema: []byte(`{"type":"object","required":["summary"]}`), Endpoint: "https://direct:resolved-secret-value@llm.example.test/v1/chat?api_key=resolved-secret-value", ParametersJSON: []byte(`{"temperature":0.2,"api_key":"resolved-secret-value"}`), }) if err != nil { t.Fatalf("WritePreparation() error = %v", err) } wantDirectory := filepath.Join(root, "daily", "2026-05-29", "run-123") if preparationDir != wantDirectory { t.Fatalf("WritePreparation() directory = %q, want %q", preparationDir, wantDirectory) } executionDir, err := writer.WriteExecution(ref, promptDebugExecutionFixture()) if err != nil { t.Fatalf("WriteExecution() error = %v", err) } if executionDir != preparationDir { t.Fatalf("WriteExecution() directory = %q, want %q", executionDir, preparationDir) } preparationData := readPromptDebugFile(t, filepath.Join(preparationDir, "preparation.json")) for _, want := range []string{"weatherreporter.prompt_preparation_debug.v2", "Use the supplied weather facts.", `"type": "object"`, "https://llm.example.test", `"temperature": 0.2`} { if !strings.Contains(string(preparationData), want) { t.Fatalf("preparation debug artifact missing %q:\n%s", want, preparationData) } } executionData := readPromptDebugFile(t, filepath.Join(executionDir, "execution.json")) for _, want := range []string{"weatherreporter.prompt_execution_debug.v2", "Generated forecast prose.", "validation details", `"status": "passed"`} { if !strings.Contains(string(executionData), want) { t.Fatalf("execution debug artifact missing %q:\n%s", want, executionData) } } for _, data := range [][]byte{preparationData, executionData} { if strings.Contains(string(data), "credential") || strings.Contains(string(data), "resolved-secret-value") || strings.Contains(string(data), "dataPackagePath") { t.Fatalf("debug artifact contains credentials:\n%s", data) } } assertPromptDebugMode(t, root, debugDirectoryMode) assertPromptDebugMode(t, preparationDir, debugDirectoryMode) assertPromptDebugMode(t, filepath.Join(preparationDir, "preparation.json"), debugFileMode) assertPromptDebugMode(t, filepath.Join(executionDir, "execution.json"), debugFileMode) } func TestPromptDebugWriterProjectsProviderConfigurationSafely(t *testing.T) { const marker = "private-debug-marker" tests := []struct { name string endpoint string }{ { name: "user info signed path and access key query", endpoint: "https://operator:" + marker + "@llm.example.test/signed/" + marker + "?access-key=" + marker + "&Signature=" + marker, }, { name: "case and separator query aliases", endpoint: "https://llm.example.test/" + marker + "?X-Amz-Signature=" + marker + "&AUTH=" + marker + "&session_cookie=" + marker, }, } parameters := []byte(`{ "temperature": 0.2, "max_tokens": 400, "top_p": 0.9, "timeout_seconds": 30, "service_tier": "flex", "reasoning_effort": "high", "access-key": "private-debug-marker", "signature": "private-debug-marker", "auth": "private-debug-marker", "cookie": "private-debug-marker", "nested": {"X Api Key": "private-debug-marker"}, "extra_params": {"authorization": "private-debug-marker"} }`) for index, tt := range tests { t.Run(tt.name, func(t *testing.T) { writer, err := NewPromptDebugWriter(filepath.Join(t.TempDir(), "debug")) if err != nil { t.Fatalf("NewPromptDebugWriter() error = %v", err) } ref := promptDebugRef() ref.RunID = fmt.Sprintf("run-%d", index) directory, err := writer.WritePreparation(ref, promptDebugPreparationFixture(), &promptexec.PreparationDebug{ Endpoint: tt.endpoint, ParametersJSON: parameters, }) if err != nil { t.Fatalf("WritePreparation() error = %v", err) } data := readPromptDebugFile(t, filepath.Join(directory, "preparation.json")) text := string(data) if strings.Contains(text, marker) || strings.Contains(text, "signed") || strings.Contains(text, "access-key") || strings.Contains(text, "signature") || strings.Contains(text, "cookie") { t.Fatalf("preparation debug artifact leaked provider configuration:\n%s", data) } for _, want := range []string{"https://llm.example.test", `"temperature": 0.2`, `"max_tokens": 400`, `"top_p": 0.9`, `"timeout_seconds": 30`, `"service_tier": "flex"`, `"reasoning_effort": "high"`} { if !strings.Contains(text, want) { t.Fatalf("preparation debug artifact missing safe value %q:\n%s", want, data) } } }) } } func TestPromptDebugWriterDoesNotProjectParameterValuesInErrors(t *testing.T) { const marker = "private-debug-marker" writer, err := NewPromptDebugWriter(filepath.Join(t.TempDir(), "debug")) if err != nil { t.Fatalf("NewPromptDebugWriter() error = %v", err) } _, err = writer.WritePreparation(promptDebugRef(), promptDebugPreparationFixture(), &promptexec.PreparationDebug{ ParametersJSON: []byte(`{"access-key":"` + marker), }) if err == nil || strings.Contains(err.Error(), marker) { t.Fatalf("WritePreparation() error = %v, want safe parameter projection error", err) } } func TestPromptDebugWriterAtomicallyReplacesArtifacts(t *testing.T) { writer, err := NewPromptDebugWriter(filepath.Join(t.TempDir(), "debug")) if err != nil { t.Fatalf("NewPromptDebugWriter() error = %v", err) } ref := promptDebugRef() first := promptDebugPreparationFixture() first.PromptHash = "old-hash" if _, err := writer.WritePreparation(ref, first, nil); err != nil { t.Fatalf("first WritePreparation() error = %v", err) } second := promptDebugPreparationFixture() second.PromptHash = "new-hash" directory, err := writer.WritePreparation(ref, second, nil) if err != nil { t.Fatalf("second WritePreparation() error = %v", err) } data := readPromptDebugFile(t, filepath.Join(directory, "preparation.json")) if strings.Contains(string(data), "old-hash") || !strings.Contains(string(data), "new-hash") { t.Fatalf("replacement artifact = %s", data) } temporary, err := filepath.Glob(filepath.Join(directory, ".preparation.json.*.tmp")) if err != nil || len(temporary) != 0 { t.Fatalf("temporary files = %v, %v", temporary, err) } } func TestPromptDebugWriterCreatesSharedMissingAncestorsConcurrently(t *testing.T) { root := filepath.Join(t.TempDir(), "debug") writer, err := NewPromptDebugWriter(root) if err != nil { t.Fatalf("NewPromptDebugWriter() error = %v", err) } const writerCount = 8 start := make(chan struct{}) type writeResult struct { directory string err error } results := make(chan writeResult, writerCount) var writers sync.WaitGroup for index := 0; index < writerCount; index++ { writers.Add(1) go func(index int) { defer writers.Done() <-start directory, err := writer.WritePreparation(PromptDebugRef{ ReportID: report.Daily, ValidDate: "2026-05-29", RunID: fmt.Sprintf("run-%02d", index), }, promptDebugPreparationFixture(), nil) results <- writeResult{directory: directory, err: err} }(index) } close(start) finished := make(chan struct{}) go func() { writers.Wait() close(finished) }() select { case <-finished: case <-time.After(5 * time.Second): t.Fatal("concurrent prompt debug writes did not finish") } close(results) directories := map[string]struct{}{} for result := range results { if result.err != nil { t.Fatalf("WritePreparation() error = %v", result.err) } if _, duplicate := directories[result.directory]; duplicate { t.Fatalf("duplicate debug directory %q", result.directory) } directories[result.directory] = struct{}{} if _, err := os.Stat(filepath.Join(result.directory, "preparation.json")); err != nil { t.Fatalf("preparation artifact %q: %v", result.directory, err) } } if len(directories) != writerCount { t.Fatalf("debug directories = %#v", directories) } assertPromptDebugMode(t, root, debugDirectoryMode) assertPromptDebugMode(t, filepath.Join(root, "daily"), debugDirectoryMode) assertPromptDebugMode(t, filepath.Join(root, "daily", "2026-05-29"), debugDirectoryMode) for directory := range directories { assertPromptDebugMode(t, directory, debugDirectoryMode) assertPromptDebugMode(t, filepath.Join(directory, "preparation.json"), debugFileMode) } } func TestPromptDebugWriterKeepsWritesAnchoredToOpenedRoot(t *testing.T) { parent := t.TempDir() root := filepath.Join(parent, "debug") writer, err := NewPromptDebugWriter(root) if err != nil { t.Fatalf("NewPromptDebugWriter() error = %v", err) } defer writer.Close() anchoredRoot := filepath.Join(parent, "anchored-debug") outside := filepath.Join(parent, "outside") if err := os.Mkdir(outside, debugDirectoryMode); err != nil { t.Fatalf("create outside directory: %v", err) } if err := os.Rename(root, anchoredRoot); err != nil { t.Fatalf("replace opened root: %v", err) } testutil.RequireSymlink(t, outside, root) ref := promptDebugRef() ref.RunID = "run-anchored" if _, err := writer.WritePreparation(ref, promptDebugPreparationFixture(), nil); err != nil { t.Fatalf("WritePreparation() error = %v", err) } anchoredArtifact := filepath.Join(anchoredRoot, "daily", "2026-05-29", "run-anchored", "preparation.json") if _, err := os.Stat(anchoredArtifact); err != nil { t.Fatalf("anchored preparation artifact: %v", err) } entries, err := os.ReadDir(outside) if err != nil { t.Fatalf("read outside directory: %v", err) } if len(entries) != 0 { t.Fatalf("outside directory received debug artifacts: %#v", entries) } } func TestPromptDebugWriterDisabledDoesNotAccessFilesystem(t *testing.T) { writer, err := NewPromptDebugWriter("") if err != nil { t.Fatalf("NewPromptDebugWriter() error = %v", err) } if writer.Enabled() { t.Fatal("disabled writer reports enabled") } directory, err := writer.WritePreparation(PromptDebugRef{ReportID: report.ID("../unsafe")}, promptDebugPreparationFixture(), nil) if err != nil || directory != "" { t.Fatalf("disabled WritePreparation() = %q, %v", directory, err) } } func TestPromptDebugWriterRejectsUnsafeRootsAndReferences(t *testing.T) { root := t.TempDir() if _, err := NewPromptDebugWriter("relative-debug"); err == nil { t.Fatal("NewPromptDebugWriter(relative) error = nil") } nonDirectory := filepath.Join(root, "not-a-directory") if err := os.WriteFile(nonDirectory, []byte("x"), 0o600); err != nil { t.Fatalf("write non-directory root: %v", err) } if _, err := NewPromptDebugWriter(nonDirectory); err == nil { t.Fatal("NewPromptDebugWriter(file) error = nil") } target := filepath.Join(root, "target") link := filepath.Join(root, "root-link") if err := os.Mkdir(target, debugDirectoryMode); err != nil { t.Fatalf("create root symlink target: %v", err) } testutil.RequireSymlink(t, target, link) if _, err := NewPromptDebugWriter(link); err == nil { t.Fatal("NewPromptDebugWriter(symlink) error = nil") } writer, err := NewPromptDebugWriter(filepath.Join(root, "debug")) if err != nil { t.Fatalf("NewPromptDebugWriter() error = %v", err) } for _, ref := range []PromptDebugRef{ {ReportID: report.ID("../daily"), ValidDate: "2026-05-29", RunID: "run-123"}, {ReportID: report.Daily, ValidDate: "2026/05/29", RunID: "run-123"}, {ReportID: report.Daily, ValidDate: "2026-05-29", RunID: "/run-123"}, {ReportID: report.Daily, ValidDate: "not-a-date", RunID: "run-123"}, } { if _, err := writer.WritePreparation(ref, promptDebugPreparationFixture(), nil); err == nil { t.Fatalf("WritePreparation(%#v) error = nil", ref) } } outside := filepath.Join(root, "outside") if err := os.Mkdir(outside, debugDirectoryMode); err != nil { t.Fatalf("create symlink component target: %v", err) } testutil.RequireSymlink(t, outside, filepath.Join(root, "debug", "daily")) if _, err := writer.WritePreparation(promptDebugRef(), promptDebugPreparationFixture(), nil); err == nil { t.Fatal("WritePreparation(symlink component) error = nil") } } func promptDebugRef() PromptDebugRef { return PromptDebugRef{ReportID: report.Daily, ValidDate: "2026-05-29", RunID: "run-123"} } func promptDebugPreparationFixture() promptexec.Preparation { startedAt := time.Date(2026, 5, 29, 12, 0, 0, 0, time.UTC) return promptexec.Preparation{ PromptID: "weather.daily", PromptVersion: "v1", PromptHash: "prompt-hash", RenderedPromptHash: "rendered-hash", InputHashes: map[string]string{"data_package": "input-hash"}, ProfileID: "local", BackendID: "local", ModelName: "weather-model", Output: promptexec.OutputContract{Format: "json_schema", ValidationMode: "strict", SchemaPath: "schemas/daily.json"}, StartedAt: startedAt, EndedAt: startedAt.Add(time.Second), Duration: time.Second, } } func promptDebugExecutionFixture() promptexec.Execution { startedAt := time.Date(2026, 5, 29, 12, 0, 0, 0, time.UTC) return promptexec.Execution{ RunID: "run-123", PromptID: "weather.daily", PromptVersion: "v1", PromptHash: "prompt-hash", RenderedPromptHash: "rendered-hash", InputHashes: map[string]string{"data_package": "input-hash"}, ProfileID: "local", BackendID: "local", ModelName: "weather-model", GeneratedHash: "generated-hash", Usage: promptexec.TokenUsage{PromptTokens: 10, CompletionTokens: 5, TotalTokens: 15}, StartedAt: startedAt, EndedAt: startedAt.Add(time.Second), Duration: time.Second, Validation: promptexec.NewValidation(promptexec.ValidationPassed, "strict", "schemas/daily.json", []string{"validation details"}), RawOutput: []byte("Generated forecast prose."), Debug: &promptexec.ExecutionDebug{ValidationDiagnostics: []string{"validation details"}}, } } func readPromptDebugFile(t *testing.T, path string) []byte { t.Helper() data, err := os.ReadFile(path) if err != nil { t.Fatalf("read %q: %v", path, err) } return data } func assertPromptDebugMode(t *testing.T, path string, want os.FileMode) { t.Helper() info, err := os.Stat(path) if err != nil { t.Fatalf("stat %q: %v", path, err) } if got := info.Mode().Perm(); got != want { t.Fatalf("mode for %q = %#o, want %#o", path, got, want) } }