package promptdebug import ( "os" "path/filepath" "runtime" "strings" "testing" "time" "gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec" "gitea.maximumdirect.net/eric/weatherreporter/internal/report" ) func TestPromptDebugWriterWritesIsolatedArtifacts(t *testing.T) { root := filepath.Join(t.TempDir(), "operator-debug") writer, err := NewPromptDebugWriter(root) if err != nil { t.Fatalf("NewPromptDebugWriter() error = %v", err) } ref := promptDebugRef() preparationDir, err := writer.WritePreparation(ref, promptDebugPreparationFixture(), &promptexec.PreparationDebug{ RenderedMessages: []promptexec.RenderedMessage{{Role: "system", Content: "Use the supplied weather facts."}}, StructuredSchema: []byte(`{"type":"object","required":["summary"]}`), Endpoint: "https://direct:resolved-secret-value@llm.example.test/v1/chat?api_key=resolved-secret-value", ParametersJSON: []byte(`{"temperature":0.2,"api_key":"resolved-secret-value"}`), }) if err != nil { t.Fatalf("WritePreparation() error = %v", err) } wantDirectory := filepath.Join(root, "daily", "2026-05-29", "run-123") if preparationDir != wantDirectory { t.Fatalf("WritePreparation() directory = %q, want %q", preparationDir, wantDirectory) } executionDir, err := writer.WriteExecution(ref, promptDebugExecutionFixture()) if err != nil { t.Fatalf("WriteExecution() error = %v", err) } if executionDir != preparationDir { t.Fatalf("WriteExecution() directory = %q, want %q", executionDir, preparationDir) } preparationData := readPromptDebugFile(t, filepath.Join(preparationDir, "preparation.json")) for _, want := range []string{"Use the supplied weather facts.", `"type": "object"`, "https://llm.example.test/v1/chat?api_key=%5Bredacted%5D", `"temperature": 0.2`, `"api_key": "[redacted]"`} { if !strings.Contains(string(preparationData), want) { t.Fatalf("preparation debug artifact missing %q:\n%s", want, preparationData) } } executionData := readPromptDebugFile(t, filepath.Join(executionDir, "execution.json")) for _, want := range []string{"Generated forecast prose.", "validation details", `"status": "passed"`} { if !strings.Contains(string(executionData), want) { t.Fatalf("execution debug artifact missing %q:\n%s", want, executionData) } } for _, data := range [][]byte{preparationData, executionData} { if strings.Contains(string(data), "credential") || strings.Contains(string(data), "resolved-secret-value") { t.Fatalf("debug artifact contains credentials:\n%s", data) } } if runtime.GOOS != "windows" { assertPromptDebugMode(t, root, debugDirectoryMode) assertPromptDebugMode(t, preparationDir, debugDirectoryMode) assertPromptDebugMode(t, filepath.Join(preparationDir, "preparation.json"), debugFileMode) assertPromptDebugMode(t, filepath.Join(executionDir, "execution.json"), debugFileMode) } } func TestPromptDebugWriterAtomicallyReplacesArtifacts(t *testing.T) { writer, err := NewPromptDebugWriter(filepath.Join(t.TempDir(), "debug")) if err != nil { t.Fatalf("NewPromptDebugWriter() error = %v", err) } ref := promptDebugRef() first := promptDebugPreparationFixture() first.PromptHash = "old-hash" if _, err := writer.WritePreparation(ref, first, nil); err != nil { t.Fatalf("first WritePreparation() error = %v", err) } second := promptDebugPreparationFixture() second.PromptHash = "new-hash" directory, err := writer.WritePreparation(ref, second, nil) if err != nil { t.Fatalf("second WritePreparation() error = %v", err) } data := readPromptDebugFile(t, filepath.Join(directory, "preparation.json")) if strings.Contains(string(data), "old-hash") || !strings.Contains(string(data), "new-hash") { t.Fatalf("replacement artifact = %s", data) } temporary, err := filepath.Glob(filepath.Join(directory, ".preparation.json.*.tmp")) if err != nil || len(temporary) != 0 { t.Fatalf("temporary files = %v, %v", temporary, err) } } func TestPromptDebugWriterDisabledDoesNotAccessFilesystem(t *testing.T) { writer, err := NewPromptDebugWriter("") if err != nil { t.Fatalf("NewPromptDebugWriter() error = %v", err) } if writer.Enabled() { t.Fatal("disabled writer reports enabled") } directory, err := writer.WritePreparation(PromptDebugRef{ReportID: report.ID("../unsafe")}, promptDebugPreparationFixture(), nil) if err != nil || directory != "" { t.Fatalf("disabled WritePreparation() = %q, %v", directory, err) } } func TestPromptDebugWriterRejectsUnsafeRootsAndReferences(t *testing.T) { root := t.TempDir() if _, err := NewPromptDebugWriter("relative-debug"); err == nil { t.Fatal("NewPromptDebugWriter(relative) error = nil") } nonDirectory := filepath.Join(root, "not-a-directory") if err := os.WriteFile(nonDirectory, []byte("x"), 0o600); err != nil { t.Fatalf("write non-directory root: %v", err) } if _, err := NewPromptDebugWriter(nonDirectory); err == nil { t.Fatal("NewPromptDebugWriter(file) error = nil") } if runtime.GOOS != "windows" { target := filepath.Join(root, "target") link := filepath.Join(root, "root-link") if err := os.Mkdir(target, debugDirectoryMode); err != nil { t.Fatalf("create root symlink target: %v", err) } if err := os.Symlink(target, link); err != nil { t.Fatalf("create root symlink: %v", err) } if _, err := NewPromptDebugWriter(link); err == nil { t.Fatal("NewPromptDebugWriter(symlink) error = nil") } } writer, err := NewPromptDebugWriter(filepath.Join(root, "debug")) if err != nil { t.Fatalf("NewPromptDebugWriter() error = %v", err) } for _, ref := range []PromptDebugRef{ {ReportID: report.ID("../daily"), ValidDate: "2026-05-29", RunID: "run-123"}, {ReportID: report.Daily, ValidDate: "2026/05/29", RunID: "run-123"}, {ReportID: report.Daily, ValidDate: "2026-05-29", RunID: "/run-123"}, {ReportID: report.Daily, ValidDate: "not-a-date", RunID: "run-123"}, } { if _, err := writer.WritePreparation(ref, promptDebugPreparationFixture(), nil); err == nil { t.Fatalf("WritePreparation(%#v) error = nil", ref) } } if runtime.GOOS != "windows" { outside := filepath.Join(root, "outside") if err := os.Mkdir(outside, debugDirectoryMode); err != nil { t.Fatalf("create symlink component target: %v", err) } if err := os.Symlink(outside, filepath.Join(root, "debug", "daily")); err != nil { t.Fatalf("create symlink component: %v", err) } if _, err := writer.WritePreparation(promptDebugRef(), promptDebugPreparationFixture(), nil); err == nil { t.Fatal("WritePreparation(symlink component) error = nil") } } } func promptDebugRef() PromptDebugRef { return PromptDebugRef{ReportID: report.Daily, ValidDate: "2026-05-29", RunID: "run-123"} } func promptDebugPreparationFixture() promptexec.Preparation { startedAt := time.Date(2026, 5, 29, 12, 0, 0, 0, time.UTC) return promptexec.Preparation{ PromptID: "weather.daily", PromptVersion: "v1", PromptHash: "prompt-hash", RenderedPromptHash: "rendered-hash", InputHashes: map[string]string{"data_package": "input-hash"}, ProfileID: "local", BackendID: "local", ModelName: "weather-model", Output: promptexec.OutputContract{Format: "json_schema", ValidationMode: "strict", SchemaPath: "schemas/daily.json"}, StartedAt: startedAt, EndedAt: startedAt.Add(time.Second), Duration: time.Second, DataPackagePath: "/packages/daily.yaml", } } func promptDebugExecutionFixture() promptexec.Execution { startedAt := time.Date(2026, 5, 29, 12, 0, 0, 0, time.UTC) return promptexec.Execution{ RunID: "run-123", PromptID: "weather.daily", PromptVersion: "v1", PromptHash: "prompt-hash", RenderedPromptHash: "rendered-hash", InputHashes: map[string]string{"data_package": "input-hash"}, ProfileID: "local", BackendID: "local", ModelName: "weather-model", GeneratedHash: "generated-hash", Usage: promptexec.TokenUsage{PromptTokens: 10, CompletionTokens: 5, TotalTokens: 15}, StartedAt: startedAt, EndedAt: startedAt.Add(time.Second), Duration: time.Second, Validation: promptexec.NewValidation(promptexec.ValidationPassed, "strict", "schemas/daily.json", []string{"validation details"}), DataPackagePath: "/packages/daily.yaml", RawOutput: []byte("Generated forecast prose."), Debug: &promptexec.ExecutionDebug{ValidationDiagnostics: []string{"validation details"}}, } } func readPromptDebugFile(t *testing.T, path string) []byte { t.Helper() data, err := os.ReadFile(path) if err != nil { t.Fatalf("read %q: %v", path, err) } return data } func assertPromptDebugMode(t *testing.T, path string, want os.FileMode) { t.Helper() info, err := os.Stat(path) if err != nil { t.Fatalf("stat %q: %v", path, err) } if got := info.Mode().Perm(); got != want { t.Fatalf("mode for %q = %#o, want %#o", path, got, want) } }