Move prompt debug capture out of state
This commit is contained in:
444
internal/promptdebug/debug_writer.go
Normal file
444
internal/promptdebug/debug_writer.go
Normal file
@@ -0,0 +1,444 @@
|
||||
// Package promptdebug writes explicitly requested prompt diagnostics outside
|
||||
// ordinary application state.
|
||||
package promptdebug
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec"
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/report"
|
||||
)
|
||||
|
||||
const (
|
||||
promptPreparationDebugSchemaVersion = "weatherreporter.prompt_preparation_debug.v1"
|
||||
promptExecutionDebugSchemaVersion = "weatherreporter.prompt_execution_debug.v1"
|
||||
debugDirectoryMode = 0o700
|
||||
debugFileMode = 0o600
|
||||
)
|
||||
|
||||
// PromptDebugWriter stores explicitly requested content-rich diagnostics outside
|
||||
// the managed report workspace. A writer created without a root is disabled.
|
||||
type PromptDebugWriter struct {
|
||||
root string
|
||||
}
|
||||
|
||||
// PromptDebugRef identifies one debug capture directory.
|
||||
type PromptDebugRef struct {
|
||||
ReportID report.ID
|
||||
ValidDate string
|
||||
RunID string
|
||||
}
|
||||
|
||||
// PromptDebugMessage is one rendered prompt message retained only in the
|
||||
// explicitly enabled debug store.
|
||||
type PromptDebugMessage struct {
|
||||
Role string `json:"role"`
|
||||
Content string `json:"content"`
|
||||
}
|
||||
|
||||
// PromptDebugOutput records the declared structured-output contract.
|
||||
type PromptDebugOutput struct {
|
||||
Format string `json:"format"`
|
||||
ValidationMode string `json:"validationMode"`
|
||||
SchemaPath string `json:"schemaPath"`
|
||||
}
|
||||
|
||||
// PromptDebugPreparation is the explicit, content-safe mapping of preparation
|
||||
// provenance. It intentionally has no fields for credentials or dependencies.
|
||||
type PromptDebugPreparation struct {
|
||||
PromptID string `json:"promptId"`
|
||||
PromptVersion string `json:"promptVersion"`
|
||||
PromptHash string `json:"promptHash"`
|
||||
RenderedPromptHash string `json:"renderedPromptHash"`
|
||||
InputHashes map[string]string `json:"inputHashes,omitempty"`
|
||||
ProfileID string `json:"profileId"`
|
||||
BackendID string `json:"backendId"`
|
||||
ModelName string `json:"modelName"`
|
||||
Output PromptDebugOutput `json:"output"`
|
||||
StartedAt time.Time `json:"startedAt"`
|
||||
EndedAt time.Time `json:"endedAt"`
|
||||
Duration time.Duration `json:"duration"`
|
||||
DataPackagePath string `json:"dataPackagePath"`
|
||||
}
|
||||
|
||||
// PromptPreparationDebugArtifact is the on-disk preparation debug record.
|
||||
type PromptPreparationDebugArtifact struct {
|
||||
SchemaVersion string `json:"schemaVersion"`
|
||||
ReportID report.ID `json:"reportId"`
|
||||
ValidDate string `json:"validDate"`
|
||||
RunID string `json:"runId"`
|
||||
Preparation PromptDebugPreparation `json:"preparation"`
|
||||
RenderedMessages []PromptDebugMessage `json:"renderedMessages,omitempty"`
|
||||
StructuredSchema json.RawMessage `json:"structuredSchema,omitempty"`
|
||||
Endpoint string `json:"endpoint,omitempty"`
|
||||
Parameters json.RawMessage `json:"parameters,omitempty"`
|
||||
}
|
||||
|
||||
// PromptDebugUsage is the explicit mapping of provider token accounting.
|
||||
type PromptDebugUsage struct {
|
||||
PromptTokens int `json:"promptTokens"`
|
||||
CompletionTokens int `json:"completionTokens"`
|
||||
TotalTokens int `json:"totalTokens"`
|
||||
CachedTokens int `json:"cachedTokens"`
|
||||
CacheWriteTokens int `json:"cacheWriteTokens"`
|
||||
}
|
||||
|
||||
// PromptDebugValidation is the completed validation detail retained in the
|
||||
// explicitly enabled debug store.
|
||||
type PromptDebugValidation struct {
|
||||
Status string `json:"status"`
|
||||
Mode string `json:"mode"`
|
||||
SchemaPath string `json:"schemaPath"`
|
||||
Diagnostics []string `json:"diagnostics,omitempty"`
|
||||
}
|
||||
|
||||
// PromptDebugExecution is the explicit mapping of execution provenance.
|
||||
type PromptDebugExecution struct {
|
||||
RunID string `json:"runId"`
|
||||
PromptID string `json:"promptId"`
|
||||
PromptVersion string `json:"promptVersion"`
|
||||
PromptHash string `json:"promptHash"`
|
||||
RenderedPromptHash string `json:"renderedPromptHash"`
|
||||
InputHashes map[string]string `json:"inputHashes,omitempty"`
|
||||
ProfileID string `json:"profileId"`
|
||||
BackendID string `json:"backendId"`
|
||||
ModelName string `json:"modelName"`
|
||||
GeneratedHash string `json:"generatedHash,omitempty"`
|
||||
Usage PromptDebugUsage `json:"usage"`
|
||||
StartedAt time.Time `json:"startedAt"`
|
||||
EndedAt time.Time `json:"endedAt"`
|
||||
Duration time.Duration `json:"duration"`
|
||||
DataPackagePath string `json:"dataPackagePath"`
|
||||
}
|
||||
|
||||
// PromptExecutionDebugArtifact is the on-disk execution debug record.
|
||||
type PromptExecutionDebugArtifact struct {
|
||||
SchemaVersion string `json:"schemaVersion"`
|
||||
ReportID report.ID `json:"reportId"`
|
||||
ValidDate string `json:"validDate"`
|
||||
RunID string `json:"runId"`
|
||||
Execution PromptDebugExecution `json:"execution"`
|
||||
Validation PromptDebugValidation `json:"validation"`
|
||||
RawOutput string `json:"rawOutput,omitempty"`
|
||||
DebugValidationDetails []string `json:"debugValidationDetails,omitempty"`
|
||||
}
|
||||
|
||||
// NewPromptDebugWriter initializes an explicitly rooted writer. An empty root
|
||||
// creates a disabled writer without touching the filesystem.
|
||||
func NewPromptDebugWriter(root string) (*PromptDebugWriter, error) {
|
||||
if strings.TrimSpace(root) == "" {
|
||||
return &PromptDebugWriter{}, nil
|
||||
}
|
||||
if !filepath.IsAbs(root) {
|
||||
return nil, fmt.Errorf("prompt debug root must be absolute")
|
||||
}
|
||||
cleaned := filepath.Clean(root)
|
||||
if cleaned == string(filepath.Separator) {
|
||||
return nil, fmt.Errorf("prompt debug root must not be the filesystem root")
|
||||
}
|
||||
if err := ensureSecureDirectory(cleaned); err != nil {
|
||||
return nil, fmt.Errorf("initialize prompt debug root %q: %w", cleaned, err)
|
||||
}
|
||||
return &PromptDebugWriter{root: cleaned}, nil
|
||||
}
|
||||
|
||||
func (w *PromptDebugWriter) Enabled() bool {
|
||||
return w != nil && w.root != ""
|
||||
}
|
||||
|
||||
// WritePreparation stores the explicitly captured preparation details and
|
||||
// returns the per-run debug directory. Disabled writers do no filesystem work.
|
||||
func (w *PromptDebugWriter) WritePreparation(ref PromptDebugRef, preparation promptexec.Preparation, debug *promptexec.PreparationDebug) (string, error) {
|
||||
if !w.Enabled() {
|
||||
return "", nil
|
||||
}
|
||||
directory, err := w.runDirectory(ref)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
artifact := PromptPreparationDebugArtifact{
|
||||
SchemaVersion: promptPreparationDebugSchemaVersion,
|
||||
ReportID: ref.ReportID,
|
||||
ValidDate: ref.ValidDate,
|
||||
RunID: ref.RunID,
|
||||
Preparation: promptDebugPreparation(preparation),
|
||||
}
|
||||
if debug != nil {
|
||||
artifact.RenderedMessages = promptDebugMessages(debug.RenderedMessages)
|
||||
artifact.StructuredSchema = copyRawJSON(debug.StructuredSchema)
|
||||
artifact.Endpoint = safePromptDebugEndpoint(debug.Endpoint)
|
||||
parameters, err := redactPromptDebugParameters(debug.ParametersJSON)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
artifact.Parameters = parameters
|
||||
}
|
||||
if err := writeSecureJSON(filepath.Join(directory, "preparation.json"), artifact); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return directory, nil
|
||||
}
|
||||
|
||||
// WriteExecution stores the explicitly captured execution details and returns
|
||||
// the per-run debug directory. Disabled writers do no filesystem work.
|
||||
func (w *PromptDebugWriter) WriteExecution(ref PromptDebugRef, execution promptexec.Execution) (string, error) {
|
||||
if !w.Enabled() {
|
||||
return "", nil
|
||||
}
|
||||
directory, err := w.runDirectory(ref)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
artifact := PromptExecutionDebugArtifact{
|
||||
SchemaVersion: promptExecutionDebugSchemaVersion,
|
||||
ReportID: ref.ReportID,
|
||||
ValidDate: ref.ValidDate,
|
||||
RunID: ref.RunID,
|
||||
Execution: promptDebugExecution(execution),
|
||||
Validation: promptDebugValidation(execution.Validation),
|
||||
RawOutput: string(execution.RawOutput),
|
||||
}
|
||||
if execution.Debug != nil {
|
||||
if len(execution.Debug.RawOutput) > 0 {
|
||||
artifact.RawOutput = string(execution.Debug.RawOutput)
|
||||
}
|
||||
artifact.DebugValidationDetails = append([]string(nil), execution.Debug.ValidationDiagnostics...)
|
||||
}
|
||||
if err := writeSecureJSON(filepath.Join(directory, "execution.json"), artifact); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return directory, nil
|
||||
}
|
||||
|
||||
func (w *PromptDebugWriter) runDirectory(ref PromptDebugRef) (string, error) {
|
||||
if err := validatePromptDebugRef(ref); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if err := ensureSecureDirectory(w.root); err != nil {
|
||||
return "", fmt.Errorf("validate prompt debug root %q: %w", w.root, err)
|
||||
}
|
||||
directory := filepath.Join(w.root, string(ref.ReportID), ref.ValidDate, ref.RunID)
|
||||
if !isWithinDirectory(w.root, directory) {
|
||||
return "", fmt.Errorf("prompt debug path escapes root")
|
||||
}
|
||||
if err := ensureSecureDirectory(directory); err != nil {
|
||||
return "", fmt.Errorf("create prompt debug directory %q: %w", directory, err)
|
||||
}
|
||||
return directory, nil
|
||||
}
|
||||
|
||||
func validatePromptDebugRef(ref PromptDebugRef) error {
|
||||
if err := validatePromptDebugSegment("report id", string(ref.ReportID)); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := validatePromptDebugSegment("valid date", ref.ValidDate); err != nil {
|
||||
return err
|
||||
}
|
||||
if parsed, err := time.Parse("2006-01-02", ref.ValidDate); err != nil || parsed.Format("2006-01-02") != ref.ValidDate {
|
||||
return fmt.Errorf("valid date must use YYYY-MM-DD")
|
||||
}
|
||||
return validatePromptDebugSegment("run id", ref.RunID)
|
||||
}
|
||||
|
||||
func validatePromptDebugSegment(name string, value string) error {
|
||||
if strings.TrimSpace(value) == "" {
|
||||
return fmt.Errorf("prompt debug %s is required", name)
|
||||
}
|
||||
if filepath.IsAbs(value) || strings.ContainsAny(value, `/\\`) || value == "." || value == ".." {
|
||||
return fmt.Errorf("prompt debug %s must be a safe path segment", name)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func ensureSecureDirectory(path string) error {
|
||||
if !filepath.IsAbs(path) {
|
||||
return fmt.Errorf("directory must be absolute")
|
||||
}
|
||||
cleaned := filepath.Clean(path)
|
||||
volume := filepath.VolumeName(cleaned)
|
||||
current := volume + string(filepath.Separator)
|
||||
for _, component := range strings.Split(strings.TrimPrefix(cleaned, current), string(filepath.Separator)) {
|
||||
if component == "" {
|
||||
continue
|
||||
}
|
||||
current = filepath.Join(current, component)
|
||||
info, err := os.Lstat(current)
|
||||
if os.IsNotExist(err) {
|
||||
if err := os.Mkdir(current, debugDirectoryMode); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.Chmod(current, debugDirectoryMode); err != nil {
|
||||
return err
|
||||
}
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if info.Mode()&os.ModeSymlink != 0 {
|
||||
return fmt.Errorf("directory component %q must not be a symlink", current)
|
||||
}
|
||||
if !info.IsDir() {
|
||||
return fmt.Errorf("directory component %q is not a directory", current)
|
||||
}
|
||||
}
|
||||
if err := os.Chmod(cleaned, debugDirectoryMode); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func isWithinDirectory(root string, path string) bool {
|
||||
relative, err := filepath.Rel(root, path)
|
||||
return err == nil && relative != ".." && !strings.HasPrefix(relative, ".."+string(filepath.Separator)) && !filepath.IsAbs(relative)
|
||||
}
|
||||
|
||||
func writeSecureJSON(path string, value any) error {
|
||||
data, err := json.MarshalIndent(value, "", " ")
|
||||
if err != nil {
|
||||
return fmt.Errorf("marshal %q: %w", path, err)
|
||||
}
|
||||
if info, err := os.Lstat(path); err == nil {
|
||||
if info.Mode()&os.ModeSymlink != 0 || !info.Mode().IsRegular() {
|
||||
return fmt.Errorf("prompt debug file %q is not a regular file", path)
|
||||
}
|
||||
} else if !os.IsNotExist(err) {
|
||||
return err
|
||||
}
|
||||
temporary, err := os.CreateTemp(filepath.Dir(path), "."+filepath.Base(path)+".*.tmp")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
temporaryPath := temporary.Name()
|
||||
defer os.Remove(temporaryPath)
|
||||
if err := temporary.Chmod(debugFileMode); err != nil {
|
||||
temporary.Close()
|
||||
return err
|
||||
}
|
||||
if _, err := temporary.Write(data); err != nil {
|
||||
temporary.Close()
|
||||
return err
|
||||
}
|
||||
if err := temporary.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.Rename(temporaryPath, path); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func promptDebugPreparation(value promptexec.Preparation) PromptDebugPreparation {
|
||||
return PromptDebugPreparation{
|
||||
PromptID: value.PromptID, PromptVersion: value.PromptVersion, PromptHash: value.PromptHash,
|
||||
RenderedPromptHash: value.RenderedPromptHash, InputHashes: copyPromptDebugMap(value.InputHashes),
|
||||
ProfileID: value.ProfileID, BackendID: value.BackendID, ModelName: value.ModelName,
|
||||
Output: PromptDebugOutput{Format: value.Output.Format, ValidationMode: value.Output.ValidationMode, SchemaPath: value.Output.SchemaPath},
|
||||
StartedAt: value.StartedAt, EndedAt: value.EndedAt, Duration: value.Duration, DataPackagePath: value.DataPackagePath,
|
||||
}
|
||||
}
|
||||
|
||||
func promptDebugExecution(value promptexec.Execution) PromptDebugExecution {
|
||||
return PromptDebugExecution{
|
||||
RunID: value.RunID, PromptID: value.PromptID, PromptVersion: value.PromptVersion,
|
||||
PromptHash: value.PromptHash, RenderedPromptHash: value.RenderedPromptHash,
|
||||
InputHashes: copyPromptDebugMap(value.InputHashes), ProfileID: value.ProfileID,
|
||||
BackendID: value.BackendID, ModelName: value.ModelName, GeneratedHash: value.GeneratedHash,
|
||||
Usage: PromptDebugUsage{PromptTokens: value.Usage.PromptTokens, CompletionTokens: value.Usage.CompletionTokens, TotalTokens: value.Usage.TotalTokens, CachedTokens: value.Usage.CachedTokens, CacheWriteTokens: value.Usage.CacheWriteTokens},
|
||||
StartedAt: value.StartedAt, EndedAt: value.EndedAt, Duration: value.Duration, DataPackagePath: value.DataPackagePath,
|
||||
}
|
||||
}
|
||||
|
||||
func promptDebugValidation(value promptexec.Validation) PromptDebugValidation {
|
||||
return PromptDebugValidation{Status: string(value.Status), Mode: value.Mode, SchemaPath: value.SchemaPath, Diagnostics: append([]string(nil), value.Diagnostics...)}
|
||||
}
|
||||
|
||||
func promptDebugMessages(values []promptexec.RenderedMessage) []PromptDebugMessage {
|
||||
if len(values) == 0 {
|
||||
return nil
|
||||
}
|
||||
result := make([]PromptDebugMessage, len(values))
|
||||
for index, value := range values {
|
||||
result[index] = PromptDebugMessage{Role: value.Role, Content: value.Content}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func copyPromptDebugMap(values map[string]string) map[string]string {
|
||||
if values == nil {
|
||||
return nil
|
||||
}
|
||||
result := make(map[string]string, len(values))
|
||||
for key, value := range values {
|
||||
result[key] = value
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func copyRawJSON(value []byte) json.RawMessage {
|
||||
if len(value) == 0 {
|
||||
return nil
|
||||
}
|
||||
return append(json.RawMessage(nil), value...)
|
||||
}
|
||||
|
||||
func safePromptDebugEndpoint(value string) string {
|
||||
parsed, err := url.Parse(value)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
parsed.User = nil
|
||||
parameters := parsed.Query()
|
||||
for key := range parameters {
|
||||
if isPromptDebugSecretKey(key) {
|
||||
parameters[key] = []string{"[redacted]"}
|
||||
}
|
||||
}
|
||||
parsed.RawQuery = parameters.Encode()
|
||||
parsed.Fragment = ""
|
||||
return parsed.String()
|
||||
}
|
||||
|
||||
func redactPromptDebugParameters(value []byte) (json.RawMessage, error) {
|
||||
if len(value) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
var decoded any
|
||||
if err := json.Unmarshal(value, &decoded); err != nil {
|
||||
return nil, fmt.Errorf("decode prompt debug parameters: %w", err)
|
||||
}
|
||||
redactPromptDebugValue(decoded)
|
||||
encoded, err := json.Marshal(decoded)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("encode prompt debug parameters: %w", err)
|
||||
}
|
||||
return encoded, nil
|
||||
}
|
||||
|
||||
func redactPromptDebugValue(value any) {
|
||||
switch typed := value.(type) {
|
||||
case map[string]any:
|
||||
for key, item := range typed {
|
||||
if isPromptDebugSecretKey(key) {
|
||||
typed[key] = "[redacted]"
|
||||
continue
|
||||
}
|
||||
redactPromptDebugValue(item)
|
||||
}
|
||||
case []any:
|
||||
for _, item := range typed {
|
||||
redactPromptDebugValue(item)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func isPromptDebugSecretKey(key string) bool {
|
||||
normalized := strings.NewReplacer("_", "", "-", "", " ", "").Replace(strings.ToLower(key))
|
||||
return strings.Contains(normalized, "credential") || strings.Contains(normalized, "secret") || strings.Contains(normalized, "password") || strings.Contains(normalized, "token") || strings.Contains(normalized, "apikey") || strings.Contains(normalized, "authorization")
|
||||
}
|
||||
207
internal/promptdebug/debug_writer_test.go
Normal file
207
internal/promptdebug/debug_writer_test.go
Normal file
@@ -0,0 +1,207 @@
|
||||
package promptdebug
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec"
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/report"
|
||||
)
|
||||
|
||||
func TestPromptDebugWriterWritesIsolatedArtifacts(t *testing.T) {
|
||||
root := filepath.Join(t.TempDir(), "operator-debug")
|
||||
writer, err := NewPromptDebugWriter(root)
|
||||
if err != nil {
|
||||
t.Fatalf("NewPromptDebugWriter() error = %v", err)
|
||||
}
|
||||
ref := promptDebugRef()
|
||||
preparationDir, err := writer.WritePreparation(ref, promptDebugPreparationFixture(), &promptexec.PreparationDebug{
|
||||
RenderedMessages: []promptexec.RenderedMessage{{Role: "system", Content: "Use the supplied weather facts."}},
|
||||
StructuredSchema: []byte(`{"type":"object","required":["summary"]}`),
|
||||
Endpoint: "https://direct:resolved-secret-value@llm.example.test/v1/chat?api_key=resolved-secret-value",
|
||||
ParametersJSON: []byte(`{"temperature":0.2,"api_key":"resolved-secret-value"}`),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("WritePreparation() error = %v", err)
|
||||
}
|
||||
wantDirectory := filepath.Join(root, "daily", "2026-05-29", "run-123")
|
||||
if preparationDir != wantDirectory {
|
||||
t.Fatalf("WritePreparation() directory = %q, want %q", preparationDir, wantDirectory)
|
||||
}
|
||||
executionDir, err := writer.WriteExecution(ref, promptDebugExecutionFixture())
|
||||
if err != nil {
|
||||
t.Fatalf("WriteExecution() error = %v", err)
|
||||
}
|
||||
if executionDir != preparationDir {
|
||||
t.Fatalf("WriteExecution() directory = %q, want %q", executionDir, preparationDir)
|
||||
}
|
||||
preparationData := readPromptDebugFile(t, filepath.Join(preparationDir, "preparation.json"))
|
||||
for _, want := range []string{"Use the supplied weather facts.", `"type": "object"`, "https://llm.example.test/v1/chat?api_key=%5Bredacted%5D", `"temperature": 0.2`, `"api_key": "[redacted]"`} {
|
||||
if !strings.Contains(string(preparationData), want) {
|
||||
t.Fatalf("preparation debug artifact missing %q:\n%s", want, preparationData)
|
||||
}
|
||||
}
|
||||
executionData := readPromptDebugFile(t, filepath.Join(executionDir, "execution.json"))
|
||||
for _, want := range []string{"Generated forecast prose.", "validation details", `"status": "passed"`} {
|
||||
if !strings.Contains(string(executionData), want) {
|
||||
t.Fatalf("execution debug artifact missing %q:\n%s", want, executionData)
|
||||
}
|
||||
}
|
||||
for _, data := range [][]byte{preparationData, executionData} {
|
||||
if strings.Contains(string(data), "credential") || strings.Contains(string(data), "resolved-secret-value") {
|
||||
t.Fatalf("debug artifact contains credentials:\n%s", data)
|
||||
}
|
||||
}
|
||||
if runtime.GOOS != "windows" {
|
||||
assertPromptDebugMode(t, root, debugDirectoryMode)
|
||||
assertPromptDebugMode(t, preparationDir, debugDirectoryMode)
|
||||
assertPromptDebugMode(t, filepath.Join(preparationDir, "preparation.json"), debugFileMode)
|
||||
assertPromptDebugMode(t, filepath.Join(executionDir, "execution.json"), debugFileMode)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPromptDebugWriterAtomicallyReplacesArtifacts(t *testing.T) {
|
||||
writer, err := NewPromptDebugWriter(filepath.Join(t.TempDir(), "debug"))
|
||||
if err != nil {
|
||||
t.Fatalf("NewPromptDebugWriter() error = %v", err)
|
||||
}
|
||||
ref := promptDebugRef()
|
||||
first := promptDebugPreparationFixture()
|
||||
first.PromptHash = "old-hash"
|
||||
if _, err := writer.WritePreparation(ref, first, nil); err != nil {
|
||||
t.Fatalf("first WritePreparation() error = %v", err)
|
||||
}
|
||||
second := promptDebugPreparationFixture()
|
||||
second.PromptHash = "new-hash"
|
||||
directory, err := writer.WritePreparation(ref, second, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("second WritePreparation() error = %v", err)
|
||||
}
|
||||
data := readPromptDebugFile(t, filepath.Join(directory, "preparation.json"))
|
||||
if strings.Contains(string(data), "old-hash") || !strings.Contains(string(data), "new-hash") {
|
||||
t.Fatalf("replacement artifact = %s", data)
|
||||
}
|
||||
temporary, err := filepath.Glob(filepath.Join(directory, ".preparation.json.*.tmp"))
|
||||
if err != nil || len(temporary) != 0 {
|
||||
t.Fatalf("temporary files = %v, %v", temporary, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPromptDebugWriterDisabledDoesNotAccessFilesystem(t *testing.T) {
|
||||
writer, err := NewPromptDebugWriter("")
|
||||
if err != nil {
|
||||
t.Fatalf("NewPromptDebugWriter() error = %v", err)
|
||||
}
|
||||
if writer.Enabled() {
|
||||
t.Fatal("disabled writer reports enabled")
|
||||
}
|
||||
directory, err := writer.WritePreparation(PromptDebugRef{ReportID: report.ID("../unsafe")}, promptDebugPreparationFixture(), nil)
|
||||
if err != nil || directory != "" {
|
||||
t.Fatalf("disabled WritePreparation() = %q, %v", directory, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPromptDebugWriterRejectsUnsafeRootsAndReferences(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
if _, err := NewPromptDebugWriter("relative-debug"); err == nil {
|
||||
t.Fatal("NewPromptDebugWriter(relative) error = nil")
|
||||
}
|
||||
nonDirectory := filepath.Join(root, "not-a-directory")
|
||||
if err := os.WriteFile(nonDirectory, []byte("x"), 0o600); err != nil {
|
||||
t.Fatalf("write non-directory root: %v", err)
|
||||
}
|
||||
if _, err := NewPromptDebugWriter(nonDirectory); err == nil {
|
||||
t.Fatal("NewPromptDebugWriter(file) error = nil")
|
||||
}
|
||||
if runtime.GOOS != "windows" {
|
||||
target := filepath.Join(root, "target")
|
||||
link := filepath.Join(root, "root-link")
|
||||
if err := os.Mkdir(target, debugDirectoryMode); err != nil {
|
||||
t.Fatalf("create root symlink target: %v", err)
|
||||
}
|
||||
if err := os.Symlink(target, link); err != nil {
|
||||
t.Fatalf("create root symlink: %v", err)
|
||||
}
|
||||
if _, err := NewPromptDebugWriter(link); err == nil {
|
||||
t.Fatal("NewPromptDebugWriter(symlink) error = nil")
|
||||
}
|
||||
}
|
||||
|
||||
writer, err := NewPromptDebugWriter(filepath.Join(root, "debug"))
|
||||
if err != nil {
|
||||
t.Fatalf("NewPromptDebugWriter() error = %v", err)
|
||||
}
|
||||
for _, ref := range []PromptDebugRef{
|
||||
{ReportID: report.ID("../daily"), ValidDate: "2026-05-29", RunID: "run-123"},
|
||||
{ReportID: report.Daily, ValidDate: "2026/05/29", RunID: "run-123"},
|
||||
{ReportID: report.Daily, ValidDate: "2026-05-29", RunID: "/run-123"},
|
||||
{ReportID: report.Daily, ValidDate: "not-a-date", RunID: "run-123"},
|
||||
} {
|
||||
if _, err := writer.WritePreparation(ref, promptDebugPreparationFixture(), nil); err == nil {
|
||||
t.Fatalf("WritePreparation(%#v) error = nil", ref)
|
||||
}
|
||||
}
|
||||
if runtime.GOOS != "windows" {
|
||||
outside := filepath.Join(root, "outside")
|
||||
if err := os.Mkdir(outside, debugDirectoryMode); err != nil {
|
||||
t.Fatalf("create symlink component target: %v", err)
|
||||
}
|
||||
if err := os.Symlink(outside, filepath.Join(root, "debug", "daily")); err != nil {
|
||||
t.Fatalf("create symlink component: %v", err)
|
||||
}
|
||||
if _, err := writer.WritePreparation(promptDebugRef(), promptDebugPreparationFixture(), nil); err == nil {
|
||||
t.Fatal("WritePreparation(symlink component) error = nil")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func promptDebugRef() PromptDebugRef {
|
||||
return PromptDebugRef{ReportID: report.Daily, ValidDate: "2026-05-29", RunID: "run-123"}
|
||||
}
|
||||
|
||||
func promptDebugPreparationFixture() promptexec.Preparation {
|
||||
startedAt := time.Date(2026, 5, 29, 12, 0, 0, 0, time.UTC)
|
||||
return promptexec.Preparation{
|
||||
PromptID: "weather.daily", PromptVersion: "v1", PromptHash: "prompt-hash", RenderedPromptHash: "rendered-hash",
|
||||
InputHashes: map[string]string{"data_package": "input-hash"}, ProfileID: "local", BackendID: "local", ModelName: "weather-model",
|
||||
Output: promptexec.OutputContract{Format: "json_schema", ValidationMode: "strict", SchemaPath: "schemas/daily.json"},
|
||||
StartedAt: startedAt, EndedAt: startedAt.Add(time.Second), Duration: time.Second, DataPackagePath: "/packages/daily.yaml",
|
||||
}
|
||||
}
|
||||
|
||||
func promptDebugExecutionFixture() promptexec.Execution {
|
||||
startedAt := time.Date(2026, 5, 29, 12, 0, 0, 0, time.UTC)
|
||||
return promptexec.Execution{
|
||||
RunID: "run-123", PromptID: "weather.daily", PromptVersion: "v1", PromptHash: "prompt-hash", RenderedPromptHash: "rendered-hash",
|
||||
InputHashes: map[string]string{"data_package": "input-hash"}, ProfileID: "local", BackendID: "local", ModelName: "weather-model",
|
||||
GeneratedHash: "generated-hash", Usage: promptexec.TokenUsage{PromptTokens: 10, CompletionTokens: 5, TotalTokens: 15},
|
||||
StartedAt: startedAt, EndedAt: startedAt.Add(time.Second), Duration: time.Second,
|
||||
Validation: promptexec.NewValidation(promptexec.ValidationPassed, "strict", "schemas/daily.json", []string{"validation details"}),
|
||||
DataPackagePath: "/packages/daily.yaml", RawOutput: []byte("Generated forecast prose."),
|
||||
Debug: &promptexec.ExecutionDebug{ValidationDiagnostics: []string{"validation details"}},
|
||||
}
|
||||
}
|
||||
|
||||
func readPromptDebugFile(t *testing.T, path string) []byte {
|
||||
t.Helper()
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatalf("read %q: %v", path, err)
|
||||
}
|
||||
return data
|
||||
}
|
||||
|
||||
func assertPromptDebugMode(t *testing.T, path string, want os.FileMode) {
|
||||
t.Helper()
|
||||
info, err := os.Stat(path)
|
||||
if err != nil {
|
||||
t.Fatalf("stat %q: %v", path, err)
|
||||
}
|
||||
if got := info.Mode().Perm(); got != want {
|
||||
t.Fatalf("mode for %q = %#o, want %#o", path, got, want)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user