389 lines
14 KiB
Go
389 lines
14 KiB
Go
// Package promptdebug writes explicitly requested prompt diagnostics outside
|
|
// ordinary application state.
|
|
package promptdebug
|
|
|
|
import (
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"net/url"
|
|
"path/filepath"
|
|
"strings"
|
|
"time"
|
|
|
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec"
|
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/report"
|
|
)
|
|
|
|
// ErrSecureCaptureUnsupported reports that the host cannot provide the
|
|
// handle-relative, no-follow filesystem operations required for prompt debug
|
|
// artifacts.
|
|
var ErrSecureCaptureUnsupported = errors.New("secure prompt debug capture is unavailable on this platform")
|
|
|
|
const (
|
|
promptPreparationDebugSchemaVersion = "weatherreporter.prompt_preparation_debug.v2"
|
|
promptExecutionDebugSchemaVersion = "weatherreporter.prompt_execution_debug.v2"
|
|
debugDirectoryMode = 0o700
|
|
debugFileMode = 0o600
|
|
)
|
|
|
|
// PromptDebugWriter stores explicitly requested content-rich diagnostics. A
|
|
// writer created without a root is disabled.
|
|
type PromptDebugWriter struct {
|
|
root string
|
|
directory *secureDirectory
|
|
}
|
|
|
|
// PromptDebugRef identifies one debug capture directory.
|
|
type PromptDebugRef struct {
|
|
ReportID report.ID
|
|
ValidDate string
|
|
RunID string
|
|
}
|
|
|
|
// PromptDebugMessage is one rendered prompt message retained only in the
|
|
// explicitly enabled debug store.
|
|
type PromptDebugMessage struct {
|
|
Role string `json:"role"`
|
|
Content string `json:"content"`
|
|
}
|
|
|
|
// PromptDebugOutput records the declared structured-output contract.
|
|
type PromptDebugOutput struct {
|
|
Format string `json:"format"`
|
|
ValidationMode string `json:"validationMode"`
|
|
SchemaPath string `json:"schemaPath"`
|
|
}
|
|
|
|
// PromptDebugPreparation is the explicit, content-safe mapping of preparation
|
|
// provenance. It intentionally has no fields for credentials or dependencies.
|
|
type PromptDebugPreparation struct {
|
|
PromptID string `json:"promptId"`
|
|
PromptVersion string `json:"promptVersion"`
|
|
PromptHash string `json:"promptHash"`
|
|
RenderedPromptHash string `json:"renderedPromptHash"`
|
|
InputHashes map[string]string `json:"inputHashes,omitempty"`
|
|
ProfileID string `json:"profileId"`
|
|
BackendID string `json:"backendId"`
|
|
ModelName string `json:"modelName"`
|
|
Output PromptDebugOutput `json:"output"`
|
|
StartedAt time.Time `json:"startedAt"`
|
|
EndedAt time.Time `json:"endedAt"`
|
|
Duration time.Duration `json:"duration"`
|
|
}
|
|
|
|
// PromptPreparationDebugArtifact is the on-disk preparation debug record.
|
|
type PromptPreparationDebugArtifact struct {
|
|
SchemaVersion string `json:"schemaVersion"`
|
|
ReportID report.ID `json:"reportId"`
|
|
ValidDate string `json:"validDate"`
|
|
RunID string `json:"runId"`
|
|
Preparation PromptDebugPreparation `json:"preparation"`
|
|
RenderedMessages []PromptDebugMessage `json:"renderedMessages,omitempty"`
|
|
StructuredSchema json.RawMessage `json:"structuredSchema,omitempty"`
|
|
Endpoint string `json:"endpoint,omitempty"`
|
|
Parameters json.RawMessage `json:"parameters,omitempty"`
|
|
}
|
|
|
|
// PromptDebugUsage is the explicit mapping of provider token accounting.
|
|
type PromptDebugUsage struct {
|
|
PromptTokens int `json:"promptTokens"`
|
|
CompletionTokens int `json:"completionTokens"`
|
|
TotalTokens int `json:"totalTokens"`
|
|
CachedTokens int `json:"cachedTokens"`
|
|
CacheWriteTokens int `json:"cacheWriteTokens"`
|
|
}
|
|
|
|
// PromptDebugValidation is the completed validation detail retained in the
|
|
// explicitly enabled debug store.
|
|
type PromptDebugValidation struct {
|
|
Status string `json:"status"`
|
|
Mode string `json:"mode"`
|
|
SchemaPath string `json:"schemaPath"`
|
|
Diagnostics []string `json:"diagnostics,omitempty"`
|
|
}
|
|
|
|
// PromptDebugExecution is the explicit mapping of execution provenance.
|
|
type PromptDebugExecution struct {
|
|
RunID string `json:"runId"`
|
|
PromptID string `json:"promptId"`
|
|
PromptVersion string `json:"promptVersion"`
|
|
PromptHash string `json:"promptHash"`
|
|
RenderedPromptHash string `json:"renderedPromptHash"`
|
|
InputHashes map[string]string `json:"inputHashes,omitempty"`
|
|
ProfileID string `json:"profileId"`
|
|
BackendID string `json:"backendId"`
|
|
ModelName string `json:"modelName"`
|
|
GeneratedHash string `json:"generatedHash,omitempty"`
|
|
Usage PromptDebugUsage `json:"usage"`
|
|
StartedAt time.Time `json:"startedAt"`
|
|
EndedAt time.Time `json:"endedAt"`
|
|
Duration time.Duration `json:"duration"`
|
|
}
|
|
|
|
// PromptExecutionDebugArtifact is the on-disk execution debug record.
|
|
type PromptExecutionDebugArtifact struct {
|
|
SchemaVersion string `json:"schemaVersion"`
|
|
ReportID report.ID `json:"reportId"`
|
|
ValidDate string `json:"validDate"`
|
|
RunID string `json:"runId"`
|
|
Execution PromptDebugExecution `json:"execution"`
|
|
Validation PromptDebugValidation `json:"validation"`
|
|
RawOutput string `json:"rawOutput,omitempty"`
|
|
DebugValidationDetails []string `json:"debugValidationDetails,omitempty"`
|
|
}
|
|
|
|
// NewPromptDebugWriter initializes an explicitly rooted writer. An empty root
|
|
// creates a disabled writer without touching the filesystem.
|
|
func NewPromptDebugWriter(root string) (*PromptDebugWriter, error) {
|
|
if strings.TrimSpace(root) == "" {
|
|
return &PromptDebugWriter{}, nil
|
|
}
|
|
if !filepath.IsAbs(root) {
|
|
return nil, fmt.Errorf("prompt debug root must be absolute")
|
|
}
|
|
cleaned := filepath.Clean(root)
|
|
if cleaned == string(filepath.Separator) {
|
|
return nil, fmt.Errorf("prompt debug root must not be the filesystem root")
|
|
}
|
|
directory, err := openSecureDirectory(cleaned)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("initialize prompt debug root %q: %w", cleaned, err)
|
|
}
|
|
return &PromptDebugWriter{root: cleaned, directory: directory}, nil
|
|
}
|
|
|
|
func (w *PromptDebugWriter) Enabled() bool {
|
|
return w != nil && w.root != "" && w.directory != nil
|
|
}
|
|
|
|
// Close releases the secure directory handle retained for an enabled writer.
|
|
// It is safe to call on disabled writers.
|
|
func (w *PromptDebugWriter) Close() error {
|
|
if w == nil || w.directory == nil {
|
|
return nil
|
|
}
|
|
directory := w.directory
|
|
w.directory = nil
|
|
w.root = ""
|
|
return directory.Close()
|
|
}
|
|
|
|
// WritePreparation stores the explicitly captured preparation details and
|
|
// returns the per-run debug directory. Disabled writers do no filesystem work.
|
|
func (w *PromptDebugWriter) WritePreparation(ref PromptDebugRef, preparation promptexec.Preparation, debug *promptexec.PreparationDebug) (string, error) {
|
|
if !w.Enabled() {
|
|
return "", nil
|
|
}
|
|
directory, secureDirectory, err := w.runDirectory(ref)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
defer secureDirectory.Close()
|
|
artifact := PromptPreparationDebugArtifact{
|
|
SchemaVersion: promptPreparationDebugSchemaVersion,
|
|
ReportID: ref.ReportID,
|
|
ValidDate: ref.ValidDate,
|
|
RunID: ref.RunID,
|
|
Preparation: promptDebugPreparation(preparation),
|
|
}
|
|
if debug != nil {
|
|
artifact.RenderedMessages = promptDebugMessages(debug.RenderedMessages)
|
|
artifact.StructuredSchema = copyRawJSON(debug.StructuredSchema)
|
|
artifact.Endpoint = safePromptDebugEndpoint(debug.Endpoint)
|
|
parameters, err := safePromptDebugParameters(debug.ParametersJSON)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
artifact.Parameters = parameters
|
|
}
|
|
if err := secureDirectory.writeJSON("preparation.json", artifact); err != nil {
|
|
return "", err
|
|
}
|
|
return directory, nil
|
|
}
|
|
|
|
// WriteExecution stores the explicitly captured execution details and returns
|
|
// the per-run debug directory. Disabled writers do no filesystem work.
|
|
func (w *PromptDebugWriter) WriteExecution(ref PromptDebugRef, execution promptexec.Execution) (string, error) {
|
|
if !w.Enabled() {
|
|
return "", nil
|
|
}
|
|
directory, secureDirectory, err := w.runDirectory(ref)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
defer secureDirectory.Close()
|
|
artifact := PromptExecutionDebugArtifact{
|
|
SchemaVersion: promptExecutionDebugSchemaVersion,
|
|
ReportID: ref.ReportID,
|
|
ValidDate: ref.ValidDate,
|
|
RunID: ref.RunID,
|
|
Execution: promptDebugExecution(execution),
|
|
Validation: promptDebugValidation(execution.Validation),
|
|
RawOutput: string(execution.RawOutput),
|
|
}
|
|
if execution.Debug != nil {
|
|
if len(execution.Debug.RawOutput) > 0 {
|
|
artifact.RawOutput = string(execution.Debug.RawOutput)
|
|
}
|
|
artifact.DebugValidationDetails = append([]string(nil), execution.Debug.ValidationDiagnostics...)
|
|
}
|
|
if err := secureDirectory.writeJSON("execution.json", artifact); err != nil {
|
|
return "", err
|
|
}
|
|
return directory, nil
|
|
}
|
|
|
|
func (w *PromptDebugWriter) runDirectory(ref PromptDebugRef) (string, *secureDirectory, error) {
|
|
if err := validatePromptDebugRef(ref); err != nil {
|
|
return "", nil, err
|
|
}
|
|
directory := filepath.Join(w.root, string(ref.ReportID), ref.ValidDate, ref.RunID)
|
|
if !isWithinDirectory(w.root, directory) {
|
|
return "", nil, fmt.Errorf("prompt debug path escapes root")
|
|
}
|
|
secureDirectory, err := w.directory.openDirectory(string(ref.ReportID), ref.ValidDate, ref.RunID)
|
|
if err != nil {
|
|
return "", nil, fmt.Errorf("create prompt debug directory %q: %w", directory, err)
|
|
}
|
|
return directory, secureDirectory, nil
|
|
}
|
|
|
|
func validatePromptDebugRef(ref PromptDebugRef) error {
|
|
if err := validatePromptDebugSegment("report id", string(ref.ReportID)); err != nil {
|
|
return err
|
|
}
|
|
if err := validatePromptDebugSegment("valid date", ref.ValidDate); err != nil {
|
|
return err
|
|
}
|
|
if parsed, err := time.Parse("2006-01-02", ref.ValidDate); err != nil || parsed.Format("2006-01-02") != ref.ValidDate {
|
|
return fmt.Errorf("valid date must use YYYY-MM-DD")
|
|
}
|
|
return validatePromptDebugSegment("run id", ref.RunID)
|
|
}
|
|
|
|
func validatePromptDebugSegment(name string, value string) error {
|
|
if strings.TrimSpace(value) == "" {
|
|
return fmt.Errorf("prompt debug %s is required", name)
|
|
}
|
|
if filepath.IsAbs(value) || strings.ContainsAny(value, `/\\`) || value == "." || value == ".." {
|
|
return fmt.Errorf("prompt debug %s must be a safe path segment", name)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func isWithinDirectory(root string, path string) bool {
|
|
relative, err := filepath.Rel(root, path)
|
|
return err == nil && relative != ".." && !strings.HasPrefix(relative, ".."+string(filepath.Separator)) && !filepath.IsAbs(relative)
|
|
}
|
|
|
|
func promptDebugPreparation(value promptexec.Preparation) PromptDebugPreparation {
|
|
return PromptDebugPreparation{
|
|
PromptID: value.PromptID, PromptVersion: value.PromptVersion, PromptHash: value.PromptHash,
|
|
RenderedPromptHash: value.RenderedPromptHash, InputHashes: copyPromptDebugMap(value.InputHashes),
|
|
ProfileID: value.ProfileID, BackendID: value.BackendID, ModelName: value.ModelName,
|
|
Output: PromptDebugOutput{Format: value.Output.Format, ValidationMode: value.Output.ValidationMode, SchemaPath: value.Output.SchemaPath},
|
|
StartedAt: value.StartedAt, EndedAt: value.EndedAt, Duration: value.Duration,
|
|
}
|
|
}
|
|
|
|
func promptDebugExecution(value promptexec.Execution) PromptDebugExecution {
|
|
return PromptDebugExecution{
|
|
RunID: value.RunID, PromptID: value.PromptID, PromptVersion: value.PromptVersion,
|
|
PromptHash: value.PromptHash, RenderedPromptHash: value.RenderedPromptHash,
|
|
InputHashes: copyPromptDebugMap(value.InputHashes), ProfileID: value.ProfileID,
|
|
BackendID: value.BackendID, ModelName: value.ModelName, GeneratedHash: value.GeneratedHash,
|
|
Usage: PromptDebugUsage{PromptTokens: value.Usage.PromptTokens, CompletionTokens: value.Usage.CompletionTokens, TotalTokens: value.Usage.TotalTokens, CachedTokens: value.Usage.CachedTokens, CacheWriteTokens: value.Usage.CacheWriteTokens},
|
|
StartedAt: value.StartedAt, EndedAt: value.EndedAt, Duration: value.Duration,
|
|
}
|
|
}
|
|
|
|
func promptDebugValidation(value promptexec.Validation) PromptDebugValidation {
|
|
return PromptDebugValidation{Status: string(value.Status), Mode: value.Mode, SchemaPath: value.SchemaPath, Diagnostics: append([]string(nil), value.Diagnostics...)}
|
|
}
|
|
|
|
func promptDebugMessages(values []promptexec.RenderedMessage) []PromptDebugMessage {
|
|
if len(values) == 0 {
|
|
return nil
|
|
}
|
|
result := make([]PromptDebugMessage, len(values))
|
|
for index, value := range values {
|
|
result[index] = PromptDebugMessage{Role: value.Role, Content: value.Content}
|
|
}
|
|
return result
|
|
}
|
|
|
|
func copyPromptDebugMap(values map[string]string) map[string]string {
|
|
if values == nil {
|
|
return nil
|
|
}
|
|
result := make(map[string]string, len(values))
|
|
for key, value := range values {
|
|
result[key] = value
|
|
}
|
|
return result
|
|
}
|
|
|
|
func copyRawJSON(value []byte) json.RawMessage {
|
|
if len(value) == 0 {
|
|
return nil
|
|
}
|
|
return append(json.RawMessage(nil), value...)
|
|
}
|
|
|
|
func safePromptDebugEndpoint(value string) string {
|
|
parsed, err := url.Parse(value)
|
|
if err != nil || parsed.Scheme == "" || parsed.Host == "" {
|
|
return ""
|
|
}
|
|
return (&url.URL{Scheme: parsed.Scheme, Host: parsed.Host}).String()
|
|
}
|
|
|
|
func safePromptDebugParameters(value []byte) (json.RawMessage, error) {
|
|
if len(value) == 0 {
|
|
return nil, nil
|
|
}
|
|
var decoded map[string]json.RawMessage
|
|
if err := json.Unmarshal(value, &decoded); err != nil {
|
|
return nil, fmt.Errorf("prompt debug parameters must be a JSON object")
|
|
}
|
|
parameters := make(map[string]json.RawMessage)
|
|
for _, field := range []struct {
|
|
name string
|
|
value func(json.RawMessage) bool
|
|
}{
|
|
{name: "temperature", value: promptDebugJSONNumber},
|
|
{name: "max_tokens", value: promptDebugJSONInteger},
|
|
{name: "top_p", value: promptDebugJSONNumber},
|
|
{name: "timeout_seconds", value: promptDebugJSONInteger},
|
|
{name: "service_tier", value: promptDebugJSONString},
|
|
{name: "reasoning_effort", value: promptDebugJSONString},
|
|
} {
|
|
value, ok := decoded[field.name]
|
|
if ok && field.value(value) {
|
|
parameters[field.name] = append(json.RawMessage(nil), value...)
|
|
}
|
|
}
|
|
encoded, err := json.Marshal(parameters)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("encode safe prompt debug parameters: %w", err)
|
|
}
|
|
return encoded, nil
|
|
}
|
|
|
|
func promptDebugJSONNumber(value json.RawMessage) bool {
|
|
var decoded float64
|
|
return json.Unmarshal(value, &decoded) == nil
|
|
}
|
|
|
|
func promptDebugJSONInteger(value json.RawMessage) bool {
|
|
var decoded int
|
|
return json.Unmarshal(value, &decoded) == nil
|
|
}
|
|
|
|
func promptDebugJSONString(value json.RawMessage) bool {
|
|
var decoded string
|
|
return json.Unmarshal(value, &decoded) == nil
|
|
}
|