Add prompt inspection command

This commit is contained in:
2026-08-29 14:29:14 +00:00
parent 157209f097
commit 36c7c5a358
9 changed files with 280 additions and 13 deletions

View File

@@ -0,0 +1,101 @@
package cli
import (
"context"
"errors"
"flag"
"fmt"
"io"
"path/filepath"
"strings"
appconfig "gitea.maximumdirect.net/eric/scriptorium/internal/config"
appformat "gitea.maximumdirect.net/eric/scriptorium/internal/format"
)
type promptInspectionConfig struct {
configPath, promptDir, promptID, promptVersion, outputPath string
outputFormat appformat.OutputFormat
}
func inspectCommand(args []string, stdout, stderr io.Writer) int {
if len(args) == 0 || args[0] != "prompt" {
fmt.Fprintln(stderr, "inspect parse error: inspection mode must be prompt")
return ExitRuntimeError
}
cfg, err := parsePromptInspectionArgs(args[1:])
if err != nil {
fmt.Fprintf(stderr, "inspect parse error: %v\n", err)
return ExitRuntimeError
}
settings, err := resolveAppSettingsForPromptInspection(cfg)
if err != nil {
fmt.Fprintf(stderr, "inspect error: %v\n", err)
return ExitRuntimeError
}
engine, err := newEngine(settings)
if err != nil {
fmt.Fprintf(stderr, "engine error: %v\n", err)
return ExitRuntimeError
}
inspection, err := engine.InspectPrompt(context.Background(), cfg.promptID, cfg.promptVersion)
if err != nil {
fmt.Fprintf(stderr, "inspect error: %v\n", err)
return ExitRuntimeError
}
data, err := appformat.FormatPromptInspection(inspection, cfg.outputFormat)
if err != nil {
fmt.Fprintf(stderr, "inspect error: %v\n", err)
return ExitRuntimeError
}
if err := writeOutput(stdout, cfg.outputPath, data); err != nil {
fmt.Fprintf(stderr, "output write error: %v\n", err)
return ExitRuntimeError
}
return ExitOK
}
func parsePromptInspectionArgs(args []string) (*promptInspectionConfig, error) {
cfg := &promptInspectionConfig{outputFormat: appformat.DefaultOutputFormat}
fs := flag.NewFlagSet("inspect prompt", flag.ContinueOnError)
fs.SetOutput(io.Discard)
registerConfigPathFlag(fs, &cfg.configPath)
fs.StringVar(&cfg.promptDir, "prompt-dir", "", "directory containing prompt definitions")
fs.StringVar(&cfg.promptID, "prompt", "", "prompt ID to inspect")
fs.StringVar(&cfg.promptVersion, "prompt-version", "", "optional prompt version")
rawFormat := ""
fs.StringVar(&rawFormat, "format", "", "output format: text or json")
fs.StringVar(&cfg.outputPath, "out", "", "optional output file path")
if err := fs.Parse(args); err != nil {
return nil, err
}
if fs.NArg() > 0 {
return nil, fmt.Errorf("unexpected positional args: %v", fs.Args())
}
if strings.TrimSpace(cfg.promptID) == "" {
return nil, errors.New("--prompt is required")
}
format, err := appformat.ParseOutputFormat(rawFormat)
if err != nil {
return nil, err
}
cfg.outputFormat = format
if cfg.outputPath != "" {
cfg.outputPath = filepath.Clean(cfg.outputPath)
}
return cfg, nil
}
func resolveAppSettingsForPromptInspection(cfg *promptInspectionConfig) (engineSettings, error) {
settings, err := appconfig.LoadConfig(cfg.configPath, cfg.configPath != "")
if err != nil {
return engineSettings{}, fmt.Errorf("application config error: %w", err)
}
if strings.TrimSpace(cfg.promptDir) != "" {
settings.PromptDir = filepath.Clean(cfg.promptDir)
}
if strings.TrimSpace(settings.PromptDir) == "" {
return engineSettings{}, errors.New(errPromptDirRequired)
}
return engineSettings{promptDir: settings.PromptDir, backends: settings.Backends}, nil
}

View File

@@ -120,6 +120,8 @@ func Run(args []string, stdout, stderr io.Writer) int {
return renderCommand(args[1:], stdout, stderr)
case "serve":
return serveCommand(args[1:], stderr)
case "inspect":
return inspectCommand(args[1:], stdout, stderr)
default:
fmt.Fprintf(stderr, "unknown command %q\n", args[0])
printUsage(stderr)
@@ -746,8 +748,9 @@ func runErrorMessage(err error) string {
}
func printUsage(w io.Writer) {
fmt.Fprintln(w, "usage: scriptorium <run|render|serve> ...")
fmt.Fprintln(w, "usage: scriptorium <run|render|serve|inspect> ...")
fmt.Fprintln(w, " run: scriptorium run [--config PATH] [--prompt-dir DIR] [--profile-dir DIR] --prompt ID [--prompt-version VERSION] [--input name=path] [--profile ID] [--session-id ID] [--llm-base-url URL] [--model NAME] [--api-key-env ENV] [--temperature N] [--max-tokens N] [--top-p N] [--reasoning-effort VALUE] [--var k=v] [--out path] [--timeout 10m]")
fmt.Fprintln(w, " render: scriptorium render [--config PATH] [--prompt-dir DIR] [--profile-dir DIR] --prompt ID [--prompt-version VERSION] [--input name=path] [--profile ID] [--session-id ID] [--llm-base-url URL] [--model NAME] [--api-key-env ENV] [--temperature N] [--max-tokens N] [--top-p N] [--reasoning-effort VALUE] [--var k=v] [--format text|json] [--out path] [--timeout 10m]")
fmt.Fprintf(w, " serve: scriptorium serve [--config PATH] [--addr %s] [--prompt-dir DIR] [--profile-dir DIR] [--schema-dir DIR] [--artifact-root DIR] [--max-request-bytes N] [--max-artifact-bytes N] [--max-response-bytes N]\n", defaults.HTTPAddrDefault)
fmt.Fprintln(w, " inspect prompt: scriptorium inspect prompt --prompt ID [--prompt-version VERSION] [--config PATH] [--prompt-dir DIR] [--format text|json] [--out path]")
}

View File

@@ -1157,6 +1157,30 @@ output:
}
}
func TestInspectPromptCommandFormatsDefinitionWithoutProfileOrGeneration(t *testing.T) {
lib := newCLITestLibrary(t)
writePromptDefinition(t, lib.promptDir, "inspect.yaml", `id: inspect
version: "1"
messages:
- role: user
content: "hello"
output:
format: text
validation_mode: none
`)
code, stdout, stderr := runCLICommand(t, inspectCommand, []string{"prompt", "--prompt-dir", lib.promptDir, "--prompt", "inspect", "--format", "json"})
if code != ExitOK {
t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr)
}
if !strings.Contains(stdout, `"prompt_id": "inspect"`) || !strings.Contains(stdout, `"inputs": []`) {
t.Fatalf("unexpected inspection output: %s", stdout)
}
code, stdout, stderr = runCLICommand(t, inspectCommand, []string{"prompt", "--prompt-dir", lib.promptDir, "--prompt", "missing"})
if code != ExitRuntimeError || stdout != "" || !strings.Contains(stderr, "inspect error") {
t.Fatalf("expected failed inspection without output, got code=%d stdout=%q stderr=%q", code, stdout, stderr)
}
}
func TestConfiguredBackendValidationComesFromPromptkit(t *testing.T) {
lib := newCLITestLibrary(t)