eric/scriptorium
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 7 Wiki Activity

Add prompt inspection command

Browse Source
This commit is contained in:
Eric Rakestraw
2026-08-29 14:29:14 +00:00
parent 157209f097
commit 36c7c5a358
9 changed files with 280 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

101
internal/adapter/cli/inspect.go Normal file
View File

@@ -0,0 +1,101 @@
package cli
import (
"context"
"errors"
"flag"
"fmt"
"io"
"path/filepath"
"strings"
appconfig "gitea.maximumdirect.net/eric/scriptorium/internal/config"
appformat "gitea.maximumdirect.net/eric/scriptorium/internal/format"
)
type promptInspectionConfig struct {
configPath, promptDir, promptID, promptVersion, outputPath string
outputFormat appformat.OutputFormat
}
func inspectCommand(args []string, stdout, stderr io.Writer) int {
if len(args) == 0 || args[0] != "prompt" {
fmt.Fprintln(stderr, "inspect parse error: inspection mode must be prompt")
return ExitRuntimeError
}
cfg, err := parsePromptInspectionArgs(args[1:])
if err != nil {
fmt.Fprintf(stderr, "inspect parse error: %v\n", err)
return ExitRuntimeError
}
settings, err := resolveAppSettingsForPromptInspection(cfg)
if err != nil {
fmt.Fprintf(stderr, "inspect error: %v\n", err)
return ExitRuntimeError
}
engine, err := newEngine(settings)
if err != nil {
fmt.Fprintf(stderr, "engine error: %v\n", err)
return ExitRuntimeError
}
inspection, err := engine.InspectPrompt(context.Background(), cfg.promptID, cfg.promptVersion)
if err != nil {
fmt.Fprintf(stderr, "inspect error: %v\n", err)
return ExitRuntimeError
}
data, err := appformat.FormatPromptInspection(inspection, cfg.outputFormat)
if err != nil {
fmt.Fprintf(stderr, "inspect error: %v\n", err)
return ExitRuntimeError
}
if err := writeOutput(stdout, cfg.outputPath, data); err != nil {
fmt.Fprintf(stderr, "output write error: %v\n", err)
return ExitRuntimeError
}
return ExitOK
}
func parsePromptInspectionArgs(args []string) (*promptInspectionConfig, error) {
cfg := &promptInspectionConfig{outputFormat: appformat.DefaultOutputFormat}
fs := flag.NewFlagSet("inspect prompt", flag.ContinueOnError)
fs.SetOutput(io.Discard)
registerConfigPathFlag(fs, &cfg.configPath)
fs.StringVar(&cfg.promptDir, "prompt-dir", "", "directory containing prompt definitions")
fs.StringVar(&cfg.promptID, "prompt", "", "prompt ID to inspect")
fs.StringVar(&cfg.promptVersion, "prompt-version", "", "optional prompt version")
rawFormat := ""
fs.StringVar(&rawFormat, "format", "", "output format: text or json")
fs.StringVar(&cfg.outputPath, "out", "", "optional output file path")
if err := fs.Parse(args); err != nil {
return nil, err
}
if fs.NArg() > 0 {
return nil, fmt.Errorf("unexpected positional args: %v", fs.Args())
}
if strings.TrimSpace(cfg.promptID) == "" {
return nil, errors.New("--prompt is required")
}
format, err := appformat.ParseOutputFormat(rawFormat)
if err != nil {
return nil, err
}
cfg.outputFormat = format
if cfg.outputPath != "" {
cfg.outputPath = filepath.Clean(cfg.outputPath)
}
return cfg, nil
}
func resolveAppSettingsForPromptInspection(cfg *promptInspectionConfig) (engineSettings, error) {
settings, err := appconfig.LoadConfig(cfg.configPath, cfg.configPath != "")
if err != nil {
return engineSettings{}, fmt.Errorf("application config error: %w", err)
}
if strings.TrimSpace(cfg.promptDir) != "" {
settings.PromptDir = filepath.Clean(cfg.promptDir)
}
if strings.TrimSpace(settings.PromptDir) == "" {
return engineSettings{}, errors.New(errPromptDirRequired)
}
return engineSettings{promptDir: settings.PromptDir, backends: settings.Backends}, nil
}

5
internal/adapter/cli/run.go
View File

@@ -120,6 +120,8 @@ func Run(args []string, stdout, stderr io.Writer) int {
return renderCommand(args[1:], stdout, stderr)
case "serve":
return serveCommand(args[1:], stderr)
case "inspect":
return inspectCommand(args[1:], stdout, stderr)
default:
fmt.Fprintf(stderr, "unknown command %q\n", args[0])
printUsage(stderr)
@@ -746,8 +748,9 @@ func runErrorMessage(err error) string {
}
func printUsage(w io.Writer) {
fmt.Fprintln(w, "usage: scriptorium <run|render|serve> ...")
fmt.Fprintln(w, "usage: scriptorium <run|render|serve|inspect> ...")
fmt.Fprintln(w, " run: scriptorium run [--config PATH] [--prompt-dir DIR] [--profile-dir DIR] --prompt ID [--prompt-version VERSION] [--input name=path] [--profile ID] [--session-id ID] [--llm-base-url URL] [--model NAME] [--api-key-env ENV] [--temperature N] [--max-tokens N] [--top-p N] [--reasoning-effort VALUE] [--var k=v] [--out path] [--timeout 10m]")
fmt.Fprintln(w, " render: scriptorium render [--config PATH] [--prompt-dir DIR] [--profile-dir DIR] --prompt ID [--prompt-version VERSION] [--input name=path] [--profile ID] [--session-id ID] [--llm-base-url URL] [--model NAME] [--api-key-env ENV] [--temperature N] [--max-tokens N] [--top-p N] [--reasoning-effort VALUE] [--var k=v] [--format text|json] [--out path] [--timeout 10m]")
fmt.Fprintf(w, " serve: scriptorium serve [--config PATH] [--addr %s] [--prompt-dir DIR] [--profile-dir DIR] [--schema-dir DIR] [--artifact-root DIR] [--max-request-bytes N] [--max-artifact-bytes N] [--max-response-bytes N]\n", defaults.HTTPAddrDefault)
fmt.Fprintln(w, " inspect prompt: scriptorium inspect prompt --prompt ID [--prompt-version VERSION] [--config PATH] [--prompt-dir DIR] [--format text|json] [--out path]")
}

24
internal/adapter/cli/run_test.go
View File

@@ -1157,6 +1157,30 @@ output:
}
}
func TestInspectPromptCommandFormatsDefinitionWithoutProfileOrGeneration(t *testing.T) {
lib := newCLITestLibrary(t)
writePromptDefinition(t, lib.promptDir, "inspect.yaml", `id: inspect
version: "1"
messages:
- role: user
content: "hello"
output:
format: text
validation_mode: none
`)
code, stdout, stderr := runCLICommand(t, inspectCommand, []string{"prompt", "--prompt-dir", lib.promptDir, "--prompt", "inspect", "--format", "json"})
if code != ExitOK {
t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr)
}
if !strings.Contains(stdout, `"prompt_id": "inspect"`) || !strings.Contains(stdout, `"inputs": []`) {
t.Fatalf("unexpected inspection output: %s", stdout)
}
code, stdout, stderr = runCLICommand(t, inspectCommand, []string{"prompt", "--prompt-dir", lib.promptDir, "--prompt", "missing"})
if code != ExitRuntimeError || stdout != "" || !strings.Contains(stderr, "inspect error") {
t.Fatalf("expected failed inspection without output, got code=%d stdout=%q stderr=%q", code, stdout, stderr)
}
}
func TestConfiguredBackendValidationComesFromPromptkit(t *testing.T) {
lib := newCLITestLibrary(t)

71
internal/format/inspection.go Normal file
View File

@@ -0,0 +1,71 @@
package format
import (
"bytes"
"encoding/json"
"errors"
"fmt"
"strconv"
"gitea.maximumdirect.net/eric/promptkit"
)
type PromptInspection struct {
PromptID string `json:"prompt_id"`
PromptVersion string `json:"prompt_version"`
PromptHash string `json:"prompt_hash"`
DefaultProfileID string `json:"default_profile_id"`
Inputs []PromptInspectionInput `json:"inputs"`
OutputContract OutputContract `json:"output_contract"`
}
type PromptInspectionInput struct {
Name string `json:"name"`
Required bool `json:"required"`
ContentType string `json:"content_type"`
Description string `json:"description"`
}
type OutputContract struct {
Format string `json:"format"`
ValidationMode string `json:"validation_mode"`
SchemaPath string `json:"schema_path"`
RepairAttempts int `json:"repair_attempts"`
}
func FormatPromptInspection(value *promptkit.PromptInspection, outputFormat OutputFormat) ([]byte, error) {
if value == nil {
return nil, errors.New("prompt inspection is nil")
}
dto := PromptInspection{
PromptID: value.PromptID, PromptVersion: value.PromptVersion, PromptHash: value.PromptHash, DefaultProfileID: value.DefaultProfileID,
Inputs: make([]PromptInspectionInput, len(value.Inputs)),
OutputContract: OutputContract{Format: string(value.OutputContract.Format), ValidationMode: string(value.OutputContract.ValidationMode), SchemaPath: value.OutputContract.SchemaPath, RepairAttempts: value.OutputContract.RepairAttempts},
}
for i, input := range value.Inputs {
dto.Inputs[i] = PromptInspectionInput{Name: input.Name, Required: input.Required, ContentType: input.ContentType, Description: input.Description}
}
switch outputFormat {
case OutputFormatJSON:
data, err := json.MarshalIndent(dto, "", " ")
if err != nil {
return nil, err
}
return append(data, '\n'), nil
case OutputFormatText:
var b bytes.Buffer
fmt.Fprintf(&b, "prompt_id: %s\nprompt_version: %s\nprompt_hash: %s\ndefault_profile_id: %s\ninputs:", dto.PromptID, dto.PromptVersion, dto.PromptHash, dto.DefaultProfileID)
if len(dto.Inputs) == 0 {
fmt.Fprintln(&b, " []")
} else {
fmt.Fprintln(&b)
for _, input := range dto.Inputs {
fmt.Fprintf(&b, " - name: %s\n required: %t\n content_type: %s\n description: %s\n", input.Name, input.Required, input.ContentType, strconv.Quote(input.Description))
}
}
fmt.Fprintf(&b, "output_contract:\n format: %s\n validation_mode: %s\n schema_path: %s\n repair_attempts: %d\n", dto.OutputContract.Format, dto.OutputContract.ValidationMode, dto.OutputContract.SchemaPath, dto.OutputContract.RepairAttempts)
return b.Bytes(), nil
default:
return nil, fmt.Errorf("%w: %q", ErrUnknownPreparedRunFormat, outputFormat)
}
}

32
internal/format/inspection_test.go Normal file
View File

@@ -0,0 +1,32 @@
package format
import (
"strings"
"testing"
"gitea.maximumdirect.net/eric/promptkit"
)
func TestFormatPromptInspectionPreservesDeclaredOrderAndEmptyValues(t *testing.T) {
value := &promptkit.PromptInspection{PromptID: "p", PromptVersion: "1", PromptHash: "hash", Inputs: []promptkit.PromptInputDefinition{{Name: "first", Required: true, ContentType: "text/plain", Description: "first input"}, {Name: "second"}}, OutputContract: promptkit.OutputContract{Format: "text", ValidationMode: promptkit.ValidationNone}}
text, err := FormatPromptInspection(value, OutputFormatText)
if err != nil {
t.Fatal(err)
}
if !strings.Contains(string(text), "default_profile_id: \ninputs:\n - name: first") || strings.Index(string(text), "name: first") > strings.Index(string(text), "name: second") {
t.Fatalf("unexpected text inspection: %s", text)
}
jsonOutput, err := FormatPromptInspection(value, OutputFormatJSON)
if err != nil {
t.Fatal(err)
}
if !strings.HasSuffix(string(jsonOutput), "\n") || !strings.Contains(string(jsonOutput), `"default_profile_id": ""`) {
t.Fatalf("unexpected json inspection: %s", jsonOutput)
}
}
func TestFormatPromptInspectionRejectsNil(t *testing.T) {
if _, err := FormatPromptInspection(nil, OutputFormatText); err == nil {
t.Fatal("expected nil inspection error")
}
}

39
internal/format/prepared_run.go
View File

@@ -14,14 +14,24 @@ import (
var ErrUnknownPreparedRunFormat = errors.New("unknown prepared run format")
// PreparedRunOutputFormat is the output format for prepared render data.
type PreparedRunOutputFormat string
// OutputFormat selects an application-owned textual or JSON representation.
type OutputFormat string
const (
PreparedRunFormatText PreparedRunOutputFormat = "text"
PreparedRunFormatJSON PreparedRunOutputFormat = "json"
OutputFormatText OutputFormat = "text"
OutputFormatJSON OutputFormat = "json"
DefaultPreparedRunOutputFormat PreparedRunOutputFormat = PreparedRunFormatText
DefaultOutputFormat OutputFormat = OutputFormatText
)
// PreparedRunOutputFormat is the output format for prepared render data.
type PreparedRunOutputFormat = OutputFormat
const (
PreparedRunFormatText = OutputFormatText
PreparedRunFormatJSON = OutputFormatJSON
DefaultPreparedRunOutputFormat = DefaultOutputFormat
)
// PreparedRunFormatter serializes a prepared run without performing use case work.
@@ -31,15 +41,20 @@ type PreparedRunFormatter interface {
// ParsePreparedRunOutputFormat parses a format name.
func ParsePreparedRunOutputFormat(raw string) (PreparedRunOutputFormat, error) {
switch PreparedRunOutputFormat(strings.ToLower(strings.TrimSpace(raw))) {
return ParseOutputFormat(raw)
}
// ParseOutputFormat parses the shared inspection and prepared-run format names.
func ParseOutputFormat(raw string) (OutputFormat, error) {
switch OutputFormat(strings.ToLower(strings.TrimSpace(raw))) {
case "":
return DefaultPreparedRunOutputFormat, nil
case PreparedRunFormatText:
return PreparedRunFormatText, nil
case PreparedRunFormatJSON:
return PreparedRunFormatJSON, nil
return DefaultOutputFormat, nil
case OutputFormatText:
return OutputFormatText, nil
case OutputFormatJSON:
return OutputFormatJSON, nil
default:
return "", fmt.Errorf("%w: %q (supported: %s, %s)", ErrUnknownPreparedRunFormat, raw, PreparedRunFormatText, PreparedRunFormatJSON)
return "", fmt.Errorf("%w: %q (supported: %s, %s)", ErrUnknownPreparedRunFormat, raw, OutputFormatText, OutputFormatJSON)
}
}