From 36c7c5a358a057b04f3f61afbf2bb0159df29ae3 Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Sat, 29 Aug 2026 14:29:14 +0000 Subject: [PATCH] Add prompt inspection command --- docs/cli.md | 14 ++++ docs/internal/adapters.md | 5 ++ docs/roadmap/implementation.md | 2 + internal/adapter/cli/inspect.go | 101 +++++++++++++++++++++++++++++ internal/adapter/cli/run.go | 5 +- internal/adapter/cli/run_test.go | 24 +++++++ internal/format/inspection.go | 71 ++++++++++++++++++++ internal/format/inspection_test.go | 32 +++++++++ internal/format/prepared_run.go | 39 +++++++---- 9 files changed, 280 insertions(+), 13 deletions(-) create mode 100644 internal/adapter/cli/inspect.go create mode 100644 internal/format/inspection.go create mode 100644 internal/format/inspection_test.go diff --git a/docs/cli.md b/docs/cli.md index d0cf847..211853d 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -23,6 +23,7 @@ go run ./cmd/scriptorium render \ generated artifact. - `scriptorium render`: prepare a prompt and write prepared-run output. - `scriptorium serve`: start the HTTP server. +- `scriptorium inspect prompt`: inspect one prompt definition without model execution. All commands accept `--config ` and reject positional arguments. An effective `prompt_dir` is required for every command. Supply it through the @@ -137,6 +138,19 @@ Optional flags: `serve` accepts no runtime model override flags. HTTP request fields, response schemas, and error codes are defined in the [HTTP API reference](api.md). +## `scriptorium inspect prompt` + +```text +scriptorium inspect prompt --prompt ID [--prompt-version VERSION] + [--config PATH] [--prompt-dir DIR] [--format text|json] [--out PATH] +``` + +`--prompt` is required. Inspection uses normal configuration discovery and a +`--prompt-dir` override, defaults to text regardless of `defaults.render_format`, +and writes to stdout unless `--out` is supplied. It loads and normalizes the +selected definition but does not resolve a profile, load a schema, render a +template, reserve backend capacity, or call a model. + ## Input And Variable Syntax `--input name=path` maps an input name to a local file; `--var name=value` diff --git a/docs/internal/adapters.md b/docs/internal/adapters.md index 0fe0db3..cb99e71 100644 --- a/docs/internal/adapters.md +++ b/docs/internal/adapters.md @@ -34,6 +34,11 @@ and profile selection, optional file inputs, variables, and presence-aware execution overrides. Omitted framework settings remain zero values so Promptkit resolves its own defaults and definition-required inputs. +`inspect prompt` maps the selected ID and optional version to +`promptkit.Engine.InspectPrompt`, then formats Scriptorium-owned inspection +data. It constructs the same configuration-aware engine but does not perform +preparation or generation. + `serve` constructs Scriptorium's restricted HTTP artifact reader, injects it with `promptkit.WithArtifactReader`, passes the engine through the HTTP adapter's consumer-owned `Runner` interface, and starts the server. diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index d789d88..da30cbb 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -338,6 +338,8 @@ backend capacity across concurrent requests. ## Stage 6: Add Stable Prompt Inspection Output And CLI +**Completion: Complete.** + Introduce reusable application-owned inspection presentation, then expose prompt inspection without model execution. diff --git a/internal/adapter/cli/inspect.go b/internal/adapter/cli/inspect.go new file mode 100644 index 0000000..1e2d48e --- /dev/null +++ b/internal/adapter/cli/inspect.go @@ -0,0 +1,101 @@ +package cli + +import ( + "context" + "errors" + "flag" + "fmt" + "io" + "path/filepath" + "strings" + + appconfig "gitea.maximumdirect.net/eric/scriptorium/internal/config" + appformat "gitea.maximumdirect.net/eric/scriptorium/internal/format" +) + +type promptInspectionConfig struct { + configPath, promptDir, promptID, promptVersion, outputPath string + outputFormat appformat.OutputFormat +} + +func inspectCommand(args []string, stdout, stderr io.Writer) int { + if len(args) == 0 || args[0] != "prompt" { + fmt.Fprintln(stderr, "inspect parse error: inspection mode must be prompt") + return ExitRuntimeError + } + cfg, err := parsePromptInspectionArgs(args[1:]) + if err != nil { + fmt.Fprintf(stderr, "inspect parse error: %v\n", err) + return ExitRuntimeError + } + settings, err := resolveAppSettingsForPromptInspection(cfg) + if err != nil { + fmt.Fprintf(stderr, "inspect error: %v\n", err) + return ExitRuntimeError + } + engine, err := newEngine(settings) + if err != nil { + fmt.Fprintf(stderr, "engine error: %v\n", err) + return ExitRuntimeError + } + inspection, err := engine.InspectPrompt(context.Background(), cfg.promptID, cfg.promptVersion) + if err != nil { + fmt.Fprintf(stderr, "inspect error: %v\n", err) + return ExitRuntimeError + } + data, err := appformat.FormatPromptInspection(inspection, cfg.outputFormat) + if err != nil { + fmt.Fprintf(stderr, "inspect error: %v\n", err) + return ExitRuntimeError + } + if err := writeOutput(stdout, cfg.outputPath, data); err != nil { + fmt.Fprintf(stderr, "output write error: %v\n", err) + return ExitRuntimeError + } + return ExitOK +} + +func parsePromptInspectionArgs(args []string) (*promptInspectionConfig, error) { + cfg := &promptInspectionConfig{outputFormat: appformat.DefaultOutputFormat} + fs := flag.NewFlagSet("inspect prompt", flag.ContinueOnError) + fs.SetOutput(io.Discard) + registerConfigPathFlag(fs, &cfg.configPath) + fs.StringVar(&cfg.promptDir, "prompt-dir", "", "directory containing prompt definitions") + fs.StringVar(&cfg.promptID, "prompt", "", "prompt ID to inspect") + fs.StringVar(&cfg.promptVersion, "prompt-version", "", "optional prompt version") + rawFormat := "" + fs.StringVar(&rawFormat, "format", "", "output format: text or json") + fs.StringVar(&cfg.outputPath, "out", "", "optional output file path") + if err := fs.Parse(args); err != nil { + return nil, err + } + if fs.NArg() > 0 { + return nil, fmt.Errorf("unexpected positional args: %v", fs.Args()) + } + if strings.TrimSpace(cfg.promptID) == "" { + return nil, errors.New("--prompt is required") + } + format, err := appformat.ParseOutputFormat(rawFormat) + if err != nil { + return nil, err + } + cfg.outputFormat = format + if cfg.outputPath != "" { + cfg.outputPath = filepath.Clean(cfg.outputPath) + } + return cfg, nil +} + +func resolveAppSettingsForPromptInspection(cfg *promptInspectionConfig) (engineSettings, error) { + settings, err := appconfig.LoadConfig(cfg.configPath, cfg.configPath != "") + if err != nil { + return engineSettings{}, fmt.Errorf("application config error: %w", err) + } + if strings.TrimSpace(cfg.promptDir) != "" { + settings.PromptDir = filepath.Clean(cfg.promptDir) + } + if strings.TrimSpace(settings.PromptDir) == "" { + return engineSettings{}, errors.New(errPromptDirRequired) + } + return engineSettings{promptDir: settings.PromptDir, backends: settings.Backends}, nil +} diff --git a/internal/adapter/cli/run.go b/internal/adapter/cli/run.go index cc297ed..c384079 100644 --- a/internal/adapter/cli/run.go +++ b/internal/adapter/cli/run.go @@ -120,6 +120,8 @@ func Run(args []string, stdout, stderr io.Writer) int { return renderCommand(args[1:], stdout, stderr) case "serve": return serveCommand(args[1:], stderr) + case "inspect": + return inspectCommand(args[1:], stdout, stderr) default: fmt.Fprintf(stderr, "unknown command %q\n", args[0]) printUsage(stderr) @@ -746,8 +748,9 @@ func runErrorMessage(err error) string { } func printUsage(w io.Writer) { - fmt.Fprintln(w, "usage: scriptorium ...") + fmt.Fprintln(w, "usage: scriptorium ...") fmt.Fprintln(w, " run: scriptorium run [--config PATH] [--prompt-dir DIR] [--profile-dir DIR] --prompt ID [--prompt-version VERSION] [--input name=path] [--profile ID] [--session-id ID] [--llm-base-url URL] [--model NAME] [--api-key-env ENV] [--temperature N] [--max-tokens N] [--top-p N] [--reasoning-effort VALUE] [--var k=v] [--out path] [--timeout 10m]") fmt.Fprintln(w, " render: scriptorium render [--config PATH] [--prompt-dir DIR] [--profile-dir DIR] --prompt ID [--prompt-version VERSION] [--input name=path] [--profile ID] [--session-id ID] [--llm-base-url URL] [--model NAME] [--api-key-env ENV] [--temperature N] [--max-tokens N] [--top-p N] [--reasoning-effort VALUE] [--var k=v] [--format text|json] [--out path] [--timeout 10m]") fmt.Fprintf(w, " serve: scriptorium serve [--config PATH] [--addr %s] [--prompt-dir DIR] [--profile-dir DIR] [--schema-dir DIR] [--artifact-root DIR] [--max-request-bytes N] [--max-artifact-bytes N] [--max-response-bytes N]\n", defaults.HTTPAddrDefault) + fmt.Fprintln(w, " inspect prompt: scriptorium inspect prompt --prompt ID [--prompt-version VERSION] [--config PATH] [--prompt-dir DIR] [--format text|json] [--out path]") } diff --git a/internal/adapter/cli/run_test.go b/internal/adapter/cli/run_test.go index 3ab138e..b76733c 100644 --- a/internal/adapter/cli/run_test.go +++ b/internal/adapter/cli/run_test.go @@ -1157,6 +1157,30 @@ output: } } +func TestInspectPromptCommandFormatsDefinitionWithoutProfileOrGeneration(t *testing.T) { + lib := newCLITestLibrary(t) + writePromptDefinition(t, lib.promptDir, "inspect.yaml", `id: inspect +version: "1" +messages: + - role: user + content: "hello" +output: + format: text + validation_mode: none +`) + code, stdout, stderr := runCLICommand(t, inspectCommand, []string{"prompt", "--prompt-dir", lib.promptDir, "--prompt", "inspect", "--format", "json"}) + if code != ExitOK { + t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr) + } + if !strings.Contains(stdout, `"prompt_id": "inspect"`) || !strings.Contains(stdout, `"inputs": []`) { + t.Fatalf("unexpected inspection output: %s", stdout) + } + code, stdout, stderr = runCLICommand(t, inspectCommand, []string{"prompt", "--prompt-dir", lib.promptDir, "--prompt", "missing"}) + if code != ExitRuntimeError || stdout != "" || !strings.Contains(stderr, "inspect error") { + t.Fatalf("expected failed inspection without output, got code=%d stdout=%q stderr=%q", code, stdout, stderr) + } +} + func TestConfiguredBackendValidationComesFromPromptkit(t *testing.T) { lib := newCLITestLibrary(t) diff --git a/internal/format/inspection.go b/internal/format/inspection.go new file mode 100644 index 0000000..9c4b4d8 --- /dev/null +++ b/internal/format/inspection.go @@ -0,0 +1,71 @@ +package format + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "strconv" + + "gitea.maximumdirect.net/eric/promptkit" +) + +type PromptInspection struct { + PromptID string `json:"prompt_id"` + PromptVersion string `json:"prompt_version"` + PromptHash string `json:"prompt_hash"` + DefaultProfileID string `json:"default_profile_id"` + Inputs []PromptInspectionInput `json:"inputs"` + OutputContract OutputContract `json:"output_contract"` +} + +type PromptInspectionInput struct { + Name string `json:"name"` + Required bool `json:"required"` + ContentType string `json:"content_type"` + Description string `json:"description"` +} + +type OutputContract struct { + Format string `json:"format"` + ValidationMode string `json:"validation_mode"` + SchemaPath string `json:"schema_path"` + RepairAttempts int `json:"repair_attempts"` +} + +func FormatPromptInspection(value *promptkit.PromptInspection, outputFormat OutputFormat) ([]byte, error) { + if value == nil { + return nil, errors.New("prompt inspection is nil") + } + dto := PromptInspection{ + PromptID: value.PromptID, PromptVersion: value.PromptVersion, PromptHash: value.PromptHash, DefaultProfileID: value.DefaultProfileID, + Inputs: make([]PromptInspectionInput, len(value.Inputs)), + OutputContract: OutputContract{Format: string(value.OutputContract.Format), ValidationMode: string(value.OutputContract.ValidationMode), SchemaPath: value.OutputContract.SchemaPath, RepairAttempts: value.OutputContract.RepairAttempts}, + } + for i, input := range value.Inputs { + dto.Inputs[i] = PromptInspectionInput{Name: input.Name, Required: input.Required, ContentType: input.ContentType, Description: input.Description} + } + switch outputFormat { + case OutputFormatJSON: + data, err := json.MarshalIndent(dto, "", " ") + if err != nil { + return nil, err + } + return append(data, '\n'), nil + case OutputFormatText: + var b bytes.Buffer + fmt.Fprintf(&b, "prompt_id: %s\nprompt_version: %s\nprompt_hash: %s\ndefault_profile_id: %s\ninputs:", dto.PromptID, dto.PromptVersion, dto.PromptHash, dto.DefaultProfileID) + if len(dto.Inputs) == 0 { + fmt.Fprintln(&b, " []") + } else { + fmt.Fprintln(&b) + for _, input := range dto.Inputs { + fmt.Fprintf(&b, " - name: %s\n required: %t\n content_type: %s\n description: %s\n", input.Name, input.Required, input.ContentType, strconv.Quote(input.Description)) + } + } + fmt.Fprintf(&b, "output_contract:\n format: %s\n validation_mode: %s\n schema_path: %s\n repair_attempts: %d\n", dto.OutputContract.Format, dto.OutputContract.ValidationMode, dto.OutputContract.SchemaPath, dto.OutputContract.RepairAttempts) + return b.Bytes(), nil + default: + return nil, fmt.Errorf("%w: %q", ErrUnknownPreparedRunFormat, outputFormat) + } +} diff --git a/internal/format/inspection_test.go b/internal/format/inspection_test.go new file mode 100644 index 0000000..9280cbf --- /dev/null +++ b/internal/format/inspection_test.go @@ -0,0 +1,32 @@ +package format + +import ( + "strings" + "testing" + + "gitea.maximumdirect.net/eric/promptkit" +) + +func TestFormatPromptInspectionPreservesDeclaredOrderAndEmptyValues(t *testing.T) { + value := &promptkit.PromptInspection{PromptID: "p", PromptVersion: "1", PromptHash: "hash", Inputs: []promptkit.PromptInputDefinition{{Name: "first", Required: true, ContentType: "text/plain", Description: "first input"}, {Name: "second"}}, OutputContract: promptkit.OutputContract{Format: "text", ValidationMode: promptkit.ValidationNone}} + text, err := FormatPromptInspection(value, OutputFormatText) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(text), "default_profile_id: \ninputs:\n - name: first") || strings.Index(string(text), "name: first") > strings.Index(string(text), "name: second") { + t.Fatalf("unexpected text inspection: %s", text) + } + jsonOutput, err := FormatPromptInspection(value, OutputFormatJSON) + if err != nil { + t.Fatal(err) + } + if !strings.HasSuffix(string(jsonOutput), "\n") || !strings.Contains(string(jsonOutput), `"default_profile_id": ""`) { + t.Fatalf("unexpected json inspection: %s", jsonOutput) + } +} + +func TestFormatPromptInspectionRejectsNil(t *testing.T) { + if _, err := FormatPromptInspection(nil, OutputFormatText); err == nil { + t.Fatal("expected nil inspection error") + } +} diff --git a/internal/format/prepared_run.go b/internal/format/prepared_run.go index e4e8c56..2516c5e 100644 --- a/internal/format/prepared_run.go +++ b/internal/format/prepared_run.go @@ -14,14 +14,24 @@ import ( var ErrUnknownPreparedRunFormat = errors.New("unknown prepared run format") -// PreparedRunOutputFormat is the output format for prepared render data. -type PreparedRunOutputFormat string +// OutputFormat selects an application-owned textual or JSON representation. +type OutputFormat string const ( - PreparedRunFormatText PreparedRunOutputFormat = "text" - PreparedRunFormatJSON PreparedRunOutputFormat = "json" + OutputFormatText OutputFormat = "text" + OutputFormatJSON OutputFormat = "json" - DefaultPreparedRunOutputFormat PreparedRunOutputFormat = PreparedRunFormatText + DefaultOutputFormat OutputFormat = OutputFormatText +) + +// PreparedRunOutputFormat is the output format for prepared render data. +type PreparedRunOutputFormat = OutputFormat + +const ( + PreparedRunFormatText = OutputFormatText + PreparedRunFormatJSON = OutputFormatJSON + + DefaultPreparedRunOutputFormat = DefaultOutputFormat ) // PreparedRunFormatter serializes a prepared run without performing use case work. @@ -31,15 +41,20 @@ type PreparedRunFormatter interface { // ParsePreparedRunOutputFormat parses a format name. func ParsePreparedRunOutputFormat(raw string) (PreparedRunOutputFormat, error) { - switch PreparedRunOutputFormat(strings.ToLower(strings.TrimSpace(raw))) { + return ParseOutputFormat(raw) +} + +// ParseOutputFormat parses the shared inspection and prepared-run format names. +func ParseOutputFormat(raw string) (OutputFormat, error) { + switch OutputFormat(strings.ToLower(strings.TrimSpace(raw))) { case "": - return DefaultPreparedRunOutputFormat, nil - case PreparedRunFormatText: - return PreparedRunFormatText, nil - case PreparedRunFormatJSON: - return PreparedRunFormatJSON, nil + return DefaultOutputFormat, nil + case OutputFormatText: + return OutputFormatText, nil + case OutputFormatJSON: + return OutputFormatJSON, nil default: - return "", fmt.Errorf("%w: %q (supported: %s, %s)", ErrUnknownPreparedRunFormat, raw, PreparedRunFormatText, PreparedRunFormatJSON) + return "", fmt.Errorf("%w: %q (supported: %s, %s)", ErrUnknownPreparedRunFormat, raw, OutputFormatText, OutputFormatJSON) } }