Add profile inspection command

This commit is contained in:
2026-08-29 14:31:13 +00:00
parent 36c7c5a358
commit 68ebe9ee50
5 changed files with 158 additions and 2 deletions

View File

@@ -24,6 +24,7 @@ go run ./cmd/scriptorium render \
- `scriptorium render`: prepare a prompt and write prepared-run output. - `scriptorium render`: prepare a prompt and write prepared-run output.
- `scriptorium serve`: start the HTTP server. - `scriptorium serve`: start the HTTP server.
- `scriptorium inspect prompt`: inspect one prompt definition without model execution. - `scriptorium inspect prompt`: inspect one prompt definition without model execution.
- `scriptorium inspect profile`: inspect one effective profile without model execution.
All commands accept `--config <path>` and reject positional arguments. An All commands accept `--config <path>` and reject positional arguments. An
effective `prompt_dir` is required for every command. Supply it through the effective `prompt_dir` is required for every command. Supply it through the
@@ -151,6 +152,18 @@ and writes to stdout unless `--out` is supplied. It loads and normalizes the
selected definition but does not resolve a profile, load a schema, render a selected definition but does not resolve a profile, load a schema, render a
template, reserve backend capacity, or call a model. template, reserve backend capacity, or call a model.
## `scriptorium inspect profile`
```text
scriptorium inspect profile --profile ID
[--config PATH] [--profile-dir DIR] [--format text|json] [--out PATH]
```
`--profile` is required; built-in profiles need no prompt directory. Inspection
resolves profile inheritance and backend defaults, but never reads a credential
value, loads a prompt, reserves capacity, or calls a model. Unset provider
controls are shown as zero values when Promptkit leaves them unspecified.
## Input And Variable Syntax ## Input And Variable Syntax
`--input name=path` maps an input name to a local file; `--var name=value` `--input name=path` maps an input name to a local file; `--var name=value`

View File

@@ -39,6 +39,10 @@ resolves its own defaults and definition-required inputs.
data. It constructs the same configuration-aware engine but does not perform data. It constructs the same configuration-aware engine but does not perform
preparation or generation. preparation or generation.
`inspect profile` maps an explicit profile ID to `Engine.InspectProfile` and
formats a safe application-owned effective profile view. It intentionally does
not require prompt, schema, or artifact sources.
`serve` constructs Scriptorium's restricted HTTP artifact reader, injects it `serve` constructs Scriptorium's restricted HTTP artifact reader, injects it
with `promptkit.WithArtifactReader`, passes the engine through the HTTP with `promptkit.WithArtifactReader`, passes the engine through the HTTP
adapter's consumer-owned `Runner` interface, and starts the server. adapter's consumer-owned `Runner` interface, and starts the server.

View File

@@ -432,6 +432,8 @@ prepared-run presentation remains backward compatible.
## Stage 7: Add Stable Profile Inspection Output And CLI ## Stage 7: Add Stable Profile Inspection Output And CLI
**Completion: Complete.**
Complete the inspection command family with inherited, built-in, custom, and Complete the inspection command family with inherited, built-in, custom, and
endpoint-only profile support. endpoint-only profile support.

View File

@@ -18,9 +18,21 @@ type promptInspectionConfig struct {
outputFormat appformat.OutputFormat outputFormat appformat.OutputFormat
} }
type profileInspectionConfig struct {
configPath, profileDir, profileID, outputPath string
outputFormat appformat.OutputFormat
}
func inspectCommand(args []string, stdout, stderr io.Writer) int { func inspectCommand(args []string, stdout, stderr io.Writer) int {
if len(args) == 0 || args[0] != "prompt" { if len(args) == 0 {
fmt.Fprintln(stderr, "inspect parse error: inspection mode must be prompt") fmt.Fprintln(stderr, "inspect parse error: inspection mode is required")
return ExitRuntimeError
}
if args[0] == "profile" {
return inspectProfileCommand(args[1:], stdout, stderr)
}
if args[0] != "prompt" {
fmt.Fprintln(stderr, "inspect parse error: unknown inspection mode")
return ExitRuntimeError return ExitRuntimeError
} }
cfg, err := parsePromptInspectionArgs(args[1:]) cfg, err := parsePromptInspectionArgs(args[1:])
@@ -55,6 +67,80 @@ func inspectCommand(args []string, stdout, stderr io.Writer) int {
return ExitOK return ExitOK
} }
func inspectProfileCommand(args []string, stdout, stderr io.Writer) int {
cfg, err := parseProfileInspectionArgs(args)
if err != nil {
fmt.Fprintf(stderr, "inspect parse error: %v\n", err)
return ExitRuntimeError
}
settings, err := resolveAppSettingsForProfileInspection(cfg)
if err != nil {
fmt.Fprintf(stderr, "inspect error: %v\n", err)
return ExitRuntimeError
}
engine, err := newEngine(settings)
if err != nil {
fmt.Fprintf(stderr, "engine error: %v\n", err)
return ExitRuntimeError
}
inspection, err := engine.InspectProfile(context.Background(), cfg.profileID)
if err != nil {
fmt.Fprintf(stderr, "inspect error: %v\n", err)
return ExitRuntimeError
}
data, err := appformat.FormatProfileInspection(inspection, cfg.outputFormat)
if err != nil {
fmt.Fprintf(stderr, "inspect error: %v\n", err)
return ExitRuntimeError
}
if err := writeOutput(stdout, cfg.outputPath, data); err != nil {
fmt.Fprintf(stderr, "output write error: %v\n", err)
return ExitRuntimeError
}
return ExitOK
}
func parseProfileInspectionArgs(args []string) (*profileInspectionConfig, error) {
cfg := &profileInspectionConfig{outputFormat: appformat.DefaultOutputFormat}
fs := flag.NewFlagSet("inspect profile", flag.ContinueOnError)
fs.SetOutput(io.Discard)
registerConfigPathFlag(fs, &cfg.configPath)
fs.StringVar(&cfg.profileDir, "profile-dir", "", "directory containing execution profiles")
fs.StringVar(&cfg.profileID, "profile", "", "profile ID to inspect")
rawFormat := ""
fs.StringVar(&rawFormat, "format", "", "output format: text or json")
fs.StringVar(&cfg.outputPath, "out", "", "optional output file path")
if err := fs.Parse(args); err != nil {
return nil, err
}
if fs.NArg() > 0 {
return nil, fmt.Errorf("unexpected positional args: %v", fs.Args())
}
if strings.TrimSpace(cfg.profileID) == "" {
return nil, errors.New("--profile is required")
}
format, err := appformat.ParseOutputFormat(rawFormat)
if err != nil {
return nil, err
}
cfg.outputFormat = format
if cfg.outputPath != "" {
cfg.outputPath = filepath.Clean(cfg.outputPath)
}
return cfg, nil
}
func resolveAppSettingsForProfileInspection(cfg *profileInspectionConfig) (engineSettings, error) {
settings, err := appconfig.LoadConfig(cfg.configPath, cfg.configPath != "")
if err != nil {
return engineSettings{}, fmt.Errorf("application config error: %w", err)
}
if strings.TrimSpace(cfg.profileDir) != "" {
settings.ProfileDir = filepath.Clean(cfg.profileDir)
}
return engineSettings{promptDir: ".", profileDir: settings.ProfileDir, backends: settings.Backends}, nil
}
func parsePromptInspectionArgs(args []string) (*promptInspectionConfig, error) { func parsePromptInspectionArgs(args []string) (*promptInspectionConfig, error) {
cfg := &promptInspectionConfig{outputFormat: appformat.DefaultOutputFormat} cfg := &promptInspectionConfig{outputFormat: appformat.DefaultOutputFormat}
fs := flag.NewFlagSet("inspect prompt", flag.ContinueOnError) fs := flag.NewFlagSet("inspect prompt", flag.ContinueOnError)

View File

@@ -33,6 +33,26 @@ type OutputContract struct {
RepairAttempts int `json:"repair_attempts"` RepairAttempts int `json:"repair_attempts"`
} }
type ProfileInspection struct {
ProfileID string `json:"profile_id"`
EffectiveModelParams ProfileModelParams `json:"effective_model_params"`
APIKeyRequired bool `json:"api_key_required"`
}
type ProfileModelParams struct {
BackendID string `json:"backend_id"`
Endpoint string `json:"endpoint"`
Model string `json:"model"`
Temperature float64 `json:"temperature"`
MaxTokens int `json:"max_tokens"`
TopP float64 `json:"top_p"`
TimeoutSeconds int `json:"timeout_seconds"`
ServiceTier string `json:"service_tier"`
ReasoningEffort string `json:"reasoning_effort"`
APIKeyEnv string `json:"api_key_env"`
ExtraParams map[string]any `json:"extra_params"`
}
func FormatPromptInspection(value *promptkit.PromptInspection, outputFormat OutputFormat) ([]byte, error) { func FormatPromptInspection(value *promptkit.PromptInspection, outputFormat OutputFormat) ([]byte, error) {
if value == nil { if value == nil {
return nil, errors.New("prompt inspection is nil") return nil, errors.New("prompt inspection is nil")
@@ -69,3 +89,34 @@ func FormatPromptInspection(value *promptkit.PromptInspection, outputFormat Outp
return nil, fmt.Errorf("%w: %q", ErrUnknownPreparedRunFormat, outputFormat) return nil, fmt.Errorf("%w: %q", ErrUnknownPreparedRunFormat, outputFormat)
} }
} }
func FormatProfileInspection(value *promptkit.ProfileInspection, outputFormat OutputFormat) ([]byte, error) {
if value == nil {
return nil, errors.New("profile inspection is nil")
}
target := value.EffectiveModelParams
params := map[string]any{}
for key, item := range target.ExtraParams {
params[key] = item
}
dto := ProfileInspection{ProfileID: value.ProfileID, APIKeyRequired: value.APIKeyRequired, EffectiveModelParams: ProfileModelParams{BackendID: target.BackendID, Endpoint: target.Endpoint, Model: target.Model, Temperature: target.Temperature, MaxTokens: target.MaxTokens, TopP: target.TopP, TimeoutSeconds: target.TimeoutSeconds, ServiceTier: target.ServiceTier, ReasoningEffort: target.ReasoningEffort, APIKeyEnv: target.APIKeyEnv, ExtraParams: params}}
switch outputFormat {
case OutputFormatJSON:
data, err := json.MarshalIndent(dto, "", " ")
if err != nil {
return nil, err
}
return append(data, '\n'), nil
case OutputFormatText:
var b bytes.Buffer
fmt.Fprintf(&b, "profile_id: %s\neffective_model_params:\n backend_id: %s\n endpoint: %s\n model: %s\n temperature: %g\n max_tokens: %d\n top_p: %g\n timeout_seconds: %d\n service_tier: %s\n reasoning_effort: %s\n api_key_env: %s\n extra_params: ", dto.ProfileID, dto.EffectiveModelParams.BackendID, dto.EffectiveModelParams.Endpoint, dto.EffectiveModelParams.Model, dto.EffectiveModelParams.Temperature, dto.EffectiveModelParams.MaxTokens, dto.EffectiveModelParams.TopP, dto.EffectiveModelParams.TimeoutSeconds, dto.EffectiveModelParams.ServiceTier, dto.EffectiveModelParams.ReasoningEffort, dto.EffectiveModelParams.APIKeyEnv)
paramsJSON, err := json.Marshal(dto.EffectiveModelParams.ExtraParams)
if err != nil {
return nil, err
}
fmt.Fprintf(&b, "%s\napi_key_required: %t\n", paramsJSON, dto.APIKeyRequired)
return b.Bytes(), nil
default:
return nil, fmt.Errorf("%w: %q", ErrUnknownPreparedRunFormat, outputFormat)
}
}