Add profile inspection command
This commit is contained in:
13
docs/cli.md
13
docs/cli.md
@@ -24,6 +24,7 @@ go run ./cmd/scriptorium render \
|
|||||||
- `scriptorium render`: prepare a prompt and write prepared-run output.
|
- `scriptorium render`: prepare a prompt and write prepared-run output.
|
||||||
- `scriptorium serve`: start the HTTP server.
|
- `scriptorium serve`: start the HTTP server.
|
||||||
- `scriptorium inspect prompt`: inspect one prompt definition without model execution.
|
- `scriptorium inspect prompt`: inspect one prompt definition without model execution.
|
||||||
|
- `scriptorium inspect profile`: inspect one effective profile without model execution.
|
||||||
|
|
||||||
All commands accept `--config <path>` and reject positional arguments. An
|
All commands accept `--config <path>` and reject positional arguments. An
|
||||||
effective `prompt_dir` is required for every command. Supply it through the
|
effective `prompt_dir` is required for every command. Supply it through the
|
||||||
@@ -151,6 +152,18 @@ and writes to stdout unless `--out` is supplied. It loads and normalizes the
|
|||||||
selected definition but does not resolve a profile, load a schema, render a
|
selected definition but does not resolve a profile, load a schema, render a
|
||||||
template, reserve backend capacity, or call a model.
|
template, reserve backend capacity, or call a model.
|
||||||
|
|
||||||
|
## `scriptorium inspect profile`
|
||||||
|
|
||||||
|
```text
|
||||||
|
scriptorium inspect profile --profile ID
|
||||||
|
[--config PATH] [--profile-dir DIR] [--format text|json] [--out PATH]
|
||||||
|
```
|
||||||
|
|
||||||
|
`--profile` is required; built-in profiles need no prompt directory. Inspection
|
||||||
|
resolves profile inheritance and backend defaults, but never reads a credential
|
||||||
|
value, loads a prompt, reserves capacity, or calls a model. Unset provider
|
||||||
|
controls are shown as zero values when Promptkit leaves them unspecified.
|
||||||
|
|
||||||
## Input And Variable Syntax
|
## Input And Variable Syntax
|
||||||
|
|
||||||
`--input name=path` maps an input name to a local file; `--var name=value`
|
`--input name=path` maps an input name to a local file; `--var name=value`
|
||||||
|
|||||||
@@ -39,6 +39,10 @@ resolves its own defaults and definition-required inputs.
|
|||||||
data. It constructs the same configuration-aware engine but does not perform
|
data. It constructs the same configuration-aware engine but does not perform
|
||||||
preparation or generation.
|
preparation or generation.
|
||||||
|
|
||||||
|
`inspect profile` maps an explicit profile ID to `Engine.InspectProfile` and
|
||||||
|
formats a safe application-owned effective profile view. It intentionally does
|
||||||
|
not require prompt, schema, or artifact sources.
|
||||||
|
|
||||||
`serve` constructs Scriptorium's restricted HTTP artifact reader, injects it
|
`serve` constructs Scriptorium's restricted HTTP artifact reader, injects it
|
||||||
with `promptkit.WithArtifactReader`, passes the engine through the HTTP
|
with `promptkit.WithArtifactReader`, passes the engine through the HTTP
|
||||||
adapter's consumer-owned `Runner` interface, and starts the server.
|
adapter's consumer-owned `Runner` interface, and starts the server.
|
||||||
|
|||||||
@@ -432,6 +432,8 @@ prepared-run presentation remains backward compatible.
|
|||||||
|
|
||||||
## Stage 7: Add Stable Profile Inspection Output And CLI
|
## Stage 7: Add Stable Profile Inspection Output And CLI
|
||||||
|
|
||||||
|
**Completion: Complete.**
|
||||||
|
|
||||||
Complete the inspection command family with inherited, built-in, custom, and
|
Complete the inspection command family with inherited, built-in, custom, and
|
||||||
endpoint-only profile support.
|
endpoint-only profile support.
|
||||||
|
|
||||||
|
|||||||
@@ -18,9 +18,21 @@ type promptInspectionConfig struct {
|
|||||||
outputFormat appformat.OutputFormat
|
outputFormat appformat.OutputFormat
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type profileInspectionConfig struct {
|
||||||
|
configPath, profileDir, profileID, outputPath string
|
||||||
|
outputFormat appformat.OutputFormat
|
||||||
|
}
|
||||||
|
|
||||||
func inspectCommand(args []string, stdout, stderr io.Writer) int {
|
func inspectCommand(args []string, stdout, stderr io.Writer) int {
|
||||||
if len(args) == 0 || args[0] != "prompt" {
|
if len(args) == 0 {
|
||||||
fmt.Fprintln(stderr, "inspect parse error: inspection mode must be prompt")
|
fmt.Fprintln(stderr, "inspect parse error: inspection mode is required")
|
||||||
|
return ExitRuntimeError
|
||||||
|
}
|
||||||
|
if args[0] == "profile" {
|
||||||
|
return inspectProfileCommand(args[1:], stdout, stderr)
|
||||||
|
}
|
||||||
|
if args[0] != "prompt" {
|
||||||
|
fmt.Fprintln(stderr, "inspect parse error: unknown inspection mode")
|
||||||
return ExitRuntimeError
|
return ExitRuntimeError
|
||||||
}
|
}
|
||||||
cfg, err := parsePromptInspectionArgs(args[1:])
|
cfg, err := parsePromptInspectionArgs(args[1:])
|
||||||
@@ -55,6 +67,80 @@ func inspectCommand(args []string, stdout, stderr io.Writer) int {
|
|||||||
return ExitOK
|
return ExitOK
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func inspectProfileCommand(args []string, stdout, stderr io.Writer) int {
|
||||||
|
cfg, err := parseProfileInspectionArgs(args)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Fprintf(stderr, "inspect parse error: %v\n", err)
|
||||||
|
return ExitRuntimeError
|
||||||
|
}
|
||||||
|
settings, err := resolveAppSettingsForProfileInspection(cfg)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Fprintf(stderr, "inspect error: %v\n", err)
|
||||||
|
return ExitRuntimeError
|
||||||
|
}
|
||||||
|
engine, err := newEngine(settings)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Fprintf(stderr, "engine error: %v\n", err)
|
||||||
|
return ExitRuntimeError
|
||||||
|
}
|
||||||
|
inspection, err := engine.InspectProfile(context.Background(), cfg.profileID)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Fprintf(stderr, "inspect error: %v\n", err)
|
||||||
|
return ExitRuntimeError
|
||||||
|
}
|
||||||
|
data, err := appformat.FormatProfileInspection(inspection, cfg.outputFormat)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Fprintf(stderr, "inspect error: %v\n", err)
|
||||||
|
return ExitRuntimeError
|
||||||
|
}
|
||||||
|
if err := writeOutput(stdout, cfg.outputPath, data); err != nil {
|
||||||
|
fmt.Fprintf(stderr, "output write error: %v\n", err)
|
||||||
|
return ExitRuntimeError
|
||||||
|
}
|
||||||
|
return ExitOK
|
||||||
|
}
|
||||||
|
|
||||||
|
func parseProfileInspectionArgs(args []string) (*profileInspectionConfig, error) {
|
||||||
|
cfg := &profileInspectionConfig{outputFormat: appformat.DefaultOutputFormat}
|
||||||
|
fs := flag.NewFlagSet("inspect profile", flag.ContinueOnError)
|
||||||
|
fs.SetOutput(io.Discard)
|
||||||
|
registerConfigPathFlag(fs, &cfg.configPath)
|
||||||
|
fs.StringVar(&cfg.profileDir, "profile-dir", "", "directory containing execution profiles")
|
||||||
|
fs.StringVar(&cfg.profileID, "profile", "", "profile ID to inspect")
|
||||||
|
rawFormat := ""
|
||||||
|
fs.StringVar(&rawFormat, "format", "", "output format: text or json")
|
||||||
|
fs.StringVar(&cfg.outputPath, "out", "", "optional output file path")
|
||||||
|
if err := fs.Parse(args); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if fs.NArg() > 0 {
|
||||||
|
return nil, fmt.Errorf("unexpected positional args: %v", fs.Args())
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(cfg.profileID) == "" {
|
||||||
|
return nil, errors.New("--profile is required")
|
||||||
|
}
|
||||||
|
format, err := appformat.ParseOutputFormat(rawFormat)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
cfg.outputFormat = format
|
||||||
|
if cfg.outputPath != "" {
|
||||||
|
cfg.outputPath = filepath.Clean(cfg.outputPath)
|
||||||
|
}
|
||||||
|
return cfg, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func resolveAppSettingsForProfileInspection(cfg *profileInspectionConfig) (engineSettings, error) {
|
||||||
|
settings, err := appconfig.LoadConfig(cfg.configPath, cfg.configPath != "")
|
||||||
|
if err != nil {
|
||||||
|
return engineSettings{}, fmt.Errorf("application config error: %w", err)
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(cfg.profileDir) != "" {
|
||||||
|
settings.ProfileDir = filepath.Clean(cfg.profileDir)
|
||||||
|
}
|
||||||
|
return engineSettings{promptDir: ".", profileDir: settings.ProfileDir, backends: settings.Backends}, nil
|
||||||
|
}
|
||||||
|
|
||||||
func parsePromptInspectionArgs(args []string) (*promptInspectionConfig, error) {
|
func parsePromptInspectionArgs(args []string) (*promptInspectionConfig, error) {
|
||||||
cfg := &promptInspectionConfig{outputFormat: appformat.DefaultOutputFormat}
|
cfg := &promptInspectionConfig{outputFormat: appformat.DefaultOutputFormat}
|
||||||
fs := flag.NewFlagSet("inspect prompt", flag.ContinueOnError)
|
fs := flag.NewFlagSet("inspect prompt", flag.ContinueOnError)
|
||||||
|
|||||||
@@ -33,6 +33,26 @@ type OutputContract struct {
|
|||||||
RepairAttempts int `json:"repair_attempts"`
|
RepairAttempts int `json:"repair_attempts"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type ProfileInspection struct {
|
||||||
|
ProfileID string `json:"profile_id"`
|
||||||
|
EffectiveModelParams ProfileModelParams `json:"effective_model_params"`
|
||||||
|
APIKeyRequired bool `json:"api_key_required"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type ProfileModelParams struct {
|
||||||
|
BackendID string `json:"backend_id"`
|
||||||
|
Endpoint string `json:"endpoint"`
|
||||||
|
Model string `json:"model"`
|
||||||
|
Temperature float64 `json:"temperature"`
|
||||||
|
MaxTokens int `json:"max_tokens"`
|
||||||
|
TopP float64 `json:"top_p"`
|
||||||
|
TimeoutSeconds int `json:"timeout_seconds"`
|
||||||
|
ServiceTier string `json:"service_tier"`
|
||||||
|
ReasoningEffort string `json:"reasoning_effort"`
|
||||||
|
APIKeyEnv string `json:"api_key_env"`
|
||||||
|
ExtraParams map[string]any `json:"extra_params"`
|
||||||
|
}
|
||||||
|
|
||||||
func FormatPromptInspection(value *promptkit.PromptInspection, outputFormat OutputFormat) ([]byte, error) {
|
func FormatPromptInspection(value *promptkit.PromptInspection, outputFormat OutputFormat) ([]byte, error) {
|
||||||
if value == nil {
|
if value == nil {
|
||||||
return nil, errors.New("prompt inspection is nil")
|
return nil, errors.New("prompt inspection is nil")
|
||||||
@@ -69,3 +89,34 @@ func FormatPromptInspection(value *promptkit.PromptInspection, outputFormat Outp
|
|||||||
return nil, fmt.Errorf("%w: %q", ErrUnknownPreparedRunFormat, outputFormat)
|
return nil, fmt.Errorf("%w: %q", ErrUnknownPreparedRunFormat, outputFormat)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func FormatProfileInspection(value *promptkit.ProfileInspection, outputFormat OutputFormat) ([]byte, error) {
|
||||||
|
if value == nil {
|
||||||
|
return nil, errors.New("profile inspection is nil")
|
||||||
|
}
|
||||||
|
target := value.EffectiveModelParams
|
||||||
|
params := map[string]any{}
|
||||||
|
for key, item := range target.ExtraParams {
|
||||||
|
params[key] = item
|
||||||
|
}
|
||||||
|
dto := ProfileInspection{ProfileID: value.ProfileID, APIKeyRequired: value.APIKeyRequired, EffectiveModelParams: ProfileModelParams{BackendID: target.BackendID, Endpoint: target.Endpoint, Model: target.Model, Temperature: target.Temperature, MaxTokens: target.MaxTokens, TopP: target.TopP, TimeoutSeconds: target.TimeoutSeconds, ServiceTier: target.ServiceTier, ReasoningEffort: target.ReasoningEffort, APIKeyEnv: target.APIKeyEnv, ExtraParams: params}}
|
||||||
|
switch outputFormat {
|
||||||
|
case OutputFormatJSON:
|
||||||
|
data, err := json.MarshalIndent(dto, "", " ")
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return append(data, '\n'), nil
|
||||||
|
case OutputFormatText:
|
||||||
|
var b bytes.Buffer
|
||||||
|
fmt.Fprintf(&b, "profile_id: %s\neffective_model_params:\n backend_id: %s\n endpoint: %s\n model: %s\n temperature: %g\n max_tokens: %d\n top_p: %g\n timeout_seconds: %d\n service_tier: %s\n reasoning_effort: %s\n api_key_env: %s\n extra_params: ", dto.ProfileID, dto.EffectiveModelParams.BackendID, dto.EffectiveModelParams.Endpoint, dto.EffectiveModelParams.Model, dto.EffectiveModelParams.Temperature, dto.EffectiveModelParams.MaxTokens, dto.EffectiveModelParams.TopP, dto.EffectiveModelParams.TimeoutSeconds, dto.EffectiveModelParams.ServiceTier, dto.EffectiveModelParams.ReasoningEffort, dto.EffectiveModelParams.APIKeyEnv)
|
||||||
|
paramsJSON, err := json.Marshal(dto.EffectiveModelParams.ExtraParams)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
fmt.Fprintf(&b, "%s\napi_key_required: %t\n", paramsJSON, dto.APIKeyRequired)
|
||||||
|
return b.Bytes(), nil
|
||||||
|
default:
|
||||||
|
return nil, fmt.Errorf("%w: %q", ErrUnknownPreparedRunFormat, outputFormat)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user