From 03cdd2416ed4cd590bba5878e5874b67a8c3a527 Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Wed, 6 May 2026 15:59:22 +0000 Subject: [PATCH] Updated documentation and tests to reflect the new render command --- README.md | 126 +++++++++++++++++++++++++++++-- internal/adapter/cli/run_test.go | 80 ++++++++++++++++++++ 2 files changed, 201 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 3aa79db..0a52675 100644 --- a/README.md +++ b/README.md @@ -10,9 +10,8 @@ It takes: - optional runtime overrides It returns: -- generated artifact -- validation result -- metadata +- for `run`: generated artifact, validation result, metadata +- for `render`: prepared/rendered prompt data (no model output) ## Prompt vs Profile @@ -61,6 +60,11 @@ To ensure security, Scriptorium does not support raw API keys in configuration f ## CLI Usage +Available CLI commands: +- `scriptorium run` +- `scriptorium render` +- `scriptorium serve` + ### `scriptorium run` Runs a single prompt execution. @@ -82,12 +86,12 @@ Runs a single prompt execution. - `--max-tokens`: Override max tokens. - `--top-p`: Override top_p. - `--timeout`: Override request timeout (e.g., `30s`, `1m`). +- `--schema-dir`: Base directory for validation schemas. **Examples:** Using the prompt's `default_profile`: ```bash -export SCRIPTORIUM_API_KEY="sk-..." scriptorium run \ --prompt-dir ./prompts \ --profile-dir ./profiles \ @@ -127,6 +131,119 @@ scriptorium run \ --input transcript=./examples/fixtures/transcript.md ``` +### `scriptorium render` + +Prepares and renders a prompt without calling the LLM. + +`render` uses the same prompt/profile/input/variable/runtime override resolution as `run`: +- Profile selection precedence: `--profile` -> prompt `default_profile` -> error. +- Runtime precedence: CLI runtime overrides -> selected profile -> built-in defaults. + +`render` is useful for debugging: +- prompt template rendering +- input mappings +- selected profile behavior +- runtime override behavior + +`render` does not: +- call the LLM +- validate model output +- perform repair +- expose resolved API key values + +It may include `api_key_env` names where relevant. + +**Required Flags:** +- `--prompt-dir`: Directory containing prompt YAML files. +- `--profile-dir`: Directory containing profile YAML files. +- `--prompt`: The prompt ID to render. +- `--input`: Input mapping `name=path` (repeatable). + +**Optional Flags:** +- `--profile`: Override the prompt's default profile. +- `--var`: Template variable `name=value` (repeatable). +- `--out`: Write output to a file instead of stdout. +- `--format`: Render output format (`text` or `json`). Default: `text`. +- `--llm-base-url`: Runtime override for endpoint. +- `--model`: Runtime override for model name. +- `--api-key-env`: Runtime override for API key environment variable name. +- `--temperature`: Runtime override for temperature. +- `--max-tokens`: Runtime override for max tokens. +- `--top-p`: Runtime override for top_p. +- `--timeout`: Runtime override for timeout (e.g., `30s`, `1m`). + +**Render Output Formats:** +- `text`: Human-readable output (default). +- `json`: Machine-readable structured output. + +Render formatting is modular; additional output formats can be added later without changing prepare/run core logic. + +**Examples:** + +Default text output: +```bash +scriptorium render \ + --prompt-dir ./prompts \ + --profile-dir ./profiles \ + --prompt generic.markdown_summary \ + --input transcript=./examples/fixtures/transcript.md +``` + +Explicit JSON output: +```bash +scriptorium render \ + --prompt-dir ./prompts \ + --profile-dir ./profiles \ + --prompt generic.markdown_summary \ + --input transcript=./examples/fixtures/transcript.md \ + --format json +``` + +Using prompt `default_profile` (omit `--profile`): +```bash +scriptorium render \ + --prompt-dir ./prompts \ + --profile-dir ./profiles \ + --prompt generic.markdown_summary \ + --input transcript=./examples/fixtures/transcript.md +``` + +Overriding profile selection: +```bash +scriptorium render \ + --prompt-dir ./prompts \ + --profile-dir ./profiles \ + --prompt generic.markdown_summary \ + --profile local-quality \ + --input transcript=./examples/fixtures/transcript.md +``` + +Overriding runtime settings: +```bash +scriptorium render \ + --prompt-dir ./prompts \ + --profile-dir ./profiles \ + --prompt generic.markdown_summary \ + --input transcript=./examples/fixtures/transcript.md \ + --llm-base-url http://localhost:8000/v1 \ + --model gpt-4o-mini \ + --temperature 0.2 \ + --max-tokens 800 \ + --top-p 1.0 \ + --timeout 45s +``` + +Writing rendered output to a file: +```bash +scriptorium render \ + --prompt-dir ./prompts \ + --profile-dir ./profiles \ + --prompt generic.markdown_summary \ + --input transcript=./examples/fixtures/transcript.md \ + --format text \ + --out ./rendered_prompt.txt +``` + ### `scriptorium serve` Starts the HTTP API. @@ -247,7 +364,6 @@ api_key_env: SCRIPTORIUM_API_KEY - **Execution Profiles**: `profiles/` - **Schemas**: `schemas/` - **Fixtures**: `examples/fixtures/` -- **Local Experimentation**: `local-test/` ## Build and Test diff --git a/internal/adapter/cli/run_test.go b/internal/adapter/cli/run_test.go index ad4cdc1..15b9659 100644 --- a/internal/adapter/cli/run_test.go +++ b/internal/adapter/cli/run_test.go @@ -522,6 +522,86 @@ func TestRenderCommandOutWritesToFile(t *testing.T) { } } +func TestRenderCommandPromptDefaultProfileWorksThroughCLIPath(t *testing.T) { + tmp := t.TempDir() + promptDir := filepath.Join(tmp, "prompts") + profileDir := filepath.Join(tmp, "profiles") + if err := os.MkdirAll(promptDir, 0o755); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(profileDir, 0o755); err != nil { + t.Fatal(err) + } + inputPath := filepath.Join(tmp, "transcript.md") + if err := os.WriteFile(inputPath, []byte("hello"), 0o644); err != nil { + t.Fatal(err) + } + + writePromptFile(t, promptDir, "prompt.default", "local-default") + writeProfileFile(t, profileDir, "local-default", "http://127.0.0.1:1/v1", "default-model") + + var stdout bytes.Buffer + var stderr bytes.Buffer + code := renderCommand([]string{ + "--prompt-dir", promptDir, + "--profile-dir", profileDir, + "--prompt", "prompt.default", + "--input", "transcript=" + inputPath, + }, &stdout, &stderr) + + if code != ExitOK { + t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr.String()) + } + out := stdout.String() + if !strings.Contains(out, "selected_profile_id: local-default") { + t.Fatalf("expected prompt default profile in output, got %q", out) + } + if !strings.Contains(out, "model: default-model") { + t.Fatalf("expected model from default profile in output, got %q", out) + } +} + +func TestRenderCommandExplicitProfileOverridesPromptDefault(t *testing.T) { + tmp := t.TempDir() + promptDir := filepath.Join(tmp, "prompts") + profileDir := filepath.Join(tmp, "profiles") + if err := os.MkdirAll(promptDir, 0o755); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(profileDir, 0o755); err != nil { + t.Fatal(err) + } + inputPath := filepath.Join(tmp, "transcript.md") + if err := os.WriteFile(inputPath, []byte("hello"), 0o644); err != nil { + t.Fatal(err) + } + + writePromptFile(t, promptDir, "prompt.default", "local-default") + writeProfileFile(t, profileDir, "local-default", "http://127.0.0.1:1/v1", "default-model") + writeProfileFile(t, profileDir, "quality", "http://127.0.0.1:1/v1", "quality-model") + + var stdout bytes.Buffer + var stderr bytes.Buffer + code := renderCommand([]string{ + "--prompt-dir", promptDir, + "--profile-dir", profileDir, + "--prompt", "prompt.default", + "--profile", "quality", + "--input", "transcript=" + inputPath, + }, &stdout, &stderr) + + if code != ExitOK { + t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr.String()) + } + out := stdout.String() + if !strings.Contains(out, "selected_profile_id: quality") { + t.Fatalf("expected explicit profile in output, got %q", out) + } + if !strings.Contains(out, "model: quality-model") { + t.Fatalf("expected model from explicit profile in output, got %q", out) + } +} + func TestRunCommandPromptDefaultProfileWorksThroughCLIPath(t *testing.T) { tmp := t.TempDir() promptDir := filepath.Join(tmp, "prompts")