1390 lines
43 KiB
Go
1390 lines
43 KiB
Go
package cli
|
|
|
|
import (
|
|
"bytes"
|
|
"encoding/json"
|
|
"errors"
|
|
"flag"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"os"
|
|
"path/filepath"
|
|
"reflect"
|
|
"strings"
|
|
"sync/atomic"
|
|
"testing"
|
|
"time"
|
|
|
|
"gitea.maximumdirect.net/eric/promptkit"
|
|
appconfig "gitea.maximumdirect.net/eric/scriptorium/internal/config"
|
|
"gitea.maximumdirect.net/eric/scriptorium/internal/defaults"
|
|
renderformat "gitea.maximumdirect.net/eric/scriptorium/internal/format"
|
|
)
|
|
|
|
func TestParseMappingsSingleAndRepeated(t *testing.T) {
|
|
got, err := parseMappings([]string{"transcript=./t.md", "glossary=./g.yml"}, false)
|
|
if err != nil {
|
|
t.Fatalf("unexpected error: %v", err)
|
|
}
|
|
if got["transcript"] != "./t.md" || got["glossary"] != "./g.yml" {
|
|
t.Fatalf("unexpected mappings: %#v", got)
|
|
}
|
|
}
|
|
|
|
func TestParseMappingsCommaSeparated(t *testing.T) {
|
|
got, err := parseMappings([]string{"transcript=./t.md,glossary=./g.yml"}, false)
|
|
if err != nil {
|
|
t.Fatalf("unexpected error: %v", err)
|
|
}
|
|
if got["transcript"] != "./t.md" || got["glossary"] != "./g.yml" {
|
|
t.Fatalf("unexpected mappings: %#v", got)
|
|
}
|
|
}
|
|
|
|
func TestParseMappingsVarWithEqualsInValue(t *testing.T) {
|
|
got, err := parseMappings([]string{"session_note=a=b=c"}, false)
|
|
if err != nil {
|
|
t.Fatalf("unexpected error: %v", err)
|
|
}
|
|
if got["session_note"] != "a=b=c" {
|
|
t.Fatalf("unexpected variable value: %#v", got)
|
|
}
|
|
}
|
|
|
|
func TestParseMappingsMalformed(t *testing.T) {
|
|
tests := []string{"", "novalue", "=emptyname", "name="}
|
|
for _, tc := range tests {
|
|
_, err := parseMappings([]string{tc}, false)
|
|
if err == nil {
|
|
t.Fatalf("expected error for %q", tc)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestParseRunArgsRequiredFlags(t *testing.T) {
|
|
configPath := writeAppConfigFile(t, "")
|
|
|
|
_, err := parseRunArgs([]string{"--config", configPath, "--profile-dir", "./profiles", "--prompt", "p", "--input", "a=b"})
|
|
if err == nil {
|
|
t.Fatal("expected missing --prompt-dir error")
|
|
}
|
|
if !strings.Contains(err.Error(), "prompt directory is required") {
|
|
t.Fatalf("expected clear prompt-dir guidance, got %v", err)
|
|
}
|
|
|
|
cfg, err := parseRunArgs([]string{"--config", configPath, "--prompt-dir", "./prompts", "--prompt", "p", "--input", "a=b"})
|
|
if err != nil {
|
|
t.Fatalf("expected missing --profile-dir to be accepted, got %v", err)
|
|
}
|
|
if cfg.profileDir != "" {
|
|
t.Fatalf("expected empty profile dir for built-ins, got %q", cfg.profileDir)
|
|
}
|
|
|
|
_, err = parseRunArgs([]string{"--config", configPath, "--prompt-dir", "./prompts", "--profile-dir", "./profiles", "--input", "a=b"})
|
|
if err == nil {
|
|
t.Fatal("expected missing --prompt error")
|
|
}
|
|
|
|
_, err = parseRunArgs([]string{"--config", configPath, "--prompt-dir", "./prompts", "--profile-dir", "./profiles", "--prompt", "p"})
|
|
if err == nil {
|
|
t.Fatal("expected missing --input error")
|
|
}
|
|
}
|
|
|
|
func TestParseRunArgsFlagMapping(t *testing.T) {
|
|
cfg, err := parseRunArgs([]string{
|
|
"--prompt-dir", "./prompts",
|
|
"--profile-dir", "./profiles",
|
|
"--prompt", "prompt.a",
|
|
"--profile", "profile.a",
|
|
"--input", "a=b",
|
|
"--llm-base-url", "http://x/v1",
|
|
"--model", "m",
|
|
"--temperature", "0.7",
|
|
"--max-tokens", "111",
|
|
"--top-p", "0.8",
|
|
"--timeout", "30s",
|
|
"--api-key-env", "SCRIPTORIUM_API_KEY",
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("expected valid args, got %v", err)
|
|
}
|
|
if cfg.promptDir != filepath.Clean("./prompts") || cfg.profileDir != filepath.Clean("./profiles") {
|
|
t.Fatalf("unexpected dirs: prompt=%q profile=%q", cfg.promptDir, cfg.profileDir)
|
|
}
|
|
if cfg.promptID != "prompt.a" || cfg.profileID != "profile.a" {
|
|
t.Fatalf("unexpected prompt/profile ids: %q %q", cfg.promptID, cfg.profileID)
|
|
}
|
|
if !cfg.llmBaseURLSet || !cfg.modelSet || !cfg.temperatureSet || !cfg.maxTokensSet || !cfg.topPSet || !cfg.timeoutSet || !cfg.apiKeyEnvSet {
|
|
t.Fatalf("expected override flags set, got %+v", cfg)
|
|
}
|
|
}
|
|
|
|
func TestParseRunArgsAllowsOmittedModelAndBaseURL(t *testing.T) {
|
|
cfg, err := parseRunArgs([]string{
|
|
"--prompt-dir", "./prompts",
|
|
"--profile-dir", "./profiles",
|
|
"--prompt", "p",
|
|
"--input", "a=b",
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("expected valid args without model/base url, got %v", err)
|
|
}
|
|
if cfg.llmBaseURL != "" || cfg.model != "" {
|
|
t.Fatalf("expected empty model/baseurl, got model=%q base=%q", cfg.model, cfg.llmBaseURL)
|
|
}
|
|
}
|
|
|
|
func TestParseRunArgsRejectsRawLLMAPIKeyFlag(t *testing.T) {
|
|
_, err := parseRunArgs([]string{
|
|
"--prompt-dir", "./prompts",
|
|
"--profile-dir", "./profiles",
|
|
"--prompt", "p",
|
|
"--input", "a=b",
|
|
"--llm-api-key", "secret",
|
|
})
|
|
if err == nil {
|
|
t.Fatal("expected unknown flag error for --llm-api-key")
|
|
}
|
|
}
|
|
|
|
func TestParseServeArgsRequiredFlags(t *testing.T) {
|
|
configPath := writeAppConfigFile(t, "")
|
|
|
|
_, err := parseServeArgs([]string{"--config", configPath, "--profile-dir", "./profiles"})
|
|
if err == nil {
|
|
t.Fatal("expected missing --prompt-dir error")
|
|
}
|
|
if !strings.Contains(err.Error(), "prompt directory is required") {
|
|
t.Fatalf("expected clear prompt-dir guidance, got %v", err)
|
|
}
|
|
|
|
cfg, err := parseServeArgs([]string{"--config", configPath, "--prompt-dir", "./prompts"})
|
|
if err != nil {
|
|
t.Fatalf("expected missing --profile-dir to be accepted, got %v", err)
|
|
}
|
|
if cfg.profileDir != "" {
|
|
t.Fatalf("expected empty profile dir for built-ins, got %q", cfg.profileDir)
|
|
}
|
|
if cfg.addr != defaults.HTTPAddrDefault {
|
|
t.Fatalf("expected default addr %s, got %q", defaults.HTTPAddrDefault, cfg.addr)
|
|
}
|
|
if cfg.schemaDir != defaults.SchemaDirDefault {
|
|
t.Fatalf("expected default schema dir %q, got %q", defaults.SchemaDirDefault, cfg.schemaDir)
|
|
}
|
|
}
|
|
|
|
func TestParseServeArgsRejectsRuntimeOverrideFlags(t *testing.T) {
|
|
base := []string{"--prompt-dir", "./prompts", "--profile-dir", "./profiles"}
|
|
tests := [][]string{
|
|
{"--llm-base-url", "http://localhost:8000/v1"},
|
|
{"--model", "gpt-4o-mini"},
|
|
{"--timeout", "30s"},
|
|
}
|
|
|
|
for _, tc := range tests {
|
|
_, err := parseServeArgs(append(base, tc...))
|
|
if err == nil {
|
|
t.Fatalf("expected unknown flag error for %q", tc[0])
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestUsageIncludesServeFileAndSizeLimitFlags(t *testing.T) {
|
|
var stderr bytes.Buffer
|
|
code := Run(nil, io.Discard, &stderr)
|
|
if code != ExitRuntimeError {
|
|
t.Fatalf("expected usage path to return runtime error, got %d", code)
|
|
}
|
|
|
|
usage := stderr.String()
|
|
for _, want := range []string{
|
|
"--artifact-root",
|
|
"--max-request-bytes",
|
|
"--max-artifact-bytes",
|
|
"--max-response-bytes",
|
|
} {
|
|
if !strings.Contains(usage, want) {
|
|
t.Fatalf("expected usage to include %q, got %q", want, usage)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestParseRunArgsTimeout(t *testing.T) {
|
|
cfg, err := parseRunArgs([]string{
|
|
"--prompt-dir", "./prompts",
|
|
"--profile-dir", "./profiles",
|
|
"--prompt", "p",
|
|
"--input", "a=b",
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("expected valid run args, got %v", err)
|
|
}
|
|
if cfg.timeout != 0 {
|
|
t.Fatalf("expected omitted timeout to remain unset, got %s", cfg.timeout)
|
|
}
|
|
|
|
cfg, err = parseRunArgs([]string{
|
|
"--prompt-dir", "./prompts",
|
|
"--profile-dir", "./profiles",
|
|
"--prompt", "p",
|
|
"--input", "a=b",
|
|
"--timeout", "2m30s",
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("expected valid run args with timeout override, got %v", err)
|
|
}
|
|
if cfg.timeout != 2*time.Minute+30*time.Second {
|
|
t.Fatalf("expected timeout override 2m30s, got %s", cfg.timeout)
|
|
}
|
|
}
|
|
|
|
func TestParseRenderArgsDefaultsAndFormat(t *testing.T) {
|
|
cfg, err := parseRenderArgs([]string{
|
|
"--prompt-dir", "./prompts",
|
|
"--profile-dir", "./profiles",
|
|
"--prompt", "p",
|
|
"--input", "a=b",
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("expected valid render args, got %v", err)
|
|
}
|
|
if cfg.outputFormat != renderformat.DefaultPreparedRunOutputFormat {
|
|
t.Fatalf("expected default render format %q, got %q", renderformat.DefaultPreparedRunOutputFormat, cfg.outputFormat)
|
|
}
|
|
}
|
|
|
|
func TestParseRenderArgsExplicitFormatsAndUnknown(t *testing.T) {
|
|
cfg, err := parseRenderArgs([]string{
|
|
"--prompt-dir", "./prompts",
|
|
"--profile-dir", "./profiles",
|
|
"--prompt", "p",
|
|
"--input", "a=b",
|
|
"--format", "text",
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("expected valid text format, got %v", err)
|
|
}
|
|
if cfg.outputFormat != renderformat.PreparedRunFormatText {
|
|
t.Fatalf("expected text format, got %q", cfg.outputFormat)
|
|
}
|
|
|
|
cfg, err = parseRenderArgs([]string{
|
|
"--prompt-dir", "./prompts",
|
|
"--profile-dir", "./profiles",
|
|
"--prompt", "p",
|
|
"--input", "a=b",
|
|
"--format", "json",
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("expected valid json format, got %v", err)
|
|
}
|
|
if cfg.outputFormat != renderformat.PreparedRunFormatJSON {
|
|
t.Fatalf("expected json format, got %q", cfg.outputFormat)
|
|
}
|
|
|
|
_, err = parseRenderArgs([]string{
|
|
"--prompt-dir", "./prompts",
|
|
"--profile-dir", "./profiles",
|
|
"--prompt", "p",
|
|
"--input", "a=b",
|
|
"--format", "yaml",
|
|
})
|
|
if err == nil {
|
|
t.Fatal("expected unknown format error")
|
|
}
|
|
if !strings.Contains(err.Error(), "unknown prepared run format") {
|
|
t.Fatalf("expected clear unknown format error, got %v", err)
|
|
}
|
|
}
|
|
|
|
func TestParseRunArgsWithExplicitConfigLoadsDirectories(t *testing.T) {
|
|
configPath := writeAppConfigFile(t, `
|
|
prompt_dir: ./from-config/prompts
|
|
profile_dir: ./from-config/profiles
|
|
schema_dir: ./from-config/schemas
|
|
`)
|
|
|
|
cfg, err := parseRunArgs([]string{
|
|
"--config", configPath,
|
|
"--prompt", "p",
|
|
"--input", "a=b",
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("expected valid args, got %v", err)
|
|
}
|
|
|
|
if cfg.promptDir != filepath.Clean("./from-config/prompts") {
|
|
t.Fatalf("expected prompt dir from config, got %q", cfg.promptDir)
|
|
}
|
|
if cfg.profileDir != filepath.Clean("./from-config/profiles") {
|
|
t.Fatalf("expected profile dir from config, got %q", cfg.profileDir)
|
|
}
|
|
if cfg.schemaDir != filepath.Clean("./from-config/schemas") {
|
|
t.Fatalf("expected schema dir from config, got %q", cfg.schemaDir)
|
|
}
|
|
}
|
|
|
|
func TestParseRunArgsMissingExplicitConfigReturnsError(t *testing.T) {
|
|
_, err := parseRunArgs([]string{
|
|
"--config", filepath.Join(t.TempDir(), "missing.yml"),
|
|
"--prompt", "p",
|
|
"--input", "a=b",
|
|
})
|
|
if err == nil {
|
|
t.Fatal("expected explicit config missing error")
|
|
}
|
|
}
|
|
|
|
func TestParseRunArgsInvalidExplicitConfigReturnsError(t *testing.T) {
|
|
configPath := writeAppConfigFile(t, "api_key: secret\n")
|
|
|
|
_, err := parseRunArgs([]string{
|
|
"--config", configPath,
|
|
"--prompt", "p",
|
|
"--input", "a=b",
|
|
})
|
|
if err == nil {
|
|
t.Fatal("expected invalid explicit config error")
|
|
}
|
|
if !strings.Contains(err.Error(), "application config error") {
|
|
t.Fatalf("expected application config context, got %v", err)
|
|
}
|
|
}
|
|
|
|
func TestResolveAppSettingsMissingImplicitConfigDoesNotError(t *testing.T) {
|
|
fs := flag.NewFlagSet("test", flag.ContinueOnError)
|
|
settings, err := resolveAppSettings(fs, filepath.Join(t.TempDir(), "missing.yml"), appconfig.CLIOverrides{})
|
|
if err != nil {
|
|
t.Fatalf("expected no error, got %v", err)
|
|
}
|
|
if settings.SchemaDir != defaults.SchemaDirDefault {
|
|
t.Fatalf("expected built-in schema dir, got %q", settings.SchemaDir)
|
|
}
|
|
if settings.ServerAddr != defaults.HTTPAddrDefault {
|
|
t.Fatalf("expected built-in server addr, got %q", settings.ServerAddr)
|
|
}
|
|
}
|
|
|
|
func TestParseRunArgsCLIOverridesConfigDirectories(t *testing.T) {
|
|
configPath := writeAppConfigFile(t, `
|
|
prompt_dir: ./from-config/prompts
|
|
profile_dir: ./from-config/profiles
|
|
schema_dir: ./from-config/schemas
|
|
`)
|
|
|
|
cfg, err := parseRunArgs([]string{
|
|
"--config", configPath,
|
|
"--prompt-dir", "./from-cli/prompts",
|
|
"--profile-dir", "./from-cli/profiles",
|
|
"--schema-dir", "./from-cli/schemas",
|
|
"--prompt", "p",
|
|
"--input", "a=b",
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("expected valid args, got %v", err)
|
|
}
|
|
|
|
if cfg.promptDir != filepath.Clean("./from-cli/prompts") {
|
|
t.Fatalf("expected CLI prompt dir override, got %q", cfg.promptDir)
|
|
}
|
|
if cfg.profileDir != filepath.Clean("./from-cli/profiles") {
|
|
t.Fatalf("expected CLI profile dir override, got %q", cfg.profileDir)
|
|
}
|
|
if cfg.schemaDir != filepath.Clean("./from-cli/schemas") {
|
|
t.Fatalf("expected CLI schema dir override, got %q", cfg.schemaDir)
|
|
}
|
|
}
|
|
|
|
func TestParseRenderArgsWithExplicitConfigLoadsDirectoriesAndFormat(t *testing.T) {
|
|
configPath := writeAppConfigFile(t, `
|
|
prompt_dir: ./from-config/prompts
|
|
profile_dir: ./from-config/profiles
|
|
defaults:
|
|
render_format: json
|
|
`)
|
|
|
|
cfg, err := parseRenderArgs([]string{
|
|
"--config", configPath,
|
|
"--prompt", "p",
|
|
"--input", "a=b",
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("expected valid args, got %v", err)
|
|
}
|
|
|
|
if cfg.promptDir != filepath.Clean("./from-config/prompts") {
|
|
t.Fatalf("expected prompt dir from config, got %q", cfg.promptDir)
|
|
}
|
|
if cfg.profileDir != filepath.Clean("./from-config/profiles") {
|
|
t.Fatalf("expected profile dir from config, got %q", cfg.profileDir)
|
|
}
|
|
if cfg.outputFormat != renderformat.PreparedRunFormatJSON {
|
|
t.Fatalf("expected render format from config, got %q", cfg.outputFormat)
|
|
}
|
|
}
|
|
|
|
func TestParseRenderArgsExplicitFormatOverridesConfigDefaultFormat(t *testing.T) {
|
|
configPath := writeAppConfigFile(t, `
|
|
prompt_dir: ./from-config/prompts
|
|
profile_dir: ./from-config/profiles
|
|
defaults:
|
|
render_format: json
|
|
`)
|
|
|
|
cfg, err := parseRenderArgs([]string{
|
|
"--config", configPath,
|
|
"--prompt", "p",
|
|
"--input", "a=b",
|
|
"--format", "text",
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("expected valid args, got %v", err)
|
|
}
|
|
if cfg.outputFormat != renderformat.PreparedRunFormatText {
|
|
t.Fatalf("expected explicit --format text to override config default, got %q", cfg.outputFormat)
|
|
}
|
|
}
|
|
|
|
func TestParseServeArgsWithExplicitConfigLoadsSettingsAndCLIAddrOverrides(t *testing.T) {
|
|
configPath := writeAppConfigFile(t, `
|
|
prompt_dir: ./from-config/prompts
|
|
profile_dir: ./from-config/profiles
|
|
schema_dir: ./from-config/schemas
|
|
server:
|
|
addr: 127.0.0.1:9000
|
|
artifact_root: ./from-config/artifacts
|
|
max_request_bytes: 1024
|
|
max_artifact_bytes: 2048
|
|
max_response_bytes: 4096
|
|
`)
|
|
|
|
cfg, err := parseServeArgs([]string{
|
|
"--config", configPath,
|
|
"--addr", ":7777",
|
|
"--artifact-root", "./from-cli/artifacts",
|
|
"--max-request-bytes", "0",
|
|
"--max-artifact-bytes", "8192",
|
|
"--max-response-bytes", "16384",
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("expected valid args, got %v", err)
|
|
}
|
|
|
|
if cfg.promptDir != filepath.Clean("./from-config/prompts") {
|
|
t.Fatalf("expected prompt dir from config, got %q", cfg.promptDir)
|
|
}
|
|
if cfg.profileDir != filepath.Clean("./from-config/profiles") {
|
|
t.Fatalf("expected profile dir from config, got %q", cfg.profileDir)
|
|
}
|
|
if cfg.schemaDir != filepath.Clean("./from-config/schemas") {
|
|
t.Fatalf("expected schema dir from config, got %q", cfg.schemaDir)
|
|
}
|
|
if cfg.addr != ":7777" {
|
|
t.Fatalf("expected CLI addr override, got %q", cfg.addr)
|
|
}
|
|
if cfg.artifactRoot != filepath.Clean("./from-cli/artifacts") {
|
|
t.Fatalf("expected CLI artifact root override, got %q", cfg.artifactRoot)
|
|
}
|
|
if cfg.maxRequestBytes != 0 {
|
|
t.Fatalf("expected CLI max request bytes override, got %d", cfg.maxRequestBytes)
|
|
}
|
|
if cfg.maxArtifactBytes != 8192 {
|
|
t.Fatalf("expected CLI max artifact bytes override, got %d", cfg.maxArtifactBytes)
|
|
}
|
|
if cfg.maxResponseBytes != 16384 {
|
|
t.Fatalf("expected CLI max response bytes override, got %d", cfg.maxResponseBytes)
|
|
}
|
|
}
|
|
|
|
func TestParseServeArgsWithConfigProvidesRequiredDirectoriesAndAddr(t *testing.T) {
|
|
configPath := writeAppConfigFile(t, `
|
|
prompt_dir: ./from-config/prompts
|
|
profile_dir: ./from-config/profiles
|
|
schema_dir: ./from-config/schemas
|
|
server:
|
|
addr: 127.0.0.1:9000
|
|
artifact_root: ./from-config/artifacts
|
|
max_request_bytes: 1024
|
|
max_artifact_bytes: 2048
|
|
max_response_bytes: 4096
|
|
`)
|
|
|
|
cfg, err := parseServeArgs([]string{
|
|
"--config", configPath,
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("expected valid args, got %v", err)
|
|
}
|
|
|
|
if cfg.promptDir != filepath.Clean("./from-config/prompts") {
|
|
t.Fatalf("expected prompt dir from config, got %q", cfg.promptDir)
|
|
}
|
|
if cfg.profileDir != filepath.Clean("./from-config/profiles") {
|
|
t.Fatalf("expected profile dir from config, got %q", cfg.profileDir)
|
|
}
|
|
if cfg.schemaDir != filepath.Clean("./from-config/schemas") {
|
|
t.Fatalf("expected schema dir from config, got %q", cfg.schemaDir)
|
|
}
|
|
if cfg.addr != "127.0.0.1:9000" {
|
|
t.Fatalf("expected addr from config, got %q", cfg.addr)
|
|
}
|
|
if cfg.artifactRoot != filepath.Clean("./from-config/artifacts") {
|
|
t.Fatalf("expected artifact root from config, got %q", cfg.artifactRoot)
|
|
}
|
|
if cfg.maxRequestBytes != 1024 {
|
|
t.Fatalf("expected max request bytes from config, got %d", cfg.maxRequestBytes)
|
|
}
|
|
if cfg.maxArtifactBytes != 2048 {
|
|
t.Fatalf("expected max artifact bytes from config, got %d", cfg.maxArtifactBytes)
|
|
}
|
|
if cfg.maxResponseBytes != 4096 {
|
|
t.Fatalf("expected max response bytes from config, got %d", cfg.maxResponseBytes)
|
|
}
|
|
}
|
|
|
|
func TestParseServeArgsRejectsNegativeSizeLimits(t *testing.T) {
|
|
tests := []struct {
|
|
name string
|
|
flag string
|
|
}{
|
|
{name: "request", flag: "--max-request-bytes"},
|
|
{name: "artifact", flag: "--max-artifact-bytes"},
|
|
{name: "response", flag: "--max-response-bytes"},
|
|
}
|
|
|
|
for _, tc := range tests {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
_, err := parseServeArgs([]string{
|
|
"--prompt-dir", "./prompts",
|
|
tc.flag, "-1",
|
|
})
|
|
if err == nil {
|
|
t.Fatal("expected negative size limit error")
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestRunAndRenderRejectServeSizeLimitFlags(t *testing.T) {
|
|
for _, tc := range []struct {
|
|
name string
|
|
parse func([]string) error
|
|
}{
|
|
{
|
|
name: "run",
|
|
parse: func(args []string) error {
|
|
_, err := parseRunArgs(args)
|
|
return err
|
|
},
|
|
},
|
|
{
|
|
name: "render",
|
|
parse: func(args []string) error {
|
|
_, err := parseRenderArgs(args)
|
|
return err
|
|
},
|
|
},
|
|
} {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
err := tc.parse([]string{
|
|
"--prompt-dir", "./prompts",
|
|
"--prompt", "p",
|
|
"--input", "a=b",
|
|
"--max-request-bytes", "1024",
|
|
})
|
|
if err == nil {
|
|
t.Fatal("expected unsupported flag error")
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestRunAndRenderBuildEquivalentRuntimeOverrideRequestsForSharedFlags(t *testing.T) {
|
|
runCfg, err := parseRunArgs([]string{
|
|
"--prompt-dir", "./prompts",
|
|
"--profile-dir", "./profiles",
|
|
"--prompt", "prompt-1",
|
|
"--profile", "profile-1",
|
|
"--input", "transcript=./transcript.md",
|
|
"--var", "session_date=2026-05-01",
|
|
"--llm-base-url", "http://localhost:8000/v1",
|
|
"--model", "model-x",
|
|
"--temperature", "0.8",
|
|
"--max-tokens", "123",
|
|
"--top-p", "0.6",
|
|
"--timeout", "90s",
|
|
"--api-key-env", "SCRIPTORIUM_API_KEY",
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("expected valid run args, got %v", err)
|
|
}
|
|
|
|
renderCfg, err := parseRenderArgs([]string{
|
|
"--prompt-dir", "./prompts",
|
|
"--profile-dir", "./profiles",
|
|
"--prompt", "prompt-1",
|
|
"--profile", "profile-1",
|
|
"--input", "transcript=./transcript.md",
|
|
"--var", "session_date=2026-05-01",
|
|
"--llm-base-url", "http://localhost:8000/v1",
|
|
"--model", "model-x",
|
|
"--temperature", "0.8",
|
|
"--max-tokens", "123",
|
|
"--top-p", "0.6",
|
|
"--timeout", "90s",
|
|
"--api-key-env", "SCRIPTORIUM_API_KEY",
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("expected valid render args, got %v", err)
|
|
}
|
|
|
|
runReq, err := buildRunRequestFromConfig(runCfg)
|
|
if err != nil {
|
|
t.Fatalf("expected run request build success, got %v", err)
|
|
}
|
|
renderReq, err := buildRunRequestFromConfig(&renderCfg.runConfig)
|
|
if err != nil {
|
|
t.Fatalf("expected render request build success, got %v", err)
|
|
}
|
|
|
|
if !reflect.DeepEqual(runReq, renderReq) {
|
|
t.Fatalf("expected run/render shared flag requests to match.\nrun=%#v\nrender=%#v", runReq, renderReq)
|
|
}
|
|
}
|
|
|
|
func TestBuildRunRequestPreservesNumericOverridePresence(t *testing.T) {
|
|
omitted, err := buildRunRequestFromConfig(&runConfig{
|
|
promptID: "prompt-1",
|
|
inputRaw: []string{"transcript=./transcript.md"},
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("expected omitted override request to build, got %v", err)
|
|
}
|
|
if omitted.Execution != nil {
|
|
t.Fatalf("expected omitted numeric flags to leave execution override nil, got %#v", omitted.Execution)
|
|
}
|
|
|
|
explicitZeros, err := buildRunRequestFromConfig(&runConfig{
|
|
promptID: "prompt-1",
|
|
inputRaw: []string{"transcript=./transcript.md"},
|
|
temperatureSet: true,
|
|
maxTokensSet: true,
|
|
topPSet: true,
|
|
timeoutSet: true,
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("expected explicit zero override request to build, got %v", err)
|
|
}
|
|
if explicitZeros.Execution == nil {
|
|
t.Fatal("expected explicit numeric flags to create execution override")
|
|
}
|
|
if explicitZeros.Execution.Temperature == nil || explicitZeros.Execution.MaxTokens == nil || explicitZeros.Execution.TopP == nil || explicitZeros.Execution.TimeoutSeconds == nil {
|
|
t.Fatalf("expected explicit zero numeric overrides to remain non-nil, got %#v", explicitZeros.Execution)
|
|
}
|
|
if *explicitZeros.Execution.Temperature != 0 || *explicitZeros.Execution.MaxTokens != 0 || *explicitZeros.Execution.TopP != 0 || *explicitZeros.Execution.TimeoutSeconds != 0 {
|
|
t.Fatalf("expected explicit numeric overrides to retain zero values, got %#v", explicitZeros.Execution)
|
|
}
|
|
}
|
|
|
|
func TestParseRunArgsFailsClearlyWhenNoEffectivePromptDir(t *testing.T) {
|
|
configPath := writeAppConfigFile(t, `
|
|
profile_dir: ./profiles
|
|
`)
|
|
|
|
_, err := parseRunArgs([]string{
|
|
"--config", configPath,
|
|
"--prompt", "p",
|
|
"--input", "a=b",
|
|
})
|
|
if err == nil {
|
|
t.Fatal("expected missing prompt_dir error")
|
|
}
|
|
if !strings.Contains(err.Error(), "prompt directory is required") || !strings.Contains(err.Error(), "config.yml prompt_dir") {
|
|
t.Fatalf("expected clear prompt_dir guidance, got %v", err)
|
|
}
|
|
}
|
|
|
|
func TestParseRunArgsAcceptsMissingEffectiveProfileDir(t *testing.T) {
|
|
configPath := writeAppConfigFile(t, `
|
|
prompt_dir: ./prompts
|
|
`)
|
|
|
|
cfg, err := parseRunArgs([]string{
|
|
"--config", configPath,
|
|
"--prompt", "p",
|
|
"--input", "a=b",
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("expected missing profile_dir to be accepted, got %v", err)
|
|
}
|
|
if cfg.profileDir != "" {
|
|
t.Fatalf("expected empty profile dir for built-ins, got %q", cfg.profileDir)
|
|
}
|
|
}
|
|
|
|
func TestParseRenderArgsFailsClearlyWhenNoEffectivePromptDir(t *testing.T) {
|
|
configPath := writeAppConfigFile(t, `
|
|
profile_dir: ./profiles
|
|
`)
|
|
|
|
_, err := parseRenderArgs([]string{
|
|
"--config", configPath,
|
|
"--prompt", "p",
|
|
"--input", "a=b",
|
|
})
|
|
if err == nil {
|
|
t.Fatal("expected missing prompt_dir error")
|
|
}
|
|
if !strings.Contains(err.Error(), "prompt directory is required") || !strings.Contains(err.Error(), "config.yml prompt_dir") {
|
|
t.Fatalf("expected clear prompt_dir guidance, got %v", err)
|
|
}
|
|
}
|
|
|
|
func TestParseRenderArgsAcceptsMissingEffectiveProfileDir(t *testing.T) {
|
|
configPath := writeAppConfigFile(t, `
|
|
prompt_dir: ./prompts
|
|
`)
|
|
|
|
cfg, err := parseRenderArgs([]string{
|
|
"--config", configPath,
|
|
"--prompt", "p",
|
|
"--input", "a=b",
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("expected missing profile_dir to be accepted, got %v", err)
|
|
}
|
|
if cfg.profileDir != "" {
|
|
t.Fatalf("expected empty profile dir for built-ins, got %q", cfg.profileDir)
|
|
}
|
|
}
|
|
|
|
func TestDetermineExitCode(t *testing.T) {
|
|
if got := determineExitCode(errors.New("boom"), nil); got != ExitRuntimeError {
|
|
t.Fatalf("expected runtime exit code, got %d", got)
|
|
}
|
|
if got := determineExitCode(nil, &promptkit.RunResult{Validation: promptkit.ValidationResult{Status: promptkit.ValidationFailed}}); got != ExitValidationFailed {
|
|
t.Fatalf("expected validation exit code, got %d", got)
|
|
}
|
|
if got := determineExitCode(nil, &promptkit.RunResult{Validation: promptkit.ValidationResult{Status: promptkit.ValidationPassed}}); got != ExitOK {
|
|
t.Fatalf("expected success exit code for passed validation, got %d", got)
|
|
}
|
|
if got := determineExitCode(nil, &promptkit.RunResult{Validation: promptkit.ValidationResult{Status: promptkit.ValidationSkipped}}); got != ExitOK {
|
|
t.Fatalf("expected success exit code for skipped validation, got %d", got)
|
|
}
|
|
}
|
|
|
|
func TestRunCommandVarsOptional(t *testing.T) {
|
|
var stdout bytes.Buffer
|
|
var stderr bytes.Buffer
|
|
|
|
code := runCommand([]string{
|
|
"--prompt-dir", "./prompts",
|
|
"--profile-dir", "./profiles",
|
|
"--prompt", "p",
|
|
"--input", "transcript=./t.md",
|
|
"--llm-base-url", "http://[::1",
|
|
"--model", "m",
|
|
}, &stdout, &stderr)
|
|
|
|
if code != ExitRuntimeError {
|
|
t.Fatalf("expected runtime error exit code, got %d", code)
|
|
}
|
|
if strings.Contains(stderr.String(), "var parse error") {
|
|
t.Fatalf("expected --var to be optional, got stderr=%q", stderr.String())
|
|
}
|
|
if !strings.Contains(stderr.String(), "llm client error") && !strings.Contains(stderr.String(), "run error") {
|
|
t.Fatalf("expected post-parse execution error, got stderr=%q", stderr.String())
|
|
}
|
|
if stdout.Len() != 0 {
|
|
t.Fatalf("expected no stdout output on error, got %q", stdout.String())
|
|
}
|
|
}
|
|
|
|
func TestRunCommandSucceedsWithPromptAndProfileDirsFromConfig(t *testing.T) {
|
|
lib := newCLITestLibrary(t)
|
|
inputPath := lib.writeInputFile(t, "transcript.md", "hello")
|
|
|
|
ts := newTestLLMServer("from-config-dirs", nil)
|
|
defer ts.Close()
|
|
|
|
writePromptFile(t, lib.promptDir, "prompt.default", "local-default")
|
|
writeProfileFile(t, lib.profileDir, "local-default", ts.URL+"/v1", "profile-model")
|
|
configPath := writeAppConfigFile(t, fmt.Sprintf(`
|
|
prompt_dir: %s
|
|
profile_dir: %s
|
|
`, lib.promptDir, lib.profileDir))
|
|
|
|
code, stdout, stderr := runCLICommand(t, runCommand, []string{
|
|
"--config", configPath,
|
|
"--prompt", "prompt.default",
|
|
"--input", "transcript=" + inputPath,
|
|
})
|
|
if code != ExitOK {
|
|
t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr)
|
|
}
|
|
if stdout != "from-config-dirs" {
|
|
t.Fatalf("unexpected stdout output: %q", stdout)
|
|
}
|
|
}
|
|
|
|
func TestRenderCommandDefaultFormatTextIncludesPreparedDetailsAndNoSecrets(t *testing.T) {
|
|
const envName = "SCRIPTORIUM_RENDER_TEST_API_KEY"
|
|
const secret = "super-secret-render-key"
|
|
t.Setenv(envName, secret)
|
|
|
|
lib := newCLITestLibrary(t)
|
|
inputPath := lib.writeInputFile(t, "transcript.md", "hello transcript")
|
|
|
|
writePromptFileWithTemplate(t, lib.promptDir, "prompt.render", "local-default", "Date {{.session_date}} - Summarize: {{input \"transcript\"}}")
|
|
writeProfileFile(t, lib.profileDir, "local-default", "http://127.0.0.1:1/v1", "profile-model")
|
|
|
|
code, stdout, stderr := runCLICommand(t, renderCommand, []string{
|
|
"--prompt-dir", lib.promptDir,
|
|
"--profile-dir", lib.profileDir,
|
|
"--prompt", "prompt.render",
|
|
"--profile", "local-default",
|
|
"--input", "transcript=" + inputPath,
|
|
"--var", "session_date=2026-05-04",
|
|
"--llm-base-url", "http://override.local/v1",
|
|
"--model", "override-model",
|
|
"--temperature", "0.7",
|
|
"--max-tokens", "55",
|
|
"--top-p", "0.2",
|
|
"--timeout", "20s",
|
|
"--api-key-env", envName,
|
|
})
|
|
if code != ExitOK {
|
|
t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr)
|
|
}
|
|
if stderr != "" {
|
|
t.Fatalf("expected empty stderr on success, got %q", stderr)
|
|
}
|
|
|
|
out := stdout
|
|
for _, want := range []string{
|
|
"prompt: prompt.render",
|
|
"selected_profile_id: local-default",
|
|
"endpoint: http://override.local/v1",
|
|
"model: override-model",
|
|
"temperature: 0.7",
|
|
"max_tokens: 55",
|
|
"top_p: 0.2",
|
|
"timeout_seconds: 20",
|
|
"api_key_env: " + envName,
|
|
"rendered_prompt_hash:",
|
|
"messages:",
|
|
"Date 2026-05-04",
|
|
"Summarize:",
|
|
"hello transcript",
|
|
} {
|
|
if !strings.Contains(out, want) {
|
|
t.Fatalf("expected render text output to include %q, got:\n%s", want, out)
|
|
}
|
|
}
|
|
if strings.Contains(out, secret) {
|
|
t.Fatalf("render output unexpectedly contained secret api key value: %s", out)
|
|
}
|
|
}
|
|
|
|
func TestRenderCommandExplicitZeroTemperatureReachesEffectiveSettings(t *testing.T) {
|
|
lib := newCLITestLibrary(t)
|
|
inputPath := lib.writeInputFile(t, "transcript.md", "hello transcript")
|
|
|
|
writePromptFile(t, lib.promptDir, "prompt.render", "local-default")
|
|
profile := `id: local-default
|
|
endpoint: http://127.0.0.1:1/v1
|
|
model: profile-model
|
|
temperature: 0.7
|
|
`
|
|
if err := os.WriteFile(filepath.Join(lib.profileDir, "local-default.yaml"), []byte(profile), 0o644); err != nil {
|
|
t.Fatalf("failed to write profile fixture: %v", err)
|
|
}
|
|
|
|
code, stdout, stderr := runCLICommand(t, renderCommand, []string{
|
|
"--prompt-dir", lib.promptDir,
|
|
"--profile-dir", lib.profileDir,
|
|
"--prompt", "prompt.render",
|
|
"--input", "transcript=" + inputPath,
|
|
"--temperature", "0",
|
|
})
|
|
if code != ExitOK {
|
|
t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr)
|
|
}
|
|
if !strings.Contains(stdout, "\n temperature: 0\n") {
|
|
t.Fatalf("expected explicit zero temperature in effective settings, got:\n%s", stdout)
|
|
}
|
|
}
|
|
|
|
func TestRenderCommandSucceedsWithPromptAndProfileDirsFromConfig(t *testing.T) {
|
|
lib := newCLITestLibrary(t)
|
|
inputPath := lib.writeInputFile(t, "transcript.md", "hello transcript")
|
|
|
|
writePromptFile(t, lib.promptDir, "prompt.render", "local-default")
|
|
writeProfileFile(t, lib.profileDir, "local-default", "http://127.0.0.1:1/v1", "profile-model")
|
|
configPath := writeAppConfigFile(t, fmt.Sprintf(`
|
|
prompt_dir: %s
|
|
profile_dir: %s
|
|
`, lib.promptDir, lib.profileDir))
|
|
|
|
code, stdout, stderr := runCLICommand(t, renderCommand, []string{
|
|
"--config", configPath,
|
|
"--prompt", "prompt.render",
|
|
"--input", "transcript=" + inputPath,
|
|
})
|
|
if code != ExitOK {
|
|
t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr)
|
|
}
|
|
if !strings.Contains(stdout, "prompt: prompt.render") {
|
|
t.Fatalf("expected rendered output, got %q", stdout)
|
|
}
|
|
}
|
|
|
|
func TestRenderCommandExplicitTextFormatWorks(t *testing.T) {
|
|
lib := newCLITestLibrary(t)
|
|
inputPath := lib.writeInputFile(t, "transcript.md", "hello transcript")
|
|
|
|
writePromptFile(t, lib.promptDir, "prompt.render", "local-default")
|
|
writeProfileFile(t, lib.profileDir, "local-default", "http://127.0.0.1:1/v1", "profile-model")
|
|
|
|
code, stdout, stderr := runCLICommand(t, renderCommand, []string{
|
|
"--prompt-dir", lib.promptDir,
|
|
"--profile-dir", lib.profileDir,
|
|
"--prompt", "prompt.render",
|
|
"--input", "transcript=" + inputPath,
|
|
"--format", "text",
|
|
})
|
|
if code != ExitOK {
|
|
t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr)
|
|
}
|
|
if !strings.Contains(stdout, "prompt: prompt.render") {
|
|
t.Fatalf("expected text output for explicit --format text, got %q", stdout)
|
|
}
|
|
}
|
|
|
|
func TestRenderCommandExplicitJSONFormatOutputsValidJSON(t *testing.T) {
|
|
lib := newCLITestLibrary(t)
|
|
inputPath := lib.writeInputFile(t, "transcript.md", "hello transcript")
|
|
|
|
writePromptFile(t, lib.promptDir, "prompt.render", "local-default")
|
|
writeProfileFile(t, lib.profileDir, "local-default", "http://127.0.0.1:1/v1", "profile-model")
|
|
|
|
code, stdout, stderr := runCLICommand(t, renderCommand, []string{
|
|
"--prompt-dir", lib.promptDir,
|
|
"--profile-dir", lib.profileDir,
|
|
"--prompt", "prompt.render",
|
|
"--input", "transcript=" + inputPath,
|
|
"--format", "json",
|
|
})
|
|
if code != ExitOK {
|
|
t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr)
|
|
}
|
|
|
|
var payload map[string]any
|
|
if err := json.Unmarshal([]byte(stdout), &payload); err != nil {
|
|
t.Fatalf("expected valid json output, got %v\nbody=%s", err, stdout)
|
|
}
|
|
if payload["prompt_id"] != "prompt.render" {
|
|
t.Fatalf("expected prompt_id, got %#v", payload["prompt_id"])
|
|
}
|
|
if payload["selected_profile_id"] != "local-default" {
|
|
t.Fatalf("expected selected_profile_id, got %#v", payload["selected_profile_id"])
|
|
}
|
|
if _, ok := payload["messages"]; !ok {
|
|
t.Fatalf("expected messages in render json output, got %#v", payload)
|
|
}
|
|
}
|
|
|
|
func TestRenderCommandUnknownFormatFailsClearly(t *testing.T) {
|
|
var stdout bytes.Buffer
|
|
var stderr bytes.Buffer
|
|
code := renderCommand([]string{
|
|
"--prompt-dir", "./prompts",
|
|
"--profile-dir", "./profiles",
|
|
"--prompt", "p",
|
|
"--input", "transcript=./x.md",
|
|
"--format", "yaml",
|
|
}, &stdout, &stderr)
|
|
if code != ExitRuntimeError {
|
|
t.Fatalf("expected ExitRuntimeError, got %d", code)
|
|
}
|
|
if !strings.Contains(stderr.String(), "render parse error") || !strings.Contains(stderr.String(), "unknown prepared run format") {
|
|
t.Fatalf("expected clear unknown-format parse error, got %q", stderr.String())
|
|
}
|
|
}
|
|
|
|
func TestRenderCommandOutWritesToFile(t *testing.T) {
|
|
lib := newCLITestLibrary(t)
|
|
inputPath := lib.writeInputFile(t, "transcript.md", "hello transcript")
|
|
outPath := filepath.Join(lib.rootDir, "render.txt")
|
|
|
|
writePromptFile(t, lib.promptDir, "prompt.render", "local-default")
|
|
writeProfileFile(t, lib.profileDir, "local-default", "http://127.0.0.1:1/v1", "profile-model")
|
|
|
|
code, stdout, stderr := runCLICommand(t, renderCommand, []string{
|
|
"--prompt-dir", lib.promptDir,
|
|
"--profile-dir", lib.profileDir,
|
|
"--prompt", "prompt.render",
|
|
"--input", "transcript=" + inputPath,
|
|
"--out", outPath,
|
|
})
|
|
if code != ExitOK {
|
|
t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr)
|
|
}
|
|
if stdout != "" {
|
|
t.Fatalf("expected empty stdout when --out is set, got %q", stdout)
|
|
}
|
|
out, err := os.ReadFile(outPath)
|
|
if err != nil {
|
|
t.Fatalf("failed reading render output file: %v", err)
|
|
}
|
|
if !strings.Contains(string(out), "prompt: prompt.render") {
|
|
t.Fatalf("expected render output in file, got %q", string(out))
|
|
}
|
|
}
|
|
|
|
func TestRenderCommandPromptDefaultProfileWorksThroughCLIPath(t *testing.T) {
|
|
lib := newCLITestLibrary(t)
|
|
inputPath := lib.writeInputFile(t, "transcript.md", "hello")
|
|
|
|
writePromptFile(t, lib.promptDir, "prompt.default", "local-default")
|
|
writeProfileFile(t, lib.profileDir, "local-default", "http://127.0.0.1:1/v1", "default-model")
|
|
|
|
code, stdout, stderr := runCLICommand(t, renderCommand, []string{
|
|
"--prompt-dir", lib.promptDir,
|
|
"--profile-dir", lib.profileDir,
|
|
"--prompt", "prompt.default",
|
|
"--input", "transcript=" + inputPath,
|
|
})
|
|
|
|
if code != ExitOK {
|
|
t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr)
|
|
}
|
|
out := stdout
|
|
if !strings.Contains(out, "selected_profile_id: local-default") {
|
|
t.Fatalf("expected prompt default profile in output, got %q", out)
|
|
}
|
|
if !strings.Contains(out, "model: default-model") {
|
|
t.Fatalf("expected model from default profile in output, got %q", out)
|
|
}
|
|
}
|
|
|
|
func TestRenderCommandUsesBuiltInProfileWithoutProfileDir(t *testing.T) {
|
|
t.Setenv("OPENROUTER_API_KEY", "test-key")
|
|
lib := newCLITestLibrary(t)
|
|
inputPath := lib.writeInputFile(t, "transcript.md", "hello")
|
|
|
|
writePromptFile(t, lib.promptDir, "prompt.builtin", "mistral-small-3")
|
|
|
|
code, stdout, stderr := runCLICommand(t, renderCommand, []string{
|
|
"--prompt-dir", lib.promptDir,
|
|
"--prompt", "prompt.builtin",
|
|
"--input", "transcript=" + inputPath,
|
|
})
|
|
if code != ExitOK {
|
|
t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr)
|
|
}
|
|
if !strings.Contains(stdout, "selected_profile_id: mistral-small-3") {
|
|
t.Fatalf("expected built-in selected profile, got %q", stdout)
|
|
}
|
|
if !strings.Contains(stdout, "model: mistralai/mistral-small-3.2-24b-instruct") {
|
|
t.Fatalf("expected built-in model, got %q", stdout)
|
|
}
|
|
}
|
|
|
|
func TestRenderCommandExplicitProfileOverridesPromptDefault(t *testing.T) {
|
|
lib := newCLITestLibrary(t)
|
|
inputPath := lib.writeInputFile(t, "transcript.md", "hello")
|
|
|
|
writePromptFile(t, lib.promptDir, "prompt.default", "local-default")
|
|
writeProfileFile(t, lib.profileDir, "local-default", "http://127.0.0.1:1/v1", "default-model")
|
|
writeProfileFile(t, lib.profileDir, "quality", "http://127.0.0.1:1/v1", "quality-model")
|
|
|
|
code, stdout, stderr := runCLICommand(t, renderCommand, []string{
|
|
"--prompt-dir", lib.promptDir,
|
|
"--profile-dir", lib.profileDir,
|
|
"--prompt", "prompt.default",
|
|
"--profile", "quality",
|
|
"--input", "transcript=" + inputPath,
|
|
})
|
|
|
|
if code != ExitOK {
|
|
t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr)
|
|
}
|
|
out := stdout
|
|
if !strings.Contains(out, "selected_profile_id: quality") {
|
|
t.Fatalf("expected explicit profile in output, got %q", out)
|
|
}
|
|
if !strings.Contains(out, "model: quality-model") {
|
|
t.Fatalf("expected model from explicit profile in output, got %q", out)
|
|
}
|
|
}
|
|
|
|
func TestRunCommandPromptDefaultProfileWorksThroughCLIPath(t *testing.T) {
|
|
lib := newCLITestLibrary(t)
|
|
inputPath := lib.writeInputFile(t, "transcript.md", "hello")
|
|
|
|
ts := newTestLLMServer("default-output", nil)
|
|
defer ts.Close()
|
|
|
|
writePromptFile(t, lib.promptDir, "prompt.default", "local-default")
|
|
writeProfileFile(t, lib.profileDir, "local-default", ts.URL+"/v1", "profile-model")
|
|
|
|
code, stdout, stderr := runCLICommand(t, runCommand, []string{
|
|
"--prompt-dir", lib.promptDir,
|
|
"--profile-dir", lib.profileDir,
|
|
"--prompt", "prompt.default",
|
|
"--input", "transcript=" + inputPath,
|
|
})
|
|
|
|
if code != ExitOK {
|
|
t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr)
|
|
}
|
|
if stdout != "default-output" {
|
|
t.Fatalf("unexpected stdout output: %q", stdout)
|
|
}
|
|
if !strings.Contains(stderr, "selected_profile=local-default") {
|
|
t.Fatalf("expected selected profile in summary, got %q", stderr)
|
|
}
|
|
}
|
|
|
|
func TestRunCommandExplicitProfileOverridesPromptDefault(t *testing.T) {
|
|
lib := newCLITestLibrary(t)
|
|
inputPath := lib.writeInputFile(t, "transcript.md", "hello")
|
|
|
|
defaultServer := newTestLLMServer("from-default", nil)
|
|
defer defaultServer.Close()
|
|
overrideServer := newTestLLMServer("from-override", nil)
|
|
defer overrideServer.Close()
|
|
|
|
writePromptFile(t, lib.promptDir, "prompt.default", "local-default")
|
|
writeProfileFile(t, lib.profileDir, "local-default", defaultServer.URL+"/v1", "default-model")
|
|
writeProfileFile(t, lib.profileDir, "quality", overrideServer.URL+"/v1", "quality-model")
|
|
|
|
code, stdout, stderr := runCLICommand(t, runCommand, []string{
|
|
"--prompt-dir", lib.promptDir,
|
|
"--profile-dir", lib.profileDir,
|
|
"--prompt", "prompt.default",
|
|
"--profile", "quality",
|
|
"--input", "transcript=" + inputPath,
|
|
})
|
|
if code != ExitOK {
|
|
t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr)
|
|
}
|
|
if stdout != "from-override" {
|
|
t.Fatalf("expected explicit profile output, got %q", stdout)
|
|
}
|
|
if !strings.Contains(stderr, "selected_profile=quality") {
|
|
t.Fatalf("expected selected profile quality, got %q", stderr)
|
|
}
|
|
}
|
|
|
|
func TestRunCommandRuntimeFlagsOverrideSelectedProfileValues(t *testing.T) {
|
|
lib := newCLITestLibrary(t)
|
|
inputPath := lib.writeInputFile(t, "transcript.md", "hello")
|
|
|
|
var baseHits int32
|
|
baseServer := newTestLLMServer("base", &baseHits)
|
|
defer baseServer.Close()
|
|
|
|
var overrideHits int32
|
|
var observedBody string
|
|
overrideServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
atomic.AddInt32(&overrideHits, 1)
|
|
body, _ := io.ReadAll(r.Body)
|
|
observedBody = string(body)
|
|
w.Header().Set("Content-Type", "application/json")
|
|
_, _ = w.Write([]byte(`{"choices":[{"message":{"role":"assistant","content":"override"}}],"usage":{"prompt_tokens":1,"completion_tokens":1,"total_tokens":2}}`))
|
|
}))
|
|
defer overrideServer.Close()
|
|
|
|
writePromptFile(t, lib.promptDir, "prompt.default", "local-default")
|
|
writeProfileFile(t, lib.profileDir, "local-default", baseServer.URL+"/v1", "profile-model")
|
|
|
|
code, stdout, stderr := runCLICommand(t, runCommand, []string{
|
|
"--prompt-dir", lib.promptDir,
|
|
"--profile-dir", lib.profileDir,
|
|
"--prompt", "prompt.default",
|
|
"--input", "transcript=" + inputPath,
|
|
"--llm-base-url", overrideServer.URL + "/v1",
|
|
"--model", "override-model",
|
|
"--temperature", "0.7",
|
|
"--max-tokens", "55",
|
|
"--top-p", "0.2",
|
|
"--timeout", "20s",
|
|
})
|
|
if code != ExitOK {
|
|
t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr)
|
|
}
|
|
if atomic.LoadInt32(&baseHits) != 0 {
|
|
t.Fatalf("expected base profile endpoint not to be hit, got %d", baseHits)
|
|
}
|
|
if atomic.LoadInt32(&overrideHits) != 1 {
|
|
t.Fatalf("expected override endpoint to be hit once, got %d", overrideHits)
|
|
}
|
|
if stdout != "override" {
|
|
t.Fatalf("unexpected stdout output: %q", stdout)
|
|
}
|
|
if !strings.Contains(observedBody, `"model":"override-model"`) {
|
|
t.Fatalf("expected override model in request body, got %s", observedBody)
|
|
}
|
|
if !strings.Contains(observedBody, `"temperature":0.7`) || !strings.Contains(observedBody, `"max_tokens":55`) || !strings.Contains(observedBody, `"top_p":0.2`) {
|
|
t.Fatalf("expected override generation params in request body, got %s", observedBody)
|
|
}
|
|
}
|
|
|
|
func TestWriteOutputAndSummaryUseSeparateWriters(t *testing.T) {
|
|
var stdout bytes.Buffer
|
|
var stderr bytes.Buffer
|
|
|
|
if err := writeOutput(&stdout, "", []byte("artifact-body")); err != nil {
|
|
t.Fatalf("unexpected writeOutput error: %v", err)
|
|
}
|
|
printSummary(&stderr, &promptkit.RunResult{
|
|
PromptID: "p",
|
|
PromptVersion: "1",
|
|
SelectedProfileID: "exec",
|
|
ModelName: "m",
|
|
Validation: promptkit.ValidationResult{Status: promptkit.ValidationPassed, Mode: promptkit.ValidationBasic},
|
|
RenderedPromptHash: "h",
|
|
InputHashes: map[string]string{"in": "x"},
|
|
})
|
|
|
|
if stdout.String() != "artifact-body" {
|
|
t.Fatalf("expected artifact output on stdout, got %q", stdout.String())
|
|
}
|
|
if !strings.Contains(stderr.String(), "prompt=p@1") {
|
|
t.Fatalf("expected summary on stderr, got %q", stderr.String())
|
|
}
|
|
if strings.Contains(stderr.String(), "cached_tokens=") || strings.Contains(stderr.String(), "cache_write_tokens=") {
|
|
t.Fatalf("expected zero cache usage to be omitted from summary, got %q", stderr.String())
|
|
}
|
|
}
|
|
|
|
func TestPrintSummaryIncludesCacheUsageWhenPresent(t *testing.T) {
|
|
var stderr bytes.Buffer
|
|
|
|
printSummary(&stderr, &promptkit.RunResult{
|
|
PromptID: "p",
|
|
PromptVersion: "1",
|
|
SelectedProfileID: "exec",
|
|
ModelName: "m",
|
|
Validation: promptkit.ValidationResult{Status: promptkit.ValidationPassed, Mode: promptkit.ValidationBasic},
|
|
RenderedPromptHash: "h",
|
|
InputHashes: map[string]string{"in": "x"},
|
|
Usage: promptkit.TokenUsage{
|
|
PromptTokens: 10,
|
|
CompletionTokens: 5,
|
|
TotalTokens: 15,
|
|
CachedTokens: 0,
|
|
CacheWriteTokens: 3,
|
|
},
|
|
})
|
|
|
|
summary := stderr.String()
|
|
if !strings.Contains(summary, "usage=10/5/15") {
|
|
t.Fatalf("expected base usage summary, got %q", summary)
|
|
}
|
|
if !strings.Contains(summary, "cached_tokens=0 cache_write_tokens=3") {
|
|
t.Fatalf("expected cache usage in summary, got %q", summary)
|
|
}
|
|
}
|
|
|
|
type cliTestLibrary struct {
|
|
rootDir string
|
|
promptDir string
|
|
profileDir string
|
|
}
|
|
|
|
func newCLITestLibrary(t *testing.T) *cliTestLibrary {
|
|
t.Helper()
|
|
root := t.TempDir()
|
|
lib := &cliTestLibrary{
|
|
rootDir: root,
|
|
promptDir: filepath.Join(root, "prompts"),
|
|
profileDir: filepath.Join(root, "profiles"),
|
|
}
|
|
if err := os.MkdirAll(lib.promptDir, 0o755); err != nil {
|
|
t.Fatalf("failed to create prompt fixture directory: %v", err)
|
|
}
|
|
if err := os.MkdirAll(lib.profileDir, 0o755); err != nil {
|
|
t.Fatalf("failed to create profile fixture directory: %v", err)
|
|
}
|
|
return lib
|
|
}
|
|
|
|
func (l *cliTestLibrary) writeInputFile(t *testing.T, name, body string) string {
|
|
t.Helper()
|
|
path := filepath.Join(l.rootDir, name)
|
|
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
|
t.Fatalf("failed to create input fixture directory: %v", err)
|
|
}
|
|
if err := os.WriteFile(path, []byte(body), 0o644); err != nil {
|
|
t.Fatalf("failed to write input fixture: %v", err)
|
|
}
|
|
return path
|
|
}
|
|
|
|
func runCLICommand(t *testing.T, command func([]string, io.Writer, io.Writer) int, args []string) (int, string, string) {
|
|
t.Helper()
|
|
var stdout bytes.Buffer
|
|
var stderr bytes.Buffer
|
|
code := command(args, &stdout, &stderr)
|
|
return code, stdout.String(), stderr.String()
|
|
}
|
|
|
|
func writePromptFile(t *testing.T, dir, id, defaultProfile string) {
|
|
t.Helper()
|
|
writePromptFileWithTemplate(t, dir, id, defaultProfile, "Summarize: {{input \"transcript\"}}")
|
|
}
|
|
|
|
func writePromptFileWithTemplate(t *testing.T, dir, id, defaultProfile, templateContent string) {
|
|
t.Helper()
|
|
data := fmt.Sprintf(`id: %s
|
|
version: "1.0.0"
|
|
default_profile: %s
|
|
inputs:
|
|
- name: transcript
|
|
required: true
|
|
messages:
|
|
- role: user
|
|
content: %q
|
|
output:
|
|
format: text
|
|
validation_mode: none
|
|
repair_attempts: 0
|
|
`, id, defaultProfile, templateContent)
|
|
if err := os.WriteFile(filepath.Join(dir, id+".yaml"), []byte(data), 0o644); err != nil {
|
|
t.Fatalf("failed to write prompt fixture: %v", err)
|
|
}
|
|
}
|
|
|
|
func writeProfileFile(t *testing.T, dir, id, endpoint, model string) {
|
|
t.Helper()
|
|
data := fmt.Sprintf("id: %s\nendpoint: %s\nmodel: %s\n", id, endpoint, model)
|
|
if err := os.WriteFile(filepath.Join(dir, id+".yaml"), []byte(data), 0o644); err != nil {
|
|
t.Fatalf("failed to write profile fixture: %v", err)
|
|
}
|
|
}
|
|
|
|
func newTestLLMServer(content string, hitCounter *int32) *httptest.Server {
|
|
return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
if hitCounter != nil {
|
|
atomic.AddInt32(hitCounter, 1)
|
|
}
|
|
w.Header().Set("Content-Type", "application/json")
|
|
_, _ = w.Write([]byte(fmt.Sprintf(`{"choices":[{"message":{"role":"assistant","content":%q}}],"usage":{"prompt_tokens":1,"completion_tokens":1,"total_tokens":2}}`, content)))
|
|
}))
|
|
}
|
|
|
|
func writeAppConfigFile(t *testing.T, content string) string {
|
|
t.Helper()
|
|
path := filepath.Join(t.TempDir(), "config.yml")
|
|
if err := os.WriteFile(path, []byte(content), 0o644); err != nil {
|
|
t.Fatalf("failed to write app config fixture: %v", err)
|
|
}
|
|
return path
|
|
}
|