package cli import ( "bytes" "encoding/json" "errors" "flag" "fmt" "io" "net/http" "net/http/httptest" "os" "path/filepath" "reflect" "strings" "sync/atomic" "testing" "time" appconfig "gitea.maximumdirect.net/eric/scriptorium/internal/config" "gitea.maximumdirect.net/eric/scriptorium/internal/defaults" "gitea.maximumdirect.net/eric/scriptorium/internal/domain" renderformat "gitea.maximumdirect.net/eric/scriptorium/internal/format" ) func TestParseMappingsSingleAndRepeated(t *testing.T) { got, err := parseMappings([]string{"transcript=./t.md", "glossary=./g.yml"}, false) if err != nil { t.Fatalf("unexpected error: %v", err) } if got["transcript"] != "./t.md" || got["glossary"] != "./g.yml" { t.Fatalf("unexpected mappings: %#v", got) } } func TestParseMappingsCommaSeparated(t *testing.T) { got, err := parseMappings([]string{"transcript=./t.md,glossary=./g.yml"}, false) if err != nil { t.Fatalf("unexpected error: %v", err) } if got["transcript"] != "./t.md" || got["glossary"] != "./g.yml" { t.Fatalf("unexpected mappings: %#v", got) } } func TestParseMappingsVarWithEqualsInValue(t *testing.T) { got, err := parseMappings([]string{"session_note=a=b=c"}, false) if err != nil { t.Fatalf("unexpected error: %v", err) } if got["session_note"] != "a=b=c" { t.Fatalf("unexpected variable value: %#v", got) } } func TestParseMappingsMalformed(t *testing.T) { tests := []string{"", "novalue", "=emptyname", "name="} for _, tc := range tests { _, err := parseMappings([]string{tc}, false) if err == nil { t.Fatalf("expected error for %q", tc) } } } func TestParseRunArgsRequiredFlags(t *testing.T) { configPath := writeAppConfigFile(t, "") _, err := parseRunArgs([]string{"--config", configPath, "--profile-dir", "./profiles", "--prompt", "p", "--input", "a=b"}) if err == nil { t.Fatal("expected missing --prompt-dir error") } if !strings.Contains(err.Error(), "prompt directory is required") { t.Fatalf("expected clear prompt-dir guidance, got %v", err) } cfg, err := parseRunArgs([]string{"--config", configPath, "--prompt-dir", "./prompts", "--prompt", "p", "--input", "a=b"}) if err != nil { t.Fatalf("expected missing --profile-dir to be accepted, got %v", err) } if cfg.profileDir != "" { t.Fatalf("expected empty profile dir for built-ins, got %q", cfg.profileDir) } _, err = parseRunArgs([]string{"--config", configPath, "--prompt-dir", "./prompts", "--profile-dir", "./profiles", "--input", "a=b"}) if err == nil { t.Fatal("expected missing --prompt error") } _, err = parseRunArgs([]string{"--config", configPath, "--prompt-dir", "./prompts", "--profile-dir", "./profiles", "--prompt", "p"}) if err == nil { t.Fatal("expected missing --input error") } } func TestParseRunArgsFlagMapping(t *testing.T) { cfg, err := parseRunArgs([]string{ "--prompt-dir", "./prompts", "--profile-dir", "./profiles", "--prompt", "prompt.a", "--profile", "profile.a", "--input", "a=b", "--llm-base-url", "http://x/v1", "--model", "m", "--temperature", "0.7", "--max-tokens", "111", "--top-p", "0.8", "--timeout", "30s", "--api-key-env", "SCRIPTORIUM_API_KEY", }) if err != nil { t.Fatalf("expected valid args, got %v", err) } if cfg.promptDir != filepath.Clean("./prompts") || cfg.profileDir != filepath.Clean("./profiles") { t.Fatalf("unexpected dirs: prompt=%q profile=%q", cfg.promptDir, cfg.profileDir) } if cfg.promptID != "prompt.a" || cfg.profileID != "profile.a" { t.Fatalf("unexpected prompt/profile ids: %q %q", cfg.promptID, cfg.profileID) } if !cfg.llmBaseURLSet || !cfg.modelSet || !cfg.temperatureSet || !cfg.maxTokensSet || !cfg.topPSet || !cfg.timeoutSet || !cfg.apiKeyEnvSet { t.Fatalf("expected override flags set, got %+v", cfg) } } func TestParseRunArgsAllowsOmittedModelAndBaseURL(t *testing.T) { cfg, err := parseRunArgs([]string{ "--prompt-dir", "./prompts", "--profile-dir", "./profiles", "--prompt", "p", "--input", "a=b", }) if err != nil { t.Fatalf("expected valid args without model/base url, got %v", err) } if cfg.llmBaseURL != "" || cfg.model != "" { t.Fatalf("expected empty model/baseurl, got model=%q base=%q", cfg.model, cfg.llmBaseURL) } } func TestParseRunArgsRejectsRawLLMAPIKeyFlag(t *testing.T) { _, err := parseRunArgs([]string{ "--prompt-dir", "./prompts", "--profile-dir", "./profiles", "--prompt", "p", "--input", "a=b", "--llm-api-key", "secret", }) if err == nil { t.Fatal("expected unknown flag error for --llm-api-key") } } func TestParseServeArgsRequiredFlags(t *testing.T) { configPath := writeAppConfigFile(t, "") _, err := parseServeArgs([]string{"--config", configPath, "--profile-dir", "./profiles"}) if err == nil { t.Fatal("expected missing --prompt-dir error") } if !strings.Contains(err.Error(), "prompt directory is required") { t.Fatalf("expected clear prompt-dir guidance, got %v", err) } cfg, err := parseServeArgs([]string{"--config", configPath, "--prompt-dir", "./prompts"}) if err != nil { t.Fatalf("expected missing --profile-dir to be accepted, got %v", err) } if cfg.profileDir != "" { t.Fatalf("expected empty profile dir for built-ins, got %q", cfg.profileDir) } if cfg.addr != defaults.HTTPAddrDefault { t.Fatalf("expected default addr %s, got %q", defaults.HTTPAddrDefault, cfg.addr) } if cfg.schemaDir != defaults.SchemaDirDefault { t.Fatalf("expected default schema dir %q, got %q", defaults.SchemaDirDefault, cfg.schemaDir) } } func TestParseServeArgsRejectsRuntimeOverrideFlags(t *testing.T) { base := []string{"--prompt-dir", "./prompts", "--profile-dir", "./profiles"} tests := [][]string{ {"--llm-base-url", "http://localhost:8000/v1"}, {"--model", "gpt-4o-mini"}, {"--timeout", "30s"}, } for _, tc := range tests { _, err := parseServeArgs(append(base, tc...)) if err == nil { t.Fatalf("expected unknown flag error for %q", tc[0]) } } } func TestParseRunArgsTimeout(t *testing.T) { cfg, err := parseRunArgs([]string{ "--prompt-dir", "./prompts", "--profile-dir", "./profiles", "--prompt", "p", "--input", "a=b", }) if err != nil { t.Fatalf("expected valid run args, got %v", err) } if cfg.timeout != defaults.LLMRequestTimeoutDefault { t.Fatalf("expected default timeout %s, got %s", defaults.LLMRequestTimeoutDefault, cfg.timeout) } cfg, err = parseRunArgs([]string{ "--prompt-dir", "./prompts", "--profile-dir", "./profiles", "--prompt", "p", "--input", "a=b", "--timeout", "2m30s", }) if err != nil { t.Fatalf("expected valid run args with timeout override, got %v", err) } if cfg.timeout != 2*time.Minute+30*time.Second { t.Fatalf("expected timeout override 2m30s, got %s", cfg.timeout) } } func TestParseRenderArgsDefaultsAndFormat(t *testing.T) { cfg, err := parseRenderArgs([]string{ "--prompt-dir", "./prompts", "--profile-dir", "./profiles", "--prompt", "p", "--input", "a=b", }) if err != nil { t.Fatalf("expected valid render args, got %v", err) } if cfg.outputFormat != renderformat.DefaultPreparedRunOutputFormat { t.Fatalf("expected default render format %q, got %q", renderformat.DefaultPreparedRunOutputFormat, cfg.outputFormat) } } func TestParseRenderArgsExplicitFormatsAndUnknown(t *testing.T) { cfg, err := parseRenderArgs([]string{ "--prompt-dir", "./prompts", "--profile-dir", "./profiles", "--prompt", "p", "--input", "a=b", "--format", "text", }) if err != nil { t.Fatalf("expected valid text format, got %v", err) } if cfg.outputFormat != renderformat.PreparedRunFormatText { t.Fatalf("expected text format, got %q", cfg.outputFormat) } cfg, err = parseRenderArgs([]string{ "--prompt-dir", "./prompts", "--profile-dir", "./profiles", "--prompt", "p", "--input", "a=b", "--format", "json", }) if err != nil { t.Fatalf("expected valid json format, got %v", err) } if cfg.outputFormat != renderformat.PreparedRunFormatJSON { t.Fatalf("expected json format, got %q", cfg.outputFormat) } _, err = parseRenderArgs([]string{ "--prompt-dir", "./prompts", "--profile-dir", "./profiles", "--prompt", "p", "--input", "a=b", "--format", "yaml", }) if err == nil { t.Fatal("expected unknown format error") } if !strings.Contains(err.Error(), "unknown prepared run format") { t.Fatalf("expected clear unknown format error, got %v", err) } } func TestParseRunArgsWithExplicitConfigLoadsDirectories(t *testing.T) { configPath := writeAppConfigFile(t, ` prompt_dir: ./from-config/prompts profile_dir: ./from-config/profiles schema_dir: ./from-config/schemas `) cfg, err := parseRunArgs([]string{ "--config", configPath, "--prompt", "p", "--input", "a=b", }) if err != nil { t.Fatalf("expected valid args, got %v", err) } if cfg.promptDir != filepath.Clean("./from-config/prompts") { t.Fatalf("expected prompt dir from config, got %q", cfg.promptDir) } if cfg.profileDir != filepath.Clean("./from-config/profiles") { t.Fatalf("expected profile dir from config, got %q", cfg.profileDir) } if cfg.schemaDir != filepath.Clean("./from-config/schemas") { t.Fatalf("expected schema dir from config, got %q", cfg.schemaDir) } } func TestParseRunArgsMissingExplicitConfigReturnsError(t *testing.T) { _, err := parseRunArgs([]string{ "--config", filepath.Join(t.TempDir(), "missing.yml"), "--prompt", "p", "--input", "a=b", }) if err == nil { t.Fatal("expected explicit config missing error") } } func TestParseRunArgsInvalidExplicitConfigReturnsError(t *testing.T) { configPath := writeAppConfigFile(t, "api_key: secret\n") _, err := parseRunArgs([]string{ "--config", configPath, "--prompt", "p", "--input", "a=b", }) if err == nil { t.Fatal("expected invalid explicit config error") } if !strings.Contains(err.Error(), "application config error") { t.Fatalf("expected application config context, got %v", err) } } func TestResolveAppSettingsMissingImplicitConfigDoesNotError(t *testing.T) { fs := flag.NewFlagSet("test", flag.ContinueOnError) settings, err := resolveAppSettings(fs, filepath.Join(t.TempDir(), "missing.yml"), appconfig.CLIOverrides{}) if err != nil { t.Fatalf("expected no error, got %v", err) } if settings.SchemaDir != defaults.SchemaDirDefault { t.Fatalf("expected built-in schema dir, got %q", settings.SchemaDir) } if settings.ServerAddr != defaults.HTTPAddrDefault { t.Fatalf("expected built-in server addr, got %q", settings.ServerAddr) } } func TestParseRunArgsCLIOverridesConfigDirectories(t *testing.T) { configPath := writeAppConfigFile(t, ` prompt_dir: ./from-config/prompts profile_dir: ./from-config/profiles schema_dir: ./from-config/schemas `) cfg, err := parseRunArgs([]string{ "--config", configPath, "--prompt-dir", "./from-cli/prompts", "--profile-dir", "./from-cli/profiles", "--schema-dir", "./from-cli/schemas", "--prompt", "p", "--input", "a=b", }) if err != nil { t.Fatalf("expected valid args, got %v", err) } if cfg.promptDir != filepath.Clean("./from-cli/prompts") { t.Fatalf("expected CLI prompt dir override, got %q", cfg.promptDir) } if cfg.profileDir != filepath.Clean("./from-cli/profiles") { t.Fatalf("expected CLI profile dir override, got %q", cfg.profileDir) } if cfg.schemaDir != filepath.Clean("./from-cli/schemas") { t.Fatalf("expected CLI schema dir override, got %q", cfg.schemaDir) } } func TestParseRenderArgsWithExplicitConfigLoadsDirectoriesAndFormat(t *testing.T) { configPath := writeAppConfigFile(t, ` prompt_dir: ./from-config/prompts profile_dir: ./from-config/profiles defaults: render_format: json `) cfg, err := parseRenderArgs([]string{ "--config", configPath, "--prompt", "p", "--input", "a=b", }) if err != nil { t.Fatalf("expected valid args, got %v", err) } if cfg.promptDir != filepath.Clean("./from-config/prompts") { t.Fatalf("expected prompt dir from config, got %q", cfg.promptDir) } if cfg.profileDir != filepath.Clean("./from-config/profiles") { t.Fatalf("expected profile dir from config, got %q", cfg.profileDir) } if cfg.outputFormat != renderformat.PreparedRunFormatJSON { t.Fatalf("expected render format from config, got %q", cfg.outputFormat) } } func TestParseRenderArgsExplicitFormatOverridesConfigDefaultFormat(t *testing.T) { configPath := writeAppConfigFile(t, ` prompt_dir: ./from-config/prompts profile_dir: ./from-config/profiles defaults: render_format: json `) cfg, err := parseRenderArgs([]string{ "--config", configPath, "--prompt", "p", "--input", "a=b", "--format", "text", }) if err != nil { t.Fatalf("expected valid args, got %v", err) } if cfg.outputFormat != renderformat.PreparedRunFormatText { t.Fatalf("expected explicit --format text to override config default, got %q", cfg.outputFormat) } } func TestParseServeArgsWithExplicitConfigLoadsSettingsAndCLIAddrOverrides(t *testing.T) { configPath := writeAppConfigFile(t, ` prompt_dir: ./from-config/prompts profile_dir: ./from-config/profiles schema_dir: ./from-config/schemas server: addr: 127.0.0.1:9000 `) cfg, err := parseServeArgs([]string{ "--config", configPath, "--addr", ":7777", }) if err != nil { t.Fatalf("expected valid args, got %v", err) } if cfg.promptDir != filepath.Clean("./from-config/prompts") { t.Fatalf("expected prompt dir from config, got %q", cfg.promptDir) } if cfg.profileDir != filepath.Clean("./from-config/profiles") { t.Fatalf("expected profile dir from config, got %q", cfg.profileDir) } if cfg.schemaDir != filepath.Clean("./from-config/schemas") { t.Fatalf("expected schema dir from config, got %q", cfg.schemaDir) } if cfg.addr != ":7777" { t.Fatalf("expected CLI addr override, got %q", cfg.addr) } } func TestParseServeArgsWithConfigProvidesRequiredDirectoriesAndAddr(t *testing.T) { configPath := writeAppConfigFile(t, ` prompt_dir: ./from-config/prompts profile_dir: ./from-config/profiles schema_dir: ./from-config/schemas server: addr: 127.0.0.1:9000 `) cfg, err := parseServeArgs([]string{ "--config", configPath, }) if err != nil { t.Fatalf("expected valid args, got %v", err) } if cfg.promptDir != filepath.Clean("./from-config/prompts") { t.Fatalf("expected prompt dir from config, got %q", cfg.promptDir) } if cfg.profileDir != filepath.Clean("./from-config/profiles") { t.Fatalf("expected profile dir from config, got %q", cfg.profileDir) } if cfg.schemaDir != filepath.Clean("./from-config/schemas") { t.Fatalf("expected schema dir from config, got %q", cfg.schemaDir) } if cfg.addr != "127.0.0.1:9000" { t.Fatalf("expected addr from config, got %q", cfg.addr) } } func TestRunAndRenderBuildEquivalentRuntimeOverrideRequestsForSharedFlags(t *testing.T) { runCfg, err := parseRunArgs([]string{ "--prompt-dir", "./prompts", "--profile-dir", "./profiles", "--prompt", "prompt-1", "--profile", "profile-1", "--input", "transcript=./transcript.md", "--var", "session_date=2026-05-01", "--llm-base-url", "http://localhost:8000/v1", "--model", "model-x", "--temperature", "0.8", "--max-tokens", "123", "--top-p", "0.6", "--timeout", "90s", "--api-key-env", "SCRIPTORIUM_API_KEY", }) if err != nil { t.Fatalf("expected valid run args, got %v", err) } renderCfg, err := parseRenderArgs([]string{ "--prompt-dir", "./prompts", "--profile-dir", "./profiles", "--prompt", "prompt-1", "--profile", "profile-1", "--input", "transcript=./transcript.md", "--var", "session_date=2026-05-01", "--llm-base-url", "http://localhost:8000/v1", "--model", "model-x", "--temperature", "0.8", "--max-tokens", "123", "--top-p", "0.6", "--timeout", "90s", "--api-key-env", "SCRIPTORIUM_API_KEY", }) if err != nil { t.Fatalf("expected valid render args, got %v", err) } runReq, err := buildRunRequestFromConfig(runCfg) if err != nil { t.Fatalf("expected run request build success, got %v", err) } renderReq, err := buildRunRequestFromConfig(&renderCfg.runConfig) if err != nil { t.Fatalf("expected render request build success, got %v", err) } if !reflect.DeepEqual(runReq, renderReq) { t.Fatalf("expected run/render shared flag requests to match.\nrun=%#v\nrender=%#v", runReq, renderReq) } } func TestParseRunArgsFailsClearlyWhenNoEffectivePromptDir(t *testing.T) { configPath := writeAppConfigFile(t, ` profile_dir: ./profiles `) _, err := parseRunArgs([]string{ "--config", configPath, "--prompt", "p", "--input", "a=b", }) if err == nil { t.Fatal("expected missing prompt_dir error") } if !strings.Contains(err.Error(), "prompt directory is required") || !strings.Contains(err.Error(), "config.yml prompt_dir") { t.Fatalf("expected clear prompt_dir guidance, got %v", err) } } func TestParseRunArgsAcceptsMissingEffectiveProfileDir(t *testing.T) { configPath := writeAppConfigFile(t, ` prompt_dir: ./prompts `) cfg, err := parseRunArgs([]string{ "--config", configPath, "--prompt", "p", "--input", "a=b", }) if err != nil { t.Fatalf("expected missing profile_dir to be accepted, got %v", err) } if cfg.profileDir != "" { t.Fatalf("expected empty profile dir for built-ins, got %q", cfg.profileDir) } } func TestParseRenderArgsFailsClearlyWhenNoEffectivePromptDir(t *testing.T) { configPath := writeAppConfigFile(t, ` profile_dir: ./profiles `) _, err := parseRenderArgs([]string{ "--config", configPath, "--prompt", "p", "--input", "a=b", }) if err == nil { t.Fatal("expected missing prompt_dir error") } if !strings.Contains(err.Error(), "prompt directory is required") || !strings.Contains(err.Error(), "config.yml prompt_dir") { t.Fatalf("expected clear prompt_dir guidance, got %v", err) } } func TestParseRenderArgsAcceptsMissingEffectiveProfileDir(t *testing.T) { configPath := writeAppConfigFile(t, ` prompt_dir: ./prompts `) cfg, err := parseRenderArgs([]string{ "--config", configPath, "--prompt", "p", "--input", "a=b", }) if err != nil { t.Fatalf("expected missing profile_dir to be accepted, got %v", err) } if cfg.profileDir != "" { t.Fatalf("expected empty profile dir for built-ins, got %q", cfg.profileDir) } } func TestDetermineExitCode(t *testing.T) { if got := determineExitCode(errors.New("boom"), nil); got != ExitRuntimeError { t.Fatalf("expected runtime exit code, got %d", got) } if got := determineExitCode(nil, &domain.RunResult{Validation: domain.ValidationResult{Status: domain.ValidationFailed}}); got != ExitValidationFailed { t.Fatalf("expected validation exit code, got %d", got) } if got := determineExitCode(nil, &domain.RunResult{Validation: domain.ValidationResult{Status: domain.ValidationPassed}}); got != ExitOK { t.Fatalf("expected success exit code for passed validation, got %d", got) } if got := determineExitCode(nil, &domain.RunResult{Validation: domain.ValidationResult{Status: domain.ValidationSkipped}}); got != ExitOK { t.Fatalf("expected success exit code for skipped validation, got %d", got) } } func TestRunCommandVarsOptional(t *testing.T) { var stdout bytes.Buffer var stderr bytes.Buffer code := runCommand([]string{ "--prompt-dir", "./prompts", "--profile-dir", "./profiles", "--prompt", "p", "--input", "transcript=./t.md", "--llm-base-url", "http://[::1", "--model", "m", }, &stdout, &stderr) if code != ExitRuntimeError { t.Fatalf("expected runtime error exit code, got %d", code) } if strings.Contains(stderr.String(), "var parse error") { t.Fatalf("expected --var to be optional, got stderr=%q", stderr.String()) } if !strings.Contains(stderr.String(), "llm client error") && !strings.Contains(stderr.String(), "run error") { t.Fatalf("expected post-parse execution error, got stderr=%q", stderr.String()) } if stdout.Len() != 0 { t.Fatalf("expected no stdout output on error, got %q", stdout.String()) } } func TestRunCommandSucceedsWithPromptAndProfileDirsFromConfig(t *testing.T) { lib := newCLITestLibrary(t) inputPath := lib.writeInputFile(t, "transcript.md", "hello") ts := newTestLLMServer("from-config-dirs", nil) defer ts.Close() writePromptFile(t, lib.promptDir, "prompt.default", "local-default") writeProfileFile(t, lib.profileDir, "local-default", ts.URL+"/v1", "profile-model") configPath := writeAppConfigFile(t, fmt.Sprintf(` prompt_dir: %s profile_dir: %s `, lib.promptDir, lib.profileDir)) code, stdout, stderr := runCLICommand(t, runCommand, []string{ "--config", configPath, "--prompt", "prompt.default", "--input", "transcript=" + inputPath, }) if code != ExitOK { t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr) } if stdout != "from-config-dirs" { t.Fatalf("unexpected stdout output: %q", stdout) } } func TestRenderCommandDefaultFormatTextIncludesPreparedDetailsAndNoSecrets(t *testing.T) { const envName = "SCRIPTORIUM_RENDER_TEST_API_KEY" const secret = "super-secret-render-key" t.Setenv(envName, secret) lib := newCLITestLibrary(t) inputPath := lib.writeInputFile(t, "transcript.md", "hello transcript") writePromptFileWithTemplate(t, lib.promptDir, "prompt.render", "local-default", "Date {{.session_date}} - Summarize: {{input \"transcript\"}}") writeProfileFile(t, lib.profileDir, "local-default", "http://127.0.0.1:1/v1", "profile-model") code, stdout, stderr := runCLICommand(t, renderCommand, []string{ "--prompt-dir", lib.promptDir, "--profile-dir", lib.profileDir, "--prompt", "prompt.render", "--profile", "local-default", "--input", "transcript=" + inputPath, "--var", "session_date=2026-05-04", "--llm-base-url", "http://override.local/v1", "--model", "override-model", "--temperature", "0.7", "--max-tokens", "55", "--top-p", "0.2", "--timeout", "20s", "--api-key-env", envName, }) if code != ExitOK { t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr) } if stderr != "" { t.Fatalf("expected empty stderr on success, got %q", stderr) } out := stdout for _, want := range []string{ "prompt: prompt.render", "selected_profile_id: local-default", "endpoint: http://override.local/v1", "model: override-model", "temperature: 0.7", "max_tokens: 55", "top_p: 0.2", "timeout_seconds: 20", "api_key_env: " + envName, "rendered_prompt_hash:", "messages:", "Date 2026-05-04", "Summarize:", "hello transcript", } { if !strings.Contains(out, want) { t.Fatalf("expected render text output to include %q, got:\n%s", want, out) } } if strings.Contains(out, secret) { t.Fatalf("render output unexpectedly contained secret api key value: %s", out) } } func TestRenderCommandExplicitZeroTemperatureReachesEffectiveSettings(t *testing.T) { lib := newCLITestLibrary(t) inputPath := lib.writeInputFile(t, "transcript.md", "hello transcript") writePromptFile(t, lib.promptDir, "prompt.render", "local-default") profile := `id: local-default endpoint: http://127.0.0.1:1/v1 model: profile-model temperature: 0.7 ` if err := os.WriteFile(filepath.Join(lib.profileDir, "local-default.yaml"), []byte(profile), 0o644); err != nil { t.Fatalf("failed to write profile fixture: %v", err) } code, stdout, stderr := runCLICommand(t, renderCommand, []string{ "--prompt-dir", lib.promptDir, "--profile-dir", lib.profileDir, "--prompt", "prompt.render", "--input", "transcript=" + inputPath, "--temperature", "0", }) if code != ExitOK { t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr) } if !strings.Contains(stdout, "\n temperature: 0\n") { t.Fatalf("expected explicit zero temperature in effective settings, got:\n%s", stdout) } } func TestRenderCommandSucceedsWithPromptAndProfileDirsFromConfig(t *testing.T) { lib := newCLITestLibrary(t) inputPath := lib.writeInputFile(t, "transcript.md", "hello transcript") writePromptFile(t, lib.promptDir, "prompt.render", "local-default") writeProfileFile(t, lib.profileDir, "local-default", "http://127.0.0.1:1/v1", "profile-model") configPath := writeAppConfigFile(t, fmt.Sprintf(` prompt_dir: %s profile_dir: %s `, lib.promptDir, lib.profileDir)) code, stdout, stderr := runCLICommand(t, renderCommand, []string{ "--config", configPath, "--prompt", "prompt.render", "--input", "transcript=" + inputPath, }) if code != ExitOK { t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr) } if !strings.Contains(stdout, "prompt: prompt.render") { t.Fatalf("expected rendered output, got %q", stdout) } } func TestRenderCommandExplicitTextFormatWorks(t *testing.T) { lib := newCLITestLibrary(t) inputPath := lib.writeInputFile(t, "transcript.md", "hello transcript") writePromptFile(t, lib.promptDir, "prompt.render", "local-default") writeProfileFile(t, lib.profileDir, "local-default", "http://127.0.0.1:1/v1", "profile-model") code, stdout, stderr := runCLICommand(t, renderCommand, []string{ "--prompt-dir", lib.promptDir, "--profile-dir", lib.profileDir, "--prompt", "prompt.render", "--input", "transcript=" + inputPath, "--format", "text", }) if code != ExitOK { t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr) } if !strings.Contains(stdout, "prompt: prompt.render") { t.Fatalf("expected text output for explicit --format text, got %q", stdout) } } func TestRenderCommandExplicitJSONFormatOutputsValidJSON(t *testing.T) { lib := newCLITestLibrary(t) inputPath := lib.writeInputFile(t, "transcript.md", "hello transcript") writePromptFile(t, lib.promptDir, "prompt.render", "local-default") writeProfileFile(t, lib.profileDir, "local-default", "http://127.0.0.1:1/v1", "profile-model") code, stdout, stderr := runCLICommand(t, renderCommand, []string{ "--prompt-dir", lib.promptDir, "--profile-dir", lib.profileDir, "--prompt", "prompt.render", "--input", "transcript=" + inputPath, "--format", "json", }) if code != ExitOK { t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr) } var payload map[string]any if err := json.Unmarshal([]byte(stdout), &payload); err != nil { t.Fatalf("expected valid json output, got %v\nbody=%s", err, stdout) } if payload["prompt_id"] != "prompt.render" { t.Fatalf("expected prompt_id, got %#v", payload["prompt_id"]) } if payload["selected_profile_id"] != "local-default" { t.Fatalf("expected selected_profile_id, got %#v", payload["selected_profile_id"]) } if _, ok := payload["messages"]; !ok { t.Fatalf("expected messages in render json output, got %#v", payload) } } func TestRenderCommandUnknownFormatFailsClearly(t *testing.T) { var stdout bytes.Buffer var stderr bytes.Buffer code := renderCommand([]string{ "--prompt-dir", "./prompts", "--profile-dir", "./profiles", "--prompt", "p", "--input", "transcript=./x.md", "--format", "yaml", }, &stdout, &stderr) if code != ExitRuntimeError { t.Fatalf("expected ExitRuntimeError, got %d", code) } if !strings.Contains(stderr.String(), "render parse error") || !strings.Contains(stderr.String(), "unknown prepared run format") { t.Fatalf("expected clear unknown-format parse error, got %q", stderr.String()) } } func TestRenderCommandOutWritesToFile(t *testing.T) { lib := newCLITestLibrary(t) inputPath := lib.writeInputFile(t, "transcript.md", "hello transcript") outPath := filepath.Join(lib.rootDir, "render.txt") writePromptFile(t, lib.promptDir, "prompt.render", "local-default") writeProfileFile(t, lib.profileDir, "local-default", "http://127.0.0.1:1/v1", "profile-model") code, stdout, stderr := runCLICommand(t, renderCommand, []string{ "--prompt-dir", lib.promptDir, "--profile-dir", lib.profileDir, "--prompt", "prompt.render", "--input", "transcript=" + inputPath, "--out", outPath, }) if code != ExitOK { t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr) } if stdout != "" { t.Fatalf("expected empty stdout when --out is set, got %q", stdout) } out, err := os.ReadFile(outPath) if err != nil { t.Fatalf("failed reading render output file: %v", err) } if !strings.Contains(string(out), "prompt: prompt.render") { t.Fatalf("expected render output in file, got %q", string(out)) } } func TestRenderCommandPromptDefaultProfileWorksThroughCLIPath(t *testing.T) { lib := newCLITestLibrary(t) inputPath := lib.writeInputFile(t, "transcript.md", "hello") writePromptFile(t, lib.promptDir, "prompt.default", "local-default") writeProfileFile(t, lib.profileDir, "local-default", "http://127.0.0.1:1/v1", "default-model") code, stdout, stderr := runCLICommand(t, renderCommand, []string{ "--prompt-dir", lib.promptDir, "--profile-dir", lib.profileDir, "--prompt", "prompt.default", "--input", "transcript=" + inputPath, }) if code != ExitOK { t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr) } out := stdout if !strings.Contains(out, "selected_profile_id: local-default") { t.Fatalf("expected prompt default profile in output, got %q", out) } if !strings.Contains(out, "model: default-model") { t.Fatalf("expected model from default profile in output, got %q", out) } } func TestRenderCommandUsesBuiltInProfileWithoutProfileDir(t *testing.T) { t.Setenv("OPENROUTER_API_KEY", "test-key") lib := newCLITestLibrary(t) inputPath := lib.writeInputFile(t, "transcript.md", "hello") writePromptFile(t, lib.promptDir, "prompt.builtin", "mistral-small-3") code, stdout, stderr := runCLICommand(t, renderCommand, []string{ "--prompt-dir", lib.promptDir, "--prompt", "prompt.builtin", "--input", "transcript=" + inputPath, }) if code != ExitOK { t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr) } if !strings.Contains(stdout, "selected_profile_id: mistral-small-3") { t.Fatalf("expected built-in selected profile, got %q", stdout) } if !strings.Contains(stdout, "model: mistralai/mistral-small-3.2-24b-instruct") { t.Fatalf("expected built-in model, got %q", stdout) } } func TestRenderCommandExplicitProfileOverridesPromptDefault(t *testing.T) { lib := newCLITestLibrary(t) inputPath := lib.writeInputFile(t, "transcript.md", "hello") writePromptFile(t, lib.promptDir, "prompt.default", "local-default") writeProfileFile(t, lib.profileDir, "local-default", "http://127.0.0.1:1/v1", "default-model") writeProfileFile(t, lib.profileDir, "quality", "http://127.0.0.1:1/v1", "quality-model") code, stdout, stderr := runCLICommand(t, renderCommand, []string{ "--prompt-dir", lib.promptDir, "--profile-dir", lib.profileDir, "--prompt", "prompt.default", "--profile", "quality", "--input", "transcript=" + inputPath, }) if code != ExitOK { t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr) } out := stdout if !strings.Contains(out, "selected_profile_id: quality") { t.Fatalf("expected explicit profile in output, got %q", out) } if !strings.Contains(out, "model: quality-model") { t.Fatalf("expected model from explicit profile in output, got %q", out) } } func TestRunCommandPromptDefaultProfileWorksThroughCLIPath(t *testing.T) { lib := newCLITestLibrary(t) inputPath := lib.writeInputFile(t, "transcript.md", "hello") ts := newTestLLMServer("default-output", nil) defer ts.Close() writePromptFile(t, lib.promptDir, "prompt.default", "local-default") writeProfileFile(t, lib.profileDir, "local-default", ts.URL+"/v1", "profile-model") code, stdout, stderr := runCLICommand(t, runCommand, []string{ "--prompt-dir", lib.promptDir, "--profile-dir", lib.profileDir, "--prompt", "prompt.default", "--input", "transcript=" + inputPath, }) if code != ExitOK { t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr) } if stdout != "default-output" { t.Fatalf("unexpected stdout output: %q", stdout) } if !strings.Contains(stderr, "selected_profile=local-default") { t.Fatalf("expected selected profile in summary, got %q", stderr) } } func TestRunCommandExplicitProfileOverridesPromptDefault(t *testing.T) { lib := newCLITestLibrary(t) inputPath := lib.writeInputFile(t, "transcript.md", "hello") defaultServer := newTestLLMServer("from-default", nil) defer defaultServer.Close() overrideServer := newTestLLMServer("from-override", nil) defer overrideServer.Close() writePromptFile(t, lib.promptDir, "prompt.default", "local-default") writeProfileFile(t, lib.profileDir, "local-default", defaultServer.URL+"/v1", "default-model") writeProfileFile(t, lib.profileDir, "quality", overrideServer.URL+"/v1", "quality-model") code, stdout, stderr := runCLICommand(t, runCommand, []string{ "--prompt-dir", lib.promptDir, "--profile-dir", lib.profileDir, "--prompt", "prompt.default", "--profile", "quality", "--input", "transcript=" + inputPath, }) if code != ExitOK { t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr) } if stdout != "from-override" { t.Fatalf("expected explicit profile output, got %q", stdout) } if !strings.Contains(stderr, "selected_profile=quality") { t.Fatalf("expected selected profile quality, got %q", stderr) } } func TestRunCommandRuntimeFlagsOverrideSelectedProfileValues(t *testing.T) { lib := newCLITestLibrary(t) inputPath := lib.writeInputFile(t, "transcript.md", "hello") var baseHits int32 baseServer := newTestLLMServer("base", &baseHits) defer baseServer.Close() var overrideHits int32 var observedBody string overrideServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { atomic.AddInt32(&overrideHits, 1) body, _ := io.ReadAll(r.Body) observedBody = string(body) w.Header().Set("Content-Type", "application/json") _, _ = w.Write([]byte(`{"choices":[{"message":{"role":"assistant","content":"override"}}],"usage":{"prompt_tokens":1,"completion_tokens":1,"total_tokens":2}}`)) })) defer overrideServer.Close() writePromptFile(t, lib.promptDir, "prompt.default", "local-default") writeProfileFile(t, lib.profileDir, "local-default", baseServer.URL+"/v1", "profile-model") code, stdout, stderr := runCLICommand(t, runCommand, []string{ "--prompt-dir", lib.promptDir, "--profile-dir", lib.profileDir, "--prompt", "prompt.default", "--input", "transcript=" + inputPath, "--llm-base-url", overrideServer.URL + "/v1", "--model", "override-model", "--temperature", "0.7", "--max-tokens", "55", "--top-p", "0.2", "--timeout", "20s", }) if code != ExitOK { t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr) } if atomic.LoadInt32(&baseHits) != 0 { t.Fatalf("expected base profile endpoint not to be hit, got %d", baseHits) } if atomic.LoadInt32(&overrideHits) != 1 { t.Fatalf("expected override endpoint to be hit once, got %d", overrideHits) } if stdout != "override" { t.Fatalf("unexpected stdout output: %q", stdout) } if !strings.Contains(observedBody, `"model":"override-model"`) { t.Fatalf("expected override model in request body, got %s", observedBody) } if !strings.Contains(observedBody, `"temperature":0.7`) || !strings.Contains(observedBody, `"max_tokens":55`) || !strings.Contains(observedBody, `"top_p":0.2`) { t.Fatalf("expected override generation params in request body, got %s", observedBody) } } func TestWriteOutputAndSummaryUseSeparateWriters(t *testing.T) { var stdout bytes.Buffer var stderr bytes.Buffer if err := writeOutput(&stdout, "", []byte("artifact-body")); err != nil { t.Fatalf("unexpected writeOutput error: %v", err) } printSummary(&stderr, &domain.RunResult{ PromptID: "p", PromptVersion: "1", SelectedProfileID: "exec", ModelName: "m", Validation: domain.ValidationResult{Status: domain.ValidationPassed, Mode: domain.ValidationBasic}, RenderedPromptHash: "h", InputHashes: map[string]string{"in": "x"}, }) if stdout.String() != "artifact-body" { t.Fatalf("expected artifact output on stdout, got %q", stdout.String()) } if !strings.Contains(stderr.String(), "prompt=p@1") { t.Fatalf("expected summary on stderr, got %q", stderr.String()) } if strings.Contains(stderr.String(), "cached_tokens=") || strings.Contains(stderr.String(), "cache_write_tokens=") { t.Fatalf("expected zero cache usage to be omitted from summary, got %q", stderr.String()) } } func TestPrintSummaryIncludesCacheUsageWhenPresent(t *testing.T) { var stderr bytes.Buffer printSummary(&stderr, &domain.RunResult{ PromptID: "p", PromptVersion: "1", SelectedProfileID: "exec", ModelName: "m", Validation: domain.ValidationResult{Status: domain.ValidationPassed, Mode: domain.ValidationBasic}, RenderedPromptHash: "h", InputHashes: map[string]string{"in": "x"}, Usage: domain.TokenUsage{ PromptTokens: 10, CompletionTokens: 5, TotalTokens: 15, CachedTokens: 0, CacheWriteTokens: 3, }, }) summary := stderr.String() if !strings.Contains(summary, "usage=10/5/15") { t.Fatalf("expected base usage summary, got %q", summary) } if !strings.Contains(summary, "cached_tokens=0 cache_write_tokens=3") { t.Fatalf("expected cache usage in summary, got %q", summary) } } type cliTestLibrary struct { rootDir string promptDir string profileDir string } func newCLITestLibrary(t *testing.T) *cliTestLibrary { t.Helper() root := t.TempDir() lib := &cliTestLibrary{ rootDir: root, promptDir: filepath.Join(root, "prompts"), profileDir: filepath.Join(root, "profiles"), } if err := os.MkdirAll(lib.promptDir, 0o755); err != nil { t.Fatalf("failed to create prompt fixture directory: %v", err) } if err := os.MkdirAll(lib.profileDir, 0o755); err != nil { t.Fatalf("failed to create profile fixture directory: %v", err) } return lib } func (l *cliTestLibrary) writeInputFile(t *testing.T, name, body string) string { t.Helper() path := filepath.Join(l.rootDir, name) if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { t.Fatalf("failed to create input fixture directory: %v", err) } if err := os.WriteFile(path, []byte(body), 0o644); err != nil { t.Fatalf("failed to write input fixture: %v", err) } return path } func runCLICommand(t *testing.T, command func([]string, io.Writer, io.Writer) int, args []string) (int, string, string) { t.Helper() var stdout bytes.Buffer var stderr bytes.Buffer code := command(args, &stdout, &stderr) return code, stdout.String(), stderr.String() } func writePromptFile(t *testing.T, dir, id, defaultProfile string) { t.Helper() writePromptFileWithTemplate(t, dir, id, defaultProfile, "Summarize: {{input \"transcript\"}}") } func writePromptFileWithTemplate(t *testing.T, dir, id, defaultProfile, templateContent string) { t.Helper() data := fmt.Sprintf(`id: %s version: "1.0.0" default_profile: %s inputs: - name: transcript required: true messages: - role: user content: %q output: format: text validation_mode: none repair_attempts: 0 `, id, defaultProfile, templateContent) if err := os.WriteFile(filepath.Join(dir, id+".yaml"), []byte(data), 0o644); err != nil { t.Fatalf("failed to write prompt fixture: %v", err) } } func writeProfileFile(t *testing.T, dir, id, endpoint, model string) { t.Helper() data := fmt.Sprintf("id: %s\nendpoint: %s\nmodel: %s\n", id, endpoint, model) if err := os.WriteFile(filepath.Join(dir, id+".yaml"), []byte(data), 0o644); err != nil { t.Fatalf("failed to write profile fixture: %v", err) } } func newTestLLMServer(content string, hitCounter *int32) *httptest.Server { return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if hitCounter != nil { atomic.AddInt32(hitCounter, 1) } w.Header().Set("Content-Type", "application/json") _, _ = w.Write([]byte(fmt.Sprintf(`{"choices":[{"message":{"role":"assistant","content":%q}}],"usage":{"prompt_tokens":1,"completion_tokens":1,"total_tokens":2}}`, content))) })) } func writeAppConfigFile(t *testing.T, content string) string { t.Helper() path := filepath.Join(t.TempDir(), "config.yml") if err := os.WriteFile(path, []byte(content), 0o644); err != nil { t.Fatalf("failed to write app config fixture: %v", err) } return path }