From 32e24336289ecfb570c1b3d62b4614017bd1322a Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Sat, 4 Jul 2026 16:48:18 +0000 Subject: [PATCH] Add built-in profile repository wiring --- docs/cli.md | 9 +- docs/config.md | 12 +- docs/consumers/pkg-scriptorium.md | 2 +- docs/internal/adapters.md | 13 +- docs/troubleshooting.md | 10 +- engine.go | 7 +- engine_test.go | 155 +++++++++++++++++- internal/adapter/cli/run.go | 24 +-- internal/adapter/cli/run_test.go | 72 +++++--- .../builtin/assets/aion-labs/aion-2.yml | 9 + .../assets/anthropic/claude-fable-latest.yml | 7 + .../assets/anthropic/claude-haiku-latest.yml | 7 + .../assets/anthropic/claude-opus-latest.yml | 7 + .../assets/anthropic/claude-sonnet-latest.yml | 7 + .../builtin/assets/deepseek/deepseek-3-2.yml | 7 + .../assets/deepseek/deepseek-4-pro.yml | 7 + .../assets/google/gemini-2-flash-lite.yml | 9 + .../builtin/assets/google/gemini-2-flash.yml | 9 + .../builtin/assets/google/gemini-2-pro.yml | 9 + .../assets/google/gemini-3-flash-lite.yml | 9 + .../assets/google/gemini-flash-latest.yml | 9 + .../assets/google/gemini-pro-latest.yml | 9 + .../builtin/assets/google/gemma-4-31b.yml | 9 + .../builtin/assets/minimax/minimax-m2.yml | 9 + .../builtin/assets/minimax/minimax-m3.yml | 9 + .../assets/mistral/mistral-large-2512.yml | 7 + .../assets/mistral/mistral-medium-3-5.yml | 8 + .../assets/mistral/mistral-small-3.yml | 7 + .../assets/mistral/mistral-small-4.yml | 8 + .../assets/nvidia/nemotron-3-ultra.yml | 7 + .../builtin/assets/openai/gpt-5-mini.yml | 7 + .../builtin/assets/openai/gpt-5-nano.yml | 7 + internal/profile/builtin/repository.go | 31 ++++ internal/profile/builtin/repository_test.go | 127 ++++++++++++++ 34 files changed, 581 insertions(+), 64 deletions(-) create mode 100644 internal/profile/builtin/assets/aion-labs/aion-2.yml create mode 100644 internal/profile/builtin/assets/anthropic/claude-fable-latest.yml create mode 100644 internal/profile/builtin/assets/anthropic/claude-haiku-latest.yml create mode 100644 internal/profile/builtin/assets/anthropic/claude-opus-latest.yml create mode 100644 internal/profile/builtin/assets/anthropic/claude-sonnet-latest.yml create mode 100644 internal/profile/builtin/assets/deepseek/deepseek-3-2.yml create mode 100644 internal/profile/builtin/assets/deepseek/deepseek-4-pro.yml create mode 100644 internal/profile/builtin/assets/google/gemini-2-flash-lite.yml create mode 100644 internal/profile/builtin/assets/google/gemini-2-flash.yml create mode 100644 internal/profile/builtin/assets/google/gemini-2-pro.yml create mode 100644 internal/profile/builtin/assets/google/gemini-3-flash-lite.yml create mode 100644 internal/profile/builtin/assets/google/gemini-flash-latest.yml create mode 100644 internal/profile/builtin/assets/google/gemini-pro-latest.yml create mode 100644 internal/profile/builtin/assets/google/gemma-4-31b.yml create mode 100644 internal/profile/builtin/assets/minimax/minimax-m2.yml create mode 100644 internal/profile/builtin/assets/minimax/minimax-m3.yml create mode 100644 internal/profile/builtin/assets/mistral/mistral-large-2512.yml create mode 100644 internal/profile/builtin/assets/mistral/mistral-medium-3-5.yml create mode 100644 internal/profile/builtin/assets/mistral/mistral-small-3.yml create mode 100644 internal/profile/builtin/assets/mistral/mistral-small-4.yml create mode 100644 internal/profile/builtin/assets/nvidia/nemotron-3-ultra.yml create mode 100644 internal/profile/builtin/assets/openai/gpt-5-mini.yml create mode 100644 internal/profile/builtin/assets/openai/gpt-5-nano.yml create mode 100644 internal/profile/builtin/repository.go create mode 100644 internal/profile/builtin/repository_test.go diff --git a/docs/cli.md b/docs/cli.md index c33e37b..79f9650 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -29,8 +29,9 @@ Integration references: - `run` and `render` require: - `--prompt` - at least one `--input` - - an effective `prompt_dir` and `profile_dir` (from flags or config) -- `serve` requires an effective `prompt_dir` and `profile_dir` (from flags or config). + - an effective `prompt_dir` from flags or config +- `serve` requires an effective `prompt_dir` from flags or config. +- `profile_dir` is optional. If omitted, only built-in profiles are available; if provided, custom profiles override built-ins with the same ID. - Positional arguments are rejected. - Prompt cache control is configured in prompt YAML (`messages[].cache_control`), not with CLI flags. - Provider-specific `reasoning_effort` and `extra_params` are configured in profile YAML or HTTP model overrides, not with CLI flags. @@ -41,7 +42,7 @@ Integration references: - `--config `: app config file path. - `--prompt-dir `: prompt definition directory. -- `--profile-dir `: profile definition directory. +- `--profile-dir `: custom profile definition directory. - `--schema-dir `: schema base directory for `json_schema` validation. - `--prompt `: prompt ID to execute. Required. - `--prompt-id `: deprecated alias for `--prompt`. @@ -79,7 +80,7 @@ Notes: - `--config `: app config file path. - `--addr `: HTTP listen address. - `--prompt-dir `: prompt definition directory. -- `--profile-dir `: profile definition directory. +- `--profile-dir `: custom profile definition directory. - `--schema-dir `: schema base directory for `json_schema` validation. Notes: diff --git a/docs/config.md b/docs/config.md index 283de01..ab9db7f 100644 --- a/docs/config.md +++ b/docs/config.md @@ -21,10 +21,9 @@ When `--config ` is provided, that file is required. ```yaml prompt_dir: ./examples/prompts -profile_dir: ./examples/profiles ``` -This is enough to use `run` and `render` when prompt/profile files are valid. +This is enough to use `run` and `render` when prompts select built-in profiles. ## Production-Oriented App Config @@ -45,7 +44,7 @@ defaults: Top-level fields: - `prompt_dir` (optional): default prompt definition directory. -- `profile_dir` (optional): default profile definition directory. +- `profile_dir` (optional): default custom profile definition directory. - `schema_dir` (optional): base directory for schema files used by `json_schema` validation. - `server.addr` (optional): default listen address for `serve`. - `defaults.render_format` (optional): default `render` output format (`text` or `json`). @@ -173,7 +172,7 @@ Repair behavior boundary: ## Profile Definition Files -Execution profiles are YAML files anywhere under `profile_dir`, including nested subdirectories. +Scriptorium includes built-in execution profiles. Custom execution profiles are YAML files anywhere under `profile_dir`, including nested subdirectories. Subdirectories are organizational only. Callers still select profiles by the YAML `id`, not by file path. For example, `profiles/local/local-quality.yaml` may still declare `id: local-quality`, and callers use `--profile local-quality`. @@ -212,6 +211,9 @@ Field reference: Profile rules: +- `profile_dir` is optional. If omitted, only built-in profiles are available. +- If `profile_dir` is set, custom profiles from that directory override built-in profiles with the same `id`. +- Duplicate IDs within the custom profile directory are invalid. Matching IDs across custom and built-in profiles are valid override behavior. - Profile decoding is strict; unknown YAML fields are rejected. - Raw `api_key` is rejected; use `api_key_env`. - If `api_key_env` is set, that environment variable must be set when preparing/running. @@ -250,7 +252,7 @@ Supported artifact reference types for request inputs are `file` and `inline`. - App config: `examples/config.yml` - Prompt examples: `examples/prompts/` -- Profile examples: `examples/profiles/` +- Custom profile examples: `examples/profiles/` - Schema examples: `examples/schemas/` - Input fixtures: `examples/fixtures/` - Render example script: `examples/render-markdown-summary.sh` diff --git a/docs/consumers/pkg-scriptorium.md b/docs/consumers/pkg-scriptorium.md index 140ba8a..e4cb2dd 100644 --- a/docs/consumers/pkg-scriptorium.md +++ b/docs/consumers/pkg-scriptorium.md @@ -21,7 +21,7 @@ if err != nil { } ``` -`PromptDir` and `ProfileDir` are required. `SchemaDir` defaults to the built-in schema directory. `Timeout` and `HTTPClient` configure the default OpenAI-compatible client used by `Run` when no custom LLM client is supplied. +`PromptDir` is required. `ProfileDir` is optional; omit it to use built-in profiles only, or set it to overlay custom profiles above built-ins. `SchemaDir` defaults to the built-in schema directory. `Timeout` and `HTTPClient` configure the default OpenAI-compatible client used by `Run` when no custom LLM client is supplied. ## Prepare A Prompt diff --git a/docs/internal/adapters.md b/docs/internal/adapters.md index 0c09106..1fd185a 100644 --- a/docs/internal/adapters.md +++ b/docs/internal/adapters.md @@ -10,7 +10,8 @@ This document describes implemented adapter/repository boundaries and their curr - `internal/adapter/http`: HTTP request/response mapping for `POST /v1/runs`. - root package `scriptorium`: public Go library facade for preparing and running prompt requests. - `internal/promptdef`: filesystem prompt-definition repository. -- `internal/profile`: filesystem execution-profile repository. +- `internal/profile`: filesystem, `fs.FS`, and overlay execution-profile repositories. +- `internal/profile/builtin`: embedded built-in execution-profile repository. - `internal/artifact`: input artifact reader. - `internal/prompt`: Go-template renderer. - `internal/llm`: OpenAI-compatible LLM client implementation. @@ -43,6 +44,13 @@ Filesystem repositories: - Input: prompt/profile YAML files under configured directories. - Output: normalized domain definitions/profiles or typed errors. +Profile repository composition: + +- Built-in profiles are embedded and loaded through the same profile validation rules as filesystem profiles. +- When no custom profile directory is configured, the runner receives the built-in profile repository. +- When a custom profile directory is configured, the runner receives an overlay repository with custom profiles as primary and built-ins as fallback. +- Overlay lookup falls back only after custom profile-not-found errors; custom load/validation/raw-key errors are returned directly. + Artifact reader: - Input: `domain.ArtifactRef`. @@ -69,7 +77,7 @@ Validator: Primary app settings consumed by adapters: - `prompt_dir` -- `profile_dir` +- `profile_dir` (optional custom profile source) - `schema_dir` - `server.addr` - `defaults.render_format` @@ -94,6 +102,7 @@ Strict decoding and input checks: - prompt/profile repositories scan nested subdirectories recursively. - prompt/profile lookup uses YAML `id` values; subdirectory paths are organizational only. - duplicate prompt/profile IDs are invalid and fail instead of using first-match behavior. +- duplicate profile IDs across custom and built-in sources are allowed; the custom source overrides the built-in profile. - HTTP DTO decoder rejects unknown JSON fields. - raw API key payload fields are rejected by strict decoding in profile/http paths. diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index c88c2bd..31b1bb8 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -33,23 +33,23 @@ Relevant links: - [Configuration reference](config.md) - [CLI reference](cli.md) -## Missing Prompt/Profile Directory Settings +## Missing Prompt Directory Settings Symptom: -- CLI parse errors saying prompt directory or profile directory is required. +- CLI parse errors saying prompt directory is required. Likely cause: -- Neither CLI flags nor config provide effective `prompt_dir` / `profile_dir`. +- Neither CLI flags nor config provide an effective `prompt_dir`. Diagnostic step: -- Run the failing command with explicit `--prompt-dir` and `--profile-dir` once to verify. +- Run the failing command with explicit `--prompt-dir` once to verify. Safe fix: -- Set `prompt_dir` and `profile_dir` in config, or always pass both flags. +- Set `prompt_dir` in config, or always pass `--prompt-dir`. Relevant links: diff --git a/engine.go b/engine.go index b7281f0..7e073b0 100644 --- a/engine.go +++ b/engine.go @@ -11,7 +11,7 @@ import ( artifactadapter "gitea.maximumdirect.net/eric/scriptorium/internal/artifact" "gitea.maximumdirect.net/eric/scriptorium/internal/defaults" "gitea.maximumdirect.net/eric/scriptorium/internal/llm" - "gitea.maximumdirect.net/eric/scriptorium/internal/profile" + "gitea.maximumdirect.net/eric/scriptorium/internal/profile/builtin" "gitea.maximumdirect.net/eric/scriptorium/internal/prompt" "gitea.maximumdirect.net/eric/scriptorium/internal/promptdef" "gitea.maximumdirect.net/eric/scriptorium/internal/usecase" @@ -71,9 +71,6 @@ func NewEngine(cfg Config, opts ...Option) (*Engine, error) { if strings.TrimSpace(cfg.PromptDir) == "" { return nil, fmt.Errorf("%w: prompt directory is required", ErrInvalidConfig) } - if strings.TrimSpace(cfg.ProfileDir) == "" { - return nil, fmt.Errorf("%w: profile directory is required", ErrInvalidConfig) - } var options engineOptions for _, opt := range opts { @@ -105,7 +102,7 @@ func NewEngine(cfg Config, opts ...Option) (*Engine, error) { return &Engine{ runner: usecase.NewRunner( promptdef.NewFilesystemRepository(cfg.PromptDir), - profile.NewFilesystemRepository(cfg.ProfileDir), + builtin.NewRepositoryWithDirectory(cfg.ProfileDir), artifactadapter.NewCompositeReader(), prompt.NewGoRenderer(), llmClient, diff --git a/engine_test.go b/engine_test.go index 6fc15cc..4ea32ae 100644 --- a/engine_test.go +++ b/engine_test.go @@ -20,10 +20,10 @@ func TestNewEngineRejectsMissingPromptDir(t *testing.T) { } } -func TestNewEngineRejectsMissingProfileDir(t *testing.T) { +func TestNewEngineAcceptsMissingProfileDir(t *testing.T) { _, err := scriptorium.NewEngine(scriptorium.Config{PromptDir: "./examples/prompts"}) - if !errors.Is(err, scriptorium.ErrInvalidConfig) { - t.Fatalf("expected ErrInvalidConfig, got %v", err) + if err != nil { + t.Fatalf("expected missing profile dir to use built-ins, got %v", err) } } @@ -435,6 +435,123 @@ unknown_field: true } } +func TestPrepareUsesBuiltInProfileWithoutProfileDir(t *testing.T) { + t.Setenv("OPENROUTER_API_KEY", "test-key") + engine, err := scriptorium.NewEngine(scriptorium.Config{ + PromptDir: "./examples/prompts", + SchemaDir: "./examples/schemas", + }) + if err != nil { + t.Fatalf("expected engine construction to succeed, got %v", err) + } + + prepared, err := engine.Prepare(context.Background(), scriptorium.RunRequest{ + PromptID: "generic.markdown_summary", + ProfileID: "mistral-small-3", + Inputs: map[string]scriptorium.ArtifactRef{ + "transcript": scriptorium.Inline("Rin opens the gate."), + "glossary": scriptorium.Inline("gate: A guarded passage."), + }, + }) + if err != nil { + t.Fatalf("expected built-in profile prepare to succeed, got %v", err) + } + if prepared.SelectedProfileID != "mistral-small-3" { + t.Fatalf("unexpected selected profile: %q", prepared.SelectedProfileID) + } + if prepared.EffectiveModelParams.Model != "mistralai/mistral-small-3.2-24b-instruct" { + t.Fatalf("unexpected built-in model: %q", prepared.EffectiveModelParams.Model) + } +} + +func TestPromptDefaultProfileCanUseBuiltInProfile(t *testing.T) { + t.Setenv("OPENROUTER_API_KEY", "test-key") + promptDir := t.TempDir() + writePublicPromptFile(t, promptDir, "prompt.builtin.default", "mistral-small-3") + + engine, err := scriptorium.NewEngine(scriptorium.Config{PromptDir: promptDir}) + if err != nil { + t.Fatalf("expected engine construction to succeed, got %v", err) + } + + prepared, err := engine.Prepare(context.Background(), scriptorium.RunRequest{ + PromptID: "prompt.builtin.default", + Inputs: map[string]scriptorium.ArtifactRef{ + "transcript": scriptorium.Inline("Rin opens the gate."), + }, + }) + if err != nil { + t.Fatalf("expected built-in default profile prepare to succeed, got %v", err) + } + if prepared.SelectedProfileID != "mistral-small-3" { + t.Fatalf("unexpected selected profile: %q", prepared.SelectedProfileID) + } +} + +func TestCustomProfileOverridesBuiltInProfile(t *testing.T) { + t.Setenv("OPENROUTER_API_KEY", "test-key") + profileDir := t.TempDir() + writePublicProfileFile(t, profileDir, "mistral-small-3", "http://localhost:8000/v1", "custom-model") + + engine, err := scriptorium.NewEngine(scriptorium.Config{ + PromptDir: "./examples/prompts", + ProfileDir: profileDir, + SchemaDir: "./examples/schemas", + }) + if err != nil { + t.Fatalf("expected engine construction to succeed, got %v", err) + } + + prepared, err := engine.Prepare(context.Background(), scriptorium.RunRequest{ + PromptID: "generic.markdown_summary", + ProfileID: "mistral-small-3", + Inputs: map[string]scriptorium.ArtifactRef{ + "transcript": scriptorium.Inline("Rin opens the gate."), + "glossary": scriptorium.Inline("gate: A guarded passage."), + }, + }) + if err != nil { + t.Fatalf("expected custom profile prepare to succeed, got %v", err) + } + if prepared.EffectiveModelParams.Model != "custom-model" { + t.Fatalf("expected custom profile to override built-in, got %q", prepared.EffectiveModelParams.Model) + } +} + +func TestMalformedCustomProfileDoesNotFallbackToBuiltIn(t *testing.T) { + t.Setenv("OPENROUTER_API_KEY", "test-key") + profileDir := t.TempDir() + if err := os.WriteFile(filepath.Join(profileDir, "mistral-small-3.yml"), []byte(` +id: mistral-small-3 +endpoint: http://localhost:8000/v1 +model: custom-model +unexpected: true +`), 0o644); err != nil { + t.Fatal(err) + } + + engine, err := scriptorium.NewEngine(scriptorium.Config{ + PromptDir: "./examples/prompts", + ProfileDir: profileDir, + SchemaDir: "./examples/schemas", + }) + if err != nil { + t.Fatalf("expected engine construction to succeed, got %v", err) + } + + _, err = engine.Prepare(context.Background(), scriptorium.RunRequest{ + PromptID: "generic.markdown_summary", + ProfileID: "mistral-small-3", + Inputs: map[string]scriptorium.ArtifactRef{ + "transcript": scriptorium.Inline("Rin opens the gate."), + "glossary": scriptorium.Inline("gate: A guarded passage."), + }, + }) + if !errors.Is(err, scriptorium.ErrProfileLoad) { + t.Fatalf("expected custom profile load error, got %v", err) + } +} + func TestExtraParamsTypedNestedValuesAreCopiedAcrossPublicBoundary(t *testing.T) { fake := &fakeLLMClient{response: &scriptorium.GenerateResponse{Content: "ok"}} engine := newExampleEngineWithOptions(t, "./examples/schemas", scriptorium.WithLLMClient(fake)) @@ -544,6 +661,38 @@ func exampleConfig(schemaDir string) scriptorium.Config { } } +func writePublicPromptFile(t *testing.T, dir, id, defaultProfile string) { + t.Helper() + data := `id: ` + id + ` +version: "1.0.0" +default_profile: ` + defaultProfile + ` +inputs: + - name: transcript + required: true +messages: + - role: user + content: "Summarize: {{input \"transcript\"}}" +output: + format: text + validation_mode: none + repair_attempts: 0 +` + if err := os.WriteFile(filepath.Join(dir, id+".yaml"), []byte(data), 0o644); err != nil { + t.Fatalf("failed to write prompt fixture: %v", err) + } +} + +func writePublicProfileFile(t *testing.T, dir, id, endpoint, model string) { + t.Helper() + data := `id: ` + id + ` +endpoint: ` + endpoint + ` +model: ` + model + ` +` + if err := os.WriteFile(filepath.Join(dir, id+".yaml"), []byte(data), 0o644); err != nil { + t.Fatalf("failed to write profile fixture: %v", err) + } +} + type fakeLLMClient struct { response *scriptorium.GenerateResponse err error diff --git a/internal/adapter/cli/run.go b/internal/adapter/cli/run.go index a3de22a..18b5e8d 100644 --- a/internal/adapter/cli/run.go +++ b/internal/adapter/cli/run.go @@ -19,7 +19,7 @@ import ( "gitea.maximumdirect.net/eric/scriptorium/internal/domain" renderformat "gitea.maximumdirect.net/eric/scriptorium/internal/format" "gitea.maximumdirect.net/eric/scriptorium/internal/llm" - "gitea.maximumdirect.net/eric/scriptorium/internal/profile" + "gitea.maximumdirect.net/eric/scriptorium/internal/profile/builtin" "gitea.maximumdirect.net/eric/scriptorium/internal/prompt" "gitea.maximumdirect.net/eric/scriptorium/internal/promptdef" "gitea.maximumdirect.net/eric/scriptorium/internal/usecase" @@ -33,8 +33,7 @@ const ( ) const ( - errPromptDirRequired = "prompt directory is required; provide --prompt-dir or config.yml prompt_dir" - errProfileDirRequired = "profile directory is required; provide --profile-dir or config.yml profile_dir" + errPromptDirRequired = "prompt directory is required; provide --prompt-dir or config.yml prompt_dir" ) type runConfig struct { @@ -305,12 +304,14 @@ func parseServeArgs(args []string) (*serveConfig, error) { cfg.schemaDir = settings.schemaDir cfg.addr = settings.serverAddr - if err := validateRequiredLibraryDirs(cfg.promptDir, cfg.profileDir); err != nil { + if err := validateRequiredLibraryDirs(cfg.promptDir); err != nil { return nil, err } cfg.promptDir = filepath.Clean(cfg.promptDir) - cfg.profileDir = filepath.Clean(cfg.profileDir) + if strings.TrimSpace(cfg.profileDir) != "" { + cfg.profileDir = filepath.Clean(cfg.profileDir) + } cfg.schemaDir = filepath.Clean(cfg.schemaDir) return cfg, nil } @@ -353,7 +354,7 @@ func finalizeExecutionRequestConfig(fs *flag.FlagSet, cfg *runConfig) error { cfg.schemaDir = settings.schemaDir cfg.defaultRenderFormat = settings.defaultRenderFormat - if err := validateRequiredLibraryDirs(cfg.promptDir, cfg.profileDir); err != nil { + if err := validateRequiredLibraryDirs(cfg.promptDir); err != nil { return err } if strings.TrimSpace(cfg.promptID) == "" { @@ -363,7 +364,9 @@ func finalizeExecutionRequestConfig(fs *flag.FlagSet, cfg *runConfig) error { return errors.New("at least one --input is required") } cfg.promptDir = filepath.Clean(cfg.promptDir) - cfg.profileDir = filepath.Clean(cfg.profileDir) + if strings.TrimSpace(cfg.profileDir) != "" { + cfg.profileDir = filepath.Clean(cfg.profileDir) + } if cfg.outputPath != "" { cfg.outputPath = filepath.Clean(cfg.outputPath) } @@ -467,20 +470,17 @@ func resolveCommonSettings(fs *flag.FlagSet, configPath string, overrides appcon }, nil } -func validateRequiredLibraryDirs(promptDir, profileDir string) error { +func validateRequiredLibraryDirs(promptDir string) error { if strings.TrimSpace(promptDir) == "" { return errors.New(errPromptDirRequired) } - if strings.TrimSpace(profileDir) == "" { - return errors.New(errProfileDirRequired) - } return nil } func newRunner(promptDir, profileDir, schemaDir string, llmClient llm.Client) *usecase.Runner { return usecase.NewRunner( promptdef.NewFilesystemRepository(promptDir), - profile.NewFilesystemRepository(profileDir), + builtin.NewRepositoryWithDirectory(profileDir), artifactadapter.NewCompositeReader(), prompt.NewGoRenderer(), llmClient, diff --git a/internal/adapter/cli/run_test.go b/internal/adapter/cli/run_test.go index 3ba41cc..1dbd9d4 100644 --- a/internal/adapter/cli/run_test.go +++ b/internal/adapter/cli/run_test.go @@ -74,12 +74,12 @@ func TestParseRunArgsRequiredFlags(t *testing.T) { t.Fatalf("expected clear prompt-dir guidance, got %v", err) } - _, err = parseRunArgs([]string{"--config", configPath, "--prompt-dir", "./prompts", "--prompt", "p", "--input", "a=b"}) - if err == nil { - t.Fatal("expected missing --profile-dir error") + cfg, err := parseRunArgs([]string{"--config", configPath, "--prompt-dir", "./prompts", "--prompt", "p", "--input", "a=b"}) + if err != nil { + t.Fatalf("expected missing --profile-dir to be accepted, got %v", err) } - if !strings.Contains(err.Error(), "profile directory is required") { - t.Fatalf("expected clear profile-dir guidance, got %v", err) + if cfg.profileDir != "" { + t.Fatalf("expected empty profile dir for built-ins, got %q", cfg.profileDir) } _, err = parseRunArgs([]string{"--config", configPath, "--prompt-dir", "./prompts", "--profile-dir", "./profiles", "--input", "a=b"}) @@ -161,17 +161,12 @@ func TestParseServeArgsRequiredFlags(t *testing.T) { t.Fatalf("expected clear prompt-dir guidance, got %v", err) } - _, err = parseServeArgs([]string{"--config", configPath, "--prompt-dir", "./prompts"}) - if err == nil { - t.Fatal("expected missing --profile-dir error") - } - if !strings.Contains(err.Error(), "profile directory is required") { - t.Fatalf("expected clear profile-dir guidance, got %v", err) - } - - cfg, err := parseServeArgs([]string{"--config", configPath, "--prompt-dir", "./prompts", "--profile-dir", "./profiles"}) + cfg, err := parseServeArgs([]string{"--config", configPath, "--prompt-dir", "./prompts"}) if err != nil { - t.Fatalf("expected valid serve args, got %v", err) + t.Fatalf("expected missing --profile-dir to be accepted, got %v", err) + } + if cfg.profileDir != "" { + t.Fatalf("expected empty profile dir for built-ins, got %q", cfg.profileDir) } if cfg.addr != defaults.HTTPAddrDefault { t.Fatalf("expected default addr %s, got %q", defaults.HTTPAddrDefault, cfg.addr) @@ -565,21 +560,21 @@ profile_dir: ./profiles } } -func TestParseRunArgsFailsClearlyWhenNoEffectiveProfileDir(t *testing.T) { +func TestParseRunArgsAcceptsMissingEffectiveProfileDir(t *testing.T) { configPath := writeAppConfigFile(t, ` prompt_dir: ./prompts `) - _, err := parseRunArgs([]string{ + cfg, err := parseRunArgs([]string{ "--config", configPath, "--prompt", "p", "--input", "a=b", }) - if err == nil { - t.Fatal("expected missing profile_dir error") + if err != nil { + t.Fatalf("expected missing profile_dir to be accepted, got %v", err) } - if !strings.Contains(err.Error(), "profile directory is required") || !strings.Contains(err.Error(), "config.yml profile_dir") { - t.Fatalf("expected clear profile_dir guidance, got %v", err) + if cfg.profileDir != "" { + t.Fatalf("expected empty profile dir for built-ins, got %q", cfg.profileDir) } } @@ -601,21 +596,21 @@ profile_dir: ./profiles } } -func TestParseRenderArgsFailsClearlyWhenNoEffectiveProfileDir(t *testing.T) { +func TestParseRenderArgsAcceptsMissingEffectiveProfileDir(t *testing.T) { configPath := writeAppConfigFile(t, ` prompt_dir: ./prompts `) - _, err := parseRenderArgs([]string{ + cfg, err := parseRenderArgs([]string{ "--config", configPath, "--prompt", "p", "--input", "a=b", }) - if err == nil { - t.Fatal("expected missing profile_dir error") + if err != nil { + t.Fatalf("expected missing profile_dir to be accepted, got %v", err) } - if !strings.Contains(err.Error(), "profile directory is required") || !strings.Contains(err.Error(), "config.yml profile_dir") { - t.Fatalf("expected clear profile_dir guidance, got %v", err) + if cfg.profileDir != "" { + t.Fatalf("expected empty profile dir for built-ins, got %q", cfg.profileDir) } } @@ -929,6 +924,29 @@ func TestRenderCommandPromptDefaultProfileWorksThroughCLIPath(t *testing.T) { } } +func TestRenderCommandUsesBuiltInProfileWithoutProfileDir(t *testing.T) { + t.Setenv("OPENROUTER_API_KEY", "test-key") + lib := newCLITestLibrary(t) + inputPath := lib.writeInputFile(t, "transcript.md", "hello") + + writePromptFile(t, lib.promptDir, "prompt.builtin", "mistral-small-3") + + code, stdout, stderr := runCLICommand(t, renderCommand, []string{ + "--prompt-dir", lib.promptDir, + "--prompt", "prompt.builtin", + "--input", "transcript=" + inputPath, + }) + if code != ExitOK { + t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr) + } + if !strings.Contains(stdout, "selected_profile_id: mistral-small-3") { + t.Fatalf("expected built-in selected profile, got %q", stdout) + } + if !strings.Contains(stdout, "model: mistralai/mistral-small-3.2-24b-instruct") { + t.Fatalf("expected built-in model, got %q", stdout) + } +} + func TestRenderCommandExplicitProfileOverridesPromptDefault(t *testing.T) { lib := newCLITestLibrary(t) inputPath := lib.writeInputFile(t, "transcript.md", "hello") diff --git a/internal/profile/builtin/assets/aion-labs/aion-2.yml b/internal/profile/builtin/assets/aion-labs/aion-2.yml new file mode 100644 index 0000000..785a6fd --- /dev/null +++ b/internal/profile/builtin/assets/aion-labs/aion-2.yml @@ -0,0 +1,9 @@ +id: aion-2 +endpoint: https://openrouter.ai/api/v1 +model: aion-labs/aion-2.0 +temperature: 0.72 +reasoning_effort: high +top_p: 0.95 +timeout_seconds: 180 +api_key_env: OPENROUTER_API_KEY +service_tier: flex diff --git a/internal/profile/builtin/assets/anthropic/claude-fable-latest.yml b/internal/profile/builtin/assets/anthropic/claude-fable-latest.yml new file mode 100644 index 0000000..1a1e7aa --- /dev/null +++ b/internal/profile/builtin/assets/anthropic/claude-fable-latest.yml @@ -0,0 +1,7 @@ +id: claude-fable-latest +endpoint: https://openrouter.ai/api/v1 +model: "~anthropic/claude-fable-latest" +reasoning_effort: high +timeout_seconds: 600 +api_key_env: OPENROUTER_API_KEY +service_tier: flex diff --git a/internal/profile/builtin/assets/anthropic/claude-haiku-latest.yml b/internal/profile/builtin/assets/anthropic/claude-haiku-latest.yml new file mode 100644 index 0000000..f22bf15 --- /dev/null +++ b/internal/profile/builtin/assets/anthropic/claude-haiku-latest.yml @@ -0,0 +1,7 @@ +id: claude-haiku-latest +endpoint: https://openrouter.ai/api/v1 +model: "~anthropic/claude-haiku-latest" +reasoning_effort: medium +timeout_seconds: 240 +api_key_env: OPENROUTER_API_KEY +service_tier: flex diff --git a/internal/profile/builtin/assets/anthropic/claude-opus-latest.yml b/internal/profile/builtin/assets/anthropic/claude-opus-latest.yml new file mode 100644 index 0000000..c192c3a --- /dev/null +++ b/internal/profile/builtin/assets/anthropic/claude-opus-latest.yml @@ -0,0 +1,7 @@ +id: claude-opus-latest +endpoint: https://openrouter.ai/api/v1 +model: "~anthropic/claude-opus-latest" +reasoning_effort: high +timeout_seconds: 240 +api_key_env: OPENROUTER_API_KEY +service_tier: flex diff --git a/internal/profile/builtin/assets/anthropic/claude-sonnet-latest.yml b/internal/profile/builtin/assets/anthropic/claude-sonnet-latest.yml new file mode 100644 index 0000000..c7be449 --- /dev/null +++ b/internal/profile/builtin/assets/anthropic/claude-sonnet-latest.yml @@ -0,0 +1,7 @@ +id: claude-sonnet-latest +endpoint: https://openrouter.ai/api/v1 +model: "~anthropic/claude-sonnet-latest" +reasoning_effort: high +timeout_seconds: 240 +api_key_env: OPENROUTER_API_KEY +service_tier: flex diff --git a/internal/profile/builtin/assets/deepseek/deepseek-3-2.yml b/internal/profile/builtin/assets/deepseek/deepseek-3-2.yml new file mode 100644 index 0000000..1f27fbc --- /dev/null +++ b/internal/profile/builtin/assets/deepseek/deepseek-3-2.yml @@ -0,0 +1,7 @@ +id: deepseek-3-2 +endpoint: https://openrouter.ai/api/v1 +model: deepseek/deepseek-v3.2 +reasoning_effort: high +timeout_seconds: 180 +api_key_env: OPENROUTER_API_KEY +service_tier: flex diff --git a/internal/profile/builtin/assets/deepseek/deepseek-4-pro.yml b/internal/profile/builtin/assets/deepseek/deepseek-4-pro.yml new file mode 100644 index 0000000..c7af1ff --- /dev/null +++ b/internal/profile/builtin/assets/deepseek/deepseek-4-pro.yml @@ -0,0 +1,7 @@ +id: deepseek-4-pro +endpoint: https://openrouter.ai/api/v1 +model: deepseek/deepseek-v4-pro +reasoning_effort: high +timeout_seconds: 180 +api_key_env: OPENROUTER_API_KEY +service_tier: flex diff --git a/internal/profile/builtin/assets/google/gemini-2-flash-lite.yml b/internal/profile/builtin/assets/google/gemini-2-flash-lite.yml new file mode 100644 index 0000000..e648e70 --- /dev/null +++ b/internal/profile/builtin/assets/google/gemini-2-flash-lite.yml @@ -0,0 +1,9 @@ +id: gemini-2-flash-lite +endpoint: https://openrouter.ai/api/v1 +model: "google/gemini-2.5-flash-lite" +#temperature: 0.15 +reasoning_effort: high +#top_p: 0.98 +timeout_seconds: 240 +api_key_env: OPENROUTER_API_KEY +service_tier: flex diff --git a/internal/profile/builtin/assets/google/gemini-2-flash.yml b/internal/profile/builtin/assets/google/gemini-2-flash.yml new file mode 100644 index 0000000..3b1267e --- /dev/null +++ b/internal/profile/builtin/assets/google/gemini-2-flash.yml @@ -0,0 +1,9 @@ +id: gemini-2-flash +endpoint: https://openrouter.ai/api/v1 +model: "google/gemini-2.5-flash" +#temperature: 0.15 +reasoning_effort: high +#top_p: 0.98 +timeout_seconds: 240 +api_key_env: OPENROUTER_API_KEY +service_tier: flex diff --git a/internal/profile/builtin/assets/google/gemini-2-pro.yml b/internal/profile/builtin/assets/google/gemini-2-pro.yml new file mode 100644 index 0000000..b779db6 --- /dev/null +++ b/internal/profile/builtin/assets/google/gemini-2-pro.yml @@ -0,0 +1,9 @@ +id: gemini-2-pro +endpoint: https://openrouter.ai/api/v1 +model: "google/gemini-2.5-pro" +#temperature: 0.15 +reasoning_effort: high +#top_p: 0.98 +timeout_seconds: 240 +api_key_env: OPENROUTER_API_KEY +service_tier: flex diff --git a/internal/profile/builtin/assets/google/gemini-3-flash-lite.yml b/internal/profile/builtin/assets/google/gemini-3-flash-lite.yml new file mode 100644 index 0000000..ae44f66 --- /dev/null +++ b/internal/profile/builtin/assets/google/gemini-3-flash-lite.yml @@ -0,0 +1,9 @@ +id: gemini-3-flash-lite +endpoint: https://openrouter.ai/api/v1 +model: "google/gemini-3.1-flash-lite" +#temperature: 0.15 +reasoning_effort: high +#top_p: 0.98 +timeout_seconds: 240 +api_key_env: OPENROUTER_API_KEY +service_tier: flex diff --git a/internal/profile/builtin/assets/google/gemini-flash-latest.yml b/internal/profile/builtin/assets/google/gemini-flash-latest.yml new file mode 100644 index 0000000..2bcda6e --- /dev/null +++ b/internal/profile/builtin/assets/google/gemini-flash-latest.yml @@ -0,0 +1,9 @@ +id: gemini-flash-latest +endpoint: https://openrouter.ai/api/v1 +model: "~google/gemini-flash-latest" +#temperature: 0.15 +reasoning_effort: high +#top_p: 0.98 +timeout_seconds: 240 +api_key_env: OPENROUTER_API_KEY +service_tier: flex diff --git a/internal/profile/builtin/assets/google/gemini-pro-latest.yml b/internal/profile/builtin/assets/google/gemini-pro-latest.yml new file mode 100644 index 0000000..2e77cbc --- /dev/null +++ b/internal/profile/builtin/assets/google/gemini-pro-latest.yml @@ -0,0 +1,9 @@ +id: gemini-pro-latest +endpoint: https://openrouter.ai/api/v1 +model: "~google/gemini-pro-latest" +#temperature: 0.15 +reasoning_effort: high +#top_p: 0.98 +timeout_seconds: 240 +api_key_env: OPENROUTER_API_KEY +service_tier: flex diff --git a/internal/profile/builtin/assets/google/gemma-4-31b.yml b/internal/profile/builtin/assets/google/gemma-4-31b.yml new file mode 100644 index 0000000..f8ff113 --- /dev/null +++ b/internal/profile/builtin/assets/google/gemma-4-31b.yml @@ -0,0 +1,9 @@ +id: gemma-4-31b +endpoint: https://openrouter.ai/api/v1 +model: google/gemma-4-31b-it:exacto +temperature: 0.15 +reasoning_effort: high +top_p: 0.98 +timeout_seconds: 240 +api_key_env: OPENROUTER_API_KEY +service_tier: flex diff --git a/internal/profile/builtin/assets/minimax/minimax-m2.yml b/internal/profile/builtin/assets/minimax/minimax-m2.yml new file mode 100644 index 0000000..6ce7fdf --- /dev/null +++ b/internal/profile/builtin/assets/minimax/minimax-m2.yml @@ -0,0 +1,9 @@ +id: minimax-m2 +endpoint: https://openrouter.ai/api/v1 +model: minimax/minimax-m2.5 +temperature: 0.5 +reasoning_effort: high +top_p: 0.95 +timeout_seconds: 180 +api_key_env: OPENROUTER_API_KEY +service_tier: flex diff --git a/internal/profile/builtin/assets/minimax/minimax-m3.yml b/internal/profile/builtin/assets/minimax/minimax-m3.yml new file mode 100644 index 0000000..da9a3af --- /dev/null +++ b/internal/profile/builtin/assets/minimax/minimax-m3.yml @@ -0,0 +1,9 @@ +id: minimax-m3 +endpoint: https://openrouter.ai/api/v1 +model: minimax/minimax-m3 +#temperature: 0.5 +reasoning_effort: high +#top_p: 0.95 +timeout_seconds: 180 +api_key_env: OPENROUTER_API_KEY +service_tier: flex diff --git a/internal/profile/builtin/assets/mistral/mistral-large-2512.yml b/internal/profile/builtin/assets/mistral/mistral-large-2512.yml new file mode 100644 index 0000000..106ea2e --- /dev/null +++ b/internal/profile/builtin/assets/mistral/mistral-large-2512.yml @@ -0,0 +1,7 @@ +id: mistral-large-2512 +endpoint: https://openrouter.ai/api/v1 +model: mistralai/mistral-large-2512 +temperature: 0.15 +top_p: 0.98 +timeout_seconds: 180 +api_key_env: OPENROUTER_API_KEY diff --git a/internal/profile/builtin/assets/mistral/mistral-medium-3-5.yml b/internal/profile/builtin/assets/mistral/mistral-medium-3-5.yml new file mode 100644 index 0000000..762149f --- /dev/null +++ b/internal/profile/builtin/assets/mistral/mistral-medium-3-5.yml @@ -0,0 +1,8 @@ +id: mistral-medium-3-5 +endpoint: https://openrouter.ai/api/v1 +model: mistralai/mistral-medium-3-5 +temperature: 0.15 +reasoning_effort: high +top_p: 0.98 +timeout_seconds: 180 +api_key_env: OPENROUTER_API_KEY diff --git a/internal/profile/builtin/assets/mistral/mistral-small-3.yml b/internal/profile/builtin/assets/mistral/mistral-small-3.yml new file mode 100644 index 0000000..d077918 --- /dev/null +++ b/internal/profile/builtin/assets/mistral/mistral-small-3.yml @@ -0,0 +1,7 @@ +id: mistral-small-3 +endpoint: https://openrouter.ai/api/v1 +model: mistralai/mistral-small-3.2-24b-instruct +temperature: 0.05 +top_p: 1.0 +timeout_seconds: 180 +api_key_env: OPENROUTER_API_KEY diff --git a/internal/profile/builtin/assets/mistral/mistral-small-4.yml b/internal/profile/builtin/assets/mistral/mistral-small-4.yml new file mode 100644 index 0000000..fd0fce6 --- /dev/null +++ b/internal/profile/builtin/assets/mistral/mistral-small-4.yml @@ -0,0 +1,8 @@ +id: mistral-small-4 +endpoint: https://openrouter.ai/api/v1 +model: mistralai/mistral-small-2603 +temperature: 0.1 +reasoning_effort: high +top_p: 0.98 +timeout_seconds: 180 +api_key_env: OPENROUTER_API_KEY diff --git a/internal/profile/builtin/assets/nvidia/nemotron-3-ultra.yml b/internal/profile/builtin/assets/nvidia/nemotron-3-ultra.yml new file mode 100644 index 0000000..bb55536 --- /dev/null +++ b/internal/profile/builtin/assets/nvidia/nemotron-3-ultra.yml @@ -0,0 +1,7 @@ +id: nemotron-3-ultra +endpoint: https://openrouter.ai/api/v1 +model: nvidia/nemotron-3-ultra-550b-a55b +reasoning_effort: high +timeout_seconds: 180 +api_key_env: OPENROUTER_API_KEY +service_tier: flex diff --git a/internal/profile/builtin/assets/openai/gpt-5-mini.yml b/internal/profile/builtin/assets/openai/gpt-5-mini.yml new file mode 100644 index 0000000..72ea4b7 --- /dev/null +++ b/internal/profile/builtin/assets/openai/gpt-5-mini.yml @@ -0,0 +1,7 @@ +id: gpt-5-mini +endpoint: https://openrouter.ai/api/v1 +model: "openai/gpt-5.4-mini" +reasoning_effort: high +timeout_seconds: 240 +api_key_env: OPENROUTER_API_KEY +service_tier: flex diff --git a/internal/profile/builtin/assets/openai/gpt-5-nano.yml b/internal/profile/builtin/assets/openai/gpt-5-nano.yml new file mode 100644 index 0000000..069a3d5 --- /dev/null +++ b/internal/profile/builtin/assets/openai/gpt-5-nano.yml @@ -0,0 +1,7 @@ +id: gpt-5-nano +endpoint: https://openrouter.ai/api/v1 +model: "openai/gpt-5.4-nano" +reasoning_effort: high +timeout_seconds: 240 +api_key_env: OPENROUTER_API_KEY +service_tier: flex diff --git a/internal/profile/builtin/repository.go b/internal/profile/builtin/repository.go new file mode 100644 index 0000000..f553352 --- /dev/null +++ b/internal/profile/builtin/repository.go @@ -0,0 +1,31 @@ +package builtin + +import ( + "embed" + "strings" + + "gitea.maximumdirect.net/eric/scriptorium/internal/profile" +) + +const assetRoot = "assets" + +//go:embed assets/**/*.yml +var assets embed.FS + +func NewRepository() profile.Repository { + return profile.NewFSRepository(assets, assetRoot) +} + +func NewRepositoryWithPrimary(primary profile.Repository) profile.Repository { + if primary == nil { + return NewRepository() + } + return profile.NewOverlayRepository(primary, NewRepository()) +} + +func NewRepositoryWithDirectory(dir string) profile.Repository { + if strings.TrimSpace(dir) == "" { + return NewRepository() + } + return NewRepositoryWithPrimary(profile.NewFilesystemRepository(dir)) +} diff --git a/internal/profile/builtin/repository_test.go b/internal/profile/builtin/repository_test.go new file mode 100644 index 0000000..5973d9a --- /dev/null +++ b/internal/profile/builtin/repository_test.go @@ -0,0 +1,127 @@ +package builtin + +import ( + "context" + "errors" + "io/fs" + "strings" + "testing" + + "gitea.maximumdirect.net/eric/scriptorium/internal/domain" + "gitea.maximumdirect.net/eric/scriptorium/internal/profile" + "gopkg.in/yaml.v3" +) + +func TestBuiltInProfilesValidateThroughRepository(t *testing.T) { + repo := NewRepository() + ids := loadBuiltInProfileIDs(t) + if len(ids) == 0 { + t.Fatal("expected built-in profiles") + } + + for id := range ids { + t.Run(id, func(t *testing.T) { + p, err := repo.GetProfile(context.Background(), id) + if err != nil { + t.Fatalf("expected built-in profile %q to load, got %v", id, err) + } + if p.ID != id { + t.Fatalf("expected profile id %q, got %q", id, p.ID) + } + }) + } +} + +func TestBuiltInProfilesDoNotContainDuplicateIDsOrRawAPIKeys(t *testing.T) { + loadBuiltInProfileIDs(t) +} + +func loadBuiltInProfileIDs(t *testing.T) map[string]string { + t.Helper() + + ids := map[string]string{} + err := fs.WalkDir(assets, assetRoot, func(name string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() || !strings.HasSuffix(name, ".yml") { + return nil + } + + data, err := assets.ReadFile(name) + if err != nil { + t.Fatalf("failed to read built-in profile %s: %v", name, err) + } + + var raw map[string]any + if err := yaml.Unmarshal(data, &raw); err != nil { + t.Fatalf("failed to decode built-in profile %s: %v", name, err) + } + if _, ok := raw["api_key"]; ok { + t.Fatalf("built-in profile %s contains raw api_key", name) + } + id, ok := raw["id"].(string) + if !ok || strings.TrimSpace(id) == "" { + t.Fatalf("built-in profile %s has missing id", name) + } + if previous, ok := ids[id]; ok { + t.Fatalf("duplicate built-in profile id %q in %s and %s", id, previous, name) + } + ids[id] = name + return nil + }) + if err != nil { + t.Fatalf("failed to walk built-in profiles: %v", err) + } + return ids +} + +func TestRepositoryWithPrimaryUsesPrimaryBeforeBuiltIns(t *testing.T) { + repo := NewRepositoryWithPrimary(staticProfileRepo{ + profiles: map[string]string{"mistral-small-3": "custom-model"}, + }) + + p, err := repo.GetProfile(context.Background(), "mistral-small-3") + if err != nil { + t.Fatalf("expected profile to load, got %v", err) + } + if p.Model != "custom-model" { + t.Fatalf("expected primary profile to override built-in, got %+v", p) + } +} + +func TestRepositoryWithPrimaryFallsBackToBuiltIns(t *testing.T) { + repo := NewRepositoryWithPrimary(staticProfileRepo{}) + + p, err := repo.GetProfile(context.Background(), "mistral-small-3") + if err != nil { + t.Fatalf("expected built-in profile to load, got %v", err) + } + if p.ID != "mistral-small-3" { + t.Fatalf("unexpected profile: %+v", p) + } +} + +func TestRepositoryWithPrimaryDoesNotFallBackAfterPrimaryError(t *testing.T) { + repo := NewRepositoryWithPrimary(staticProfileRepo{err: profile.ErrInvalidProfile}) + + _, err := repo.GetProfile(context.Background(), "mistral-small-3") + if !errors.Is(err, profile.ErrInvalidProfile) { + t.Fatalf("expected primary error, got %v", err) + } +} + +type staticProfileRepo struct { + profiles map[string]string + err error +} + +func (r staticProfileRepo) GetProfile(_ context.Context, id string) (*domain.ExecutionProfile, error) { + if r.err != nil { + return nil, r.err + } + if model, ok := r.profiles[id]; ok { + return &domain.ExecutionProfile{ID: id, Endpoint: "http://primary/v1", Model: model}, nil + } + return nil, profile.ErrProfileNotFound +}