diff --git a/docs/config.md b/docs/config.md index 9959321..c88ff3d 100644 --- a/docs/config.md +++ b/docs/config.md @@ -39,11 +39,39 @@ do not override a prior value. Raw API-key fields are not accepted. | `server.max_artifact_bytes` | `16777216` | Maximum HTTP file-input artifact bytes; `0` disables the limit. | | `server.max_response_bytes` | `16777216` | Maximum encoded HTTP response bytes; `0` disables the limit. | | `defaults.render_format` | `text` | Default prepared-run output format: `text` or `json`. | +| `backends` | unset | Optional mapping of custom Promptkit backend IDs to engine-scoped connection and capacity settings. | The size fields must be zero or greater. The [HTTP API](api.md) defines how each limit is enforced and reported. `server.artifact_root` configures an HTTP deployment boundary; see [operations](operations.md) for deployment handling. +## Custom Backends + +Use `backends` when a profile selects an application-defined backend ID: + +```yaml +backends: + local-gpu: + endpoint: http://localhost:11434/v1 + api_key_env: LOCAL_GPU_API_KEY + extra_params: + provider_option: enabled + concurrency_limit: 2 + queue_capacity: 0 +``` + +Each mapping key is the case-sensitive backend ID. `endpoint` is required; +`api_key_env`, `extra_params`, `concurrency_limit`, and `queue_capacity` are +optional. `concurrency_limit: 0` leaves the backend unlimited. Omitting +`queue_capacity` lets Promptkit use its default for a limited backend, while +an explicit `queue_capacity: 0` disables queueing. + +Configuration strictly owns the YAML shape and rejects unknown fields. Promptkit +validates backend IDs, endpoints, environment-variable names, extra parameters, +and capacity relationships when Scriptorium constructs its engine. There are no +backend command-line overrides. Store only an environment-variable name in +`api_key_env`; raw API-key fields are not accepted. + ## Framework Source Mapping Scriptorium passes `prompt_dir`, `profile_dir`, and `schema_dir` to Promptkit diff --git a/docs/internal/adapters.md b/docs/internal/adapters.md index c61e8c3..0fe0db3 100644 --- a/docs/internal/adapters.md +++ b/docs/internal/adapters.md @@ -38,6 +38,12 @@ resolves its own defaults and definition-required inputs. with `promptkit.WithArtifactReader`, passes the engine through the HTTP adapter's consumer-owned `Runner` interface, and starts the server. +All three CLI paths assemble the engine from the same resolved prompt, profile, +and schema directories plus configured custom backends. Each backend is mapped +to Promptkit's public `Backend` value and registered during engine construction, +so one constructed server engine retains one immutable backend registry and its +associated capacity state. + ### HTTP The handler enforces transport limits and strict JSON decoding before mapping diff --git a/docs/internal/sources.md b/docs/internal/sources.md index 2725d04..ab572bb 100644 --- a/docs/internal/sources.md +++ b/docs/internal/sources.md @@ -10,9 +10,11 @@ owned by the tagged ## Application Source Locations `internal/config` resolves `prompt_dir`, `profile_dir`, and `schema_dir` from -Scriptorium defaults, configuration files, and CLI overrides. -`internal/adapter/cli` passes those paths into `promptkit.Config` when -constructing the engine. +Scriptorium defaults, configuration files, and CLI overrides. It also resolves +the application-owned `backends` mapping into sorted engine settings. +`internal/adapter/cli` passes the directories into `promptkit.Config` and maps +each configured backend to Promptkit's public engine registration when +constructing an engine shared by the command path. Scriptorium does not search, parse, validate, or overlay framework source files itself. Promptkit owns prompt selection, profile built-ins and overlays, schema diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index fb4cf26..8184773 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -145,6 +145,8 @@ no inputs; only Promptkit rejects missing definition-required data. ## Stage 3: Add Engine-Scoped Custom Backend Configuration +**Completion: Complete.** + Add the application-owned configuration needed for Promptkit profiles to select custom backend IDs and capacity policies. diff --git a/examples/config.full.yml b/examples/config.full.yml index 0e70e12..826f5af 100644 --- a/examples/config.full.yml +++ b/examples/config.full.yml @@ -2,6 +2,15 @@ prompt_dir: ./examples/prompts profile_dir: ./examples/profiles schema_dir: ./examples/schemas +backends: + local-gpu: + endpoint: http://localhost:11434/v1 + api_key_env: LOCAL_GPU_API_KEY + extra_params: + provider_option: enabled + concurrency_limit: 2 + queue_capacity: 0 + server: addr: 127.0.0.1:8080 artifact_root: . diff --git a/internal/adapter/cli/run.go b/internal/adapter/cli/run.go index e36d6e1..4e8b4b6 100644 --- a/internal/adapter/cli/run.go +++ b/internal/adapter/cli/run.go @@ -47,6 +47,7 @@ type runConfig struct { maxTokens int topP float64 schemaDir string + backends []appconfig.BackendSettings timeout time.Duration defaultRenderFormat renderformat.PreparedRunOutputFormat @@ -76,6 +77,7 @@ type serveConfig struct { maxRequestBytes int64 maxArtifactBytes int64 maxResponseBytes int64 + backends []appconfig.BackendSettings } type commonCommandSettings struct { @@ -88,6 +90,7 @@ type commonCommandSettings struct { maxArtifactBytes int64 maxResponseBytes int64 defaultRenderFormat renderformat.PreparedRunOutputFormat + backends []appconfig.BackendSettings } type listFlag []string @@ -134,7 +137,7 @@ func runCommand(args []string, stdout, stderr io.Writer) int { return ExitRuntimeError } - engine, err := newEngine(cfg) + engine, err := newEngine(cfg.engineSettings()) if err != nil { fmt.Fprintf(stderr, "engine error: %v\n", err) return ExitRuntimeError @@ -168,7 +171,7 @@ func renderCommand(args []string, stdout, stderr io.Writer) int { return ExitRuntimeError } - engine, err := newEngine(&cfg.runConfig) + engine, err := newEngine(cfg.runConfig.engineSettings()) if err != nil { fmt.Fprintf(stderr, "engine error: %v\n", err) return ExitRuntimeError @@ -206,11 +209,7 @@ func serveCommand(args []string, stderr io.Writer) int { return ExitRuntimeError } - engine, err := newEngine(&runConfig{ - promptDir: cfg.promptDir, - profileDir: cfg.profileDir, - schemaDir: cfg.schemaDir, - }, promptkit.WithArtifactReader(artifactReader)) + engine, err := newEngine(cfg.engineSettings(), promptkit.WithArtifactReader(artifactReader)) if err != nil { fmt.Fprintf(stderr, "engine error: %v\n", err) return ExitRuntimeError @@ -330,6 +329,7 @@ func parseServeArgs(args []string) (*serveConfig, error) { cfg.maxRequestBytes = settings.maxRequestBytes cfg.maxArtifactBytes = settings.maxArtifactBytes cfg.maxResponseBytes = settings.maxResponseBytes + cfg.backends = settings.backends if err := validateRequiredLibraryDirs(cfg.promptDir); err != nil { return nil, err @@ -384,6 +384,7 @@ func finalizeExecutionRequestConfig(fs *flag.FlagSet, cfg *runConfig) error { cfg.profileDir = settings.profileDir cfg.schemaDir = settings.schemaDir cfg.defaultRenderFormat = settings.defaultRenderFormat + cfg.backends = settings.backends if err := validateRequiredLibraryDirs(cfg.promptDir); err != nil { return err @@ -527,6 +528,7 @@ func resolveCommonSettings(fs *flag.FlagSet, configPath string, overrides appcon maxArtifactBytes: settings.MaxArtifactBytes, maxResponseBytes: settings.MaxResponseBytes, defaultRenderFormat: settings.DefaultRenderFormat, + backends: settings.Backends, }, nil } @@ -537,12 +539,43 @@ func validateRequiredLibraryDirs(promptDir string) error { return nil } -func newEngine(cfg *runConfig, options ...promptkit.Option) (*promptkit.Engine, error) { - return promptkit.NewEngine(promptkit.Config{ - PromptDir: cfg.promptDir, - ProfileDir: cfg.profileDir, - SchemaDir: cfg.schemaDir, - }, options...) +type engineSettings struct { + promptDir string + profileDir string + schemaDir string + backends []appconfig.BackendSettings +} + +func (c runConfig) engineSettings() engineSettings { + return engineSettings{promptDir: c.promptDir, profileDir: c.profileDir, schemaDir: c.schemaDir, backends: c.backends} +} + +func (c serveConfig) engineSettings() engineSettings { + return engineSettings{promptDir: c.promptDir, profileDir: c.profileDir, schemaDir: c.schemaDir, backends: c.backends} +} + +func newEngine(settings engineSettings, options ...promptkit.Option) (*promptkit.Engine, error) { + engineOptions := make([]promptkit.Option, 0, len(settings.backends)+len(options)) + for _, configured := range settings.backends { + engineOptions = append(engineOptions, promptkit.WithBackend(promptkit.Backend{ + ID: configured.ID, + Endpoint: configured.Endpoint, + APIKeyEnv: configured.APIKeyEnv, + ExtraParams: configured.ExtraParams, + ConcurrencyLimit: configured.ConcurrencyLimit, + QueueCapacity: configured.QueueCapacity, + })) + } + engineOptions = append(engineOptions, options...) + engine, err := promptkit.NewEngine(promptkit.Config{ + PromptDir: settings.promptDir, + ProfileDir: settings.profileDir, + SchemaDir: settings.schemaDir, + }, engineOptions...) + if err != nil { + return nil, fmt.Errorf("engine initialization from application configuration: %w", err) + } + return engine, nil } func buildRunRequestFromConfig(cfg *runConfig) (promptkit.RunRequest, error) { diff --git a/internal/adapter/cli/run_test.go b/internal/adapter/cli/run_test.go index c27d1de..dd5aa54 100644 --- a/internal/adapter/cli/run_test.go +++ b/internal/adapter/cli/run_test.go @@ -963,6 +963,98 @@ profile_dir: %s } } +func TestRenderCommandUsesConfiguredCustomBackend(t *testing.T) { + lib := newCLITestLibrary(t) + writePromptDefinition(t, lib.promptDir, "custom.yaml", `id: custom +version: "1" +default_profile: local-gpu +messages: + - role: user + content: "hello" +output: + format: text + validation_mode: none +`) + if err := os.WriteFile(filepath.Join(lib.profileDir, "local-gpu.yaml"), []byte(`id: local-gpu +backend: local-gpu +model: local-model +`), 0o644); err != nil { + t.Fatalf("write profile fixture: %v", err) + } + configPath := writeAppConfigFile(t, fmt.Sprintf(` +prompt_dir: %s +profile_dir: %s +backends: + local-gpu: + endpoint: http://localhost:11434/v1 + extra_params: + provider_option: enabled + concurrency_limit: 2 + queue_capacity: 0 +`, lib.promptDir, lib.profileDir)) + + code, _, stderr := runCLICommand(t, renderCommand, []string{ + "--config", configPath, + "--prompt", "custom", + }) + if code != ExitOK { + t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr) + } +} + +func TestConfiguredBackendValidationComesFromPromptkit(t *testing.T) { + lib := newCLITestLibrary(t) + + for _, tc := range []struct { + name string + backend string + }{ + { + name: "reserved ID", + backend: `openrouter: + endpoint: http://localhost:11434/v1`, + }, + { + name: "invalid endpoint", + backend: `local-gpu: + endpoint: not-a-url`, + }, + { + name: "invalid capacity relationship", + backend: `local-gpu: + endpoint: http://localhost:11434/v1 + queue_capacity: 0`, + }, + { + name: "reserved extra parameter", + backend: `local-gpu: + endpoint: http://localhost:11434/v1 + extra_params: + model: forbidden`, + }, + } { + t.Run(tc.name, func(t *testing.T) { + configPath := writeAppConfigFile(t, fmt.Sprintf(` +prompt_dir: %s +profile_dir: %s +backends: + %s +`, lib.promptDir, lib.profileDir, tc.backend)) + cfg, err := parseRenderArgs([]string{"--config", configPath, "--prompt", "custom"}) + if err != nil { + t.Fatalf("expected config decoding to succeed, got %v", err) + } + _, err = newEngine(cfg.runConfig.engineSettings()) + if !errors.Is(err, promptkit.ErrInvalidConfig) { + t.Fatalf("expected Promptkit ErrInvalidConfig, got %v", err) + } + if !strings.Contains(err.Error(), "engine initialization from application configuration") { + t.Fatalf("expected application context, got %v", err) + } + }) + } +} + func TestRenderCommandExplicitTextFormatWorks(t *testing.T) { lib := newCLITestLibrary(t) inputPath := lib.writeInputFile(t, "transcript.md", "hello transcript") diff --git a/internal/config/config.go b/internal/config/config.go index a9af376..6104837 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -7,6 +7,7 @@ import ( "io" "os" "path/filepath" + "sort" "strings" "gitea.maximumdirect.net/eric/scriptorium/internal/defaults" @@ -32,11 +33,29 @@ var ( // Config is the on-disk YAML shape for application-level settings. type Config struct { - PromptDir string `yaml:"prompt_dir"` - ProfileDir string `yaml:"profile_dir"` - SchemaDir string `yaml:"schema_dir"` - Server ServerConfig `yaml:"server"` - Defaults DefaultsConfig `yaml:"defaults"` + PromptDir string `yaml:"prompt_dir"` + ProfileDir string `yaml:"profile_dir"` + SchemaDir string `yaml:"schema_dir"` + Server ServerConfig `yaml:"server"` + Defaults DefaultsConfig `yaml:"defaults"` + Backends map[string]BackendConfig `yaml:"backends"` +} + +type BackendConfig struct { + Endpoint string `yaml:"endpoint"` + APIKeyEnv string `yaml:"api_key_env"` + ExtraParams map[string]any `yaml:"extra_params"` + ConcurrencyLimit int `yaml:"concurrency_limit"` + QueueCapacity *int `yaml:"queue_capacity"` +} + +type BackendSettings struct { + ID string + Endpoint string + APIKeyEnv string + ExtraParams map[string]any + ConcurrencyLimit int + QueueCapacity *int } type ServerConfig struct { @@ -62,6 +81,7 @@ type AppSettings struct { MaxArtifactBytes int64 MaxResponseBytes int64 DefaultRenderFormat renderformat.PreparedRunOutputFormat + Backends []BackendSettings } // CLIOverrides can be applied after config load to enforce precedence. @@ -245,6 +265,25 @@ func applyConfig(base AppSettings, cfg Config) (AppSettings, error) { } out.DefaultRenderFormat = parsed } + if len(cfg.Backends) > 0 { + ids := make([]string, 0, len(cfg.Backends)) + for id := range cfg.Backends { + ids = append(ids, id) + } + sort.Strings(ids) + out.Backends = make([]BackendSettings, 0, len(ids)) + for _, id := range ids { + backend := cfg.Backends[id] + out.Backends = append(out.Backends, BackendSettings{ + ID: id, + Endpoint: backend.Endpoint, + APIKeyEnv: backend.APIKeyEnv, + ExtraParams: backend.ExtraParams, + ConcurrencyLimit: backend.ConcurrencyLimit, + QueueCapacity: backend.QueueCapacity, + }) + } + } return out, nil } diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 0a54d1f..b2f0e39 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -4,6 +4,7 @@ import ( "errors" "os" "path/filepath" + "reflect" "testing" "gitea.maximumdirect.net/eric/scriptorium/internal/defaults" @@ -20,7 +21,7 @@ func TestLoadConfigMissingImplicitPathUsesBuiltInDefaults(t *testing.T) { } want := BuiltInDefaults() - if got != want { + if !reflect.DeepEqual(got, want) { t.Fatalf("unexpected settings: got=%+v want=%+v", got, want) } } @@ -100,6 +101,78 @@ func TestLoadConfigAPIKeyFieldIsRejectedAsUnknown(t *testing.T) { } } +func TestLoadConfigBackendsRetainsResolvedSettingsInSortedOrder(t *testing.T) { + path := writeConfigFile(t, "config.yml", ` +backends: + zebra: + endpoint: https://zebra.example/v1 + api_key_env: ZEBRA_API_KEY + extra_params: + provider_option: enabled + nested: + enabled: true + attempts: 2 + concurrency_limit: 2 + alpha: + endpoint: http://alpha.example/v1 + queue_capacity: 0 +`) + + got, err := LoadConfig(path, true) + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + if len(got.Backends) != 2 { + t.Fatalf("expected two backends, got %#v", got.Backends) + } + if got.Backends[0].ID != "alpha" || got.Backends[1].ID != "zebra" { + t.Fatalf("expected sorted backend IDs, got %#v", got.Backends) + } + if got.Backends[0].QueueCapacity == nil || *got.Backends[0].QueueCapacity != 0 { + t.Fatalf("expected explicit zero queue capacity, got %#v", got.Backends[0].QueueCapacity) + } + if got.Backends[1].QueueCapacity != nil { + t.Fatalf("expected omitted queue capacity to remain nil, got %#v", got.Backends[1].QueueCapacity) + } + wantParams := map[string]any{ + "provider_option": "enabled", + "nested": map[string]any{ + "enabled": true, + "attempts": 2, + }, + } + if !reflect.DeepEqual(got.Backends[1].ExtraParams, wantParams) { + t.Fatalf("unexpected extra params: got=%#v want=%#v", got.Backends[1].ExtraParams, wantParams) + } + if got.Backends[1].Endpoint != "https://zebra.example/v1" || got.Backends[1].APIKeyEnv != "ZEBRA_API_KEY" || got.Backends[1].ConcurrencyLimit != 2 { + t.Fatalf("unexpected zebra backend: %#v", got.Backends[1]) + } +} + +func TestLoadConfigRejectsUnknownOrSecretBackendFields(t *testing.T) { + for name, body := range map[string]string{ + "unknown": "backends:\n local:\n endpoint: http://localhost:11434/v1\n unexpected: value\n", + "secret": "backends:\n local:\n endpoint: http://localhost:11434/v1\n api_key: secret\n", + } { + t.Run(name, func(t *testing.T) { + path := writeConfigFile(t, "config.yml", body) + _, err := LoadConfig(path, true) + if !errors.Is(err, ErrInvalidConfigYAML) { + t.Fatalf("expected ErrInvalidConfigYAML, got %v", err) + } + }) + } +} + +func TestLoadConfigRejectsInvalidBackendFieldTypes(t *testing.T) { + path := writeConfigFile(t, "config.yml", "backends:\n local:\n endpoint: http://localhost:11434/v1\n queue_capacity: not-a-number\n") + + _, err := LoadConfig(path, true) + if !errors.Is(err, ErrInvalidConfigYAML) { + t.Fatalf("expected ErrInvalidConfigYAML, got %v", err) + } +} + func TestLoadConfigValidConfigSetsDirectoriesAndServerAddr(t *testing.T) { path := writeConfigFile(t, "config.yml", ` prompt_dir: ./prompts @@ -196,7 +269,7 @@ func TestLoadConfigEmptyFileResolvesToBuiltInDefaults(t *testing.T) { } want := BuiltInDefaults() - if got != want { + if !reflect.DeepEqual(got, want) { t.Fatalf("unexpected settings: got=%+v want=%+v", got, want) } }