eric/scriptorium
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 7 Wiki Activity

Add custom backend configuration

Browse Source
This commit is contained in:
Eric Rakestraw
2026-08-29 14:18:29 +00:00
parent 5f946a5a1f
commit 1a0f15e210
9 changed files with 307 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
docs/config.md
View File

@@ -39,11 +39,39 @@ do not override a prior value. Raw API-key fields are not accepted.
| `server.max_artifact_bytes` | `16777216` | Maximum HTTP file-input artifact bytes; `0` disables the limit. |
| `server.max_response_bytes` | `16777216` | Maximum encoded HTTP response bytes; `0` disables the limit. |
| `defaults.render_format` | `text` | Default prepared-run output format: `text` or `json`. |
| `backends` | unset | Optional mapping of custom Promptkit backend IDs to engine-scoped connection and capacity settings. |
The size fields must be zero or greater. The [HTTP API](api.md) defines how
each limit is enforced and reported. `server.artifact_root` configures an HTTP
deployment boundary; see [operations](operations.md) for deployment handling.
## Custom Backends
Use `backends` when a profile selects an application-defined backend ID:
```yaml
backends:
local-gpu:
endpoint: http://localhost:11434/v1
api_key_env: LOCAL_GPU_API_KEY
extra_params:
provider_option: enabled
concurrency_limit: 2
queue_capacity: 0
```
Each mapping key is the case-sensitive backend ID. `endpoint` is required;
`api_key_env`, `extra_params`, `concurrency_limit`, and `queue_capacity` are
optional. `concurrency_limit: 0` leaves the backend unlimited. Omitting
`queue_capacity` lets Promptkit use its default for a limited backend, while
an explicit `queue_capacity: 0` disables queueing.
Configuration strictly owns the YAML shape and rejects unknown fields. Promptkit
validates backend IDs, endpoints, environment-variable names, extra parameters,
and capacity relationships when Scriptorium constructs its engine. There are no
backend command-line overrides. Store only an environment-variable name in
`api_key_env`; raw API-key fields are not accepted.
## Framework Source Mapping
Scriptorium passes `prompt_dir`, `profile_dir`, and `schema_dir` to Promptkit

6
docs/internal/adapters.md
View File

@@ -38,6 +38,12 @@ resolves its own defaults and definition-required inputs.
with `promptkit.WithArtifactReader`, passes the engine through the HTTP
adapter's consumer-owned `Runner` interface, and starts the server.
All three CLI paths assemble the engine from the same resolved prompt, profile,
and schema directories plus configured custom backends. Each backend is mapped
to Promptkit's public `Backend` value and registered during engine construction,
so one constructed server engine retains one immutable backend registry and its
associated capacity state.
### HTTP
The handler enforces transport limits and strict JSON decoding before mapping

8
docs/internal/sources.md
View File

@@ -10,9 +10,11 @@ owned by the tagged
## Application Source Locations
`internal/config` resolves `prompt_dir`, `profile_dir`, and `schema_dir` from
Scriptorium defaults, configuration files, and CLI overrides.
`internal/adapter/cli` passes those paths into `promptkit.Config` when
constructing the engine.
Scriptorium defaults, configuration files, and CLI overrides. It also resolves
the application-owned `backends` mapping into sorted engine settings.
`internal/adapter/cli` passes the directories into `promptkit.Config` and maps
each configured backend to Promptkit's public engine registration when
constructing an engine shared by the command path.
Scriptorium does not search, parse, validate, or overlay framework source files
itself. Promptkit owns prompt selection, profile built-ins and overlays, schema

2
docs/roadmap/implementation.md
View File

@@ -145,6 +145,8 @@ no inputs; only Promptkit rejects missing definition-required data.
## Stage 3: Add Engine-Scoped Custom Backend Configuration
**Completion: Complete.**
Add the application-owned configuration needed for Promptkit profiles to select
custom backend IDs and capacity policies.

9
examples/config.full.yml
View File

@@ -2,6 +2,15 @@ prompt_dir: ./examples/prompts
profile_dir: ./examples/profiles
schema_dir: ./examples/schemas
backends:
local-gpu:
endpoint: http://localhost:11434/v1
api_key_env: LOCAL_GPU_API_KEY
extra_params:
provider_option: enabled
concurrency_limit: 2
queue_capacity: 0
server:
addr: 127.0.0.1:8080
artifact_root: .

59
internal/adapter/cli/run.go
View File

@@ -47,6 +47,7 @@ type runConfig struct {
maxTokens int
topP float64
schemaDir string
backends []appconfig.BackendSettings
timeout time.Duration
defaultRenderFormat renderformat.PreparedRunOutputFormat
@@ -76,6 +77,7 @@ type serveConfig struct {
maxRequestBytes int64
maxArtifactBytes int64
maxResponseBytes int64
backends []appconfig.BackendSettings
}
type commonCommandSettings struct {
@@ -88,6 +90,7 @@ type commonCommandSettings struct {
maxArtifactBytes int64
maxResponseBytes int64
defaultRenderFormat renderformat.PreparedRunOutputFormat
backends []appconfig.BackendSettings
}
type listFlag []string
@@ -134,7 +137,7 @@ func runCommand(args []string, stdout, stderr io.Writer) int {
return ExitRuntimeError
}
engine, err := newEngine(cfg)
engine, err := newEngine(cfg.engineSettings())
if err != nil {
fmt.Fprintf(stderr, "engine error: %v\n", err)
return ExitRuntimeError
@@ -168,7 +171,7 @@ func renderCommand(args []string, stdout, stderr io.Writer) int {
return ExitRuntimeError
}
engine, err := newEngine(&cfg.runConfig)
engine, err := newEngine(cfg.runConfig.engineSettings())
if err != nil {
fmt.Fprintf(stderr, "engine error: %v\n", err)
return ExitRuntimeError
@@ -206,11 +209,7 @@ func serveCommand(args []string, stderr io.Writer) int {
return ExitRuntimeError
}
engine, err := newEngine(&runConfig{
promptDir: cfg.promptDir,
profileDir: cfg.profileDir,
schemaDir: cfg.schemaDir,
}, promptkit.WithArtifactReader(artifactReader))
engine, err := newEngine(cfg.engineSettings(), promptkit.WithArtifactReader(artifactReader))
if err != nil {
fmt.Fprintf(stderr, "engine error: %v\n", err)
return ExitRuntimeError
@@ -330,6 +329,7 @@ func parseServeArgs(args []string) (*serveConfig, error) {
cfg.maxRequestBytes = settings.maxRequestBytes
cfg.maxArtifactBytes = settings.maxArtifactBytes
cfg.maxResponseBytes = settings.maxResponseBytes
cfg.backends = settings.backends
if err := validateRequiredLibraryDirs(cfg.promptDir); err != nil {
return nil, err
@@ -384,6 +384,7 @@ func finalizeExecutionRequestConfig(fs *flag.FlagSet, cfg *runConfig) error {
cfg.profileDir = settings.profileDir
cfg.schemaDir = settings.schemaDir
cfg.defaultRenderFormat = settings.defaultRenderFormat
cfg.backends = settings.backends
if err := validateRequiredLibraryDirs(cfg.promptDir); err != nil {
return err
@@ -527,6 +528,7 @@ func resolveCommonSettings(fs *flag.FlagSet, configPath string, overrides appcon
maxArtifactBytes: settings.MaxArtifactBytes,
maxResponseBytes: settings.MaxResponseBytes,
defaultRenderFormat: settings.DefaultRenderFormat,
backends: settings.Backends,
}, nil
}
@@ -537,12 +539,43 @@ func validateRequiredLibraryDirs(promptDir string) error {
return nil
}
func newEngine(cfg *runConfig, options ...promptkit.Option) (*promptkit.Engine, error) {
return promptkit.NewEngine(promptkit.Config{
PromptDir: cfg.promptDir,
ProfileDir: cfg.profileDir,
SchemaDir: cfg.schemaDir,
}, options...)
type engineSettings struct {
promptDir string
profileDir string
schemaDir string
backends []appconfig.BackendSettings
}
func (c runConfig) engineSettings() engineSettings {
return engineSettings{promptDir: c.promptDir, profileDir: c.profileDir, schemaDir: c.schemaDir, backends: c.backends}
}
func (c serveConfig) engineSettings() engineSettings {
return engineSettings{promptDir: c.promptDir, profileDir: c.profileDir, schemaDir: c.schemaDir, backends: c.backends}
}
func newEngine(settings engineSettings, options ...promptkit.Option) (*promptkit.Engine, error) {
engineOptions := make([]promptkit.Option, 0, len(settings.backends)+len(options))
for _, configured := range settings.backends {
engineOptions = append(engineOptions, promptkit.WithBackend(promptkit.Backend{
ID: configured.ID,
Endpoint: configured.Endpoint,
APIKeyEnv: configured.APIKeyEnv,
ExtraParams: configured.ExtraParams,
ConcurrencyLimit: configured.ConcurrencyLimit,
QueueCapacity: configured.QueueCapacity,
}))
}
engineOptions = append(engineOptions, options...)
engine, err := promptkit.NewEngine(promptkit.Config{
PromptDir: settings.promptDir,
ProfileDir: settings.profileDir,
SchemaDir: settings.schemaDir,
}, engineOptions...)
if err != nil {
return nil, fmt.Errorf("engine initialization from application configuration: %w", err)
}
return engine, nil
}
func buildRunRequestFromConfig(cfg *runConfig) (promptkit.RunRequest, error) {

92
internal/adapter/cli/run_test.go
View File

@@ -963,6 +963,98 @@ profile_dir: %s
}
}
func TestRenderCommandUsesConfiguredCustomBackend(t *testing.T) {
lib := newCLITestLibrary(t)
writePromptDefinition(t, lib.promptDir, "custom.yaml", `id: custom
version: "1"
default_profile: local-gpu
messages:
- role: user
content: "hello"
output:
format: text
validation_mode: none
`)
if err := os.WriteFile(filepath.Join(lib.profileDir, "local-gpu.yaml"), []byte(`id: local-gpu
backend: local-gpu
model: local-model
`), 0o644); err != nil {
t.Fatalf("write profile fixture: %v", err)
}
configPath := writeAppConfigFile(t, fmt.Sprintf(`
prompt_dir: %s
profile_dir: %s
backends:
local-gpu:
endpoint: http://localhost:11434/v1
extra_params:
provider_option: enabled
concurrency_limit: 2
queue_capacity: 0
`, lib.promptDir, lib.profileDir))
code, _, stderr := runCLICommand(t, renderCommand, []string{
"--config", configPath,
"--prompt", "custom",
})
if code != ExitOK {
t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr)
}
}
func TestConfiguredBackendValidationComesFromPromptkit(t *testing.T) {
lib := newCLITestLibrary(t)
for _, tc := range []struct {
name string
backend string
}{
{
name: "reserved ID",
backend: `openrouter:
endpoint: http://localhost:11434/v1`,
},
{
name: "invalid endpoint",
backend: `local-gpu:
endpoint: not-a-url`,
},
{
name: "invalid capacity relationship",
backend: `local-gpu:
endpoint: http://localhost:11434/v1
queue_capacity: 0`,
},
{
name: "reserved extra parameter",
backend: `local-gpu:
endpoint: http://localhost:11434/v1
extra_params:
model: forbidden`,
},
} {
t.Run(tc.name, func(t *testing.T) {
configPath := writeAppConfigFile(t, fmt.Sprintf(`
prompt_dir: %s
profile_dir: %s
backends:
%s
`, lib.promptDir, lib.profileDir, tc.backend))
cfg, err := parseRenderArgs([]string{"--config", configPath, "--prompt", "custom"})
if err != nil {
t.Fatalf("expected config decoding to succeed, got %v", err)
}
_, err = newEngine(cfg.runConfig.engineSettings())
if !errors.Is(err, promptkit.ErrInvalidConfig) {
t.Fatalf("expected Promptkit ErrInvalidConfig, got %v", err)
}
if !strings.Contains(err.Error(), "engine initialization from application configuration") {
t.Fatalf("expected application context, got %v", err)
}
})
}
}
func TestRenderCommandExplicitTextFormatWorks(t *testing.T) {
lib := newCLITestLibrary(t)
inputPath := lib.writeInputFile(t, "transcript.md", "hello transcript")

49
internal/config/config.go
View File

@@ -7,6 +7,7 @@ import (
"io"
"os"
"path/filepath"
"sort"
"strings"
"gitea.maximumdirect.net/eric/scriptorium/internal/defaults"
@@ -32,11 +33,29 @@ var (
// Config is the on-disk YAML shape for application-level settings.
type Config struct {
PromptDir string `yaml:"prompt_dir"`
ProfileDir string `yaml:"profile_dir"`
SchemaDir string `yaml:"schema_dir"`
Server ServerConfig `yaml:"server"`
Defaults DefaultsConfig `yaml:"defaults"`
PromptDir string `yaml:"prompt_dir"`
ProfileDir string `yaml:"profile_dir"`
SchemaDir string `yaml:"schema_dir"`
Server ServerConfig `yaml:"server"`
Defaults DefaultsConfig `yaml:"defaults"`
Backends map[string]BackendConfig `yaml:"backends"`
}
type BackendConfig struct {
Endpoint string `yaml:"endpoint"`
APIKeyEnv string `yaml:"api_key_env"`
ExtraParams map[string]any `yaml:"extra_params"`
ConcurrencyLimit int `yaml:"concurrency_limit"`
QueueCapacity *int `yaml:"queue_capacity"`
}
type BackendSettings struct {
ID string
Endpoint string
APIKeyEnv string
ExtraParams map[string]any
ConcurrencyLimit int
QueueCapacity *int
}
type ServerConfig struct {
@@ -62,6 +81,7 @@ type AppSettings struct {
MaxArtifactBytes int64
MaxResponseBytes int64
DefaultRenderFormat renderformat.PreparedRunOutputFormat
Backends []BackendSettings
}
// CLIOverrides can be applied after config load to enforce precedence.
@@ -245,6 +265,25 @@ func applyConfig(base AppSettings, cfg Config) (AppSettings, error) {
}
out.DefaultRenderFormat = parsed
}
if len(cfg.Backends) > 0 {
ids := make([]string, 0, len(cfg.Backends))
for id := range cfg.Backends {
ids = append(ids, id)
}
sort.Strings(ids)
out.Backends = make([]BackendSettings, 0, len(ids))
for _, id := range ids {
backend := cfg.Backends[id]
out.Backends = append(out.Backends, BackendSettings{
ID: id,
Endpoint: backend.Endpoint,
APIKeyEnv: backend.APIKeyEnv,
ExtraParams: backend.ExtraParams,
ConcurrencyLimit: backend.ConcurrencyLimit,
QueueCapacity: backend.QueueCapacity,
})
}
}
return out, nil
}

77
internal/config/config_test.go
View File

@@ -4,6 +4,7 @@ import (
"errors"
"os"
"path/filepath"
"reflect"
"testing"
"gitea.maximumdirect.net/eric/scriptorium/internal/defaults"
@@ -20,7 +21,7 @@ func TestLoadConfigMissingImplicitPathUsesBuiltInDefaults(t *testing.T) {
}
want := BuiltInDefaults()
if got != want {
if !reflect.DeepEqual(got, want) {
t.Fatalf("unexpected settings: got=%+v want=%+v", got, want)
}
}
@@ -100,6 +101,78 @@ func TestLoadConfigAPIKeyFieldIsRejectedAsUnknown(t *testing.T) {
}
}
func TestLoadConfigBackendsRetainsResolvedSettingsInSortedOrder(t *testing.T) {
path := writeConfigFile(t, "config.yml", `
backends:
zebra:
endpoint: https://zebra.example/v1
api_key_env: ZEBRA_API_KEY
extra_params:
provider_option: enabled
nested:
enabled: true
attempts: 2
concurrency_limit: 2
alpha:
endpoint: http://alpha.example/v1
queue_capacity: 0
`)
got, err := LoadConfig(path, true)
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
if len(got.Backends) != 2 {
t.Fatalf("expected two backends, got %#v", got.Backends)
}
if got.Backends[0].ID != "alpha" || got.Backends[1].ID != "zebra" {
t.Fatalf("expected sorted backend IDs, got %#v", got.Backends)
}
if got.Backends[0].QueueCapacity == nil || *got.Backends[0].QueueCapacity != 0 {
t.Fatalf("expected explicit zero queue capacity, got %#v", got.Backends[0].QueueCapacity)
}
if got.Backends[1].QueueCapacity != nil {
t.Fatalf("expected omitted queue capacity to remain nil, got %#v", got.Backends[1].QueueCapacity)
}
wantParams := map[string]any{
"provider_option": "enabled",
"nested": map[string]any{
"enabled": true,
"attempts": 2,
},
}
if !reflect.DeepEqual(got.Backends[1].ExtraParams, wantParams) {
t.Fatalf("unexpected extra params: got=%#v want=%#v", got.Backends[1].ExtraParams, wantParams)
}
if got.Backends[1].Endpoint != "https://zebra.example/v1" || got.Backends[1].APIKeyEnv != "ZEBRA_API_KEY" || got.Backends[1].ConcurrencyLimit != 2 {
t.Fatalf("unexpected zebra backend: %#v", got.Backends[1])
}
}
func TestLoadConfigRejectsUnknownOrSecretBackendFields(t *testing.T) {
for name, body := range map[string]string{
"unknown": "backends:\n local:\n endpoint: http://localhost:11434/v1\n unexpected: value\n",
"secret": "backends:\n local:\n endpoint: http://localhost:11434/v1\n api_key: secret\n",
} {
t.Run(name, func(t *testing.T) {
path := writeConfigFile(t, "config.yml", body)
_, err := LoadConfig(path, true)
if !errors.Is(err, ErrInvalidConfigYAML) {
t.Fatalf("expected ErrInvalidConfigYAML, got %v", err)
}
})
}
}
func TestLoadConfigRejectsInvalidBackendFieldTypes(t *testing.T) {
path := writeConfigFile(t, "config.yml", "backends:\n local:\n endpoint: http://localhost:11434/v1\n queue_capacity: not-a-number\n")
_, err := LoadConfig(path, true)
if !errors.Is(err, ErrInvalidConfigYAML) {
t.Fatalf("expected ErrInvalidConfigYAML, got %v", err)
}
}
func TestLoadConfigValidConfigSetsDirectoriesAndServerAddr(t *testing.T) {
path := writeConfigFile(t, "config.yml", `
prompt_dir: ./prompts
@@ -196,7 +269,7 @@ func TestLoadConfigEmptyFileResolvesToBuiltInDefaults(t *testing.T) {
}
want := BuiltInDefaults()
if got != want {
if !reflect.DeepEqual(got, want) {
t.Fatalf("unexpected settings: got=%+v want=%+v", got, want)
}
}