eric/scriptorium
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 7 Wiki Activity

Add custom backend configuration

Browse Source
This commit is contained in:
Eric Rakestraw
2026-08-29 14:18:29 +00:00
parent 5f946a5a1f
commit 1a0f15e210
9 changed files with 307 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files

59
internal/adapter/cli/run.go
View File

@@ -47,6 +47,7 @@ type runConfig struct {
maxTokens int
topP float64
schemaDir string
backends []appconfig.BackendSettings
timeout time.Duration
defaultRenderFormat renderformat.PreparedRunOutputFormat
@@ -76,6 +77,7 @@ type serveConfig struct {
maxRequestBytes int64
maxArtifactBytes int64
maxResponseBytes int64
backends []appconfig.BackendSettings
}
type commonCommandSettings struct {
@@ -88,6 +90,7 @@ type commonCommandSettings struct {
maxArtifactBytes int64
maxResponseBytes int64
defaultRenderFormat renderformat.PreparedRunOutputFormat
backends []appconfig.BackendSettings
}
type listFlag []string
@@ -134,7 +137,7 @@ func runCommand(args []string, stdout, stderr io.Writer) int {
return ExitRuntimeError
}
engine, err := newEngine(cfg)
engine, err := newEngine(cfg.engineSettings())
if err != nil {
fmt.Fprintf(stderr, "engine error: %v\n", err)
return ExitRuntimeError
@@ -168,7 +171,7 @@ func renderCommand(args []string, stdout, stderr io.Writer) int {
return ExitRuntimeError
}
engine, err := newEngine(&cfg.runConfig)
engine, err := newEngine(cfg.runConfig.engineSettings())
if err != nil {
fmt.Fprintf(stderr, "engine error: %v\n", err)
return ExitRuntimeError
@@ -206,11 +209,7 @@ func serveCommand(args []string, stderr io.Writer) int {
return ExitRuntimeError
}
engine, err := newEngine(&runConfig{
promptDir: cfg.promptDir,
profileDir: cfg.profileDir,
schemaDir: cfg.schemaDir,
}, promptkit.WithArtifactReader(artifactReader))
engine, err := newEngine(cfg.engineSettings(), promptkit.WithArtifactReader(artifactReader))
if err != nil {
fmt.Fprintf(stderr, "engine error: %v\n", err)
return ExitRuntimeError
@@ -330,6 +329,7 @@ func parseServeArgs(args []string) (*serveConfig, error) {
cfg.maxRequestBytes = settings.maxRequestBytes
cfg.maxArtifactBytes = settings.maxArtifactBytes
cfg.maxResponseBytes = settings.maxResponseBytes
cfg.backends = settings.backends
if err := validateRequiredLibraryDirs(cfg.promptDir); err != nil {
return nil, err
@@ -384,6 +384,7 @@ func finalizeExecutionRequestConfig(fs *flag.FlagSet, cfg *runConfig) error {
cfg.profileDir = settings.profileDir
cfg.schemaDir = settings.schemaDir
cfg.defaultRenderFormat = settings.defaultRenderFormat
cfg.backends = settings.backends
if err := validateRequiredLibraryDirs(cfg.promptDir); err != nil {
return err
@@ -527,6 +528,7 @@ func resolveCommonSettings(fs *flag.FlagSet, configPath string, overrides appcon
maxArtifactBytes: settings.MaxArtifactBytes,
maxResponseBytes: settings.MaxResponseBytes,
defaultRenderFormat: settings.DefaultRenderFormat,
backends: settings.Backends,
}, nil
}
@@ -537,12 +539,43 @@ func validateRequiredLibraryDirs(promptDir string) error {
return nil
}
func newEngine(cfg *runConfig, options ...promptkit.Option) (*promptkit.Engine, error) {
return promptkit.NewEngine(promptkit.Config{
PromptDir: cfg.promptDir,
ProfileDir: cfg.profileDir,
SchemaDir: cfg.schemaDir,
}, options...)
type engineSettings struct {
promptDir string
profileDir string
schemaDir string
backends []appconfig.BackendSettings
}
func (c runConfig) engineSettings() engineSettings {
return engineSettings{promptDir: c.promptDir, profileDir: c.profileDir, schemaDir: c.schemaDir, backends: c.backends}
}
func (c serveConfig) engineSettings() engineSettings {
return engineSettings{promptDir: c.promptDir, profileDir: c.profileDir, schemaDir: c.schemaDir, backends: c.backends}
}
func newEngine(settings engineSettings, options ...promptkit.Option) (*promptkit.Engine, error) {
engineOptions := make([]promptkit.Option, 0, len(settings.backends)+len(options))
for _, configured := range settings.backends {
engineOptions = append(engineOptions, promptkit.WithBackend(promptkit.Backend{
ID: configured.ID,
Endpoint: configured.Endpoint,
APIKeyEnv: configured.APIKeyEnv,
ExtraParams: configured.ExtraParams,
ConcurrencyLimit: configured.ConcurrencyLimit,
QueueCapacity: configured.QueueCapacity,
}))
}
engineOptions = append(engineOptions, options...)
engine, err := promptkit.NewEngine(promptkit.Config{
PromptDir: settings.promptDir,
ProfileDir: settings.profileDir,
SchemaDir: settings.schemaDir,
}, engineOptions...)
if err != nil {
return nil, fmt.Errorf("engine initialization from application configuration: %w", err)
}
return engine, nil
}
func buildRunRequestFromConfig(cfg *runConfig) (promptkit.RunRequest, error) {

92
internal/adapter/cli/run_test.go
View File

@@ -963,6 +963,98 @@ profile_dir: %s
}
}
func TestRenderCommandUsesConfiguredCustomBackend(t *testing.T) {
lib := newCLITestLibrary(t)
writePromptDefinition(t, lib.promptDir, "custom.yaml", `id: custom
version: "1"
default_profile: local-gpu
messages:
- role: user
content: "hello"
output:
format: text
validation_mode: none
`)
if err := os.WriteFile(filepath.Join(lib.profileDir, "local-gpu.yaml"), []byte(`id: local-gpu
backend: local-gpu
model: local-model
`), 0o644); err != nil {
t.Fatalf("write profile fixture: %v", err)
}
configPath := writeAppConfigFile(t, fmt.Sprintf(`
prompt_dir: %s
profile_dir: %s
backends:
local-gpu:
endpoint: http://localhost:11434/v1
extra_params:
provider_option: enabled
concurrency_limit: 2
queue_capacity: 0
`, lib.promptDir, lib.profileDir))
code, _, stderr := runCLICommand(t, renderCommand, []string{
"--config", configPath,
"--prompt", "custom",
})
if code != ExitOK {
t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr)
}
}
func TestConfiguredBackendValidationComesFromPromptkit(t *testing.T) {
lib := newCLITestLibrary(t)
for _, tc := range []struct {
name string
backend string
}{
{
name: "reserved ID",
backend: `openrouter:
endpoint: http://localhost:11434/v1`,
},
{
name: "invalid endpoint",
backend: `local-gpu:
endpoint: not-a-url`,
},
{
name: "invalid capacity relationship",
backend: `local-gpu:
endpoint: http://localhost:11434/v1
queue_capacity: 0`,
},
{
name: "reserved extra parameter",
backend: `local-gpu:
endpoint: http://localhost:11434/v1
extra_params:
model: forbidden`,
},
} {
t.Run(tc.name, func(t *testing.T) {
configPath := writeAppConfigFile(t, fmt.Sprintf(`
prompt_dir: %s
profile_dir: %s
backends:
%s
`, lib.promptDir, lib.profileDir, tc.backend))
cfg, err := parseRenderArgs([]string{"--config", configPath, "--prompt", "custom"})
if err != nil {
t.Fatalf("expected config decoding to succeed, got %v", err)
}
_, err = newEngine(cfg.runConfig.engineSettings())
if !errors.Is(err, promptkit.ErrInvalidConfig) {
t.Fatalf("expected Promptkit ErrInvalidConfig, got %v", err)
}
if !strings.Contains(err.Error(), "engine initialization from application configuration") {
t.Fatalf("expected application context, got %v", err)
}
})
}
}
func TestRenderCommandExplicitTextFormatWorks(t *testing.T) {
lib := newCLITestLibrary(t)
inputPath := lib.writeInputFile(t, "transcript.md", "hello transcript")

49
internal/config/config.go
View File

@@ -7,6 +7,7 @@ import (
"io"
"os"
"path/filepath"
"sort"
"strings"
"gitea.maximumdirect.net/eric/scriptorium/internal/defaults"
@@ -32,11 +33,29 @@ var (
// Config is the on-disk YAML shape for application-level settings.
type Config struct {
PromptDir string `yaml:"prompt_dir"`
ProfileDir string `yaml:"profile_dir"`
SchemaDir string `yaml:"schema_dir"`
Server ServerConfig `yaml:"server"`
Defaults DefaultsConfig `yaml:"defaults"`
PromptDir string `yaml:"prompt_dir"`
ProfileDir string `yaml:"profile_dir"`
SchemaDir string `yaml:"schema_dir"`
Server ServerConfig `yaml:"server"`
Defaults DefaultsConfig `yaml:"defaults"`
Backends map[string]BackendConfig `yaml:"backends"`
}
type BackendConfig struct {
Endpoint string `yaml:"endpoint"`
APIKeyEnv string `yaml:"api_key_env"`
ExtraParams map[string]any `yaml:"extra_params"`
ConcurrencyLimit int `yaml:"concurrency_limit"`
QueueCapacity *int `yaml:"queue_capacity"`
}
type BackendSettings struct {
ID string
Endpoint string
APIKeyEnv string
ExtraParams map[string]any
ConcurrencyLimit int
QueueCapacity *int
}
type ServerConfig struct {
@@ -62,6 +81,7 @@ type AppSettings struct {
MaxArtifactBytes int64
MaxResponseBytes int64
DefaultRenderFormat renderformat.PreparedRunOutputFormat
Backends []BackendSettings
}
// CLIOverrides can be applied after config load to enforce precedence.
@@ -245,6 +265,25 @@ func applyConfig(base AppSettings, cfg Config) (AppSettings, error) {
}
out.DefaultRenderFormat = parsed
}
if len(cfg.Backends) > 0 {
ids := make([]string, 0, len(cfg.Backends))
for id := range cfg.Backends {
ids = append(ids, id)
}
sort.Strings(ids)
out.Backends = make([]BackendSettings, 0, len(ids))
for _, id := range ids {
backend := cfg.Backends[id]
out.Backends = append(out.Backends, BackendSettings{
ID: id,
Endpoint: backend.Endpoint,
APIKeyEnv: backend.APIKeyEnv,
ExtraParams: backend.ExtraParams,
ConcurrencyLimit: backend.ConcurrencyLimit,
QueueCapacity: backend.QueueCapacity,
})
}
}
return out, nil
}

77
internal/config/config_test.go
View File

@@ -4,6 +4,7 @@ import (
"errors"
"os"
"path/filepath"
"reflect"
"testing"
"gitea.maximumdirect.net/eric/scriptorium/internal/defaults"
@@ -20,7 +21,7 @@ func TestLoadConfigMissingImplicitPathUsesBuiltInDefaults(t *testing.T) {
}
want := BuiltInDefaults()
if got != want {
if !reflect.DeepEqual(got, want) {
t.Fatalf("unexpected settings: got=%+v want=%+v", got, want)
}
}
@@ -100,6 +101,78 @@ func TestLoadConfigAPIKeyFieldIsRejectedAsUnknown(t *testing.T) {
}
}
func TestLoadConfigBackendsRetainsResolvedSettingsInSortedOrder(t *testing.T) {
path := writeConfigFile(t, "config.yml", `
backends:
zebra:
endpoint: https://zebra.example/v1
api_key_env: ZEBRA_API_KEY
extra_params:
provider_option: enabled
nested:
enabled: true
attempts: 2
concurrency_limit: 2
alpha:
endpoint: http://alpha.example/v1
queue_capacity: 0
`)
got, err := LoadConfig(path, true)
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
if len(got.Backends) != 2 {
t.Fatalf("expected two backends, got %#v", got.Backends)
}
if got.Backends[0].ID != "alpha" || got.Backends[1].ID != "zebra" {
t.Fatalf("expected sorted backend IDs, got %#v", got.Backends)
}
if got.Backends[0].QueueCapacity == nil || *got.Backends[0].QueueCapacity != 0 {
t.Fatalf("expected explicit zero queue capacity, got %#v", got.Backends[0].QueueCapacity)
}
if got.Backends[1].QueueCapacity != nil {
t.Fatalf("expected omitted queue capacity to remain nil, got %#v", got.Backends[1].QueueCapacity)
}
wantParams := map[string]any{
"provider_option": "enabled",
"nested": map[string]any{
"enabled": true,
"attempts": 2,
},
}
if !reflect.DeepEqual(got.Backends[1].ExtraParams, wantParams) {
t.Fatalf("unexpected extra params: got=%#v want=%#v", got.Backends[1].ExtraParams, wantParams)
}
if got.Backends[1].Endpoint != "https://zebra.example/v1" || got.Backends[1].APIKeyEnv != "ZEBRA_API_KEY" || got.Backends[1].ConcurrencyLimit != 2 {
t.Fatalf("unexpected zebra backend: %#v", got.Backends[1])
}
}
func TestLoadConfigRejectsUnknownOrSecretBackendFields(t *testing.T) {
for name, body := range map[string]string{
"unknown": "backends:\n local:\n endpoint: http://localhost:11434/v1\n unexpected: value\n",
"secret": "backends:\n local:\n endpoint: http://localhost:11434/v1\n api_key: secret\n",
} {
t.Run(name, func(t *testing.T) {
path := writeConfigFile(t, "config.yml", body)
_, err := LoadConfig(path, true)
if !errors.Is(err, ErrInvalidConfigYAML) {
t.Fatalf("expected ErrInvalidConfigYAML, got %v", err)
}
})
}
}
func TestLoadConfigRejectsInvalidBackendFieldTypes(t *testing.T) {
path := writeConfigFile(t, "config.yml", "backends:\n local:\n endpoint: http://localhost:11434/v1\n queue_capacity: not-a-number\n")
_, err := LoadConfig(path, true)
if !errors.Is(err, ErrInvalidConfigYAML) {
t.Fatalf("expected ErrInvalidConfigYAML, got %v", err)
}
}
func TestLoadConfigValidConfigSetsDirectoriesAndServerAddr(t *testing.T) {
path := writeConfigFile(t, "config.yml", `
prompt_dir: ./prompts
@@ -196,7 +269,7 @@ func TestLoadConfigEmptyFileResolvesToBuiltInDefaults(t *testing.T) {
}
want := BuiltInDefaults()
if got != want {
if !reflect.DeepEqual(got, want) {
t.Fatalf("unexpected settings: got=%+v want=%+v", got, want)
}
}