eric/scriptorium
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 9 Wiki Activity

Add configurable transient retries for CLI execution

Browse Source
This commit is contained in:
Eric Rakestraw
2026-09-26 15:28:30 +00:00
parent 2a06a911bc
commit acf511fc0d
10 changed files with 235 additions and 63 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
internal/adapter/cli/inspect.go
View File

@@ -151,7 +151,7 @@ func resolveAppSettingsForProfileInspection(cfg *profileInspectionConfig) (engin
if err != nil {
return engineSettings{}, err
}
return engineSettings{profileDir: settings.ProfileDir, backends: settings.Backends}, nil
return engineSettings{profileDir: settings.ProfileDir, backends: settings.Backends, transientRetries: settings.TransientRetries}, nil
}
func parsePromptInspectionArgs(args []string) (*promptInspectionConfig, error) {
@@ -196,5 +196,5 @@ func resolveAppSettingsForPromptInspection(cfg *promptInspectionConfig) (engineS
if strings.TrimSpace(settings.PromptDir) == "" {
return engineSettings{}, errors.New(errPromptDirRequired)
}
return engineSettings{promptDir: settings.PromptDir, backends: settings.Backends}, nil
return engineSettings{promptDir: settings.PromptDir, backends: settings.Backends, transientRetries: settings.TransientRetries}, nil
}

110
internal/adapter/cli/run.go
View File

@@ -32,36 +32,39 @@ const (
type runConfig struct {
configPath string
promptDir string
profileDir string
promptID string
promptVersion string
profileID string
sessionID string
inputRaw listFlag
varRaw listFlag
outputPath string
llmBaseURL string
apiKeyEnv string
model string
temperature float64
maxTokens int
topP float64
reasoningEffort string
schemaDir string
backends []appconfig.BackendSettings
timeout time.Duration
promptDir string
profileDir string
promptID string
promptVersion string
profileID string
sessionID string
inputRaw listFlag
varRaw listFlag
outputPath string
llmBaseURL string
apiKeyEnv string
model string
temperature float64
maxTokens int
topP float64
reasoningEffort string
schemaDir string
backends []appconfig.BackendSettings
timeout time.Duration
transientRetries int
configuredTransientRetries *int
defaultRenderFormat renderformat.PreparedRunOutputFormat
llmBaseURLSet bool
apiKeyEnvSet bool
modelSet bool
temperatureSet bool
maxTokensSet bool
topPSet bool
reasoningEffortSet bool
timeoutSet bool
llmBaseURLSet bool
apiKeyEnvSet bool
modelSet bool
temperatureSet bool
maxTokensSet bool
topPSet bool
reasoningEffortSet bool
timeoutSet bool
transientRetriesSet bool
}
type renderConfig struct {
@@ -81,6 +84,7 @@ type serveConfig struct {
maxArtifactBytes int64
maxResponseBytes int64
backends []appconfig.BackendSettings
transientRetries *int
}
type commonCommandSettings struct {
@@ -94,6 +98,7 @@ type commonCommandSettings struct {
maxResponseBytes int64
defaultRenderFormat renderformat.PreparedRunOutputFormat
backends []appconfig.BackendSettings
transientRetries *int
}
type listFlag []string
@@ -335,6 +340,7 @@ func parseServeArgs(args []string) (*serveConfig, error) {
cfg.maxArtifactBytes = settings.maxArtifactBytes
cfg.maxResponseBytes = settings.maxResponseBytes
cfg.backends = settings.backends
cfg.transientRetries = settings.transientRetries
if err := validateRequiredLibraryDirs(cfg.promptDir); err != nil {
return nil, err
@@ -369,6 +375,7 @@ func registerExecutionRequestFlags(fs *flag.FlagSet, cfg *runConfig) {
fs.Float64Var(&cfg.topP, "top-p", 0, "optional top_p override")
fs.StringVar(&cfg.reasoningEffort, "reasoning-effort", "", "optional reasoning effort override")
fs.DurationVar(&cfg.timeout, "timeout", 0, "LLM request timeout")
fs.IntVar(&cfg.transientRetries, "transient-retries", 0, "additional transient generation attempts")
fs.StringVar(&cfg.promptID, "prompt-id", "", "deprecated alias for --prompt")
fs.StringVar(&cfg.profileID, "profile-id", "", "deprecated alias for --profile")
}
@@ -392,6 +399,7 @@ func finalizeExecutionRequestConfig(fs *flag.FlagSet, cfg *runConfig) error {
cfg.schemaDir = settings.schemaDir
cfg.defaultRenderFormat = settings.defaultRenderFormat
cfg.backends = settings.backends
cfg.configuredTransientRetries = settings.transientRetries
if err := validateRequiredLibraryDirs(cfg.promptDir); err != nil {
return err
@@ -414,6 +422,10 @@ func finalizeExecutionRequestConfig(fs *flag.FlagSet, cfg *runConfig) error {
cfg.topPSet = flagWasSet(fs, "top-p")
cfg.reasoningEffortSet = flagWasSet(fs, "reasoning-effort")
cfg.timeoutSet = flagWasSet(fs, "timeout")
cfg.transientRetriesSet = flagWasSet(fs, "transient-retries")
if cfg.transientRetriesSet && cfg.transientRetries < 0 {
return errors.New("--transient-retries must be greater than or equal to 0")
}
return nil
}
@@ -541,6 +553,7 @@ func resolveCommonSettings(fs *flag.FlagSet, configPath string, overrides appcon
maxResponseBytes: settings.MaxResponseBytes,
defaultRenderFormat: settings.DefaultRenderFormat,
backends: settings.Backends,
transientRetries: settings.TransientRetries,
}, nil
}
@@ -552,18 +565,19 @@ func validateRequiredLibraryDirs(promptDir string) error {
}
type engineSettings struct {
promptDir string
profileDir string
schemaDir string
backends []appconfig.BackendSettings
promptDir string
profileDir string
schemaDir string
backends []appconfig.BackendSettings
transientRetries *int
}
func (c runConfig) engineSettings() engineSettings {
return engineSettings{promptDir: c.promptDir, profileDir: c.profileDir, schemaDir: c.schemaDir, backends: c.backends}
return engineSettings{promptDir: c.promptDir, profileDir: c.profileDir, schemaDir: c.schemaDir, backends: c.backends, transientRetries: c.configuredTransientRetries}
}
func (c serveConfig) engineSettings() engineSettings {
return engineSettings{promptDir: c.promptDir, profileDir: c.profileDir, schemaDir: c.schemaDir, backends: c.backends}
return engineSettings{promptDir: c.promptDir, profileDir: c.profileDir, schemaDir: c.schemaDir, backends: c.backends, transientRetries: c.transientRetries}
}
func newEngine(settings engineSettings, options ...promptkit.Option) (*promptkit.Engine, error) {
@@ -580,9 +594,10 @@ func newEngine(settings engineSettings, options ...promptkit.Option) (*promptkit
}
engineOptions = append(engineOptions, options...)
engine, err := promptkit.NewEngine(promptkit.Config{
PromptDir: settings.promptDir,
ProfileDir: settings.profileDir,
SchemaDir: settings.schemaDir,
PromptDir: settings.promptDir,
ProfileDir: settings.profileDir,
SchemaDir: settings.schemaDir,
TransientRetries: settings.transientRetries,
}, engineOptions...)
if err != nil {
return nil, fmt.Errorf("engine initialization from application configuration: %w", err)
@@ -641,14 +656,19 @@ func buildRunRequestFromConfig(cfg *runConfig) (promptkit.RunRequest, error) {
}
}
var transientRetries *int
if cfg.transientRetriesSet {
transientRetries = &cfg.transientRetries
}
return promptkit.RunRequest{
PromptID: cfg.promptID,
PromptVersion: cfg.promptVersion,
ProfileID: cfg.profileID,
SessionID: cfg.sessionID,
Inputs: inputs,
Vars: varMappings,
Execution: modelOverride,
PromptID: cfg.promptID,
PromptVersion: cfg.promptVersion,
ProfileID: cfg.profileID,
SessionID: cfg.sessionID,
Inputs: inputs,
Vars: varMappings,
Execution: modelOverride,
TransientRetries: transientRetries,
}, nil
}
@@ -753,8 +773,8 @@ func runErrorMessage(err error) string {
func printUsage(w io.Writer) {
fmt.Fprintln(w, "usage: scriptorium <run|render|serve|inspect> ...")
fmt.Fprintln(w, " run: scriptorium run [--config PATH] [--prompt-dir DIR] [--profile-dir DIR] --prompt ID [--prompt-version VERSION] [--input name=path] [--profile ID] [--session-id ID] [--llm-base-url URL] [--model NAME] [--api-key-env ENV] [--temperature N] [--max-tokens N] [--top-p N] [--reasoning-effort VALUE] [--var k=v] [--out path] [--timeout 10m]")
fmt.Fprintln(w, " render: scriptorium render [--config PATH] [--prompt-dir DIR] [--profile-dir DIR] --prompt ID [--prompt-version VERSION] [--input name=path] [--profile ID] [--session-id ID] [--llm-base-url URL] [--model NAME] [--api-key-env ENV] [--temperature N] [--max-tokens N] [--top-p N] [--reasoning-effort VALUE] [--var k=v] [--format text|json] [--out path] [--timeout 10m]")
fmt.Fprintln(w, " run: scriptorium run [--config PATH] [--prompt-dir DIR] [--profile-dir DIR] --prompt ID [--prompt-version VERSION] [--input name=path] [--profile ID] [--session-id ID] [--llm-base-url URL] [--model NAME] [--api-key-env ENV] [--temperature N] [--max-tokens N] [--top-p N] [--reasoning-effort VALUE] [--var k=v] [--out path] [--timeout 10m] [--transient-retries N]")
fmt.Fprintln(w, " render: scriptorium render [--config PATH] [--prompt-dir DIR] [--profile-dir DIR] --prompt ID [--prompt-version VERSION] [--input name=path] [--profile ID] [--session-id ID] [--llm-base-url URL] [--model NAME] [--api-key-env ENV] [--temperature N] [--max-tokens N] [--top-p N] [--reasoning-effort VALUE] [--var k=v] [--format text|json] [--out path] [--timeout 10m] [--transient-retries N]")
fmt.Fprintf(w, " serve: scriptorium serve [--config PATH] [--addr %s] [--prompt-dir DIR] [--profile-dir DIR] [--schema-dir DIR] [--artifact-root DIR] [--max-request-bytes N] [--max-artifact-bytes N] [--max-response-bytes N]\n", defaults.HTTPAddrDefault)
fmt.Fprintln(w, " inspect prompt: scriptorium inspect prompt --prompt ID [--prompt-version VERSION] [--config PATH] [--prompt-dir DIR] [--format text|json] [--out path]")
fmt.Fprintln(w, " inspect profile: scriptorium inspect profile --profile ID [--config PATH] [--profile-dir DIR] [--format text|json] [--out path]")

75
internal/adapter/cli/run_test.go
View File

@@ -1258,6 +1258,81 @@ func TestPromptkitDefinitionsRenderThroughCLI(t *testing.T) {
}
}
func TestTransientRetrySettingsAndCLIOverride(t *testing.T) {
lib := newCLITestLibrary(t)
inputPath := lib.writeInputFile(t, "transcript.md", "hello")
writePromptFile(t, lib.promptDir, "retry.prompt", "local-default")
var hits int32
provider := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if atomic.AddInt32(&hits, 1) == 1 {
http.Error(w, "temporary", http.StatusServiceUnavailable)
return
}
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{"choices":[{"message":{"role":"assistant","content":"recovered"}}]}`))
}))
defer provider.Close()
writeProfileFile(t, lib.profileDir, "local-default", provider.URL+"/v1", "local-model")
configPath := writeAppConfigFile(t, fmt.Sprintf("prompt_dir: %q\nprofile_dir: %q\ntransient_retries: 1\n", lib.promptDir, lib.profileDir))
args := []string{"--config", configPath, "--prompt", "retry.prompt", "--input", "transcript=" + inputPath}
serve, err := parseServeArgs([]string{"--config", configPath})
if err != nil || serve.transientRetries == nil || *serve.transientRetries != 1 {
t.Fatalf("server did not inherit retry setting: cfg=%+v err=%v", serve, err)
}
if _, err := newEngine(serve.engineSettings()); err != nil {
t.Fatalf("server engine configuration: %v", err)
}
code, stdout, stderr := runCLICommand(t, runCommand, args)
if code != ExitOK || stdout != "recovered" || atomic.LoadInt32(&hits) != 2 {
t.Fatalf("configured retry was not applied: code=%d stdout=%q stderr=%q hits=%d", code, stdout, stderr, atomic.LoadInt32(&hits))
}
atomic.StoreInt32(&hits, 0)
code, stdout, stderr = runCLICommand(t, runCommand, append(append([]string{}, args...), "--transient-retries", "0"))
if code != ExitRuntimeError || stdout != "" || atomic.LoadInt32(&hits) != 1 {
t.Fatalf("explicit zero did not override configuration: code=%d stdout=%q stderr=%q hits=%d", code, stdout, stderr, atomic.LoadInt32(&hits))
}
for _, tc := range []struct {
name string
args []string
want *int
}{
{name: "omitted", args: args},
{name: "zero", args: append(append([]string{}, args...), "--transient-retries", "0"), want: intPointer(0)},
{name: "positive", args: append(append([]string{}, args...), "--transient-retries", "3"), want: intPointer(3)},
} {
t.Run(tc.name, func(t *testing.T) {
cfg, err := parseRunArgs(tc.args)
if err != nil {
t.Fatal(err)
}
req, err := buildRunRequestFromConfig(cfg)
if err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(req.TransientRetries, tc.want) {
t.Fatalf("request retry override: got %v, want %v", req.TransientRetries, tc.want)
}
})
}
for _, invalid := range []string{"-1", "wrong"} {
if _, err := parseRunArgs(append(append([]string{}, args...), "--transient-retries", invalid)); err == nil {
t.Fatalf("run accepted invalid retry count %q", invalid)
}
if _, err := parseRenderArgs(append(append([]string{}, args...), "--transient-retries", invalid)); err == nil {
t.Fatalf("render accepted invalid retry count %q", invalid)
}
}
code, stdout, stderr = runCLICommand(t, renderCommand, append(append([]string{}, args...), "--transient-retries", "0", "--format", "json"))
if code != ExitOK || strings.Contains(stdout, "transient_retries") {
t.Fatalf("render retry override changed prepared output: code=%d stdout=%q stderr=%q", code, stdout, stderr)
}
}
func intPointer(value int) *int { return &value }
func TestPromptkitPromptInspectionSelectsVersionsAndContracts(t *testing.T) {
fixtureRoot := definitionFixtureRoot(t)
configPath := writeDefinitionFixtureConfig(t, fixtureRoot, true)

35
internal/config/config.go
View File

@@ -33,12 +33,13 @@ var (
// Config is the on-disk YAML shape for application-level settings.
type Config struct {
PromptDir string `yaml:"prompt_dir"`
ProfileDir string `yaml:"profile_dir"`
SchemaDir string `yaml:"schema_dir"`
Server ServerConfig `yaml:"server"`
Defaults DefaultsConfig `yaml:"defaults"`
Backends map[string]BackendConfig `yaml:"backends"`
PromptDir string `yaml:"prompt_dir"`
ProfileDir string `yaml:"profile_dir"`
SchemaDir string `yaml:"schema_dir"`
TransientRetries *strictInteger `yaml:"transient_retries"`
Server ServerConfig `yaml:"server"`
Defaults DefaultsConfig `yaml:"defaults"`
Backends map[string]BackendConfig `yaml:"backends"`
}
type BackendConfig struct {
@@ -49,6 +50,20 @@ type BackendConfig struct {
QueueCapacity *int `yaml:"queue_capacity"`
}
type strictInteger int
func (value *strictInteger) UnmarshalYAML(node *yaml.Node) error {
if node.Tag != "!!int" {
return fmt.Errorf("expected integer, got %s", node.Tag)
}
var decoded int
if err := node.Decode(&decoded); err != nil {
return err
}
*value = strictInteger(decoded)
return nil
}
type BackendSettings struct {
ID string
Endpoint string
@@ -82,6 +97,7 @@ type AppSettings struct {
MaxResponseBytes int64
DefaultRenderFormat renderformat.PreparedRunOutputFormat
Backends []BackendSettings
TransientRetries *int
}
// CLIOverrides can be applied after config load to enforce precedence.
@@ -224,6 +240,13 @@ func decodeConfig(raw []byte) (Config, error) {
func applyConfig(base AppSettings, cfg Config) (AppSettings, error) {
out := base
if cfg.TransientRetries != nil {
if *cfg.TransientRetries < 0 {
return AppSettings{}, fmt.Errorf("%w: transient_retries must be greater than or equal to 0", ErrInvalidConfig)
}
value := int(*cfg.TransientRetries)
out.TransientRetries = &value
}
if v := strings.TrimSpace(cfg.PromptDir); v != "" {
out.PromptDir = filepath.Clean(v)

38
internal/config/config_test.go
View File

@@ -239,6 +239,44 @@ server:
}
}
func TestLoadConfigPreservesTransientRetryPresence(t *testing.T) {
for _, tc := range []struct {
name string
body string
want *int
}{
{name: "omitted", body: "prompt_dir: ./prompts\n"},
{name: "null", body: "transient_retries: null\n"},
{name: "zero", body: "transient_retries: 0\n", want: intPointer(0)},
{name: "positive", body: "transient_retries: 3\n", want: intPointer(3)},
} {
t.Run(tc.name, func(t *testing.T) {
settings, err := LoadConfig(writeConfigFile(t, "config.yml", tc.body), true)
if err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(settings.TransientRetries, tc.want) {
t.Fatalf("retry setting: got %v, want %v", settings.TransientRetries, tc.want)
}
})
}
for _, tc := range []struct {
body string
want error
}{
{body: "transient_retries: -1\n", want: ErrInvalidConfig},
{body: "transient_retries: wrong\n", want: ErrInvalidConfigYAML},
{body: "transient_retries: 1.5\n", want: ErrInvalidConfigYAML},
} {
_, err := LoadConfig(writeConfigFile(t, "config.yml", tc.body), true)
if !errors.Is(err, tc.want) {
t.Fatalf("body %q: expected %v, got %v", tc.body, tc.want, err)
}
}
}
func intPointer(value int) *int { return &value }
func TestLoadConfigRejectsNegativeHTTPSizeLimits(t *testing.T) {
tests := []struct {
name string