Add a whole-run deadline to CLI execution
This commit is contained in:
@@ -64,6 +64,7 @@ Optional flags:
|
||||
| `--top-p <float>` | Runtime top-p override. |
|
||||
| `--reasoning-effort <value>` | Runtime reasoning-effort override. |
|
||||
| `--timeout <duration>` | Runtime timeout override using Go duration syntax. |
|
||||
| `--run-timeout <duration>` | Optional whole-run engine deadline using Go duration syntax; zero or omission adds no deadline, negative values are invalid. |
|
||||
| `--transient-retries <integer>` | Nonnegative additional transient attempt count for this request; `0` disables additional attempts. |
|
||||
|
||||
Deprecated aliases: `--prompt-id` for `--prompt`, and `--profile-id` for
|
||||
@@ -78,6 +79,13 @@ zero-second override. The timeout layers are defined in the
|
||||
[Promptkit outbound integration contract](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.11.0/docs/integrations/openai-compatible-chat.md#timeout-and-cancellation).
|
||||
With the built-in client, each transient retry gets a fresh generation timeout;
|
||||
this flag does not limit the complete run.
|
||||
`--run-timeout` keeps subsecond precision and starts immediately before engine
|
||||
execution. It bounds preparation, capacity waits, retry waits, generation, and
|
||||
repairs where they honor context. Parsing, engine construction, and output
|
||||
writes are outside that interval. A failed run whose execution context reaches
|
||||
this deadline exits 1 with a safe deadline diagnostic; a completed invalid
|
||||
output still exits 2. It does not promise immediate interruption of synchronous
|
||||
work.
|
||||
Omitting `--transient-retries` inherits the application setting or Promptkit's
|
||||
current default of two additional attempts. The flag overrides only this
|
||||
request; it does not alter the configured engine. Structural repair generations
|
||||
|
||||
@@ -37,5 +37,11 @@ process according to its own process-management policy. A later invocation is a
|
||||
new request and can make another model call; there is no resume or checkpoint
|
||||
protocol.
|
||||
|
||||
For `run`, callers may set the [whole-run deadline](../cli.md) to bound the
|
||||
engine invocation. It is separate from the per-attempt generation timeout and
|
||||
does not include child startup or output writing. A deadline failure exits 1;
|
||||
supervisors still own process termination when they need to stop the child
|
||||
outside that engine interval.
|
||||
|
||||
For deployment, filesystem permissions, and sensitive-artifact handling, see
|
||||
the [operations guide](../operations.md).
|
||||
|
||||
@@ -123,6 +123,12 @@ per attempt, while the caller context covers the complete operation. Provider
|
||||
[outbound integration contract](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.11.0/docs/integrations/openai-compatible-chat.md)
|
||||
for details.
|
||||
|
||||
CLI `run` can set a [whole-run deadline](cli.md) when total execution time must
|
||||
be bounded. The existing generation timeout applies afresh to each attempt and
|
||||
does not cap retry waits or the complete invocation. The whole-run interval
|
||||
covers the engine call, including capacity waiting and repairs as supported by
|
||||
context cancellation; it excludes parsing, engine setup, and output writes.
|
||||
|
||||
Before increasing a limit:
|
||||
|
||||
1. measure representative input, generated-output, and optional raw-output
|
||||
|
||||
@@ -176,7 +176,7 @@ body text exposes provider diagnostics or secrets.
|
||||
|
||||
## Stage 4: Add The CLI Whole-Run Deadline
|
||||
|
||||
**Status: Pending.**
|
||||
**Status: Complete.**
|
||||
|
||||
Selected interface: `run --run-timeout <Go duration>`. Omission or zero adds
|
||||
no deadline, a positive duration retains full duration precision, and a negative
|
||||
|
||||
@@ -51,6 +51,7 @@ type runConfig struct {
|
||||
schemaDir string
|
||||
backends []appconfig.BackendSettings
|
||||
timeout time.Duration
|
||||
runTimeout time.Duration
|
||||
transientRetries int
|
||||
configuredTransientRetries *int
|
||||
|
||||
@@ -153,8 +154,19 @@ func runCommand(args []string, stdout, stderr io.Writer) int {
|
||||
return ExitRuntimeError
|
||||
}
|
||||
|
||||
res, runErr := engine.Run(context.Background(), req)
|
||||
runContext := context.Background()
|
||||
cancel := func() {}
|
||||
if cfg.runTimeout > 0 {
|
||||
runContext, cancel = context.WithTimeout(runContext, cfg.runTimeout)
|
||||
}
|
||||
res, runErr := engine.Run(runContext, req)
|
||||
contextErr := runContext.Err()
|
||||
cancel()
|
||||
if runErr != nil {
|
||||
if errors.Is(contextErr, context.DeadlineExceeded) {
|
||||
fmt.Fprintln(stderr, "run error: execution deadline exceeded")
|
||||
return ExitRuntimeError
|
||||
}
|
||||
fmt.Fprintln(stderr, runErrorMessage(runErr))
|
||||
return ExitRuntimeError
|
||||
}
|
||||
@@ -251,6 +263,7 @@ func parseRunArgs(args []string) (*runConfig, error) {
|
||||
registerConfigPathFlag(fs, &cfg.configPath)
|
||||
registerExecutionRequestFlags(fs, cfg)
|
||||
fs.StringVar(&cfg.schemaDir, "schema-dir", "", "base directory for validation schemas")
|
||||
fs.DurationVar(&cfg.runTimeout, "run-timeout", 0, "whole-run execution deadline")
|
||||
|
||||
if err := fs.Parse(args); err != nil {
|
||||
return nil, err
|
||||
@@ -258,6 +271,9 @@ func parseRunArgs(args []string) (*runConfig, error) {
|
||||
if err := finalizeExecutionRequestConfig(fs, cfg); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if cfg.runTimeout < 0 {
|
||||
return nil, errors.New("--run-timeout must be greater than or equal to 0")
|
||||
}
|
||||
cfg.schemaDir = filepath.Clean(cfg.schemaDir)
|
||||
|
||||
return cfg, nil
|
||||
@@ -773,7 +789,7 @@ func runErrorMessage(err error) string {
|
||||
|
||||
func printUsage(w io.Writer) {
|
||||
fmt.Fprintln(w, "usage: scriptorium <run|render|serve|inspect> ...")
|
||||
fmt.Fprintln(w, " run: scriptorium run [--config PATH] [--prompt-dir DIR] [--profile-dir DIR] --prompt ID [--prompt-version VERSION] [--input name=path] [--profile ID] [--session-id ID] [--llm-base-url URL] [--model NAME] [--api-key-env ENV] [--temperature N] [--max-tokens N] [--top-p N] [--reasoning-effort VALUE] [--var k=v] [--out path] [--timeout 10m] [--transient-retries N]")
|
||||
fmt.Fprintln(w, " run: scriptorium run [--config PATH] [--prompt-dir DIR] [--profile-dir DIR] --prompt ID [--prompt-version VERSION] [--input name=path] [--profile ID] [--session-id ID] [--llm-base-url URL] [--model NAME] [--api-key-env ENV] [--temperature N] [--max-tokens N] [--top-p N] [--reasoning-effort VALUE] [--var k=v] [--out path] [--timeout 10m] [--run-timeout 10m] [--transient-retries N]")
|
||||
fmt.Fprintln(w, " render: scriptorium render [--config PATH] [--prompt-dir DIR] [--profile-dir DIR] --prompt ID [--prompt-version VERSION] [--input name=path] [--profile ID] [--session-id ID] [--llm-base-url URL] [--model NAME] [--api-key-env ENV] [--temperature N] [--max-tokens N] [--top-p N] [--reasoning-effort VALUE] [--var k=v] [--format text|json] [--out path] [--timeout 10m] [--transient-retries N]")
|
||||
fmt.Fprintf(w, " serve: scriptorium serve [--config PATH] [--addr %s] [--prompt-dir DIR] [--profile-dir DIR] [--schema-dir DIR] [--artifact-root DIR] [--max-request-bytes N] [--max-artifact-bytes N] [--max-response-bytes N]\n", defaults.HTTPAddrDefault)
|
||||
fmt.Fprintln(w, " inspect prompt: scriptorium inspect prompt --prompt ID [--prompt-version VERSION] [--config PATH] [--prompt-dir DIR] [--format text|json] [--out path]")
|
||||
|
||||
@@ -1331,6 +1331,54 @@ func TestTransientRetrySettingsAndCLIOverride(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunTimeoutParsingAndCancellation(t *testing.T) {
|
||||
lib := newCLITestLibrary(t)
|
||||
inputPath := lib.writeInputFile(t, "transcript.md", "hello")
|
||||
args := []string{"--prompt-dir", lib.promptDir, "--profile-dir", lib.profileDir, "--prompt", "timed", "--input", "transcript=" + inputPath}
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
flag []string
|
||||
want time.Duration
|
||||
}{
|
||||
{name: "omitted"},
|
||||
{name: "zero", flag: []string{"--run-timeout", "0s"}},
|
||||
{name: "subsecond", flag: []string{"--run-timeout", "125ms"}, want: 125 * time.Millisecond},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
cfg, err := parseRunArgs(append(append([]string{}, args...), tc.flag...))
|
||||
if err != nil || cfg.runTimeout != tc.want {
|
||||
t.Fatalf("run timeout: cfg=%+v err=%v", cfg, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
if _, err := parseRunArgs(append(append([]string{}, args...), "--run-timeout", "-1ms")); err == nil {
|
||||
t.Fatal("negative run timeout accepted")
|
||||
}
|
||||
if _, err := parseRenderArgs(append(append([]string{}, args...), "--run-timeout", "1s")); err == nil {
|
||||
t.Fatal("render accepted run-only timeout")
|
||||
}
|
||||
|
||||
started := make(chan struct{}, 1)
|
||||
release := make(chan struct{})
|
||||
provider := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
started <- struct{}{}
|
||||
<-release
|
||||
}))
|
||||
defer provider.Close()
|
||||
defer close(release)
|
||||
writePromptFile(t, lib.promptDir, "timed", "local-default")
|
||||
writeProfileFile(t, lib.profileDir, "local-default", provider.URL+"/v1", "local-model")
|
||||
code, stdout, stderr := runCLICommand(t, runCommand, append(append([]string{}, args...), "--run-timeout", "500ms", "--transient-retries", "0"))
|
||||
if code != ExitRuntimeError || stdout != "" || !strings.Contains(stderr, "execution deadline exceeded") {
|
||||
t.Fatalf("deadline outcome: code=%d stdout=%q stderr=%q", code, stdout, stderr)
|
||||
}
|
||||
select {
|
||||
case <-started:
|
||||
default:
|
||||
t.Fatal("provider call did not start before deadline")
|
||||
}
|
||||
}
|
||||
|
||||
func intPointer(value int) *int { return &value }
|
||||
|
||||
func TestPromptkitPromptInspectionSelectsVersionsAndContracts(t *testing.T) {
|
||||
|
||||
Reference in New Issue
Block a user