Add a whole-run deadline to CLI execution
This commit is contained in:
@@ -64,6 +64,7 @@ Optional flags:
|
|||||||
| `--top-p <float>` | Runtime top-p override. |
|
| `--top-p <float>` | Runtime top-p override. |
|
||||||
| `--reasoning-effort <value>` | Runtime reasoning-effort override. |
|
| `--reasoning-effort <value>` | Runtime reasoning-effort override. |
|
||||||
| `--timeout <duration>` | Runtime timeout override using Go duration syntax. |
|
| `--timeout <duration>` | Runtime timeout override using Go duration syntax. |
|
||||||
|
| `--run-timeout <duration>` | Optional whole-run engine deadline using Go duration syntax; zero or omission adds no deadline, negative values are invalid. |
|
||||||
| `--transient-retries <integer>` | Nonnegative additional transient attempt count for this request; `0` disables additional attempts. |
|
| `--transient-retries <integer>` | Nonnegative additional transient attempt count for this request; `0` disables additional attempts. |
|
||||||
|
|
||||||
Deprecated aliases: `--prompt-id` for `--prompt`, and `--profile-id` for
|
Deprecated aliases: `--prompt-id` for `--prompt`, and `--profile-id` for
|
||||||
@@ -78,6 +79,13 @@ zero-second override. The timeout layers are defined in the
|
|||||||
[Promptkit outbound integration contract](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.11.0/docs/integrations/openai-compatible-chat.md#timeout-and-cancellation).
|
[Promptkit outbound integration contract](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.11.0/docs/integrations/openai-compatible-chat.md#timeout-and-cancellation).
|
||||||
With the built-in client, each transient retry gets a fresh generation timeout;
|
With the built-in client, each transient retry gets a fresh generation timeout;
|
||||||
this flag does not limit the complete run.
|
this flag does not limit the complete run.
|
||||||
|
`--run-timeout` keeps subsecond precision and starts immediately before engine
|
||||||
|
execution. It bounds preparation, capacity waits, retry waits, generation, and
|
||||||
|
repairs where they honor context. Parsing, engine construction, and output
|
||||||
|
writes are outside that interval. A failed run whose execution context reaches
|
||||||
|
this deadline exits 1 with a safe deadline diagnostic; a completed invalid
|
||||||
|
output still exits 2. It does not promise immediate interruption of synchronous
|
||||||
|
work.
|
||||||
Omitting `--transient-retries` inherits the application setting or Promptkit's
|
Omitting `--transient-retries` inherits the application setting or Promptkit's
|
||||||
current default of two additional attempts. The flag overrides only this
|
current default of two additional attempts. The flag overrides only this
|
||||||
request; it does not alter the configured engine. Structural repair generations
|
request; it does not alter the configured engine. Structural repair generations
|
||||||
|
|||||||
@@ -37,5 +37,11 @@ process according to its own process-management policy. A later invocation is a
|
|||||||
new request and can make another model call; there is no resume or checkpoint
|
new request and can make another model call; there is no resume or checkpoint
|
||||||
protocol.
|
protocol.
|
||||||
|
|
||||||
|
For `run`, callers may set the [whole-run deadline](../cli.md) to bound the
|
||||||
|
engine invocation. It is separate from the per-attempt generation timeout and
|
||||||
|
does not include child startup or output writing. A deadline failure exits 1;
|
||||||
|
supervisors still own process termination when they need to stop the child
|
||||||
|
outside that engine interval.
|
||||||
|
|
||||||
For deployment, filesystem permissions, and sensitive-artifact handling, see
|
For deployment, filesystem permissions, and sensitive-artifact handling, see
|
||||||
the [operations guide](../operations.md).
|
the [operations guide](../operations.md).
|
||||||
|
|||||||
@@ -123,6 +123,12 @@ per attempt, while the caller context covers the complete operation. Provider
|
|||||||
[outbound integration contract](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.11.0/docs/integrations/openai-compatible-chat.md)
|
[outbound integration contract](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.11.0/docs/integrations/openai-compatible-chat.md)
|
||||||
for details.
|
for details.
|
||||||
|
|
||||||
|
CLI `run` can set a [whole-run deadline](cli.md) when total execution time must
|
||||||
|
be bounded. The existing generation timeout applies afresh to each attempt and
|
||||||
|
does not cap retry waits or the complete invocation. The whole-run interval
|
||||||
|
covers the engine call, including capacity waiting and repairs as supported by
|
||||||
|
context cancellation; it excludes parsing, engine setup, and output writes.
|
||||||
|
|
||||||
Before increasing a limit:
|
Before increasing a limit:
|
||||||
|
|
||||||
1. measure representative input, generated-output, and optional raw-output
|
1. measure representative input, generated-output, and optional raw-output
|
||||||
|
|||||||
@@ -176,7 +176,7 @@ body text exposes provider diagnostics or secrets.
|
|||||||
|
|
||||||
## Stage 4: Add The CLI Whole-Run Deadline
|
## Stage 4: Add The CLI Whole-Run Deadline
|
||||||
|
|
||||||
**Status: Pending.**
|
**Status: Complete.**
|
||||||
|
|
||||||
Selected interface: `run --run-timeout <Go duration>`. Omission or zero adds
|
Selected interface: `run --run-timeout <Go duration>`. Omission or zero adds
|
||||||
no deadline, a positive duration retains full duration precision, and a negative
|
no deadline, a positive duration retains full duration precision, and a negative
|
||||||
|
|||||||
@@ -51,6 +51,7 @@ type runConfig struct {
|
|||||||
schemaDir string
|
schemaDir string
|
||||||
backends []appconfig.BackendSettings
|
backends []appconfig.BackendSettings
|
||||||
timeout time.Duration
|
timeout time.Duration
|
||||||
|
runTimeout time.Duration
|
||||||
transientRetries int
|
transientRetries int
|
||||||
configuredTransientRetries *int
|
configuredTransientRetries *int
|
||||||
|
|
||||||
@@ -153,8 +154,19 @@ func runCommand(args []string, stdout, stderr io.Writer) int {
|
|||||||
return ExitRuntimeError
|
return ExitRuntimeError
|
||||||
}
|
}
|
||||||
|
|
||||||
res, runErr := engine.Run(context.Background(), req)
|
runContext := context.Background()
|
||||||
|
cancel := func() {}
|
||||||
|
if cfg.runTimeout > 0 {
|
||||||
|
runContext, cancel = context.WithTimeout(runContext, cfg.runTimeout)
|
||||||
|
}
|
||||||
|
res, runErr := engine.Run(runContext, req)
|
||||||
|
contextErr := runContext.Err()
|
||||||
|
cancel()
|
||||||
if runErr != nil {
|
if runErr != nil {
|
||||||
|
if errors.Is(contextErr, context.DeadlineExceeded) {
|
||||||
|
fmt.Fprintln(stderr, "run error: execution deadline exceeded")
|
||||||
|
return ExitRuntimeError
|
||||||
|
}
|
||||||
fmt.Fprintln(stderr, runErrorMessage(runErr))
|
fmt.Fprintln(stderr, runErrorMessage(runErr))
|
||||||
return ExitRuntimeError
|
return ExitRuntimeError
|
||||||
}
|
}
|
||||||
@@ -251,6 +263,7 @@ func parseRunArgs(args []string) (*runConfig, error) {
|
|||||||
registerConfigPathFlag(fs, &cfg.configPath)
|
registerConfigPathFlag(fs, &cfg.configPath)
|
||||||
registerExecutionRequestFlags(fs, cfg)
|
registerExecutionRequestFlags(fs, cfg)
|
||||||
fs.StringVar(&cfg.schemaDir, "schema-dir", "", "base directory for validation schemas")
|
fs.StringVar(&cfg.schemaDir, "schema-dir", "", "base directory for validation schemas")
|
||||||
|
fs.DurationVar(&cfg.runTimeout, "run-timeout", 0, "whole-run execution deadline")
|
||||||
|
|
||||||
if err := fs.Parse(args); err != nil {
|
if err := fs.Parse(args); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
@@ -258,6 +271,9 @@ func parseRunArgs(args []string) (*runConfig, error) {
|
|||||||
if err := finalizeExecutionRequestConfig(fs, cfg); err != nil {
|
if err := finalizeExecutionRequestConfig(fs, cfg); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
if cfg.runTimeout < 0 {
|
||||||
|
return nil, errors.New("--run-timeout must be greater than or equal to 0")
|
||||||
|
}
|
||||||
cfg.schemaDir = filepath.Clean(cfg.schemaDir)
|
cfg.schemaDir = filepath.Clean(cfg.schemaDir)
|
||||||
|
|
||||||
return cfg, nil
|
return cfg, nil
|
||||||
@@ -773,7 +789,7 @@ func runErrorMessage(err error) string {
|
|||||||
|
|
||||||
func printUsage(w io.Writer) {
|
func printUsage(w io.Writer) {
|
||||||
fmt.Fprintln(w, "usage: scriptorium <run|render|serve|inspect> ...")
|
fmt.Fprintln(w, "usage: scriptorium <run|render|serve|inspect> ...")
|
||||||
fmt.Fprintln(w, " run: scriptorium run [--config PATH] [--prompt-dir DIR] [--profile-dir DIR] --prompt ID [--prompt-version VERSION] [--input name=path] [--profile ID] [--session-id ID] [--llm-base-url URL] [--model NAME] [--api-key-env ENV] [--temperature N] [--max-tokens N] [--top-p N] [--reasoning-effort VALUE] [--var k=v] [--out path] [--timeout 10m] [--transient-retries N]")
|
fmt.Fprintln(w, " run: scriptorium run [--config PATH] [--prompt-dir DIR] [--profile-dir DIR] --prompt ID [--prompt-version VERSION] [--input name=path] [--profile ID] [--session-id ID] [--llm-base-url URL] [--model NAME] [--api-key-env ENV] [--temperature N] [--max-tokens N] [--top-p N] [--reasoning-effort VALUE] [--var k=v] [--out path] [--timeout 10m] [--run-timeout 10m] [--transient-retries N]")
|
||||||
fmt.Fprintln(w, " render: scriptorium render [--config PATH] [--prompt-dir DIR] [--profile-dir DIR] --prompt ID [--prompt-version VERSION] [--input name=path] [--profile ID] [--session-id ID] [--llm-base-url URL] [--model NAME] [--api-key-env ENV] [--temperature N] [--max-tokens N] [--top-p N] [--reasoning-effort VALUE] [--var k=v] [--format text|json] [--out path] [--timeout 10m] [--transient-retries N]")
|
fmt.Fprintln(w, " render: scriptorium render [--config PATH] [--prompt-dir DIR] [--profile-dir DIR] --prompt ID [--prompt-version VERSION] [--input name=path] [--profile ID] [--session-id ID] [--llm-base-url URL] [--model NAME] [--api-key-env ENV] [--temperature N] [--max-tokens N] [--top-p N] [--reasoning-effort VALUE] [--var k=v] [--format text|json] [--out path] [--timeout 10m] [--transient-retries N]")
|
||||||
fmt.Fprintf(w, " serve: scriptorium serve [--config PATH] [--addr %s] [--prompt-dir DIR] [--profile-dir DIR] [--schema-dir DIR] [--artifact-root DIR] [--max-request-bytes N] [--max-artifact-bytes N] [--max-response-bytes N]\n", defaults.HTTPAddrDefault)
|
fmt.Fprintf(w, " serve: scriptorium serve [--config PATH] [--addr %s] [--prompt-dir DIR] [--profile-dir DIR] [--schema-dir DIR] [--artifact-root DIR] [--max-request-bytes N] [--max-artifact-bytes N] [--max-response-bytes N]\n", defaults.HTTPAddrDefault)
|
||||||
fmt.Fprintln(w, " inspect prompt: scriptorium inspect prompt --prompt ID [--prompt-version VERSION] [--config PATH] [--prompt-dir DIR] [--format text|json] [--out path]")
|
fmt.Fprintln(w, " inspect prompt: scriptorium inspect prompt --prompt ID [--prompt-version VERSION] [--config PATH] [--prompt-dir DIR] [--format text|json] [--out path]")
|
||||||
|
|||||||
@@ -1331,6 +1331,54 @@ func TestTransientRetrySettingsAndCLIOverride(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestRunTimeoutParsingAndCancellation(t *testing.T) {
|
||||||
|
lib := newCLITestLibrary(t)
|
||||||
|
inputPath := lib.writeInputFile(t, "transcript.md", "hello")
|
||||||
|
args := []string{"--prompt-dir", lib.promptDir, "--profile-dir", lib.profileDir, "--prompt", "timed", "--input", "transcript=" + inputPath}
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name string
|
||||||
|
flag []string
|
||||||
|
want time.Duration
|
||||||
|
}{
|
||||||
|
{name: "omitted"},
|
||||||
|
{name: "zero", flag: []string{"--run-timeout", "0s"}},
|
||||||
|
{name: "subsecond", flag: []string{"--run-timeout", "125ms"}, want: 125 * time.Millisecond},
|
||||||
|
} {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
cfg, err := parseRunArgs(append(append([]string{}, args...), tc.flag...))
|
||||||
|
if err != nil || cfg.runTimeout != tc.want {
|
||||||
|
t.Fatalf("run timeout: cfg=%+v err=%v", cfg, err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
if _, err := parseRunArgs(append(append([]string{}, args...), "--run-timeout", "-1ms")); err == nil {
|
||||||
|
t.Fatal("negative run timeout accepted")
|
||||||
|
}
|
||||||
|
if _, err := parseRenderArgs(append(append([]string{}, args...), "--run-timeout", "1s")); err == nil {
|
||||||
|
t.Fatal("render accepted run-only timeout")
|
||||||
|
}
|
||||||
|
|
||||||
|
started := make(chan struct{}, 1)
|
||||||
|
release := make(chan struct{})
|
||||||
|
provider := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
started <- struct{}{}
|
||||||
|
<-release
|
||||||
|
}))
|
||||||
|
defer provider.Close()
|
||||||
|
defer close(release)
|
||||||
|
writePromptFile(t, lib.promptDir, "timed", "local-default")
|
||||||
|
writeProfileFile(t, lib.profileDir, "local-default", provider.URL+"/v1", "local-model")
|
||||||
|
code, stdout, stderr := runCLICommand(t, runCommand, append(append([]string{}, args...), "--run-timeout", "500ms", "--transient-retries", "0"))
|
||||||
|
if code != ExitRuntimeError || stdout != "" || !strings.Contains(stderr, "execution deadline exceeded") {
|
||||||
|
t.Fatalf("deadline outcome: code=%d stdout=%q stderr=%q", code, stdout, stderr)
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case <-started:
|
||||||
|
default:
|
||||||
|
t.Fatal("provider call did not start before deadline")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func intPointer(value int) *int { return &value }
|
func intPointer(value int) *int { return &value }
|
||||||
|
|
||||||
func TestPromptkitPromptInspectionSelectsVersionsAndContracts(t *testing.T) {
|
func TestPromptkitPromptInspectionSelectsVersionsAndContracts(t *testing.T) {
|
||||||
|
|||||||
Reference in New Issue
Block a user