From 911ca5485a646bffaa0dc47ebac43a7f6da5418b Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Sat, 26 Sep 2026 15:34:55 +0000 Subject: [PATCH] Add a whole-run deadline to CLI execution --- docs/cli.md | 8 ++++++ docs/integrations/subprocess.md | 6 ++++ docs/operations.md | 6 ++++ docs/roadmap/implementation.md | 2 +- internal/adapter/cli/run.go | 20 +++++++++++-- internal/adapter/cli/run_test.go | 48 ++++++++++++++++++++++++++++++++ 6 files changed, 87 insertions(+), 3 deletions(-) diff --git a/docs/cli.md b/docs/cli.md index d6c278b..b22e0b9 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -64,6 +64,7 @@ Optional flags: | `--top-p ` | Runtime top-p override. | | `--reasoning-effort ` | Runtime reasoning-effort override. | | `--timeout ` | Runtime timeout override using Go duration syntax. | +| `--run-timeout ` | Optional whole-run engine deadline using Go duration syntax; zero or omission adds no deadline, negative values are invalid. | | `--transient-retries ` | Nonnegative additional transient attempt count for this request; `0` disables additional attempts. | Deprecated aliases: `--prompt-id` for `--prompt`, and `--profile-id` for @@ -78,6 +79,13 @@ zero-second override. The timeout layers are defined in the [Promptkit outbound integration contract](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.11.0/docs/integrations/openai-compatible-chat.md#timeout-and-cancellation). With the built-in client, each transient retry gets a fresh generation timeout; this flag does not limit the complete run. +`--run-timeout` keeps subsecond precision and starts immediately before engine +execution. It bounds preparation, capacity waits, retry waits, generation, and +repairs where they honor context. Parsing, engine construction, and output +writes are outside that interval. A failed run whose execution context reaches +this deadline exits 1 with a safe deadline diagnostic; a completed invalid +output still exits 2. It does not promise immediate interruption of synchronous +work. Omitting `--transient-retries` inherits the application setting or Promptkit's current default of two additional attempts. The flag overrides only this request; it does not alter the configured engine. Structural repair generations diff --git a/docs/integrations/subprocess.md b/docs/integrations/subprocess.md index 3426caf..7b24b26 100644 --- a/docs/integrations/subprocess.md +++ b/docs/integrations/subprocess.md @@ -37,5 +37,11 @@ process according to its own process-management policy. A later invocation is a new request and can make another model call; there is no resume or checkpoint protocol. +For `run`, callers may set the [whole-run deadline](../cli.md) to bound the +engine invocation. It is separate from the per-attempt generation timeout and +does not include child startup or output writing. A deadline failure exits 1; +supervisors still own process termination when they need to stop the child +outside that engine interval. + For deployment, filesystem permissions, and sensitive-artifact handling, see the [operations guide](../operations.md). diff --git a/docs/operations.md b/docs/operations.md index 1209ddb..3c3bf1c 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -123,6 +123,12 @@ per attempt, while the caller context covers the complete operation. Provider [outbound integration contract](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.11.0/docs/integrations/openai-compatible-chat.md) for details. +CLI `run` can set a [whole-run deadline](cli.md) when total execution time must +be bounded. The existing generation timeout applies afresh to each attempt and +does not cap retry waits or the complete invocation. The whole-run interval +covers the engine call, including capacity waiting and repairs as supported by +context cancellation; it excludes parsing, engine setup, and output writes. + Before increasing a limit: 1. measure representative input, generated-output, and optional raw-output diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index 517e905..5162f48 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -176,7 +176,7 @@ body text exposes provider diagnostics or secrets. ## Stage 4: Add The CLI Whole-Run Deadline -**Status: Pending.** +**Status: Complete.** Selected interface: `run --run-timeout `. Omission or zero adds no deadline, a positive duration retains full duration precision, and a negative diff --git a/internal/adapter/cli/run.go b/internal/adapter/cli/run.go index f0dceea..b634b42 100644 --- a/internal/adapter/cli/run.go +++ b/internal/adapter/cli/run.go @@ -51,6 +51,7 @@ type runConfig struct { schemaDir string backends []appconfig.BackendSettings timeout time.Duration + runTimeout time.Duration transientRetries int configuredTransientRetries *int @@ -153,8 +154,19 @@ func runCommand(args []string, stdout, stderr io.Writer) int { return ExitRuntimeError } - res, runErr := engine.Run(context.Background(), req) + runContext := context.Background() + cancel := func() {} + if cfg.runTimeout > 0 { + runContext, cancel = context.WithTimeout(runContext, cfg.runTimeout) + } + res, runErr := engine.Run(runContext, req) + contextErr := runContext.Err() + cancel() if runErr != nil { + if errors.Is(contextErr, context.DeadlineExceeded) { + fmt.Fprintln(stderr, "run error: execution deadline exceeded") + return ExitRuntimeError + } fmt.Fprintln(stderr, runErrorMessage(runErr)) return ExitRuntimeError } @@ -251,6 +263,7 @@ func parseRunArgs(args []string) (*runConfig, error) { registerConfigPathFlag(fs, &cfg.configPath) registerExecutionRequestFlags(fs, cfg) fs.StringVar(&cfg.schemaDir, "schema-dir", "", "base directory for validation schemas") + fs.DurationVar(&cfg.runTimeout, "run-timeout", 0, "whole-run execution deadline") if err := fs.Parse(args); err != nil { return nil, err @@ -258,6 +271,9 @@ func parseRunArgs(args []string) (*runConfig, error) { if err := finalizeExecutionRequestConfig(fs, cfg); err != nil { return nil, err } + if cfg.runTimeout < 0 { + return nil, errors.New("--run-timeout must be greater than or equal to 0") + } cfg.schemaDir = filepath.Clean(cfg.schemaDir) return cfg, nil @@ -773,7 +789,7 @@ func runErrorMessage(err error) string { func printUsage(w io.Writer) { fmt.Fprintln(w, "usage: scriptorium ...") - fmt.Fprintln(w, " run: scriptorium run [--config PATH] [--prompt-dir DIR] [--profile-dir DIR] --prompt ID [--prompt-version VERSION] [--input name=path] [--profile ID] [--session-id ID] [--llm-base-url URL] [--model NAME] [--api-key-env ENV] [--temperature N] [--max-tokens N] [--top-p N] [--reasoning-effort VALUE] [--var k=v] [--out path] [--timeout 10m] [--transient-retries N]") + fmt.Fprintln(w, " run: scriptorium run [--config PATH] [--prompt-dir DIR] [--profile-dir DIR] --prompt ID [--prompt-version VERSION] [--input name=path] [--profile ID] [--session-id ID] [--llm-base-url URL] [--model NAME] [--api-key-env ENV] [--temperature N] [--max-tokens N] [--top-p N] [--reasoning-effort VALUE] [--var k=v] [--out path] [--timeout 10m] [--run-timeout 10m] [--transient-retries N]") fmt.Fprintln(w, " render: scriptorium render [--config PATH] [--prompt-dir DIR] [--profile-dir DIR] --prompt ID [--prompt-version VERSION] [--input name=path] [--profile ID] [--session-id ID] [--llm-base-url URL] [--model NAME] [--api-key-env ENV] [--temperature N] [--max-tokens N] [--top-p N] [--reasoning-effort VALUE] [--var k=v] [--format text|json] [--out path] [--timeout 10m] [--transient-retries N]") fmt.Fprintf(w, " serve: scriptorium serve [--config PATH] [--addr %s] [--prompt-dir DIR] [--profile-dir DIR] [--schema-dir DIR] [--artifact-root DIR] [--max-request-bytes N] [--max-artifact-bytes N] [--max-response-bytes N]\n", defaults.HTTPAddrDefault) fmt.Fprintln(w, " inspect prompt: scriptorium inspect prompt --prompt ID [--prompt-version VERSION] [--config PATH] [--prompt-dir DIR] [--format text|json] [--out path]") diff --git a/internal/adapter/cli/run_test.go b/internal/adapter/cli/run_test.go index 08856cd..3d1f8b6 100644 --- a/internal/adapter/cli/run_test.go +++ b/internal/adapter/cli/run_test.go @@ -1331,6 +1331,54 @@ func TestTransientRetrySettingsAndCLIOverride(t *testing.T) { } } +func TestRunTimeoutParsingAndCancellation(t *testing.T) { + lib := newCLITestLibrary(t) + inputPath := lib.writeInputFile(t, "transcript.md", "hello") + args := []string{"--prompt-dir", lib.promptDir, "--profile-dir", lib.profileDir, "--prompt", "timed", "--input", "transcript=" + inputPath} + for _, tc := range []struct { + name string + flag []string + want time.Duration + }{ + {name: "omitted"}, + {name: "zero", flag: []string{"--run-timeout", "0s"}}, + {name: "subsecond", flag: []string{"--run-timeout", "125ms"}, want: 125 * time.Millisecond}, + } { + t.Run(tc.name, func(t *testing.T) { + cfg, err := parseRunArgs(append(append([]string{}, args...), tc.flag...)) + if err != nil || cfg.runTimeout != tc.want { + t.Fatalf("run timeout: cfg=%+v err=%v", cfg, err) + } + }) + } + if _, err := parseRunArgs(append(append([]string{}, args...), "--run-timeout", "-1ms")); err == nil { + t.Fatal("negative run timeout accepted") + } + if _, err := parseRenderArgs(append(append([]string{}, args...), "--run-timeout", "1s")); err == nil { + t.Fatal("render accepted run-only timeout") + } + + started := make(chan struct{}, 1) + release := make(chan struct{}) + provider := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + started <- struct{}{} + <-release + })) + defer provider.Close() + defer close(release) + writePromptFile(t, lib.promptDir, "timed", "local-default") + writeProfileFile(t, lib.profileDir, "local-default", provider.URL+"/v1", "local-model") + code, stdout, stderr := runCLICommand(t, runCommand, append(append([]string{}, args...), "--run-timeout", "500ms", "--transient-retries", "0")) + if code != ExitRuntimeError || stdout != "" || !strings.Contains(stderr, "execution deadline exceeded") { + t.Fatalf("deadline outcome: code=%d stdout=%q stderr=%q", code, stdout, stderr) + } + select { + case <-started: + default: + t.Fatal("provider call did not start before deadline") + } +} + func intPointer(value int) *int { return &value } func TestPromptkitPromptInspectionSelectsVersionsAndContracts(t *testing.T) {