From edc1f8abd9a4d10719aef4ea4c983566196a0a9e Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Sat, 26 Sep 2026 15:47:26 +0000 Subject: [PATCH] Add opt-in CLI execution report files --- docs/cli.md | 46 +++++++++ docs/integrations/subprocess.md | 7 ++ docs/internal/adapters.md | 4 + docs/operations.md | 5 + docs/roadmap/implementation.md | 2 +- internal/adapter/cli/run.go | 80 ++++++++++++++-- internal/adapter/cli/run_test.go | 159 +++++++++++++++++++++++++++++++ 7 files changed, 295 insertions(+), 8 deletions(-) diff --git a/docs/cli.md b/docs/cli.md index b22e0b9..7adb077 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -56,6 +56,7 @@ Optional flags: | `--input name=path` | Optional input file mapping; repeat or use comma-separated mappings. | | `--var name=value` | Template-variable mapping; repeat or use comma-separated mappings. | | `--out ` | Write generated content to this file instead of stdout. | +| `--execution-report ` | Write an opt-in terminal execution report as newline-terminated JSON to a separate file. | | `--llm-base-url ` | Runtime endpoint override. | | `--model ` | Runtime model override. | | `--api-key-env ` | Runtime API-key environment-variable name override. | @@ -110,6 +111,51 @@ profile value, a nonblank value replaces that value, and `--reasoning-effort=` explicitly clears inherited reasoning. Promptkit treats nonblank values as provider-specific opaque strings. +### Execution report file + +`--execution-report` requires a nonempty path. It is available only for `run`; +`-` is an ordinary filename. Parent directories are not created. An existing +file is replaced through ordinary file writing, without rollback or an atomic +replacement guarantee. Generated content still goes to stdout or `--out`. +Equal output/report paths, including existing paths that refer to the same +file, are rejected before execution. + +The file is written after engine execution when Promptkit supplies a terminal +event, including for a failed run or invalid generated output. A failure before +invocation leaves the report path untouched. If no terminal event is supplied, +Scriptorium leaves the path untouched and emits a diagnostic; an existing file +may therefore contain an older report. Artifact output is attempted first on a +successful run, and report writing is attempted even if artifact writing fails. +Any output or report write failure exits 1; already-written output is not +undone. Without a write failure, the actual run keeps its usual exit status. + +The JSON object contains the following fields: + +| Field | Meaning | +| --- | --- | +| `run_id`, `prompt_id`, `prompt_version`, `profile_id`, `backend_id`, `model` | Terminal identifiers; absent values are omitted. | +| `outcome` | `success`, `invalid_output`, `error`, or `other` for an unrecognized upstream value. | +| `failure` | The known terminal failure category, `none`, or `other` for an unrecognized value. | +| `status_code` | Provider status, omitted when unavailable. | +| `retries_exhausted` | Boolean independent of the failure category. | +| `attempts`, `transient_retries`, `repair_calls` | Unsigned terminal aggregate counts. | +| `duration_ms`, `capacity_wait_ms`, `generation_duration_ms`, `retry_delay_ms` | Terminal observed durations in integer milliseconds; their sum need not equal wall time. | +| `usage` | Object containing `prompt_tokens`, `completion_tokens`, `total_tokens`, `cached_tokens`, and `cache_write_tokens`. | + +Each `usage` count contains `value` (integer), `known` (boolean), and +`complete` (boolean). Reported zero differs from an unknown count; an +incomplete total does not establish complete billing. All counts, durations, +`retries_exhausted`, and `usage` are present even when zero or unknown. The +recognized failure values are `none`, `canceled`, `deadline`, `transport`, +`provider_status`, `malformed_response`, `capacity`, `invalid_config`, +`invalid_request`, `prompt_not_found`, `profile_not_found`, `prompt_load`, +`profile_load`, `artifact_load`, `prompt_render`, `validation`, `generation`, +and `other`. + +The report excludes content, credentials, URLs, headers, and raw errors, but +its identifiers can still be sensitive. The caller owns file retention and +access. + ## `scriptorium render` ```text diff --git a/docs/integrations/subprocess.md b/docs/integrations/subprocess.md index 7b24b26..6a0e3f8 100644 --- a/docs/integrations/subprocess.md +++ b/docs/integrations/subprocess.md @@ -29,6 +29,13 @@ When using `--out`, the caller owns the output path, its permissions, and cleanup. Treat rendered prompts, generated artifacts, stdout, and stderr as potentially sensitive. +For `run`, an optional [execution report file](../cli.md) contains terminal +metadata separate from generated stdout or `--out`. The caller owns that +path's permissions and retention. Output and report files are not a +transaction: one may be written when the other fails, and a preexisting report +may remain when the invocation has no terminal event. Check the exit status +and diagnostics rather than assuming a report path contains this run's data. + ## Cancellation And Recovery A CLI invocation performs one synchronous request and creates no durable run diff --git a/docs/internal/adapters.md b/docs/internal/adapters.md index 8d818a4..8289645 100644 --- a/docs/internal/adapters.md +++ b/docs/internal/adapters.md @@ -77,6 +77,10 @@ The report collector is per invocation. It ignores nonterminal events and copies terminal aggregates once, including usage presence and completeness. It retains no history or shared state; adapters decide whether to attach it and how to present its snapshot. +CLI `run` attaches one only when a report path is selected, writes generated +output first on success, and then writes the terminal snapshot when present. +Its report path remains untouched for pre-invocation failures or absent +terminal observations; report and artifact writes are independent. ## Package-Local Guarantees diff --git a/docs/operations.md b/docs/operations.md index 0c04ec7..12af8aa 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -26,6 +26,11 @@ configuration, or environment and submitting a new request. Generated artifacts, rendered prompts, model output, and run metadata are caller-owned data. Retention, encryption, backup, and deletion are deployment responsibilities. +Optional [CLI execution reports](cli.md) also belong to the caller. They can +contain sensitive identifiers even though they omit prompt and response bodies. +Keep report paths separate from generated output and apply the same access and +retention controls. Output and report writes are not transactional; an older +report may remain when no terminal observation was produced. ## Deploy The Filesystem And Process diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index f09b51d..2eedcaa 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -302,7 +302,7 @@ no new orchestration layer or durable state. ## Stage 7: Expose CLI Execution Report Files -**Status: Pending.** +**Status: Complete.** Selected interface: run-only `--execution-report `, writing one newline-terminated JSON report to a separate file. A nonempty path opts in; diff --git a/internal/adapter/cli/run.go b/internal/adapter/cli/run.go index b634b42..f15ea09 100644 --- a/internal/adapter/cli/run.go +++ b/internal/adapter/cli/run.go @@ -2,6 +2,7 @@ package cli import ( "context" + "encoding/json" "errors" "flag" "fmt" @@ -17,6 +18,7 @@ import ( appconfig "gitea.maximumdirect.net/eric/scriptorium/internal/config" "gitea.maximumdirect.net/eric/scriptorium/internal/defaults" renderformat "gitea.maximumdirect.net/eric/scriptorium/internal/format" + executionreport "gitea.maximumdirect.net/eric/scriptorium/internal/report" ) const ( @@ -41,6 +43,7 @@ type runConfig struct { inputRaw listFlag varRaw listFlag outputPath string + executionReportPath string llmBaseURL string apiKeyEnv string model string @@ -153,26 +156,48 @@ func runCommand(args []string, stdout, stderr io.Writer) int { fmt.Fprintf(stderr, "engine error: %v\n", err) return ExitRuntimeError } + return executeRun(cfg, req, engine.Run, stdout, stderr) +} +func executeRun(cfg *runConfig, req promptkit.RunRequest, run func(context.Context, promptkit.RunRequest) (*promptkit.RunResult, error), stdout, stderr io.Writer) int { + var collector *executionreport.Collector + if cfg.executionReportPath != "" { + collector = executionreport.NewCollector() + req.Observer = collector.Observe + } runContext := context.Background() cancel := func() {} if cfg.runTimeout > 0 { runContext, cancel = context.WithTimeout(runContext, cfg.runTimeout) } - res, runErr := engine.Run(runContext, req) + res, runErr := run(runContext, req) contextErr := runContext.Err() cancel() if runErr != nil { if errors.Is(contextErr, context.DeadlineExceeded) { fmt.Fprintln(stderr, "run error: execution deadline exceeded") - return ExitRuntimeError + } else { + fmt.Fprintln(stderr, runErrorMessage(runErr)) } - fmt.Fprintln(stderr, runErrorMessage(runErr)) - return ExitRuntimeError } - if err := writeOutput(stdout, cfg.outputPath, res.Artifact.Body); err != nil { - fmt.Fprintf(stderr, "output write error: %v\n", err) + outputFailed := false + if runErr == nil { + if err := writeOutput(stdout, cfg.outputPath, res.Artifact.Body); err != nil { + fmt.Fprintf(stderr, "output write error: %v\n", err) + outputFailed = true + } + } + reportFailed := false + if collector != nil { + if value := collector.Snapshot(); value == nil { + fmt.Fprintln(stderr, "execution report not produced: no terminal observation") + } else if err := writeExecutionReport(cfg.executionReportPath, value); err != nil { + fmt.Fprintf(stderr, "execution report write error: %v\n", err) + reportFailed = true + } + } + if runErr != nil || outputFailed || reportFailed { return ExitRuntimeError } @@ -180,6 +205,14 @@ func runCommand(args []string, stdout, stderr io.Writer) int { return determineExitCode(nil, res) } +func writeExecutionReport(path string, value *executionreport.Execution) error { + data, err := json.Marshal(value) + if err != nil { + return err + } + return os.WriteFile(path, append(data, '\n'), 0644) +} + func renderCommand(args []string, stdout, stderr io.Writer) int { cfg, err := parseRenderArgs(args) if err != nil { @@ -264,6 +297,7 @@ func parseRunArgs(args []string) (*runConfig, error) { registerExecutionRequestFlags(fs, cfg) fs.StringVar(&cfg.schemaDir, "schema-dir", "", "base directory for validation schemas") fs.DurationVar(&cfg.runTimeout, "run-timeout", 0, "whole-run execution deadline") + fs.StringVar(&cfg.executionReportPath, "execution-report", "", "JSON execution report file path") if err := fs.Parse(args); err != nil { return nil, err @@ -274,11 +308,43 @@ func parseRunArgs(args []string) (*runConfig, error) { if cfg.runTimeout < 0 { return nil, errors.New("--run-timeout must be greater than or equal to 0") } + if flagWasSet(fs, "execution-report") { + if cfg.executionReportPath == "" { + return nil, errors.New("--execution-report requires a nonempty path") + } + cfg.executionReportPath = filepath.Clean(cfg.executionReportPath) + if cfg.outputPath != "" { + collision, err := outputPathsCollide(cfg.outputPath, cfg.executionReportPath) + if err != nil { + return nil, err + } + if collision { + return nil, errors.New("--out and --execution-report select the same file") + } + } + } cfg.schemaDir = filepath.Clean(cfg.schemaDir) return cfg, nil } +func outputPathsCollide(first, second string) (bool, error) { + firstAbs, err := filepath.Abs(first) + if err != nil { + return false, err + } + secondAbs, err := filepath.Abs(second) + if err != nil { + return false, err + } + if firstAbs == secondAbs { + return true, nil + } + firstInfo, firstErr := os.Stat(firstAbs) + secondInfo, secondErr := os.Stat(secondAbs) + return firstErr == nil && secondErr == nil && os.SameFile(firstInfo, secondInfo), nil +} + func parseRenderArgs(args []string) (*renderConfig, error) { cfg := &renderConfig{ outputFormat: renderformat.DefaultPreparedRunOutputFormat, @@ -789,7 +855,7 @@ func runErrorMessage(err error) string { func printUsage(w io.Writer) { fmt.Fprintln(w, "usage: scriptorium ...") - fmt.Fprintln(w, " run: scriptorium run [--config PATH] [--prompt-dir DIR] [--profile-dir DIR] --prompt ID [--prompt-version VERSION] [--input name=path] [--profile ID] [--session-id ID] [--llm-base-url URL] [--model NAME] [--api-key-env ENV] [--temperature N] [--max-tokens N] [--top-p N] [--reasoning-effort VALUE] [--var k=v] [--out path] [--timeout 10m] [--run-timeout 10m] [--transient-retries N]") + fmt.Fprintln(w, " run: scriptorium run [--config PATH] [--prompt-dir DIR] [--profile-dir DIR] --prompt ID [--prompt-version VERSION] [--input name=path] [--profile ID] [--session-id ID] [--llm-base-url URL] [--model NAME] [--api-key-env ENV] [--temperature N] [--max-tokens N] [--top-p N] [--reasoning-effort VALUE] [--var k=v] [--out path] [--execution-report path] [--timeout 10m] [--run-timeout 10m] [--transient-retries N]") fmt.Fprintln(w, " render: scriptorium render [--config PATH] [--prompt-dir DIR] [--profile-dir DIR] --prompt ID [--prompt-version VERSION] [--input name=path] [--profile ID] [--session-id ID] [--llm-base-url URL] [--model NAME] [--api-key-env ENV] [--temperature N] [--max-tokens N] [--top-p N] [--reasoning-effort VALUE] [--var k=v] [--format text|json] [--out path] [--timeout 10m] [--transient-retries N]") fmt.Fprintf(w, " serve: scriptorium serve [--config PATH] [--addr %s] [--prompt-dir DIR] [--profile-dir DIR] [--schema-dir DIR] [--artifact-root DIR] [--max-request-bytes N] [--max-artifact-bytes N] [--max-response-bytes N]\n", defaults.HTTPAddrDefault) fmt.Fprintln(w, " inspect prompt: scriptorium inspect prompt --prompt ID [--prompt-version VERSION] [--config PATH] [--prompt-dir DIR] [--format text|json] [--out path]") diff --git a/internal/adapter/cli/run_test.go b/internal/adapter/cli/run_test.go index 3d1f8b6..d21caf6 100644 --- a/internal/adapter/cli/run_test.go +++ b/internal/adapter/cli/run_test.go @@ -2,6 +2,7 @@ package cli import ( "bytes" + "context" "encoding/json" "errors" "flag" @@ -1379,6 +1380,164 @@ func TestRunTimeoutParsingAndCancellation(t *testing.T) { } } +func TestRunCommandWritesSeparateExecutionReport(t *testing.T) { + lib := newCLITestLibrary(t) + inputPath := lib.writeInputFile(t, "transcript.md", "hello") + var hits int32 + provider := newTestLLMServer("generated", &hits) + defer provider.Close() + writePromptFile(t, lib.promptDir, "reported", "local-default") + writeProfileFile(t, lib.profileDir, "local-default", provider.URL+"/v1", "local-model") + path := filepath.Join(t.TempDir(), "execution.json") + code, stdout, stderr := runCLICommand(t, runCommand, []string{ + "--prompt-dir", lib.promptDir, "--profile-dir", lib.profileDir, + "--prompt", "reported", "--input", "transcript=" + inputPath, + "--execution-report", path, + }) + if code != ExitOK || stdout != "generated" || atomic.LoadInt32(&hits) != 1 { + t.Fatalf("run with report: code=%d stdout=%q stderr=%q hits=%d", code, stdout, stderr, atomic.LoadInt32(&hits)) + } + data, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + if !bytes.HasSuffix(data, []byte("\n")) { + t.Fatalf("report is not newline-terminated: %q", data) + } + var report map[string]any + if err := json.Unmarshal(data, &report); err != nil { + t.Fatal(err) + } + if report["prompt_id"] != "reported" || report["outcome"] != "success" || report["attempts"] != float64(1) { + t.Fatalf("unexpected execution report: %#v", report) + } +} + +func TestExecutionReportFailureAndOutputHandling(t *testing.T) { + terminal := func(ctx context.Context, req promptkit.RunRequest) (*promptkit.RunResult, error) { + if req.Observer != nil { + req.Observer(promptkit.ExecutionEvent{Kind: promptkit.ExecutionFinished, PromptID: "p", Outcome: promptkit.ExecutionOutcomeSuccess, Attempts: 1}) + } + return &promptkit.RunResult{Artifact: promptkit.Artifact{Body: []byte("artifact")}, Validation: promptkit.ValidationResult{Status: promptkit.ValidationPassed}}, nil + } + for _, tc := range []struct { + name string + modify func(*runConfig, string) + run func(context.Context, promptkit.RunRequest) (*promptkit.RunResult, error) + wantCode int + wantOutput string + wantDiagnostic string + wantReport bool + }{ + {name: "default streams", run: terminal, wantCode: ExitOK, wantOutput: "artifact"}, + {name: "validation failure", run: func(ctx context.Context, req promptkit.RunRequest) (*promptkit.RunResult, error) { + req.Observer(promptkit.ExecutionEvent{Kind: promptkit.ExecutionFinished, Outcome: promptkit.ExecutionOutcomeInvalidOutput}) + return &promptkit.RunResult{Artifact: promptkit.Artifact{Body: []byte("invalid")}, Validation: promptkit.ValidationResult{Status: promptkit.ValidationFailed}}, nil + }, wantCode: ExitValidationFailed, wantOutput: "invalid", wantReport: true}, + {name: "execution failure", run: func(ctx context.Context, req promptkit.RunRequest) (*promptkit.RunResult, error) { + req.Observer(promptkit.ExecutionEvent{Kind: promptkit.ExecutionFinished, Outcome: promptkit.ExecutionOutcomeError, Failure: promptkit.ExecutionFailureProviderStatus}) + return nil, promptkit.ErrLLMGenerate + }, wantCode: ExitRuntimeError, wantDiagnostic: "run error", wantReport: true}, + {name: "artifact write failure still writes report", run: terminal, modify: func(cfg *runConfig, root string) { + cfg.outputPath = filepath.Join(root, "missing", "artifact.txt") + }, wantCode: ExitRuntimeError, wantDiagnostic: "output write error", wantReport: true}, + {name: "report write failure preserves artifact", run: terminal, modify: func(cfg *runConfig, root string) { + cfg.executionReportPath = filepath.Join(root, "missing", "execution.json") + }, wantCode: ExitRuntimeError, wantOutput: "artifact", wantDiagnostic: "execution report write error"}, + {name: "execution and report failures are both visible", run: func(ctx context.Context, req promptkit.RunRequest) (*promptkit.RunResult, error) { + req.Observer(promptkit.ExecutionEvent{Kind: promptkit.ExecutionFinished, Outcome: promptkit.ExecutionOutcomeError}) + return nil, promptkit.ErrLLMGenerate + }, modify: func(cfg *runConfig, root string) { + cfg.executionReportPath = filepath.Join(root, "missing", "execution.json") + }, wantCode: ExitRuntimeError, wantDiagnostic: "execution report write error"}, + } { + t.Run(tc.name, func(t *testing.T) { + root := t.TempDir() + cfg := &runConfig{executionReportPath: filepath.Join(root, "execution.json")} + if tc.name == "default streams" { + cfg.executionReportPath = "" + } + if tc.modify != nil { + tc.modify(cfg, root) + } + var stdout, stderr bytes.Buffer + code := executeRun(cfg, promptkit.RunRequest{PromptID: "p"}, tc.run, &stdout, &stderr) + if code != tc.wantCode || stdout.String() != tc.wantOutput || (tc.wantDiagnostic != "" && !strings.Contains(stderr.String(), tc.wantDiagnostic)) { + t.Fatalf("unexpected output: code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) + } + if tc.wantReport { + if _, err := os.Stat(cfg.executionReportPath); err != nil { + t.Fatalf("terminal report missing: %v", err) + } + } + if tc.name == "execution and report failures are both visible" && !strings.Contains(stderr.String(), "run error") { + t.Fatalf("execution failure diagnostic was lost: %s", stderr.String()) + } + }) + } +} + +func TestExecutionReportLeavesExistingFileWhenNoTerminalEvent(t *testing.T) { + path := filepath.Join(t.TempDir(), "execution.json") + if err := os.WriteFile(path, []byte("old report"), 0o644); err != nil { + t.Fatal(err) + } + cfg := &runConfig{executionReportPath: path} + var stdout, stderr bytes.Buffer + code := executeRun(cfg, promptkit.RunRequest{PromptID: "p"}, func(ctx context.Context, req promptkit.RunRequest) (*promptkit.RunResult, error) { + if req.Observer == nil { + t.Fatal("requested report did not attach observer") + } + return &promptkit.RunResult{Artifact: promptkit.Artifact{Body: []byte("artifact")}, Validation: promptkit.ValidationResult{Status: promptkit.ValidationPassed}}, nil + }, &stdout, &stderr) + data, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + if code != ExitOK || stdout.String() != "artifact" || string(data) != "old report" || !strings.Contains(stderr.String(), "no terminal observation") { + t.Fatalf("no-terminal handling: code=%d stdout=%q stderr=%q report=%q", code, stdout.String(), stderr.String(), data) + } +} + +func TestExecutionReportPathValidationAndPreInvocationFailure(t *testing.T) { + root := t.TempDir() + args := []string{"--prompt-dir", root, "--prompt", "p"} + path := filepath.Join(root, "execution.json") + for _, tail := range [][]string{ + {"--execution-report", ""}, + {"--out", path, "--execution-report", path}, + {"--out", path, "--execution-report", filepath.Join(root, "subdir", "..", "execution.json")}, + } { + if _, err := parseRunArgs(append(append([]string{}, args...), tail...)); err == nil { + t.Fatalf("accepted invalid report destination: %q", tail) + } + } + if err := os.WriteFile(path, []byte("old report"), 0o644); err != nil { + t.Fatal(err) + } + alias := filepath.Join(root, "alias.json") + if err := os.Link(path, alias); err != nil { + t.Fatal(err) + } + if _, err := parseRunArgs(append(append([]string{}, args...), "--out", path, "--execution-report", alias)); err == nil { + t.Fatal("accepted two names for the same output file") + } + if cfg, err := parseRunArgs(append(append([]string{}, args...), "--execution-report", "-")); err != nil || cfg.executionReportPath != "-" { + t.Fatalf("ordinary dash path was rejected: cfg=%+v err=%v", cfg, err) + } + if _, err := parseRenderArgs(append(append([]string{}, args...), "--execution-report", path)); err == nil { + t.Fatal("render accepted run-only report flag") + } + code, stdout, stderr := runCLICommand(t, runCommand, []string{"--config", filepath.Join(root, "missing.yml"), "--prompt", "p", "--execution-report", path}) + data, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + if code != ExitRuntimeError || stdout != "" || !strings.Contains(stderr, "run parse error") || string(data) != "old report" { + t.Fatalf("pre-invocation failure changed report: code=%d stdout=%q stderr=%q report=%q", code, stdout, stderr, data) + } +} + func intPointer(value int) *int { return &value } func TestPromptkitPromptInspectionSelectsVersionsAndContracts(t *testing.T) {