eric/scriptorium
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 9 Wiki Activity

Add opt-in CLI execution report files

Browse Source
This commit is contained in:
Eric Rakestraw
2026-09-26 15:47:26 +00:00
parent f77bd4b528
commit edc1f8abd9
7 changed files with 295 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

80
internal/adapter/cli/run.go
View File

@@ -2,6 +2,7 @@ package cli
import (
"context"
"encoding/json"
"errors"
"flag"
"fmt"
@@ -17,6 +18,7 @@ import (
appconfig "gitea.maximumdirect.net/eric/scriptorium/internal/config"
"gitea.maximumdirect.net/eric/scriptorium/internal/defaults"
renderformat "gitea.maximumdirect.net/eric/scriptorium/internal/format"
executionreport "gitea.maximumdirect.net/eric/scriptorium/internal/report"
)
const (
@@ -41,6 +43,7 @@ type runConfig struct {
inputRaw listFlag
varRaw listFlag
outputPath string
executionReportPath string
llmBaseURL string
apiKeyEnv string
model string
@@ -153,26 +156,48 @@ func runCommand(args []string, stdout, stderr io.Writer) int {
fmt.Fprintf(stderr, "engine error: %v\n", err)
return ExitRuntimeError
}
return executeRun(cfg, req, engine.Run, stdout, stderr)
}
func executeRun(cfg *runConfig, req promptkit.RunRequest, run func(context.Context, promptkit.RunRequest) (*promptkit.RunResult, error), stdout, stderr io.Writer) int {
var collector *executionreport.Collector
if cfg.executionReportPath != "" {
collector = executionreport.NewCollector()
req.Observer = collector.Observe
}
runContext := context.Background()
cancel := func() {}
if cfg.runTimeout > 0 {
runContext, cancel = context.WithTimeout(runContext, cfg.runTimeout)
}
res, runErr := engine.Run(runContext, req)
res, runErr := run(runContext, req)
contextErr := runContext.Err()
cancel()
if runErr != nil {
if errors.Is(contextErr, context.DeadlineExceeded) {
fmt.Fprintln(stderr, "run error: execution deadline exceeded")
return ExitRuntimeError
} else {
fmt.Fprintln(stderr, runErrorMessage(runErr))
}
fmt.Fprintln(stderr, runErrorMessage(runErr))
return ExitRuntimeError
}
if err := writeOutput(stdout, cfg.outputPath, res.Artifact.Body); err != nil {
fmt.Fprintf(stderr, "output write error: %v\n", err)
outputFailed := false
if runErr == nil {
if err := writeOutput(stdout, cfg.outputPath, res.Artifact.Body); err != nil {
fmt.Fprintf(stderr, "output write error: %v\n", err)
outputFailed = true
}
}
reportFailed := false
if collector != nil {
if value := collector.Snapshot(); value == nil {
fmt.Fprintln(stderr, "execution report not produced: no terminal observation")
} else if err := writeExecutionReport(cfg.executionReportPath, value); err != nil {
fmt.Fprintf(stderr, "execution report write error: %v\n", err)
reportFailed = true
}
}
if runErr != nil || outputFailed || reportFailed {
return ExitRuntimeError
}
@@ -180,6 +205,14 @@ func runCommand(args []string, stdout, stderr io.Writer) int {
return determineExitCode(nil, res)
}
func writeExecutionReport(path string, value *executionreport.Execution) error {
data, err := json.Marshal(value)
if err != nil {
return err
}
return os.WriteFile(path, append(data, '\n'), 0644)
}
func renderCommand(args []string, stdout, stderr io.Writer) int {
cfg, err := parseRenderArgs(args)
if err != nil {
@@ -264,6 +297,7 @@ func parseRunArgs(args []string) (*runConfig, error) {
registerExecutionRequestFlags(fs, cfg)
fs.StringVar(&cfg.schemaDir, "schema-dir", "", "base directory for validation schemas")
fs.DurationVar(&cfg.runTimeout, "run-timeout", 0, "whole-run execution deadline")
fs.StringVar(&cfg.executionReportPath, "execution-report", "", "JSON execution report file path")
if err := fs.Parse(args); err != nil {
return nil, err
@@ -274,11 +308,43 @@ func parseRunArgs(args []string) (*runConfig, error) {
if cfg.runTimeout < 0 {
return nil, errors.New("--run-timeout must be greater than or equal to 0")
}
if flagWasSet(fs, "execution-report") {
if cfg.executionReportPath == "" {
return nil, errors.New("--execution-report requires a nonempty path")
}
cfg.executionReportPath = filepath.Clean(cfg.executionReportPath)
if cfg.outputPath != "" {
collision, err := outputPathsCollide(cfg.outputPath, cfg.executionReportPath)
if err != nil {
return nil, err
}
if collision {
return nil, errors.New("--out and --execution-report select the same file")
}
}
}
cfg.schemaDir = filepath.Clean(cfg.schemaDir)
return cfg, nil
}
func outputPathsCollide(first, second string) (bool, error) {
firstAbs, err := filepath.Abs(first)
if err != nil {
return false, err
}
secondAbs, err := filepath.Abs(second)
if err != nil {
return false, err
}
if firstAbs == secondAbs {
return true, nil
}
firstInfo, firstErr := os.Stat(firstAbs)
secondInfo, secondErr := os.Stat(secondAbs)
return firstErr == nil && secondErr == nil && os.SameFile(firstInfo, secondInfo), nil
}
func parseRenderArgs(args []string) (*renderConfig, error) {
cfg := &renderConfig{
outputFormat: renderformat.DefaultPreparedRunOutputFormat,
@@ -789,7 +855,7 @@ func runErrorMessage(err error) string {
func printUsage(w io.Writer) {
fmt.Fprintln(w, "usage: scriptorium <run|render|serve|inspect> ...")
fmt.Fprintln(w, " run: scriptorium run [--config PATH] [--prompt-dir DIR] [--profile-dir DIR] --prompt ID [--prompt-version VERSION] [--input name=path] [--profile ID] [--session-id ID] [--llm-base-url URL] [--model NAME] [--api-key-env ENV] [--temperature N] [--max-tokens N] [--top-p N] [--reasoning-effort VALUE] [--var k=v] [--out path] [--timeout 10m] [--run-timeout 10m] [--transient-retries N]")
fmt.Fprintln(w, " run: scriptorium run [--config PATH] [--prompt-dir DIR] [--profile-dir DIR] --prompt ID [--prompt-version VERSION] [--input name=path] [--profile ID] [--session-id ID] [--llm-base-url URL] [--model NAME] [--api-key-env ENV] [--temperature N] [--max-tokens N] [--top-p N] [--reasoning-effort VALUE] [--var k=v] [--out path] [--execution-report path] [--timeout 10m] [--run-timeout 10m] [--transient-retries N]")
fmt.Fprintln(w, " render: scriptorium render [--config PATH] [--prompt-dir DIR] [--profile-dir DIR] --prompt ID [--prompt-version VERSION] [--input name=path] [--profile ID] [--session-id ID] [--llm-base-url URL] [--model NAME] [--api-key-env ENV] [--temperature N] [--max-tokens N] [--top-p N] [--reasoning-effort VALUE] [--var k=v] [--format text|json] [--out path] [--timeout 10m] [--transient-retries N]")
fmt.Fprintf(w, " serve: scriptorium serve [--config PATH] [--addr %s] [--prompt-dir DIR] [--profile-dir DIR] [--schema-dir DIR] [--artifact-root DIR] [--max-request-bytes N] [--max-artifact-bytes N] [--max-response-bytes N]\n", defaults.HTTPAddrDefault)
fmt.Fprintln(w, " inspect prompt: scriptorium inspect prompt --prompt ID [--prompt-version VERSION] [--config PATH] [--prompt-dir DIR] [--format text|json] [--out path]")