eric/scriptorium
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 9 Wiki Activity

Add opt-in CLI execution report files

Browse Source
This commit is contained in:
Eric Rakestraw
2026-09-26 15:47:26 +00:00
parent f77bd4b528
commit edc1f8abd9
7 changed files with 295 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

80
internal/adapter/cli/run.go
View File

@@ -2,6 +2,7 @@ package cli
import (
"context"
"encoding/json"
"errors"
"flag"
"fmt"
@@ -17,6 +18,7 @@ import (
appconfig "gitea.maximumdirect.net/eric/scriptorium/internal/config"
"gitea.maximumdirect.net/eric/scriptorium/internal/defaults"
renderformat "gitea.maximumdirect.net/eric/scriptorium/internal/format"
executionreport "gitea.maximumdirect.net/eric/scriptorium/internal/report"
)
const (
@@ -41,6 +43,7 @@ type runConfig struct {
inputRaw listFlag
varRaw listFlag
outputPath string
executionReportPath string
llmBaseURL string
apiKeyEnv string
model string
@@ -153,26 +156,48 @@ func runCommand(args []string, stdout, stderr io.Writer) int {
fmt.Fprintf(stderr, "engine error: %v\n", err)
return ExitRuntimeError
}
return executeRun(cfg, req, engine.Run, stdout, stderr)
}
func executeRun(cfg *runConfig, req promptkit.RunRequest, run func(context.Context, promptkit.RunRequest) (*promptkit.RunResult, error), stdout, stderr io.Writer) int {
var collector *executionreport.Collector
if cfg.executionReportPath != "" {
collector = executionreport.NewCollector()
req.Observer = collector.Observe
}
runContext := context.Background()
cancel := func() {}
if cfg.runTimeout > 0 {
runContext, cancel = context.WithTimeout(runContext, cfg.runTimeout)
}
res, runErr := engine.Run(runContext, req)
res, runErr := run(runContext, req)
contextErr := runContext.Err()
cancel()
if runErr != nil {
if errors.Is(contextErr, context.DeadlineExceeded) {
fmt.Fprintln(stderr, "run error: execution deadline exceeded")
return ExitRuntimeError
} else {
fmt.Fprintln(stderr, runErrorMessage(runErr))
}
fmt.Fprintln(stderr, runErrorMessage(runErr))
return ExitRuntimeError
}
if err := writeOutput(stdout, cfg.outputPath, res.Artifact.Body); err != nil {
fmt.Fprintf(stderr, "output write error: %v\n", err)
outputFailed := false
if runErr == nil {
if err := writeOutput(stdout, cfg.outputPath, res.Artifact.Body); err != nil {
fmt.Fprintf(stderr, "output write error: %v\n", err)
outputFailed = true
}
}
reportFailed := false
if collector != nil {
if value := collector.Snapshot(); value == nil {
fmt.Fprintln(stderr, "execution report not produced: no terminal observation")
} else if err := writeExecutionReport(cfg.executionReportPath, value); err != nil {
fmt.Fprintf(stderr, "execution report write error: %v\n", err)
reportFailed = true
}
}
if runErr != nil || outputFailed || reportFailed {
return ExitRuntimeError
}
@@ -180,6 +205,14 @@ func runCommand(args []string, stdout, stderr io.Writer) int {
return determineExitCode(nil, res)
}
func writeExecutionReport(path string, value *executionreport.Execution) error {
data, err := json.Marshal(value)
if err != nil {
return err
}
return os.WriteFile(path, append(data, '\n'), 0644)
}
func renderCommand(args []string, stdout, stderr io.Writer) int {
cfg, err := parseRenderArgs(args)
if err != nil {
@@ -264,6 +297,7 @@ func parseRunArgs(args []string) (*runConfig, error) {
registerExecutionRequestFlags(fs, cfg)
fs.StringVar(&cfg.schemaDir, "schema-dir", "", "base directory for validation schemas")
fs.DurationVar(&cfg.runTimeout, "run-timeout", 0, "whole-run execution deadline")
fs.StringVar(&cfg.executionReportPath, "execution-report", "", "JSON execution report file path")
if err := fs.Parse(args); err != nil {
return nil, err
@@ -274,11 +308,43 @@ func parseRunArgs(args []string) (*runConfig, error) {
if cfg.runTimeout < 0 {
return nil, errors.New("--run-timeout must be greater than or equal to 0")
}
if flagWasSet(fs, "execution-report") {
if cfg.executionReportPath == "" {
return nil, errors.New("--execution-report requires a nonempty path")
}
cfg.executionReportPath = filepath.Clean(cfg.executionReportPath)
if cfg.outputPath != "" {
collision, err := outputPathsCollide(cfg.outputPath, cfg.executionReportPath)
if err != nil {
return nil, err
}
if collision {
return nil, errors.New("--out and --execution-report select the same file")
}
}
}
cfg.schemaDir = filepath.Clean(cfg.schemaDir)
return cfg, nil
}
func outputPathsCollide(first, second string) (bool, error) {
firstAbs, err := filepath.Abs(first)
if err != nil {
return false, err
}
secondAbs, err := filepath.Abs(second)
if err != nil {
return false, err
}
if firstAbs == secondAbs {
return true, nil
}
firstInfo, firstErr := os.Stat(firstAbs)
secondInfo, secondErr := os.Stat(secondAbs)
return firstErr == nil && secondErr == nil && os.SameFile(firstInfo, secondInfo), nil
}
func parseRenderArgs(args []string) (*renderConfig, error) {
cfg := &renderConfig{
outputFormat: renderformat.DefaultPreparedRunOutputFormat,
@@ -789,7 +855,7 @@ func runErrorMessage(err error) string {
func printUsage(w io.Writer) {
fmt.Fprintln(w, "usage: scriptorium <run|render|serve|inspect> ...")
fmt.Fprintln(w, " run: scriptorium run [--config PATH] [--prompt-dir DIR] [--profile-dir DIR] --prompt ID [--prompt-version VERSION] [--input name=path] [--profile ID] [--session-id ID] [--llm-base-url URL] [--model NAME] [--api-key-env ENV] [--temperature N] [--max-tokens N] [--top-p N] [--reasoning-effort VALUE] [--var k=v] [--out path] [--timeout 10m] [--run-timeout 10m] [--transient-retries N]")
fmt.Fprintln(w, " run: scriptorium run [--config PATH] [--prompt-dir DIR] [--profile-dir DIR] --prompt ID [--prompt-version VERSION] [--input name=path] [--profile ID] [--session-id ID] [--llm-base-url URL] [--model NAME] [--api-key-env ENV] [--temperature N] [--max-tokens N] [--top-p N] [--reasoning-effort VALUE] [--var k=v] [--out path] [--execution-report path] [--timeout 10m] [--run-timeout 10m] [--transient-retries N]")
fmt.Fprintln(w, " render: scriptorium render [--config PATH] [--prompt-dir DIR] [--profile-dir DIR] --prompt ID [--prompt-version VERSION] [--input name=path] [--profile ID] [--session-id ID] [--llm-base-url URL] [--model NAME] [--api-key-env ENV] [--temperature N] [--max-tokens N] [--top-p N] [--reasoning-effort VALUE] [--var k=v] [--format text|json] [--out path] [--timeout 10m] [--transient-retries N]")
fmt.Fprintf(w, " serve: scriptorium serve [--config PATH] [--addr %s] [--prompt-dir DIR] [--profile-dir DIR] [--schema-dir DIR] [--artifact-root DIR] [--max-request-bytes N] [--max-artifact-bytes N] [--max-response-bytes N]\n", defaults.HTTPAddrDefault)
fmt.Fprintln(w, " inspect prompt: scriptorium inspect prompt --prompt ID [--prompt-version VERSION] [--config PATH] [--prompt-dir DIR] [--format text|json] [--out path]")

159
internal/adapter/cli/run_test.go
View File

@@ -2,6 +2,7 @@ package cli
import (
"bytes"
"context"
"encoding/json"
"errors"
"flag"
@@ -1379,6 +1380,164 @@ func TestRunTimeoutParsingAndCancellation(t *testing.T) {
}
}
func TestRunCommandWritesSeparateExecutionReport(t *testing.T) {
lib := newCLITestLibrary(t)
inputPath := lib.writeInputFile(t, "transcript.md", "hello")
var hits int32
provider := newTestLLMServer("generated", &hits)
defer provider.Close()
writePromptFile(t, lib.promptDir, "reported", "local-default")
writeProfileFile(t, lib.profileDir, "local-default", provider.URL+"/v1", "local-model")
path := filepath.Join(t.TempDir(), "execution.json")
code, stdout, stderr := runCLICommand(t, runCommand, []string{
"--prompt-dir", lib.promptDir, "--profile-dir", lib.profileDir,
"--prompt", "reported", "--input", "transcript=" + inputPath,
"--execution-report", path,
})
if code != ExitOK || stdout != "generated" || atomic.LoadInt32(&hits) != 1 {
t.Fatalf("run with report: code=%d stdout=%q stderr=%q hits=%d", code, stdout, stderr, atomic.LoadInt32(&hits))
}
data, err := os.ReadFile(path)
if err != nil {
t.Fatal(err)
}
if !bytes.HasSuffix(data, []byte("\n")) {
t.Fatalf("report is not newline-terminated: %q", data)
}
var report map[string]any
if err := json.Unmarshal(data, &report); err != nil {
t.Fatal(err)
}
if report["prompt_id"] != "reported" || report["outcome"] != "success" || report["attempts"] != float64(1) {
t.Fatalf("unexpected execution report: %#v", report)
}
}
func TestExecutionReportFailureAndOutputHandling(t *testing.T) {
terminal := func(ctx context.Context, req promptkit.RunRequest) (*promptkit.RunResult, error) {
if req.Observer != nil {
req.Observer(promptkit.ExecutionEvent{Kind: promptkit.ExecutionFinished, PromptID: "p", Outcome: promptkit.ExecutionOutcomeSuccess, Attempts: 1})
}
return &promptkit.RunResult{Artifact: promptkit.Artifact{Body: []byte("artifact")}, Validation: promptkit.ValidationResult{Status: promptkit.ValidationPassed}}, nil
}
for _, tc := range []struct {
name string
modify func(*runConfig, string)
run func(context.Context, promptkit.RunRequest) (*promptkit.RunResult, error)
wantCode int
wantOutput string
wantDiagnostic string
wantReport bool
}{
{name: "default streams", run: terminal, wantCode: ExitOK, wantOutput: "artifact"},
{name: "validation failure", run: func(ctx context.Context, req promptkit.RunRequest) (*promptkit.RunResult, error) {
req.Observer(promptkit.ExecutionEvent{Kind: promptkit.ExecutionFinished, Outcome: promptkit.ExecutionOutcomeInvalidOutput})
return &promptkit.RunResult{Artifact: promptkit.Artifact{Body: []byte("invalid")}, Validation: promptkit.ValidationResult{Status: promptkit.ValidationFailed}}, nil
}, wantCode: ExitValidationFailed, wantOutput: "invalid", wantReport: true},
{name: "execution failure", run: func(ctx context.Context, req promptkit.RunRequest) (*promptkit.RunResult, error) {
req.Observer(promptkit.ExecutionEvent{Kind: promptkit.ExecutionFinished, Outcome: promptkit.ExecutionOutcomeError, Failure: promptkit.ExecutionFailureProviderStatus})
return nil, promptkit.ErrLLMGenerate
}, wantCode: ExitRuntimeError, wantDiagnostic: "run error", wantReport: true},
{name: "artifact write failure still writes report", run: terminal, modify: func(cfg *runConfig, root string) {
cfg.outputPath = filepath.Join(root, "missing", "artifact.txt")
}, wantCode: ExitRuntimeError, wantDiagnostic: "output write error", wantReport: true},
{name: "report write failure preserves artifact", run: terminal, modify: func(cfg *runConfig, root string) {
cfg.executionReportPath = filepath.Join(root, "missing", "execution.json")
}, wantCode: ExitRuntimeError, wantOutput: "artifact", wantDiagnostic: "execution report write error"},
{name: "execution and report failures are both visible", run: func(ctx context.Context, req promptkit.RunRequest) (*promptkit.RunResult, error) {
req.Observer(promptkit.ExecutionEvent{Kind: promptkit.ExecutionFinished, Outcome: promptkit.ExecutionOutcomeError})
return nil, promptkit.ErrLLMGenerate
}, modify: func(cfg *runConfig, root string) {
cfg.executionReportPath = filepath.Join(root, "missing", "execution.json")
}, wantCode: ExitRuntimeError, wantDiagnostic: "execution report write error"},
} {
t.Run(tc.name, func(t *testing.T) {
root := t.TempDir()
cfg := &runConfig{executionReportPath: filepath.Join(root, "execution.json")}
if tc.name == "default streams" {
cfg.executionReportPath = ""
}
if tc.modify != nil {
tc.modify(cfg, root)
}
var stdout, stderr bytes.Buffer
code := executeRun(cfg, promptkit.RunRequest{PromptID: "p"}, tc.run, &stdout, &stderr)
if code != tc.wantCode || stdout.String() != tc.wantOutput || (tc.wantDiagnostic != "" && !strings.Contains(stderr.String(), tc.wantDiagnostic)) {
t.Fatalf("unexpected output: code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
}
if tc.wantReport {
if _, err := os.Stat(cfg.executionReportPath); err != nil {
t.Fatalf("terminal report missing: %v", err)
}
}
if tc.name == "execution and report failures are both visible" && !strings.Contains(stderr.String(), "run error") {
t.Fatalf("execution failure diagnostic was lost: %s", stderr.String())
}
})
}
}
func TestExecutionReportLeavesExistingFileWhenNoTerminalEvent(t *testing.T) {
path := filepath.Join(t.TempDir(), "execution.json")
if err := os.WriteFile(path, []byte("old report"), 0o644); err != nil {
t.Fatal(err)
}
cfg := &runConfig{executionReportPath: path}
var stdout, stderr bytes.Buffer
code := executeRun(cfg, promptkit.RunRequest{PromptID: "p"}, func(ctx context.Context, req promptkit.RunRequest) (*promptkit.RunResult, error) {
if req.Observer == nil {
t.Fatal("requested report did not attach observer")
}
return &promptkit.RunResult{Artifact: promptkit.Artifact{Body: []byte("artifact")}, Validation: promptkit.ValidationResult{Status: promptkit.ValidationPassed}}, nil
}, &stdout, &stderr)
data, err := os.ReadFile(path)
if err != nil {
t.Fatal(err)
}
if code != ExitOK || stdout.String() != "artifact" || string(data) != "old report" || !strings.Contains(stderr.String(), "no terminal observation") {
t.Fatalf("no-terminal handling: code=%d stdout=%q stderr=%q report=%q", code, stdout.String(), stderr.String(), data)
}
}
func TestExecutionReportPathValidationAndPreInvocationFailure(t *testing.T) {
root := t.TempDir()
args := []string{"--prompt-dir", root, "--prompt", "p"}
path := filepath.Join(root, "execution.json")
for _, tail := range [][]string{
{"--execution-report", ""},
{"--out", path, "--execution-report", path},
{"--out", path, "--execution-report", filepath.Join(root, "subdir", "..", "execution.json")},
} {
if _, err := parseRunArgs(append(append([]string{}, args...), tail...)); err == nil {
t.Fatalf("accepted invalid report destination: %q", tail)
}
}
if err := os.WriteFile(path, []byte("old report"), 0o644); err != nil {
t.Fatal(err)
}
alias := filepath.Join(root, "alias.json")
if err := os.Link(path, alias); err != nil {
t.Fatal(err)
}
if _, err := parseRunArgs(append(append([]string{}, args...), "--out", path, "--execution-report", alias)); err == nil {
t.Fatal("accepted two names for the same output file")
}
if cfg, err := parseRunArgs(append(append([]string{}, args...), "--execution-report", "-")); err != nil || cfg.executionReportPath != "-" {
t.Fatalf("ordinary dash path was rejected: cfg=%+v err=%v", cfg, err)
}
if _, err := parseRenderArgs(append(append([]string{}, args...), "--execution-report", path)); err == nil {
t.Fatal("render accepted run-only report flag")
}
code, stdout, stderr := runCLICommand(t, runCommand, []string{"--config", filepath.Join(root, "missing.yml"), "--prompt", "p", "--execution-report", path})
data, err := os.ReadFile(path)
if err != nil {
t.Fatal(err)
}
if code != ExitRuntimeError || stdout != "" || !strings.Contains(stderr, "run parse error") || string(data) != "old report" {
t.Fatalf("pre-invocation failure changed report: code=%d stdout=%q stderr=%q report=%q", code, stdout, stderr, data)
}
}
func intPointer(value int) *int { return &value }
func TestPromptkitPromptInspectionSelectsVersionsAndContracts(t *testing.T) {