Add opt-in CLI execution report files
This commit is contained in:
@@ -2,6 +2,7 @@ package cli
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
@@ -1379,6 +1380,164 @@ func TestRunTimeoutParsingAndCancellation(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunCommandWritesSeparateExecutionReport(t *testing.T) {
|
||||
lib := newCLITestLibrary(t)
|
||||
inputPath := lib.writeInputFile(t, "transcript.md", "hello")
|
||||
var hits int32
|
||||
provider := newTestLLMServer("generated", &hits)
|
||||
defer provider.Close()
|
||||
writePromptFile(t, lib.promptDir, "reported", "local-default")
|
||||
writeProfileFile(t, lib.profileDir, "local-default", provider.URL+"/v1", "local-model")
|
||||
path := filepath.Join(t.TempDir(), "execution.json")
|
||||
code, stdout, stderr := runCLICommand(t, runCommand, []string{
|
||||
"--prompt-dir", lib.promptDir, "--profile-dir", lib.profileDir,
|
||||
"--prompt", "reported", "--input", "transcript=" + inputPath,
|
||||
"--execution-report", path,
|
||||
})
|
||||
if code != ExitOK || stdout != "generated" || atomic.LoadInt32(&hits) != 1 {
|
||||
t.Fatalf("run with report: code=%d stdout=%q stderr=%q hits=%d", code, stdout, stderr, atomic.LoadInt32(&hits))
|
||||
}
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !bytes.HasSuffix(data, []byte("\n")) {
|
||||
t.Fatalf("report is not newline-terminated: %q", data)
|
||||
}
|
||||
var report map[string]any
|
||||
if err := json.Unmarshal(data, &report); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if report["prompt_id"] != "reported" || report["outcome"] != "success" || report["attempts"] != float64(1) {
|
||||
t.Fatalf("unexpected execution report: %#v", report)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecutionReportFailureAndOutputHandling(t *testing.T) {
|
||||
terminal := func(ctx context.Context, req promptkit.RunRequest) (*promptkit.RunResult, error) {
|
||||
if req.Observer != nil {
|
||||
req.Observer(promptkit.ExecutionEvent{Kind: promptkit.ExecutionFinished, PromptID: "p", Outcome: promptkit.ExecutionOutcomeSuccess, Attempts: 1})
|
||||
}
|
||||
return &promptkit.RunResult{Artifact: promptkit.Artifact{Body: []byte("artifact")}, Validation: promptkit.ValidationResult{Status: promptkit.ValidationPassed}}, nil
|
||||
}
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
modify func(*runConfig, string)
|
||||
run func(context.Context, promptkit.RunRequest) (*promptkit.RunResult, error)
|
||||
wantCode int
|
||||
wantOutput string
|
||||
wantDiagnostic string
|
||||
wantReport bool
|
||||
}{
|
||||
{name: "default streams", run: terminal, wantCode: ExitOK, wantOutput: "artifact"},
|
||||
{name: "validation failure", run: func(ctx context.Context, req promptkit.RunRequest) (*promptkit.RunResult, error) {
|
||||
req.Observer(promptkit.ExecutionEvent{Kind: promptkit.ExecutionFinished, Outcome: promptkit.ExecutionOutcomeInvalidOutput})
|
||||
return &promptkit.RunResult{Artifact: promptkit.Artifact{Body: []byte("invalid")}, Validation: promptkit.ValidationResult{Status: promptkit.ValidationFailed}}, nil
|
||||
}, wantCode: ExitValidationFailed, wantOutput: "invalid", wantReport: true},
|
||||
{name: "execution failure", run: func(ctx context.Context, req promptkit.RunRequest) (*promptkit.RunResult, error) {
|
||||
req.Observer(promptkit.ExecutionEvent{Kind: promptkit.ExecutionFinished, Outcome: promptkit.ExecutionOutcomeError, Failure: promptkit.ExecutionFailureProviderStatus})
|
||||
return nil, promptkit.ErrLLMGenerate
|
||||
}, wantCode: ExitRuntimeError, wantDiagnostic: "run error", wantReport: true},
|
||||
{name: "artifact write failure still writes report", run: terminal, modify: func(cfg *runConfig, root string) {
|
||||
cfg.outputPath = filepath.Join(root, "missing", "artifact.txt")
|
||||
}, wantCode: ExitRuntimeError, wantDiagnostic: "output write error", wantReport: true},
|
||||
{name: "report write failure preserves artifact", run: terminal, modify: func(cfg *runConfig, root string) {
|
||||
cfg.executionReportPath = filepath.Join(root, "missing", "execution.json")
|
||||
}, wantCode: ExitRuntimeError, wantOutput: "artifact", wantDiagnostic: "execution report write error"},
|
||||
{name: "execution and report failures are both visible", run: func(ctx context.Context, req promptkit.RunRequest) (*promptkit.RunResult, error) {
|
||||
req.Observer(promptkit.ExecutionEvent{Kind: promptkit.ExecutionFinished, Outcome: promptkit.ExecutionOutcomeError})
|
||||
return nil, promptkit.ErrLLMGenerate
|
||||
}, modify: func(cfg *runConfig, root string) {
|
||||
cfg.executionReportPath = filepath.Join(root, "missing", "execution.json")
|
||||
}, wantCode: ExitRuntimeError, wantDiagnostic: "execution report write error"},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
cfg := &runConfig{executionReportPath: filepath.Join(root, "execution.json")}
|
||||
if tc.name == "default streams" {
|
||||
cfg.executionReportPath = ""
|
||||
}
|
||||
if tc.modify != nil {
|
||||
tc.modify(cfg, root)
|
||||
}
|
||||
var stdout, stderr bytes.Buffer
|
||||
code := executeRun(cfg, promptkit.RunRequest{PromptID: "p"}, tc.run, &stdout, &stderr)
|
||||
if code != tc.wantCode || stdout.String() != tc.wantOutput || (tc.wantDiagnostic != "" && !strings.Contains(stderr.String(), tc.wantDiagnostic)) {
|
||||
t.Fatalf("unexpected output: code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||
}
|
||||
if tc.wantReport {
|
||||
if _, err := os.Stat(cfg.executionReportPath); err != nil {
|
||||
t.Fatalf("terminal report missing: %v", err)
|
||||
}
|
||||
}
|
||||
if tc.name == "execution and report failures are both visible" && !strings.Contains(stderr.String(), "run error") {
|
||||
t.Fatalf("execution failure diagnostic was lost: %s", stderr.String())
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecutionReportLeavesExistingFileWhenNoTerminalEvent(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "execution.json")
|
||||
if err := os.WriteFile(path, []byte("old report"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cfg := &runConfig{executionReportPath: path}
|
||||
var stdout, stderr bytes.Buffer
|
||||
code := executeRun(cfg, promptkit.RunRequest{PromptID: "p"}, func(ctx context.Context, req promptkit.RunRequest) (*promptkit.RunResult, error) {
|
||||
if req.Observer == nil {
|
||||
t.Fatal("requested report did not attach observer")
|
||||
}
|
||||
return &promptkit.RunResult{Artifact: promptkit.Artifact{Body: []byte("artifact")}, Validation: promptkit.ValidationResult{Status: promptkit.ValidationPassed}}, nil
|
||||
}, &stdout, &stderr)
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if code != ExitOK || stdout.String() != "artifact" || string(data) != "old report" || !strings.Contains(stderr.String(), "no terminal observation") {
|
||||
t.Fatalf("no-terminal handling: code=%d stdout=%q stderr=%q report=%q", code, stdout.String(), stderr.String(), data)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecutionReportPathValidationAndPreInvocationFailure(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
args := []string{"--prompt-dir", root, "--prompt", "p"}
|
||||
path := filepath.Join(root, "execution.json")
|
||||
for _, tail := range [][]string{
|
||||
{"--execution-report", ""},
|
||||
{"--out", path, "--execution-report", path},
|
||||
{"--out", path, "--execution-report", filepath.Join(root, "subdir", "..", "execution.json")},
|
||||
} {
|
||||
if _, err := parseRunArgs(append(append([]string{}, args...), tail...)); err == nil {
|
||||
t.Fatalf("accepted invalid report destination: %q", tail)
|
||||
}
|
||||
}
|
||||
if err := os.WriteFile(path, []byte("old report"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
alias := filepath.Join(root, "alias.json")
|
||||
if err := os.Link(path, alias); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := parseRunArgs(append(append([]string{}, args...), "--out", path, "--execution-report", alias)); err == nil {
|
||||
t.Fatal("accepted two names for the same output file")
|
||||
}
|
||||
if cfg, err := parseRunArgs(append(append([]string{}, args...), "--execution-report", "-")); err != nil || cfg.executionReportPath != "-" {
|
||||
t.Fatalf("ordinary dash path was rejected: cfg=%+v err=%v", cfg, err)
|
||||
}
|
||||
if _, err := parseRenderArgs(append(append([]string{}, args...), "--execution-report", path)); err == nil {
|
||||
t.Fatal("render accepted run-only report flag")
|
||||
}
|
||||
code, stdout, stderr := runCLICommand(t, runCommand, []string{"--config", filepath.Join(root, "missing.yml"), "--prompt", "p", "--execution-report", path})
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if code != ExitRuntimeError || stdout != "" || !strings.Contains(stderr, "run parse error") || string(data) != "old report" {
|
||||
t.Fatalf("pre-invocation failure changed report: code=%d stdout=%q stderr=%q report=%q", code, stdout, stderr, data)
|
||||
}
|
||||
}
|
||||
|
||||
func intPointer(value int) *int { return &value }
|
||||
|
||||
func TestPromptkitPromptInspectionSelectsVersionsAndContracts(t *testing.T) {
|
||||
|
||||
Reference in New Issue
Block a user