387 lines
12 KiB
Go
387 lines
12 KiB
Go
package scriptorium
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"os"
|
|
"path/filepath"
|
|
"runtime"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
func TestSubprocessRunnerRunSuccessBuildsDeterministicArgsAndCapturesLogs(t *testing.T) {
|
|
if runtime.GOOS == "windows" {
|
|
t.Skip("helper wrapper script uses /bin/sh")
|
|
}
|
|
|
|
t.Setenv("GO_WANT_SCRIPTORIUM_HELPER", "1")
|
|
t.Setenv("SCRIPTORIUM_HELPER_MODE", "run_success")
|
|
t.Setenv("OPENAI_KEY_SOURCE", "super-secret")
|
|
recordPath := filepath.Join(t.TempDir(), "record.json")
|
|
t.Setenv("SCRIPTORIUM_HELPER_RECORD_PATH", recordPath)
|
|
|
|
runner := NewSubprocessRunner()
|
|
wrapper := writeScriptoriumHelperWrapper(t)
|
|
dir := t.TempDir()
|
|
req := RunArtifactRequest{
|
|
Binary: wrapper,
|
|
ConfigPath: "/etc/scriptorium/config.yml",
|
|
PromptID: "dnd.session_recap",
|
|
ProfileID: "local-quality",
|
|
InputPaths: map[string]string{"transcript": filepath.Join(dir, "polished.json"), "other": filepath.Join(dir, "other.md")},
|
|
Vars: map[string]string{"session_id": "2026-05-03", "campaign_name": "Icewind Dale"},
|
|
OutputPath: filepath.Join(dir, "artifacts", "session_recap.md"),
|
|
StdoutLogPath: filepath.Join(dir, "logs", "scriptorium.run.stdout.log"),
|
|
StderrLogPath: filepath.Join(dir, "logs", "scriptorium.run.stderr.log"),
|
|
GeneratedConfigPath: filepath.Join(dir, "config", "scriptorium.run.generated.yml"),
|
|
Timeout: mustParseScriptoriumDuration(t, "2s"),
|
|
APIKeyEnv: "OPENAI_KEY_SOURCE",
|
|
}
|
|
writeScriptoriumFile(t, req.InputPaths["transcript"], `{"segments":[]}`)
|
|
writeScriptoriumFile(t, req.InputPaths["other"], "notes\n")
|
|
|
|
res, err := runner.RunArtifact(context.Background(), req)
|
|
if err != nil {
|
|
t.Fatalf("RunArtifact() error = %v", err)
|
|
}
|
|
if res.ExitCode != 0 {
|
|
t.Fatalf("ExitCode = %d, want 0", res.ExitCode)
|
|
}
|
|
if res.CommandMode != CommandModeRun {
|
|
t.Fatalf("CommandMode = %q, want %q", res.CommandMode, CommandModeRun)
|
|
}
|
|
if res.OutputPath != req.OutputPath {
|
|
t.Fatalf("OutputPath = %q, want %q", res.OutputPath, req.OutputPath)
|
|
}
|
|
if res.Duration <= 0 {
|
|
t.Fatalf("Duration = %s, want > 0", res.Duration)
|
|
}
|
|
|
|
assertFileNonEmpty(t, req.OutputPath)
|
|
assertFileContains(t, req.StdoutLogPath, "scriptorium helper stdout")
|
|
assertFileContains(t, req.StderrLogPath, "scriptorium helper stderr")
|
|
|
|
rec := readScriptoriumHelperRecord(t, recordPath)
|
|
wantArgs := []string{
|
|
"run",
|
|
"--prompt", "dnd.session_recap",
|
|
"--config", "/etc/scriptorium/config.yml",
|
|
"--profile", "local-quality",
|
|
"--input", "other=" + req.InputPaths["other"],
|
|
"--input", "transcript=" + req.InputPaths["transcript"],
|
|
"--var", "campaign_name=Icewind Dale",
|
|
"--var", "session_id=2026-05-03",
|
|
"--api-key-env", "OPENAI_KEY_SOURCE",
|
|
"--timeout", "2s",
|
|
"--out", req.OutputPath,
|
|
}
|
|
if strings.Join(rec.Args, "\n") != strings.Join(wantArgs, "\n") {
|
|
t.Fatalf("args = %#v, want %#v", rec.Args, wantArgs)
|
|
}
|
|
if rec.Env["OPENAI_KEY_SOURCE"] != "super-secret" {
|
|
t.Fatalf("OPENAI_KEY_SOURCE env = %q, want inherited value", rec.Env["OPENAI_KEY_SOURCE"])
|
|
}
|
|
|
|
cfgData, err := os.ReadFile(req.GeneratedConfigPath)
|
|
if err != nil {
|
|
t.Fatalf("read generated config: %v", err)
|
|
}
|
|
if strings.Contains(string(cfgData), "super-secret") {
|
|
t.Fatalf("generated config must not include credential value")
|
|
}
|
|
}
|
|
|
|
func TestSubprocessRunnerRunExitCodeOneFails(t *testing.T) {
|
|
if runtime.GOOS == "windows" {
|
|
t.Skip("helper wrapper script uses /bin/sh")
|
|
}
|
|
t.Setenv("GO_WANT_SCRIPTORIUM_HELPER", "1")
|
|
t.Setenv("SCRIPTORIUM_HELPER_MODE", "fail_exit1")
|
|
t.Setenv("SCRIPTORIUM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
|
|
|
runner := NewSubprocessRunner()
|
|
req := runReqForTest(t, writeScriptoriumHelperWrapper(t))
|
|
res, err := runner.RunArtifact(context.Background(), req)
|
|
if err == nil {
|
|
t.Fatal("RunArtifact() error = nil, want non-nil")
|
|
}
|
|
if res.ExitCode != 1 {
|
|
t.Fatalf("ExitCode = %d, want 1", res.ExitCode)
|
|
}
|
|
if res.ValidationFailed {
|
|
t.Fatalf("ValidationFailed = true, want false")
|
|
}
|
|
assertFileContains(t, req.StderrLogPath, "scriptorium helper failure")
|
|
}
|
|
|
|
func TestSubprocessRunnerRunExitCodeTwoReturnsValidationFailureAndPreservesOutput(t *testing.T) {
|
|
if runtime.GOOS == "windows" {
|
|
t.Skip("helper wrapper script uses /bin/sh")
|
|
}
|
|
t.Setenv("GO_WANT_SCRIPTORIUM_HELPER", "1")
|
|
t.Setenv("SCRIPTORIUM_HELPER_MODE", "fail_exit2_with_output")
|
|
t.Setenv("SCRIPTORIUM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
|
|
|
runner := NewSubprocessRunner()
|
|
req := runReqForTest(t, writeScriptoriumHelperWrapper(t))
|
|
res, err := runner.RunArtifact(context.Background(), req)
|
|
if err == nil {
|
|
t.Fatal("RunArtifact() error = nil, want non-nil")
|
|
}
|
|
if res.ExitCode != 2 {
|
|
t.Fatalf("ExitCode = %d, want 2", res.ExitCode)
|
|
}
|
|
if !res.ValidationFailed {
|
|
t.Fatalf("ValidationFailed = false, want true")
|
|
}
|
|
if res.OutputPath != req.OutputPath {
|
|
t.Fatalf("OutputPath = %q, want %q", res.OutputPath, req.OutputPath)
|
|
}
|
|
assertFileNonEmpty(t, req.OutputPath)
|
|
}
|
|
|
|
func TestSubprocessRunnerRunMissingOutputOnSuccessFails(t *testing.T) {
|
|
if runtime.GOOS == "windows" {
|
|
t.Skip("helper wrapper script uses /bin/sh")
|
|
}
|
|
t.Setenv("GO_WANT_SCRIPTORIUM_HELPER", "1")
|
|
t.Setenv("SCRIPTORIUM_HELPER_MODE", "run_success_missing_output")
|
|
t.Setenv("SCRIPTORIUM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
|
|
|
runner := NewSubprocessRunner()
|
|
req := runReqForTest(t, writeScriptoriumHelperWrapper(t))
|
|
res, err := runner.RunArtifact(context.Background(), req)
|
|
if err == nil {
|
|
t.Fatal("RunArtifact() error = nil, want non-nil")
|
|
}
|
|
if res.ExitCode != 0 {
|
|
t.Fatalf("ExitCode = %d, want 0", res.ExitCode)
|
|
}
|
|
if !strings.Contains(err.Error(), "validate scriptorium run output") {
|
|
t.Fatalf("error = %q, want output validation context", err.Error())
|
|
}
|
|
}
|
|
|
|
func TestSubprocessRunnerRenderSuccess(t *testing.T) {
|
|
if runtime.GOOS == "windows" {
|
|
t.Skip("helper wrapper script uses /bin/sh")
|
|
}
|
|
t.Setenv("GO_WANT_SCRIPTORIUM_HELPER", "1")
|
|
t.Setenv("SCRIPTORIUM_HELPER_MODE", "render_success")
|
|
recordPath := filepath.Join(t.TempDir(), "record.json")
|
|
t.Setenv("SCRIPTORIUM_HELPER_RECORD_PATH", recordPath)
|
|
|
|
runner := NewSubprocessRunner()
|
|
wrapper := writeScriptoriumHelperWrapper(t)
|
|
dir := t.TempDir()
|
|
req := RenderArtifactRequest{
|
|
Binary: wrapper,
|
|
PromptID: "dnd.session_recap",
|
|
InputPaths: map[string]string{"transcript": filepath.Join(dir, "polished.json")},
|
|
OutputPath: filepath.Join(dir, "artifacts", "session_recap.render.json"),
|
|
StdoutLogPath: filepath.Join(dir, "logs", "scriptorium.render.stdout.log"),
|
|
StderrLogPath: filepath.Join(dir, "logs", "scriptorium.render.stderr.log"),
|
|
GeneratedConfigPath: filepath.Join(dir, "config", "scriptorium.render.generated.yml"),
|
|
Timeout: mustParseScriptoriumDuration(t, "2s"),
|
|
}
|
|
writeScriptoriumFile(t, req.InputPaths["transcript"], `{"segments":[]}`)
|
|
|
|
res, err := runner.RenderArtifact(context.Background(), req)
|
|
if err != nil {
|
|
t.Fatalf("RenderArtifact() error = %v", err)
|
|
}
|
|
if res.CommandMode != CommandModeRender {
|
|
t.Fatalf("CommandMode = %q, want %q", res.CommandMode, CommandModeRender)
|
|
}
|
|
if res.ExitCode != 0 {
|
|
t.Fatalf("ExitCode = %d, want 0", res.ExitCode)
|
|
}
|
|
assertFileNonEmpty(t, req.OutputPath)
|
|
|
|
rec := readScriptoriumHelperRecord(t, recordPath)
|
|
wantArgs := []string{
|
|
"render",
|
|
"--prompt", "dnd.session_recap",
|
|
"--input", "transcript=" + req.InputPaths["transcript"],
|
|
"--timeout", "2s",
|
|
"--format", "json",
|
|
"--out", req.OutputPath,
|
|
}
|
|
if strings.Join(rec.Args, "\n") != strings.Join(wantArgs, "\n") {
|
|
t.Fatalf("args = %#v, want %#v", rec.Args, wantArgs)
|
|
}
|
|
}
|
|
|
|
func TestScriptoriumSubprocessHelper(t *testing.T) {
|
|
if os.Getenv("GO_WANT_SCRIPTORIUM_HELPER") != "1" {
|
|
return
|
|
}
|
|
|
|
args := os.Args
|
|
start := -1
|
|
for i := range args {
|
|
if args[i] == "--" {
|
|
start = i + 1
|
|
break
|
|
}
|
|
}
|
|
if start < 0 || start >= len(args) {
|
|
_, _ = os.Stderr.WriteString("missing -- args separator\n")
|
|
os.Exit(2)
|
|
}
|
|
cliArgs := args[start:]
|
|
|
|
outputPath := flagValue(cliArgs, "--out")
|
|
recordPath := os.Getenv("SCRIPTORIUM_HELPER_RECORD_PATH")
|
|
if strings.TrimSpace(recordPath) != "" {
|
|
rec := scriptoriumHelperRecord{
|
|
Args: cliArgs,
|
|
Env: map[string]string{
|
|
"OPENAI_KEY_SOURCE": os.Getenv("OPENAI_KEY_SOURCE"),
|
|
},
|
|
}
|
|
data, _ := json.Marshal(rec)
|
|
_ = os.MkdirAll(filepath.Dir(recordPath), 0o755)
|
|
_ = os.WriteFile(recordPath, data, 0o644)
|
|
}
|
|
|
|
mode := os.Getenv("SCRIPTORIUM_HELPER_MODE")
|
|
switch mode {
|
|
case "run_success":
|
|
writeScriptoriumHelperFile(outputPath, "generated artifact\n")
|
|
_, _ = os.Stdout.WriteString("scriptorium helper stdout\n")
|
|
_, _ = os.Stderr.WriteString("scriptorium helper stderr\n")
|
|
os.Exit(0)
|
|
case "run_success_missing_output":
|
|
_, _ = os.Stdout.WriteString("scriptorium helper stdout missing output\n")
|
|
_, _ = os.Stderr.WriteString("scriptorium helper stderr missing output\n")
|
|
os.Exit(0)
|
|
case "fail_exit1":
|
|
_, _ = os.Stderr.WriteString("scriptorium helper failure\n")
|
|
os.Exit(1)
|
|
case "fail_exit2_with_output":
|
|
writeScriptoriumHelperFile(outputPath, "validation failed artifact\n")
|
|
_, _ = os.Stderr.WriteString("scriptorium helper validation failure\n")
|
|
os.Exit(2)
|
|
case "render_success":
|
|
writeScriptoriumHelperFile(outputPath, "{\"rendered\":true}\n")
|
|
_, _ = os.Stdout.WriteString("scriptorium helper render stdout\n")
|
|
_, _ = os.Stderr.WriteString("scriptorium helper render stderr\n")
|
|
os.Exit(0)
|
|
default:
|
|
_, _ = os.Stderr.WriteString(fmt.Sprintf("unknown helper mode %q\n", mode))
|
|
os.Exit(2)
|
|
}
|
|
}
|
|
|
|
type scriptoriumHelperRecord struct {
|
|
Args []string `json:"args"`
|
|
Env map[string]string `json:"env"`
|
|
}
|
|
|
|
func runReqForTest(t *testing.T, binary string) RunArtifactRequest {
|
|
t.Helper()
|
|
dir := t.TempDir()
|
|
transcriptPath := filepath.Join(dir, "polished.json")
|
|
writeScriptoriumFile(t, transcriptPath, `{"segments":[]}`)
|
|
return RunArtifactRequest{
|
|
Binary: binary,
|
|
PromptID: "dnd.session_recap",
|
|
InputPaths: map[string]string{"transcript": transcriptPath},
|
|
Vars: map[string]string{"output_kind": "session_recap"},
|
|
OutputPath: filepath.Join(dir, "artifacts", "session_recap.md"),
|
|
StdoutLogPath: filepath.Join(dir, "logs", "scriptorium.run.stdout.log"),
|
|
StderrLogPath: filepath.Join(dir, "logs", "scriptorium.run.stderr.log"),
|
|
GeneratedConfigPath: filepath.Join(dir, "config", "scriptorium.generated.yml"),
|
|
Timeout: mustParseScriptoriumDuration(t, "2s"),
|
|
}
|
|
}
|
|
|
|
func writeScriptoriumHelperWrapper(t *testing.T) string {
|
|
t.Helper()
|
|
exe, err := os.Executable()
|
|
if err != nil {
|
|
t.Fatalf("os.Executable() error = %v", err)
|
|
}
|
|
path := filepath.Join(t.TempDir(), "scriptorium-helper-wrapper.sh")
|
|
content := "#!/bin/sh\nexec \"" + exe + "\" -test.run=TestScriptoriumSubprocessHelper -- \"$@\"\n"
|
|
if err := os.WriteFile(path, []byte(content), 0o755); err != nil {
|
|
t.Fatalf("WriteFile(%q) error = %v", path, err)
|
|
}
|
|
return path
|
|
}
|
|
|
|
func mustParseScriptoriumDuration(t *testing.T, v string) time.Duration {
|
|
t.Helper()
|
|
d, err := time.ParseDuration(v)
|
|
if err != nil {
|
|
t.Fatalf("time.ParseDuration(%q) error = %v", v, err)
|
|
}
|
|
return d
|
|
}
|
|
|
|
func flagValue(args []string, name string) string {
|
|
for i := 0; i < len(args)-1; i++ {
|
|
if args[i] == name {
|
|
return args[i+1]
|
|
}
|
|
}
|
|
return ""
|
|
}
|
|
|
|
func writeScriptoriumFile(t *testing.T, path, contents string) {
|
|
t.Helper()
|
|
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
|
t.Fatalf("MkdirAll(%q): %v", path, err)
|
|
}
|
|
if err := os.WriteFile(path, []byte(contents), 0o644); err != nil {
|
|
t.Fatalf("WriteFile(%q): %v", path, err)
|
|
}
|
|
}
|
|
|
|
func writeScriptoriumHelperFile(path, contents string) {
|
|
if strings.TrimSpace(path) == "" {
|
|
return
|
|
}
|
|
_ = os.MkdirAll(filepath.Dir(path), 0o755)
|
|
_ = os.WriteFile(path, []byte(contents), 0o644)
|
|
}
|
|
|
|
func assertFileNonEmpty(t *testing.T, path string) {
|
|
t.Helper()
|
|
info, err := os.Stat(path)
|
|
if err != nil {
|
|
t.Fatalf("Stat(%q): %v", path, err)
|
|
}
|
|
if info.Size() <= 0 {
|
|
t.Fatalf("file %q is empty", path)
|
|
}
|
|
}
|
|
|
|
func assertFileContains(t *testing.T, path, wantSubstring string) {
|
|
t.Helper()
|
|
data, err := os.ReadFile(path)
|
|
if err != nil {
|
|
t.Fatalf("ReadFile(%q): %v", path, err)
|
|
}
|
|
if !strings.Contains(string(data), wantSubstring) {
|
|
t.Fatalf("file %q contents = %q, want substring %q", path, string(data), wantSubstring)
|
|
}
|
|
}
|
|
|
|
func readScriptoriumHelperRecord(t *testing.T, path string) scriptoriumHelperRecord {
|
|
t.Helper()
|
|
data, err := os.ReadFile(path)
|
|
if err != nil {
|
|
t.Fatalf("ReadFile(%q): %v", path, err)
|
|
}
|
|
var rec scriptoriumHelperRecord
|
|
if err := json.Unmarshal(data, &rec); err != nil {
|
|
t.Fatalf("json unmarshal helper record: %v", err)
|
|
}
|
|
return rec
|
|
}
|