414 lines
12 KiB
Go
414 lines
12 KiB
Go
package scriptorium
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"reflect"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
func TestRenderConstructsCommand(t *testing.T) {
|
|
commands := &fakeCommands{result: CommandResult{Stdout: []byte(`{"ok":true}`)}}
|
|
runner := Runner{
|
|
Binary: "/usr/local/bin/scriptorium",
|
|
ConfigPath: "/etc/scriptorium.yml",
|
|
Profile: "weather",
|
|
Timeout: time.Minute,
|
|
Commands: commands,
|
|
}
|
|
|
|
result, err := runner.Render(context.Background(), RenderRequest{
|
|
PromptID: "weather.daily_generated_text",
|
|
DataPackagePath: "/tmp/data_package.yaml",
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("Render() error = %v", err)
|
|
}
|
|
|
|
wantArgs := []string{
|
|
"render",
|
|
"--config", "/etc/scriptorium.yml",
|
|
"--profile", "weather",
|
|
"--prompt", "weather.daily_generated_text",
|
|
"--input", "data_package=/tmp/data_package.yaml",
|
|
"--format", "json",
|
|
}
|
|
if commands.name != "/usr/local/bin/scriptorium" {
|
|
t.Fatalf("command name = %q, want custom binary", commands.name)
|
|
}
|
|
if !reflect.DeepEqual(commands.args, wantArgs) {
|
|
t.Fatalf("args = %#v, want %#v", commands.args, wantArgs)
|
|
}
|
|
if !reflect.DeepEqual(result.Command, append([]string{"/usr/local/bin/scriptorium"}, wantArgs...)) {
|
|
t.Fatalf("result command = %#v, want full argv", result.Command)
|
|
}
|
|
}
|
|
|
|
func TestRenderReturnsResultForNonzeroExit(t *testing.T) {
|
|
runner := Runner{
|
|
Commands: &fakeCommands{
|
|
result: CommandResult{
|
|
Stderr: []byte("missing input"),
|
|
ExitCode: 1,
|
|
},
|
|
},
|
|
}
|
|
|
|
result, err := runner.Render(context.Background(), RenderRequest{
|
|
PromptID: "weather.daily_generated_text",
|
|
DataPackagePath: "/tmp/data_package.yaml",
|
|
})
|
|
if err == nil {
|
|
t.Fatal("Render() error = nil, want nonzero exit error")
|
|
}
|
|
if result == nil {
|
|
t.Fatal("Render() result = nil, want captured result")
|
|
}
|
|
if result.ExitCode != 1 {
|
|
t.Fatalf("ExitCode = %d, want 1", result.ExitCode)
|
|
}
|
|
if !strings.Contains(err.Error(), "missing input") {
|
|
t.Fatalf("error = %q, want stderr context", err.Error())
|
|
}
|
|
}
|
|
|
|
func TestStructuredRunConstructsCommandWithoutSchemaFlags(t *testing.T) {
|
|
commands := &fakeCommands{result: CommandResult{
|
|
Stdout: []byte(`{"summary":"ok"}`),
|
|
Stderr: []byte("wrote generated text"),
|
|
StdoutTruncated: true,
|
|
}}
|
|
runner := Runner{
|
|
Binary: "/usr/local/bin/scriptorium",
|
|
ConfigPath: "/etc/scriptorium.yml",
|
|
Profile: "weather",
|
|
Timeout: 30 * time.Second,
|
|
Commands: commands,
|
|
}
|
|
|
|
result, err := runner.StructuredRun(context.Background(), StructuredRunRequest{
|
|
PromptID: "weather.hourly_generated_text",
|
|
DataPackagePath: "/tmp/data_package.hourly.yaml",
|
|
OutputPath: "/tmp/generated_text_raw.hourly.json",
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("StructuredRun() error = %v", err)
|
|
}
|
|
|
|
wantArgs := []string{
|
|
"run",
|
|
"--config", "/etc/scriptorium.yml",
|
|
"--profile", "weather",
|
|
"--prompt", "weather.hourly_generated_text",
|
|
"--input", "data_package=/tmp/data_package.hourly.yaml",
|
|
"--out", "/tmp/generated_text_raw.hourly.json",
|
|
}
|
|
if commands.name != "/usr/local/bin/scriptorium" {
|
|
t.Fatalf("command name = %q, want custom binary", commands.name)
|
|
}
|
|
if !reflect.DeepEqual(commands.args, wantArgs) {
|
|
t.Fatalf("args = %#v, want %#v", commands.args, wantArgs)
|
|
}
|
|
for _, disallowed := range []string{"--format", "--schema", "--schema-path", "--json-schema"} {
|
|
if containsArg(commands.args, disallowed) {
|
|
t.Fatalf("args = %#v, should not include %q", commands.args, disallowed)
|
|
}
|
|
}
|
|
if commands.timeout != 30*time.Second {
|
|
t.Fatalf("timeout = %s, want 30s", commands.timeout)
|
|
}
|
|
if !reflect.DeepEqual(result.Command, append([]string{"/usr/local/bin/scriptorium"}, wantArgs...)) {
|
|
t.Fatalf("result command = %#v, want full argv", result.Command)
|
|
}
|
|
if result.Stdout != `{"summary":"ok"}` || result.Stderr != "wrote generated text" || !result.StdoutTruncated {
|
|
t.Fatalf("result = %#v, want captured output and truncation flags", result)
|
|
}
|
|
if result.OutputPath != "/tmp/generated_text_raw.hourly.json" {
|
|
t.Fatalf("OutputPath = %q, want generated text raw path", result.OutputPath)
|
|
}
|
|
}
|
|
|
|
func TestStructuredRunReturnsResultForNonzeroExit(t *testing.T) {
|
|
runner := Runner{
|
|
Commands: &fakeCommands{
|
|
result: CommandResult{
|
|
Stdout: []byte(`{"summary":"partial"}`),
|
|
Stderr: []byte("structured output failed"),
|
|
ExitCode: 3,
|
|
},
|
|
},
|
|
}
|
|
|
|
result, err := runner.StructuredRun(context.Background(), StructuredRunRequest{
|
|
PromptID: "weather.hourly_generated_text",
|
|
DataPackagePath: "/tmp/data_package.hourly.yaml",
|
|
OutputPath: "/tmp/generated_text_raw.hourly.json",
|
|
})
|
|
if err == nil {
|
|
t.Fatal("StructuredRun() error = nil, want nonzero exit error")
|
|
}
|
|
if result == nil {
|
|
t.Fatal("StructuredRun() result = nil, want captured result")
|
|
}
|
|
if result.ExitCode != 3 {
|
|
t.Fatalf("ExitCode = %d, want 3", result.ExitCode)
|
|
}
|
|
if result.Stdout != `{"summary":"partial"}` || result.OutputPath != "/tmp/generated_text_raw.hourly.json" {
|
|
t.Fatalf("result = %#v, want captured result fields", result)
|
|
}
|
|
if !strings.Contains(err.Error(), "structured output failed") {
|
|
t.Fatalf("error = %q, want stderr context", err.Error())
|
|
}
|
|
}
|
|
|
|
func TestOutputRunsPreserveCapturedResultFields(t *testing.T) {
|
|
type commonResult struct {
|
|
Command []string
|
|
Stdout string
|
|
Stderr string
|
|
StdoutTruncated bool
|
|
StderrTruncated bool
|
|
ExitCode int
|
|
OutputPath string
|
|
}
|
|
tests := []struct {
|
|
name string
|
|
run func(Runner) (*commonResult, error)
|
|
}{
|
|
{
|
|
name: "StructuredRun",
|
|
run: func(runner Runner) (*commonResult, error) {
|
|
result, err := runner.StructuredRun(context.Background(), StructuredRunRequest{
|
|
PromptID: "weather.daily_generated_text",
|
|
DataPackagePath: "/tmp/data_package.yaml",
|
|
OutputPath: "/tmp/report.md",
|
|
})
|
|
if result == nil {
|
|
return nil, err
|
|
}
|
|
return &commonResult{
|
|
Command: result.Command,
|
|
Stdout: result.Stdout,
|
|
Stderr: result.Stderr,
|
|
StdoutTruncated: result.StdoutTruncated,
|
|
StderrTruncated: result.StderrTruncated,
|
|
ExitCode: result.ExitCode,
|
|
OutputPath: result.OutputPath,
|
|
}, err
|
|
},
|
|
},
|
|
}
|
|
|
|
for _, test := range tests {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
commands := &fakeCommands{result: CommandResult{
|
|
Stdout: []byte("captured stdout"),
|
|
Stderr: []byte("captured stderr"),
|
|
StdoutTruncated: true,
|
|
StderrTruncated: true,
|
|
}}
|
|
runner := Runner{
|
|
Binary: "/usr/local/bin/scriptorium",
|
|
ConfigPath: "/etc/scriptorium.yml",
|
|
Profile: "weather",
|
|
Timeout: 15 * time.Second,
|
|
Commands: commands,
|
|
}
|
|
|
|
result, err := test.run(runner)
|
|
if err != nil {
|
|
t.Fatalf("%s error = %v", test.name, err)
|
|
}
|
|
wantArgs := []string{
|
|
"run",
|
|
"--config", "/etc/scriptorium.yml",
|
|
"--profile", "weather",
|
|
"--prompt", "weather.daily_generated_text",
|
|
"--input", "data_package=/tmp/data_package.yaml",
|
|
"--out", "/tmp/report.md",
|
|
}
|
|
if !reflect.DeepEqual(commands.args, wantArgs) {
|
|
t.Fatalf("args = %#v, want %#v", commands.args, wantArgs)
|
|
}
|
|
if commands.timeout != 15*time.Second {
|
|
t.Fatalf("timeout = %s, want 15s", commands.timeout)
|
|
}
|
|
if !reflect.DeepEqual(result.Command, append([]string{"/usr/local/bin/scriptorium"}, wantArgs...)) {
|
|
t.Fatalf("Command = %#v, want full argv", result.Command)
|
|
}
|
|
if result.Stdout != "captured stdout" || result.Stderr != "captured stderr" {
|
|
t.Fatalf("captured output = %q/%q, want stdout/stderr", result.Stdout, result.Stderr)
|
|
}
|
|
if !result.StdoutTruncated || !result.StderrTruncated {
|
|
t.Fatalf("truncation flags = %t/%t, want both true", result.StdoutTruncated, result.StderrTruncated)
|
|
}
|
|
if result.ExitCode != 0 || result.OutputPath != "/tmp/report.md" {
|
|
t.Fatalf("result = %#v, want exit 0 and output path", result)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestOutputRunsReturnCapturedResultForNonzeroExit(t *testing.T) {
|
|
type commonResult struct {
|
|
Stdout string
|
|
Stderr string
|
|
StderrTruncated bool
|
|
ExitCode int
|
|
OutputPath string
|
|
}
|
|
tests := []struct {
|
|
name string
|
|
run func(Runner) (*commonResult, error)
|
|
wantErr string
|
|
}{
|
|
{
|
|
name: "StructuredRun",
|
|
run: func(runner Runner) (*commonResult, error) {
|
|
result, err := runner.StructuredRun(context.Background(), StructuredRunRequest{
|
|
PromptID: "weather.daily_generated_text",
|
|
DataPackagePath: "/tmp/data_package.yaml",
|
|
OutputPath: "/tmp/report.md",
|
|
})
|
|
if result == nil {
|
|
return nil, err
|
|
}
|
|
return &commonResult{
|
|
Stdout: result.Stdout,
|
|
Stderr: result.Stderr,
|
|
StderrTruncated: result.StderrTruncated,
|
|
ExitCode: result.ExitCode,
|
|
OutputPath: result.OutputPath,
|
|
}, err
|
|
},
|
|
wantErr: "scriptorium structured run exited with code 7: captured stderr",
|
|
},
|
|
}
|
|
|
|
for _, test := range tests {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
runner := Runner{
|
|
Commands: &fakeCommands{result: CommandResult{
|
|
Stdout: []byte("captured stdout"),
|
|
Stderr: []byte("captured stderr"),
|
|
StderrTruncated: true,
|
|
ExitCode: 7,
|
|
}},
|
|
}
|
|
|
|
result, err := test.run(runner)
|
|
if err == nil {
|
|
t.Fatalf("%s error = nil, want nonzero exit error", test.name)
|
|
}
|
|
if result == nil {
|
|
t.Fatalf("%s result = nil, want captured result", test.name)
|
|
}
|
|
if err.Error() != test.wantErr {
|
|
t.Fatalf("%s error = %q, want %q", test.name, err.Error(), test.wantErr)
|
|
}
|
|
if result.Stdout != "captured stdout" || result.Stderr != "captured stderr" || !result.StderrTruncated {
|
|
t.Fatalf("captured result = %#v, want stdout/stderr/truncation", result)
|
|
}
|
|
if result.ExitCode != 7 || result.OutputPath != "/tmp/report.md" {
|
|
t.Fatalf("result = %#v, want exit 7 and output path", result)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestOutputRunsValidateRequiredFieldsBeforeExecution(t *testing.T) {
|
|
tests := []struct {
|
|
name string
|
|
run func(Runner, string, string, string) error
|
|
}{
|
|
{
|
|
name: "StructuredRun",
|
|
run: func(runner Runner, promptID string, dataPackagePath string, outputPath string) error {
|
|
result, err := runner.StructuredRun(context.Background(), StructuredRunRequest{
|
|
PromptID: promptID,
|
|
DataPackagePath: dataPackagePath,
|
|
OutputPath: outputPath,
|
|
})
|
|
if result != nil {
|
|
return fmt.Errorf("result = %#v, want nil", result)
|
|
}
|
|
return err
|
|
},
|
|
},
|
|
}
|
|
cases := []struct {
|
|
name string
|
|
promptID string
|
|
dataPackagePath string
|
|
outputPath string
|
|
want string
|
|
}{
|
|
{
|
|
name: "prompt id",
|
|
dataPackagePath: "/tmp/data_package.yaml",
|
|
outputPath: "/tmp/report.md",
|
|
want: "prompt id is required",
|
|
},
|
|
{
|
|
name: "data package path",
|
|
promptID: "weather.daily_generated_text",
|
|
outputPath: "/tmp/report.md",
|
|
want: "data package path is required",
|
|
},
|
|
{
|
|
name: "output path",
|
|
promptID: "weather.daily_generated_text",
|
|
dataPackagePath: "/tmp/data_package.yaml",
|
|
want: "output path is required",
|
|
},
|
|
}
|
|
|
|
for _, test := range tests {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
for _, tc := range cases {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
commands := &fakeCommands{}
|
|
err := test.run(Runner{Commands: commands}, tc.promptID, tc.dataPackagePath, tc.outputPath)
|
|
if err == nil {
|
|
t.Fatalf("%s error = nil, want validation error", test.name)
|
|
}
|
|
if !strings.Contains(err.Error(), tc.want) {
|
|
t.Fatalf("%s error = %v, want %q", test.name, err, tc.want)
|
|
}
|
|
if commands.calls != 0 {
|
|
t.Fatalf("commands calls = %d, want no subprocess execution", commands.calls)
|
|
}
|
|
})
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
type fakeCommands struct {
|
|
name string
|
|
args []string
|
|
timeout time.Duration
|
|
result CommandResult
|
|
err error
|
|
calls int
|
|
}
|
|
|
|
func (f *fakeCommands) Run(_ context.Context, name string, args []string, timeout time.Duration) (CommandResult, error) {
|
|
f.calls++
|
|
f.name = name
|
|
f.args = append([]string{}, args...)
|
|
f.timeout = timeout
|
|
return f.result, f.err
|
|
}
|
|
|
|
func containsArg(args []string, want string) bool {
|
|
for _, arg := range args {
|
|
if arg == want {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|