package scriptorium import ( "context" "fmt" "reflect" "strings" "testing" "time" ) func TestRenderConstructsCommand(t *testing.T) { commands := &fakeCommands{result: CommandResult{Stdout: []byte(`{"ok":true}`)}} runner := Runner{ Binary: "/usr/local/bin/scriptorium", ConfigPath: "/etc/scriptorium.yml", Profile: "weather", Timeout: time.Minute, Commands: commands, } result, err := runner.Render(context.Background(), RenderRequest{ PromptID: "weather.daily_generated_text", DataPackagePath: "/tmp/data_package.yaml", }) if err != nil { t.Fatalf("Render() error = %v", err) } wantArgs := []string{ "render", "--config", "/etc/scriptorium.yml", "--profile", "weather", "--prompt", "weather.daily_generated_text", "--input", "data_package=/tmp/data_package.yaml", "--format", "json", } if commands.name != "/usr/local/bin/scriptorium" { t.Fatalf("command name = %q, want custom binary", commands.name) } if !reflect.DeepEqual(commands.args, wantArgs) { t.Fatalf("args = %#v, want %#v", commands.args, wantArgs) } if !reflect.DeepEqual(result.Command, append([]string{"/usr/local/bin/scriptorium"}, wantArgs...)) { t.Fatalf("result command = %#v, want full argv", result.Command) } } func TestRenderReturnsResultForNonzeroExit(t *testing.T) { runner := Runner{ Commands: &fakeCommands{ result: CommandResult{ Stderr: []byte("missing input"), ExitCode: 1, }, }, } result, err := runner.Render(context.Background(), RenderRequest{ PromptID: "weather.daily_generated_text", DataPackagePath: "/tmp/data_package.yaml", }) if err == nil { t.Fatal("Render() error = nil, want nonzero exit error") } if result == nil { t.Fatal("Render() result = nil, want captured result") } if result.ExitCode != 1 { t.Fatalf("ExitCode = %d, want 1", result.ExitCode) } if !strings.Contains(err.Error(), "missing input") { t.Fatalf("error = %q, want stderr context", err.Error()) } } func TestStructuredRunConstructsCommandWithoutSchemaFlags(t *testing.T) { commands := &fakeCommands{result: CommandResult{ Stdout: []byte(`{"summary":"ok"}`), Stderr: []byte("wrote generated text"), StdoutTruncated: true, }} runner := Runner{ Binary: "/usr/local/bin/scriptorium", ConfigPath: "/etc/scriptorium.yml", Profile: "weather", Timeout: 30 * time.Second, Commands: commands, } result, err := runner.StructuredRun(context.Background(), StructuredRunRequest{ PromptID: "weather.hourly_generated_text", DataPackagePath: "/tmp/data_package.hourly.yaml", OutputPath: "/tmp/generated_text_raw.hourly.json", }) if err != nil { t.Fatalf("StructuredRun() error = %v", err) } wantArgs := []string{ "run", "--config", "/etc/scriptorium.yml", "--profile", "weather", "--prompt", "weather.hourly_generated_text", "--input", "data_package=/tmp/data_package.hourly.yaml", "--out", "/tmp/generated_text_raw.hourly.json", } if commands.name != "/usr/local/bin/scriptorium" { t.Fatalf("command name = %q, want custom binary", commands.name) } if !reflect.DeepEqual(commands.args, wantArgs) { t.Fatalf("args = %#v, want %#v", commands.args, wantArgs) } for _, disallowed := range []string{"--format", "--schema", "--schema-path", "--json-schema"} { if containsArg(commands.args, disallowed) { t.Fatalf("args = %#v, should not include %q", commands.args, disallowed) } } if commands.timeout != 30*time.Second { t.Fatalf("timeout = %s, want 30s", commands.timeout) } if !reflect.DeepEqual(result.Command, append([]string{"/usr/local/bin/scriptorium"}, wantArgs...)) { t.Fatalf("result command = %#v, want full argv", result.Command) } if result.Stdout != `{"summary":"ok"}` || result.Stderr != "wrote generated text" || !result.StdoutTruncated { t.Fatalf("result = %#v, want captured output and truncation flags", result) } if result.OutputPath != "/tmp/generated_text_raw.hourly.json" { t.Fatalf("OutputPath = %q, want generated text raw path", result.OutputPath) } } func TestStructuredRunReturnsResultForNonzeroExit(t *testing.T) { runner := Runner{ Commands: &fakeCommands{ result: CommandResult{ Stdout: []byte(`{"summary":"partial"}`), Stderr: []byte("structured output failed"), ExitCode: 3, }, }, } result, err := runner.StructuredRun(context.Background(), StructuredRunRequest{ PromptID: "weather.hourly_generated_text", DataPackagePath: "/tmp/data_package.hourly.yaml", OutputPath: "/tmp/generated_text_raw.hourly.json", }) if err == nil { t.Fatal("StructuredRun() error = nil, want nonzero exit error") } if result == nil { t.Fatal("StructuredRun() result = nil, want captured result") } if result.ExitCode != 3 { t.Fatalf("ExitCode = %d, want 3", result.ExitCode) } if result.Stdout != `{"summary":"partial"}` || result.OutputPath != "/tmp/generated_text_raw.hourly.json" { t.Fatalf("result = %#v, want captured result fields", result) } if !strings.Contains(err.Error(), "structured output failed") { t.Fatalf("error = %q, want stderr context", err.Error()) } } func TestOutputRunsPreserveCapturedResultFields(t *testing.T) { type commonResult struct { Command []string Stdout string Stderr string StdoutTruncated bool StderrTruncated bool ExitCode int OutputPath string } tests := []struct { name string run func(Runner) (*commonResult, error) }{ { name: "StructuredRun", run: func(runner Runner) (*commonResult, error) { result, err := runner.StructuredRun(context.Background(), StructuredRunRequest{ PromptID: "weather.daily_generated_text", DataPackagePath: "/tmp/data_package.yaml", OutputPath: "/tmp/report.md", }) if result == nil { return nil, err } return &commonResult{ Command: result.Command, Stdout: result.Stdout, Stderr: result.Stderr, StdoutTruncated: result.StdoutTruncated, StderrTruncated: result.StderrTruncated, ExitCode: result.ExitCode, OutputPath: result.OutputPath, }, err }, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { commands := &fakeCommands{result: CommandResult{ Stdout: []byte("captured stdout"), Stderr: []byte("captured stderr"), StdoutTruncated: true, StderrTruncated: true, }} runner := Runner{ Binary: "/usr/local/bin/scriptorium", ConfigPath: "/etc/scriptorium.yml", Profile: "weather", Timeout: 15 * time.Second, Commands: commands, } result, err := test.run(runner) if err != nil { t.Fatalf("%s error = %v", test.name, err) } wantArgs := []string{ "run", "--config", "/etc/scriptorium.yml", "--profile", "weather", "--prompt", "weather.daily_generated_text", "--input", "data_package=/tmp/data_package.yaml", "--out", "/tmp/report.md", } if !reflect.DeepEqual(commands.args, wantArgs) { t.Fatalf("args = %#v, want %#v", commands.args, wantArgs) } if commands.timeout != 15*time.Second { t.Fatalf("timeout = %s, want 15s", commands.timeout) } if !reflect.DeepEqual(result.Command, append([]string{"/usr/local/bin/scriptorium"}, wantArgs...)) { t.Fatalf("Command = %#v, want full argv", result.Command) } if result.Stdout != "captured stdout" || result.Stderr != "captured stderr" { t.Fatalf("captured output = %q/%q, want stdout/stderr", result.Stdout, result.Stderr) } if !result.StdoutTruncated || !result.StderrTruncated { t.Fatalf("truncation flags = %t/%t, want both true", result.StdoutTruncated, result.StderrTruncated) } if result.ExitCode != 0 || result.OutputPath != "/tmp/report.md" { t.Fatalf("result = %#v, want exit 0 and output path", result) } }) } } func TestOutputRunsReturnCapturedResultForNonzeroExit(t *testing.T) { type commonResult struct { Stdout string Stderr string StderrTruncated bool ExitCode int OutputPath string } tests := []struct { name string run func(Runner) (*commonResult, error) wantErr string }{ { name: "StructuredRun", run: func(runner Runner) (*commonResult, error) { result, err := runner.StructuredRun(context.Background(), StructuredRunRequest{ PromptID: "weather.daily_generated_text", DataPackagePath: "/tmp/data_package.yaml", OutputPath: "/tmp/report.md", }) if result == nil { return nil, err } return &commonResult{ Stdout: result.Stdout, Stderr: result.Stderr, StderrTruncated: result.StderrTruncated, ExitCode: result.ExitCode, OutputPath: result.OutputPath, }, err }, wantErr: "scriptorium structured run exited with code 7: captured stderr", }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { runner := Runner{ Commands: &fakeCommands{result: CommandResult{ Stdout: []byte("captured stdout"), Stderr: []byte("captured stderr"), StderrTruncated: true, ExitCode: 7, }}, } result, err := test.run(runner) if err == nil { t.Fatalf("%s error = nil, want nonzero exit error", test.name) } if result == nil { t.Fatalf("%s result = nil, want captured result", test.name) } if err.Error() != test.wantErr { t.Fatalf("%s error = %q, want %q", test.name, err.Error(), test.wantErr) } if result.Stdout != "captured stdout" || result.Stderr != "captured stderr" || !result.StderrTruncated { t.Fatalf("captured result = %#v, want stdout/stderr/truncation", result) } if result.ExitCode != 7 || result.OutputPath != "/tmp/report.md" { t.Fatalf("result = %#v, want exit 7 and output path", result) } }) } } func TestOutputRunsValidateRequiredFieldsBeforeExecution(t *testing.T) { tests := []struct { name string run func(Runner, string, string, string) error }{ { name: "StructuredRun", run: func(runner Runner, promptID string, dataPackagePath string, outputPath string) error { result, err := runner.StructuredRun(context.Background(), StructuredRunRequest{ PromptID: promptID, DataPackagePath: dataPackagePath, OutputPath: outputPath, }) if result != nil { return fmt.Errorf("result = %#v, want nil", result) } return err }, }, } cases := []struct { name string promptID string dataPackagePath string outputPath string want string }{ { name: "prompt id", dataPackagePath: "/tmp/data_package.yaml", outputPath: "/tmp/report.md", want: "prompt id is required", }, { name: "data package path", promptID: "weather.daily_generated_text", outputPath: "/tmp/report.md", want: "data package path is required", }, { name: "output path", promptID: "weather.daily_generated_text", dataPackagePath: "/tmp/data_package.yaml", want: "output path is required", }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { commands := &fakeCommands{} err := test.run(Runner{Commands: commands}, tc.promptID, tc.dataPackagePath, tc.outputPath) if err == nil { t.Fatalf("%s error = nil, want validation error", test.name) } if !strings.Contains(err.Error(), tc.want) { t.Fatalf("%s error = %v, want %q", test.name, err, tc.want) } if commands.calls != 0 { t.Fatalf("commands calls = %d, want no subprocess execution", commands.calls) } }) } }) } } type fakeCommands struct { name string args []string timeout time.Duration result CommandResult err error calls int } func (f *fakeCommands) Run(_ context.Context, name string, args []string, timeout time.Duration) (CommandResult, error) { f.calls++ f.name = name f.args = append([]string{}, args...) f.timeout = timeout return f.result, f.err } func containsArg(args []string, want string) bool { for _, arg := range args { if arg == want { return true } } return false }