Add public prepare engine API
This commit is contained in:
164
engine_test.go
Normal file
164
engine_test.go
Normal file
@@ -0,0 +1,164 @@
|
||||
package scriptorium_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/scriptorium"
|
||||
)
|
||||
|
||||
func TestNewEngineRejectsMissingPromptDir(t *testing.T) {
|
||||
_, err := scriptorium.NewEngine(scriptorium.Config{ProfileDir: "./examples/profiles"})
|
||||
if !errors.Is(err, scriptorium.ErrInvalidConfig) {
|
||||
t.Fatalf("expected ErrInvalidConfig, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewEngineRejectsMissingProfileDir(t *testing.T) {
|
||||
_, err := scriptorium.NewEngine(scriptorium.Config{PromptDir: "./examples/prompts"})
|
||||
if !errors.Is(err, scriptorium.ErrInvalidConfig) {
|
||||
t.Fatalf("expected ErrInvalidConfig, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPrepareWorksWithExampleDirectoriesAndFileInputs(t *testing.T) {
|
||||
engine := newExampleEngine(t)
|
||||
|
||||
prepared, err := engine.Prepare(context.Background(), scriptorium.RunRequest{
|
||||
PromptID: "generic.markdown_summary",
|
||||
Inputs: map[string]scriptorium.ArtifactRef{
|
||||
"transcript": scriptorium.File("./examples/fixtures/transcript.md"),
|
||||
"glossary": scriptorium.File("./examples/fixtures/glossary.yml"),
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("expected prepare to succeed, got %v", err)
|
||||
}
|
||||
if prepared.PromptID != "generic.markdown_summary" {
|
||||
t.Fatalf("unexpected prompt id: %q", prepared.PromptID)
|
||||
}
|
||||
if prepared.SelectedProfileID != "local-fast" {
|
||||
t.Fatalf("unexpected selected profile: %q", prepared.SelectedProfileID)
|
||||
}
|
||||
if prepared.EffectiveModelParams.Model != "gpt-4o-mini" {
|
||||
t.Fatalf("unexpected effective model: %q", prepared.EffectiveModelParams.Model)
|
||||
}
|
||||
if len(prepared.Messages) != 2 {
|
||||
t.Fatalf("expected rendered messages, got %d", len(prepared.Messages))
|
||||
}
|
||||
if prepared.InputHashes["transcript"] == "" || prepared.InputHashes["glossary"] == "" {
|
||||
t.Fatalf("expected input hashes, got %#v", prepared.InputHashes)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPrepareWorksWithInlineInputs(t *testing.T) {
|
||||
engine := newExampleEngine(t)
|
||||
|
||||
prepared, err := engine.Prepare(context.Background(), scriptorium.RunRequest{
|
||||
PromptID: "generic.markdown_summary",
|
||||
Inputs: map[string]scriptorium.ArtifactRef{
|
||||
"transcript": scriptorium.Inline("Rin scouts the tower.\nKara lights a lantern."),
|
||||
"glossary": scriptorium.InlineWithURI("memory://glossary.yml", "party:\n - Rin\n - Kara\n"),
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("expected prepare to succeed, got %v", err)
|
||||
}
|
||||
if len(prepared.Messages) != 2 {
|
||||
t.Fatalf("expected rendered messages, got %d", len(prepared.Messages))
|
||||
}
|
||||
rendered := prepared.Messages[1].Content
|
||||
if !strings.Contains(rendered, "Rin scouts the tower.") || !strings.Contains(rendered, "party:") {
|
||||
t.Fatalf("expected inline inputs in rendered prompt, got %q", rendered)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPreparedRunJSONDoesNotExposeSecretOrTargetPresence(t *testing.T) {
|
||||
const envName = "SCRIPTORIUM_API_KEY"
|
||||
const secret = "public-api-test-secret"
|
||||
t.Setenv(envName, secret)
|
||||
|
||||
engine := newExampleEngine(t)
|
||||
prepared, err := engine.Prepare(context.Background(), scriptorium.RunRequest{
|
||||
PromptID: "generic.structured_events",
|
||||
Inputs: map[string]scriptorium.ArtifactRef{
|
||||
"transcript": scriptorium.File("./examples/fixtures/transcript.md"),
|
||||
"glossary": scriptorium.File("./examples/fixtures/glossary.yml"),
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("expected prepare to succeed, got %v", err)
|
||||
}
|
||||
|
||||
payload, err := json.Marshal(prepared)
|
||||
if err != nil {
|
||||
t.Fatalf("expected prepared run to marshal, got %v", err)
|
||||
}
|
||||
out := string(payload)
|
||||
if strings.Contains(out, secret) {
|
||||
t.Fatalf("prepared run JSON leaked raw API key value: %s", out)
|
||||
}
|
||||
if !strings.Contains(out, envName) {
|
||||
t.Fatalf("prepared run JSON should retain api_key_env name, got %s", out)
|
||||
}
|
||||
for _, forbidden := range []string{"TargetPresence", "target_presence"} {
|
||||
if strings.Contains(out, forbidden) {
|
||||
t.Fatalf("prepared run JSON exposed internal target presence metadata %q: %s", forbidden, out)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestPreparePreservesExplicitZeroExecutionOverrides(t *testing.T) {
|
||||
engine := newExampleEngine(t)
|
||||
zeroFloat := 0.0
|
||||
zeroInt := 0
|
||||
|
||||
prepared, err := engine.Prepare(context.Background(), scriptorium.RunRequest{
|
||||
PromptID: "generic.markdown_summary",
|
||||
Inputs: map[string]scriptorium.ArtifactRef{
|
||||
"transcript": scriptorium.File("./examples/fixtures/transcript.md"),
|
||||
"glossary": scriptorium.File("./examples/fixtures/glossary.yml"),
|
||||
},
|
||||
Execution: &scriptorium.ExecutionTargetOverride{
|
||||
Temperature: &zeroFloat,
|
||||
MaxTokens: &zeroInt,
|
||||
TopP: &zeroFloat,
|
||||
TimeoutSeconds: &zeroInt,
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("expected prepare to succeed, got %v", err)
|
||||
}
|
||||
target := prepared.EffectiveModelParams
|
||||
if target.Temperature != 0 || target.MaxTokens != 0 || target.TopP != 0 || target.TimeoutSeconds != 0 {
|
||||
t.Fatalf("expected explicit zero overrides in effective target, got %+v", target)
|
||||
}
|
||||
}
|
||||
|
||||
func newExampleEngine(t *testing.T) *scriptorium.Engine {
|
||||
t.Helper()
|
||||
|
||||
for _, path := range []string{
|
||||
"./examples/prompts",
|
||||
"./examples/profiles",
|
||||
"./examples/schemas",
|
||||
} {
|
||||
if _, err := os.Stat(path); err != nil {
|
||||
t.Fatalf("expected example path %s to exist: %v", path, err)
|
||||
}
|
||||
}
|
||||
|
||||
engine, err := scriptorium.NewEngine(scriptorium.Config{
|
||||
PromptDir: "./examples/prompts",
|
||||
ProfileDir: "./examples/profiles",
|
||||
SchemaDir: "./examples/schemas",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("expected engine construction to succeed, got %v", err)
|
||||
}
|
||||
return engine
|
||||
}
|
||||
Reference in New Issue
Block a user