107 lines
4.3 KiB
Go
107 lines
4.3 KiB
Go
package scenedescriptions
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"reflect"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
|
"gitea.maximumdirect.net/eric/scriptorium"
|
|
)
|
|
|
|
func TestRegisterPromptAssetsPreparesOrderedSceneDescriptionPrompt(t *testing.T) {
|
|
registry := llm.NewAssetRegistry()
|
|
if err := RegisterPromptAssets(registry); err != nil {
|
|
t.Fatalf("RegisterPromptAssets() error = %v, want nil", err)
|
|
}
|
|
options, err := registry.ScriptoriumOptions()
|
|
if err != nil {
|
|
t.Fatalf("ScriptoriumOptions() error = %v, want nil", err)
|
|
}
|
|
options = append(options, scriptorium.WithProfiles(scriptorium.OpenAICompatibleProfile(scriptorium.OpenAICompatibleProfileConfig{
|
|
ID: "scene-description-test-profile", Endpoint: "http://127.0.0.1:1/v1", Model: "scene-description-test-model",
|
|
})))
|
|
engine, err := scriptorium.NewEngine(scriptorium.Config{Timeout: time.Second}, options...)
|
|
if err != nil {
|
|
t.Fatalf("NewEngine() error = %v, want nil", err)
|
|
}
|
|
prepared, err := engine.Prepare(context.Background(), scriptorium.RunRequest{
|
|
PromptID: PromptID, PromptVersion: SchemaVersion, ProfileID: "scene-description-test-profile",
|
|
Inputs: map[string]scriptorium.ArtifactRef{
|
|
"transcript": scriptorium.InlineWithURI("file:///session.json", `{"units":[1]}`),
|
|
"players": scriptorium.Inline("Dana: Mira"),
|
|
"party": scriptorium.Inline("Mira: ranger"),
|
|
"glossary": scriptorium.Inline("Greencloak: title"),
|
|
},
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("Prepare() error = %v, want nil", err)
|
|
}
|
|
if prepared.PromptID != PromptID || prepared.OutputContract.SchemaPath != "dnd_scene_descriptions_llm.v1.json" {
|
|
t.Fatalf("prepared prompt = %#v, want scene-description prompt identity and schema wiring", prepared)
|
|
}
|
|
if got, want := messageRoles(prepared.Messages), []string{"system", "user", "user", "user", "user", "user"}; !reflect.DeepEqual(got, want) {
|
|
t.Fatalf("message roles = %#v, want %#v", got, want)
|
|
}
|
|
for _, index := range []int{1, 2, 4} {
|
|
if prepared.Messages[index].CacheControl == nil || prepared.Messages[index].CacheControl.Type != scriptorium.CacheControlEphemeral {
|
|
t.Fatalf("message %d cache control = %#v, want ephemeral", index, prepared.Messages[index].CacheControl)
|
|
}
|
|
}
|
|
for _, index := range []int{0, 3, 5} {
|
|
if prepared.Messages[index].CacheControl != nil {
|
|
t.Fatalf("message %d cache control = %#v, want nil", index, prepared.Messages[index].CacheControl)
|
|
}
|
|
}
|
|
if !strings.Contains(prepared.Messages[0].Content, "Dungeons & Dragons") {
|
|
t.Fatalf("first message does not use shared D&D system asset: %q", prepared.Messages[0].Content)
|
|
}
|
|
for index, want := range []string{"Dana: Mira", "Mira: ranger", "Greencloak: title"} {
|
|
if !strings.Contains(prepared.Messages[2].Content, want) {
|
|
t.Fatalf("reference %d not rendered in shared reference message: %q", index, prepared.Messages[2].Content)
|
|
}
|
|
}
|
|
if strings.Contains(prepared.Messages[1].Content, `{"units":[1]}`) || strings.Contains(prepared.Messages[2].Content, `{"units":[1]}`) {
|
|
t.Fatal("transcript rendered before its final message")
|
|
}
|
|
if !strings.Contains(prepared.Messages[5].Content, `{"units":[1]}`) {
|
|
t.Fatalf("final message does not render transcript: %q", prepared.Messages[5].Content)
|
|
}
|
|
transcriptMessages := 0
|
|
for _, message := range prepared.Messages {
|
|
if strings.Contains(message.Content, `{"units":[1]}`) {
|
|
transcriptMessages++
|
|
}
|
|
}
|
|
if transcriptMessages != 1 {
|
|
t.Fatalf("raw transcript rendered in %d messages, want exactly one", transcriptMessages)
|
|
}
|
|
}
|
|
|
|
func TestPromptMetadataAndDiagnosticsDoNotContainRawAssets(t *testing.T) {
|
|
hash, err := scriptoriumPromptMetadata()
|
|
if err != nil || !strings.HasPrefix(hash, "sha256:") {
|
|
t.Fatalf("scriptoriumPromptMetadata() = %q, %v; want hash", hash, err)
|
|
}
|
|
payload, err := json.Marshal(newExtractor(t, &fakeSceneDescriptionsLLMClient{}).ManifestMetadata())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for _, forbidden := range []string{"Choose exactly one kind", "common-dnd-system", "dnd_scene_descriptions_llm.v1.json"} {
|
|
if strings.Contains(string(payload), forbidden) {
|
|
t.Fatalf("metadata leaked raw prompt/schema content %q: %s", forbidden, payload)
|
|
}
|
|
}
|
|
}
|
|
|
|
func messageRoles(messages []scriptorium.RenderedMessage) []string {
|
|
roles := make([]string, len(messages))
|
|
for i, message := range messages {
|
|
roles[i] = string(message.Role)
|
|
}
|
|
return roles
|
|
}
|