124 lines
4.4 KiB
Go
124 lines
4.4 KiB
Go
package enemyevents
|
|
|
|
import (
|
|
"context"
|
|
"io/fs"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
|
"gitea.maximumdirect.net/eric/promptkit"
|
|
)
|
|
|
|
func TestRegisterPromptAssetsAndPrepareEnemyEventPrompt(t *testing.T) {
|
|
registry := llm.NewAssetRegistry()
|
|
if err := RegisterPromptAssets(registry); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
schemaFS, err := registry.SchemaFS()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := fs.ReadFile(schemaFS, "dnd_enemy_events_llm.v1.json"); err != nil {
|
|
t.Fatalf("response schema asset: %v", err)
|
|
}
|
|
|
|
prepared := prepareEnemyEventPrompt(t)
|
|
if prepared.PromptID != PromptID || prepared.OutputContract.SchemaPath != "dnd_enemy_events_llm.v1.json" || prepared.SelectedProfileID != "dnd-extraction" {
|
|
t.Fatalf("prepared prompt = %#v", prepared)
|
|
}
|
|
transcriptIndex := renderedMessageIndex(t, prepared.Messages, "enemy-transcript")
|
|
for _, sentinel := range []string{"enemy-npc", "enemy-turn", "enemy-opponent", "Extract Dungeons & Dragons enemy events"} {
|
|
if index := renderedMessageIndex(t, prepared.Messages, sentinel); index <= transcriptIndex {
|
|
t.Fatalf("message containing %q has index %d, want after transcript index %d", sentinel, index, transcriptIndex)
|
|
}
|
|
}
|
|
instructionIndex := renderedMessageIndex(t, prepared.Messages, "Return the `events` array")
|
|
if instructionIndex <= transcriptIndex {
|
|
t.Fatalf("instruction index = %d, want after transcript index %d", instructionIndex, transcriptIndex)
|
|
}
|
|
if instructionIndex != len(prepared.Messages)-1 {
|
|
t.Fatalf("instruction message index = %d, want final message", instructionIndex)
|
|
}
|
|
if cache := prepared.Messages[instructionIndex].CacheControl; cache == nil || cache.Type != promptkit.CacheControlEphemeral {
|
|
t.Fatalf("final instruction cache control = %#v", cache)
|
|
}
|
|
}
|
|
|
|
func TestEnemyEventPromptRequiresGroundingInputs(t *testing.T) {
|
|
engine := newEnemyEventPromptEngine(t)
|
|
for _, inputName := range []string{"npcs", "combat_turns", "npc_interactions"} {
|
|
t.Run(inputName, func(t *testing.T) {
|
|
inputs := enemyEventPromptInputs()
|
|
delete(inputs, inputName)
|
|
_, err := engine.Prepare(context.Background(), promptkit.RunRequest{
|
|
PromptID: PromptID, PromptVersion: SchemaVersion, Inputs: inputs,
|
|
})
|
|
if err == nil || !strings.Contains(err.Error(), inputName) {
|
|
t.Fatalf("Prepare() error = %v, want required %q input", err, inputName)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func prepareEnemyEventPrompt(t *testing.T) *promptkit.PreparedRun {
|
|
t.Helper()
|
|
prepared, err := newEnemyEventPromptEngine(t).Prepare(context.Background(), promptkit.RunRequest{
|
|
PromptID: PromptID, PromptVersion: SchemaVersion, Inputs: enemyEventPromptInputs(),
|
|
})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return prepared
|
|
}
|
|
|
|
func newEnemyEventPromptEngine(t *testing.T) *promptkit.Engine {
|
|
t.Helper()
|
|
registry := llm.NewAssetRegistry()
|
|
if err := RegisterPromptAssets(registry); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
options, err := registry.PromptKitOptions()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
options = append(options, promptkit.WithProfiles(promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{
|
|
ID: "dnd-extraction", Endpoint: "http://127.0.0.1:1/v1", Model: "enemy-test-model",
|
|
})))
|
|
engine, err := promptkit.NewEngine(promptkit.Config{Timeout: time.Second}, options...)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return engine
|
|
}
|
|
|
|
func enemyEventPromptInputs() map[string]promptkit.ArtifactRef {
|
|
return map[string]promptkit.ArtifactRef{
|
|
"transcript": promptkit.Inline(`{"value":"enemy-transcript"}`),
|
|
"players": promptkit.Inline("enemy-player"),
|
|
"party": promptkit.Inline("enemy-party"),
|
|
"glossary": promptkit.Inline("enemy-glossary"),
|
|
"npcs": promptkit.Inline(`{"npcs":[{"name":"enemy-npc"}]}`),
|
|
"combat_turns": promptkit.Inline(`{"combat_turns":[{"actor":"enemy-turn","turn_kind":"turn"}]}`),
|
|
"npc_interactions": promptkit.Inline(`{"npc_interactions":[{"name":"enemy-opponent","kind":"combat_opponent"}]}`),
|
|
}
|
|
}
|
|
|
|
func renderedMessageIndex(t *testing.T, messages []promptkit.RenderedMessage, sentinel string) int {
|
|
t.Helper()
|
|
index := -1
|
|
occurrences := 0
|
|
for messageIndex, message := range messages {
|
|
count := strings.Count(message.Content, sentinel)
|
|
if count > 0 {
|
|
index = messageIndex
|
|
occurrences += count
|
|
}
|
|
}
|
|
if occurrences != 1 {
|
|
t.Fatalf("message sentinel %q rendered %d times, want exactly once", sentinel, occurrences)
|
|
}
|
|
return index
|
|
}
|