Consolidate D&D prompt ordering tests
This commit is contained in:
@@ -12,35 +12,43 @@ import (
|
||||
)
|
||||
|
||||
func TestScriptoriumPromptPreparesTranscriptAndTaskMessages(t *testing.T) {
|
||||
transcript := []byte(`{"id":"session-1","segments":[{"id":"u1","text":"We enter the crypt."}]}`)
|
||||
prepared := prepareScenesPrompt(t, transcript, "Alice: Aria", "Aria: cleric", "Brightmantle: temple")
|
||||
transcript := []byte(`{"sentinel":"scene-transcript"}`)
|
||||
prepared := prepareScenesPrompt(t, transcript, "scene-players", "scene-party", "scene-glossary")
|
||||
|
||||
if prepared.PromptID != PromptID {
|
||||
t.Fatalf("prompt id = %q, want %q", prepared.PromptID, PromptID)
|
||||
}
|
||||
transcriptMessages := 0
|
||||
referenceMessageFound := false
|
||||
for _, message := range prepared.Messages {
|
||||
if strings.Contains(message.Content, string(transcript)) {
|
||||
transcriptMessages++
|
||||
if message.Role != "user" || message.CacheControl == nil {
|
||||
t.Fatalf("transcript message did not render as cacheable user message: %#v", message)
|
||||
}
|
||||
}
|
||||
if strings.Contains(message.Content, "Alice: Aria") &&
|
||||
strings.Contains(message.Content, "Aria: cleric") &&
|
||||
strings.Contains(message.Content, "Brightmantle: temple") {
|
||||
referenceMessageFound = true
|
||||
if message.Role != "user" || message.CacheControl == nil {
|
||||
t.Fatalf("reference message did not render as cacheable user message: %#v", message)
|
||||
}
|
||||
if len(prepared.Messages) != 5 {
|
||||
t.Fatalf("prepared messages = %d, want 5", len(prepared.Messages))
|
||||
}
|
||||
for index, role := range []string{"system", "user", "user", "user", "user"} {
|
||||
if prepared.Messages[index].Role != role {
|
||||
t.Errorf("message %d role = %q, want %q", index, prepared.Messages[index].Role, role)
|
||||
}
|
||||
}
|
||||
if transcriptMessages != 1 {
|
||||
t.Fatalf("raw transcript rendered in %d messages, want exactly one", transcriptMessages)
|
||||
for _, index := range []int{1, 4} {
|
||||
if cache := prepared.Messages[index].CacheControl; cache == nil || cache.Type != scriptorium.CacheControlEphemeral {
|
||||
t.Errorf("message %d cache control = %#v, want ephemeral", index, cache)
|
||||
}
|
||||
}
|
||||
if !referenceMessageFound {
|
||||
t.Fatalf("reference message did not render all supplied reference material")
|
||||
for _, index := range []int{0, 2, 3} {
|
||||
if cache := prepared.Messages[index].CacheControl; cache != nil {
|
||||
t.Errorf("message %d cache control = %#v, want nil", index, cache)
|
||||
}
|
||||
}
|
||||
if references := prepared.Messages[1].Content; !strings.Contains(references, "scene-players") || !strings.Contains(references, "scene-party") || !strings.Contains(references, "scene-glossary") {
|
||||
t.Fatalf("reference message = %q, want supplied reference inputs", references)
|
||||
}
|
||||
if transcriptMessage := prepared.Messages[4].Content; !strings.Contains(transcriptMessage, "scene-transcript") {
|
||||
t.Fatalf("final message = %q, want transcript input", transcriptMessage)
|
||||
}
|
||||
for index, message := range prepared.Messages {
|
||||
if index != 1 && (strings.Contains(message.Content, "scene-players") || strings.Contains(message.Content, "scene-party") || strings.Contains(message.Content, "scene-glossary")) {
|
||||
t.Errorf("message %d unexpectedly rendered reference input", index)
|
||||
}
|
||||
if index != 4 && strings.Contains(message.Content, "scene-transcript") {
|
||||
t.Errorf("message %d unexpectedly rendered transcript input", index)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -50,10 +50,10 @@ func TestScriptoriumPromptPreparesRequiredInputs(t *testing.T) {
|
||||
PromptID: PromptID, PromptVersion: SchemaVersion, ProfileID: "combat-test-profile",
|
||||
Inputs: map[string]scriptorium.ArtifactRef{
|
||||
"transcript": scriptorium.InlineWithURI("file:///session.json", transcript),
|
||||
"players": scriptorium.Inline("Dana: Mira"),
|
||||
"party": scriptorium.Inline("Mira: ranger"),
|
||||
"glossary": scriptorium.Inline("Greencloak: title"),
|
||||
"npcs": scriptorium.Inline(`{"npcs":[]}`),
|
||||
"players": scriptorium.Inline(" "),
|
||||
"party": scriptorium.Inline(" "),
|
||||
"glossary": scriptorium.Inline(" "),
|
||||
"npcs": scriptorium.Inline(" "),
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
@@ -62,48 +62,4 @@ func TestScriptoriumPromptPreparesRequiredInputs(t *testing.T) {
|
||||
if prepared.PromptID != PromptID || prepared.OutputContract.SchemaPath != "dnd_combat_turns_llm.v1.json" {
|
||||
t.Fatalf("prepared prompt = %#v, want combat prompt identity and schema", prepared)
|
||||
}
|
||||
for index, want := range []struct {
|
||||
role string
|
||||
cached bool
|
||||
marker string
|
||||
}{
|
||||
{role: "system", marker: "Dungeons & Dragons gameplay transcripts"},
|
||||
{role: "user", marker: "Transcript units are the only evidence"},
|
||||
{role: "user", cached: true, marker: "most specific supported in-world"},
|
||||
{role: "user", cached: true},
|
||||
{role: "user", cached: true},
|
||||
{role: "user", marker: "combat-turn artifacts"},
|
||||
{role: "user", cached: true, marker: "turn_kind"},
|
||||
{role: "user"},
|
||||
} {
|
||||
if index >= len(prepared.Messages) {
|
||||
t.Fatalf("prepared prompt has %d messages, want at least %d", len(prepared.Messages), index+1)
|
||||
}
|
||||
message := prepared.Messages[index]
|
||||
if message.Role != want.role {
|
||||
t.Errorf("message %d role = %q, want %q", index, message.Role, want.role)
|
||||
}
|
||||
if want.cached {
|
||||
if message.CacheControl == nil || message.CacheControl.Type != scriptorium.CacheControlEphemeral {
|
||||
t.Errorf("message %d cache control = %#v, want ephemeral", index, message.CacheControl)
|
||||
}
|
||||
} else if message.CacheControl != nil {
|
||||
t.Errorf("message %d cache control = %#v, want nil", index, message.CacheControl)
|
||||
}
|
||||
if want.marker != "" && !strings.Contains(message.Content, want.marker) {
|
||||
t.Errorf("message %d content does not contain purpose marker %q", index, want.marker)
|
||||
}
|
||||
}
|
||||
if len(prepared.Messages) != 8 {
|
||||
t.Fatalf("prepared prompt has %d messages, want 8", len(prepared.Messages))
|
||||
}
|
||||
if references := prepared.Messages[3].Content; !strings.Contains(references, "Dana: Mira") || !strings.Contains(references, "Mira: ranger") || !strings.Contains(references, "Greencloak: title") {
|
||||
t.Fatalf("campaign references message = %q, want rendered reference inputs", references)
|
||||
}
|
||||
if registry := prepared.Messages[4].Content; !strings.Contains(registry, `{"npcs":[]}`) {
|
||||
t.Fatalf("NPC registry message = %q, want registry input", registry)
|
||||
}
|
||||
if final := prepared.Messages[7].Content; !strings.Contains(final, transcript) {
|
||||
t.Fatalf("final message = %q, want transcript", final)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,7 +3,6 @@ package itemevents
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"io/fs"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
@@ -13,7 +12,7 @@ import (
|
||||
"gitea.maximumdirect.net/eric/scriptorium"
|
||||
)
|
||||
|
||||
func TestPromptAssetsUseSharedSequenceAndTranscriptLast(t *testing.T) {
|
||||
func TestPromptAssetsPrepareItemEventPrompt(t *testing.T) {
|
||||
registry := llm.NewAssetRegistry()
|
||||
if err := RegisterPromptAssets(registry); err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -33,9 +32,9 @@ func TestPromptAssetsUseSharedSequenceAndTranscriptLast(t *testing.T) {
|
||||
PromptID: PromptID, PromptVersion: SchemaVersion, ProfileID: "item-events-test-profile",
|
||||
Inputs: map[string]scriptorium.ArtifactRef{
|
||||
"transcript": scriptorium.InlineWithURI("file:///session.json", `{"segments":[1]}`),
|
||||
"players": scriptorium.Inline("Dana: Aria"),
|
||||
"party": scriptorium.Inline("Aria: ranger"),
|
||||
"glossary": scriptorium.Inline("Moonblade: heirloom"),
|
||||
"players": scriptorium.Inline(" "),
|
||||
"party": scriptorium.Inline(" "),
|
||||
"glossary": scriptorium.Inline(" "),
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
@@ -44,34 +43,6 @@ func TestPromptAssetsUseSharedSequenceAndTranscriptLast(t *testing.T) {
|
||||
if prepared.PromptID != PromptID || prepared.OutputContract.SchemaPath != "dnd_item_events_llm.v1.json" {
|
||||
t.Fatalf("prepared prompt = %#v", prepared)
|
||||
}
|
||||
want := []struct {
|
||||
role string
|
||||
cached bool
|
||||
marker string
|
||||
}{
|
||||
{"system", false, "Dungeons & Dragons gameplay transcripts"},
|
||||
{"user", false, "Transcript units are the only evidence"},
|
||||
{"user", true, "most specific supported in-world"},
|
||||
{"user", true, "Dana: Aria"},
|
||||
{"user", false, "item and currency events"},
|
||||
{"user", true, "Ordinary non-depleting use"},
|
||||
{"user", false, `{"segments":[1]}`},
|
||||
}
|
||||
if len(prepared.Messages) != len(want) {
|
||||
t.Fatalf("prompt messages = %d, want %d", len(prepared.Messages), len(want))
|
||||
}
|
||||
for index, expected := range want {
|
||||
message := prepared.Messages[index]
|
||||
if message.Role != expected.role || !strings.Contains(message.Content, expected.marker) {
|
||||
t.Fatalf("message %d = %#v", index, message)
|
||||
}
|
||||
if (message.CacheControl != nil) != expected.cached {
|
||||
t.Fatalf("message %d cache control = %#v", index, message.CacheControl)
|
||||
}
|
||||
}
|
||||
if strings.Contains(prepared.Messages[len(prepared.Messages)-1].Content, "Moonblade: heirloom") {
|
||||
t.Fatal("transcript message contains optional reference content")
|
||||
}
|
||||
}
|
||||
|
||||
func TestPromptAssetsDoNotLeakIntoMetadata(t *testing.T) {
|
||||
@@ -84,7 +55,7 @@ func TestPromptAssetsDoNotLeakIntoMetadata(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, forbidden := range []string{"meaningful Dungeons", "common-dnd-system", "start_segment", "dnd_item_events_llm.v1.json"} {
|
||||
for _, forbidden := range []string{"common-dnd-system", "dnd_item_events_llm.v1.json"} {
|
||||
if strings.Contains(string(payload), forbidden) {
|
||||
t.Fatalf("metadata leaked raw asset content %q: %s", forbidden, payload)
|
||||
}
|
||||
@@ -102,13 +73,4 @@ func TestPromptManifestReusesOnlySharedAssets(t *testing.T) {
|
||||
if !reflect.DeepEqual(promptAssetManifest.SharedFiles, want) {
|
||||
t.Fatalf("shared assets = %#v, want %#v", promptAssetManifest.SharedFiles, want)
|
||||
}
|
||||
for _, path := range []string{"assets/prompts/task.md", "assets/prompts/instructions.md"} {
|
||||
content, err := fs.ReadFile(embeddedAssets, path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if strings.Contains(string(content), "Transcript units are the only evidence") || strings.Contains(string(content), "Dungeons & Dragons gameplay transcripts") {
|
||||
t.Fatalf("module asset %q copied shared prompt text", path)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -51,50 +51,6 @@ func TestRegisterPromptAssetsAndPrepareInteractionPrompt(t *testing.T) {
|
||||
if prepared.PromptID != PromptID || prepared.OutputContract.SchemaPath != "dnd_npc_interactions_llm.v1.json" {
|
||||
t.Fatalf("prepared prompt = %#v", prepared)
|
||||
}
|
||||
for index, want := range []struct {
|
||||
role string
|
||||
cached bool
|
||||
marker string
|
||||
}{
|
||||
{role: "system", marker: "Dungeons & Dragons gameplay transcripts"},
|
||||
{role: "user", marker: "Transcript units are the only evidence"},
|
||||
{role: "user", cached: true, marker: "most specific supported in-world"},
|
||||
{role: "user", cached: true},
|
||||
{role: "user", cached: true},
|
||||
{role: "user", marker: "interaction occurrences"},
|
||||
{role: "user", cached: true, marker: "Use exactly one kind per occurrence"},
|
||||
{role: "user"},
|
||||
} {
|
||||
if index >= len(prepared.Messages) {
|
||||
t.Fatalf("prepared prompt has %d messages, want at least %d", len(prepared.Messages), index+1)
|
||||
}
|
||||
message := prepared.Messages[index]
|
||||
if message.Role != want.role {
|
||||
t.Errorf("message %d role = %q, want %q", index, message.Role, want.role)
|
||||
}
|
||||
if want.cached {
|
||||
if message.CacheControl == nil || message.CacheControl.Type != scriptorium.CacheControlEphemeral {
|
||||
t.Errorf("message %d cache control = %#v, want ephemeral", index, message.CacheControl)
|
||||
}
|
||||
} else if message.CacheControl != nil {
|
||||
t.Errorf("message %d cache control = %#v, want nil", index, message.CacheControl)
|
||||
}
|
||||
if want.marker != "" && !strings.Contains(message.Content, want.marker) {
|
||||
t.Errorf("message %d content does not contain purpose marker %q", index, want.marker)
|
||||
}
|
||||
}
|
||||
if len(prepared.Messages) != 8 {
|
||||
t.Fatalf("prepared prompt has %d messages, want 8", len(prepared.Messages))
|
||||
}
|
||||
if references := prepared.Messages[3].Content; !strings.Contains(references, "Dana: Mira") || !strings.Contains(references, "Mira: ranger") || !strings.Contains(references, "Greencloak: title") {
|
||||
t.Fatalf("campaign references message = %q, want rendered reference inputs", references)
|
||||
}
|
||||
if registry := prepared.Messages[4].Content; !strings.Contains(registry, `{"npcs":[{"name":"Mira Thorn"}]}`) {
|
||||
t.Fatalf("NPC registry message = %q, want names-only registry input", registry)
|
||||
}
|
||||
if final := prepared.Messages[7].Content; !strings.Contains(final, transcript) {
|
||||
t.Fatalf("final message = %q, want transcript", final)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPromptMetadataDoesNotExposeAssetContent(t *testing.T) {
|
||||
@@ -103,7 +59,7 @@ func TestPromptMetadataDoesNotExposeAssetContent(t *testing.T) {
|
||||
t.Fatalf("scriptoriumPromptMetadata() = %q, %v", hash, err)
|
||||
}
|
||||
metadata := newExtractor(t, &fakeInteractionsLLMClient{}).ManifestMetadata()
|
||||
for _, forbidden := range []string{"combat_opponent", "common-dnd-system", "source_refs", "dnd_npc_interactions_llm.v1.json"} {
|
||||
for _, forbidden := range []string{"common-dnd-system", "dnd_npc_interactions_llm.v1.json"} {
|
||||
if strings.Contains(strings.Join(mapValues(metadata), " "), forbidden) {
|
||||
t.Fatalf("metadata leaked prompt or schema content %q: %#v", forbidden, metadata)
|
||||
}
|
||||
|
||||
@@ -31,9 +31,9 @@ func TestRegisterPromptAssetsAndPrepareNPCPrompt(t *testing.T) {
|
||||
PromptID: PromptID, PromptVersion: SchemaVersion, ProfileID: "npc-test-profile",
|
||||
Inputs: map[string]scriptorium.ArtifactRef{
|
||||
"transcript": scriptorium.InlineWithURI("file:///session.json", `{"units":[1]}`),
|
||||
"players": scriptorium.Inline("Dana: Mira"),
|
||||
"party": scriptorium.Inline("Mira: ranger"),
|
||||
"glossary": scriptorium.Inline("Greencloak: title"),
|
||||
"players": scriptorium.Inline(" "),
|
||||
"party": scriptorium.Inline(" "),
|
||||
"glossary": scriptorium.Inline(" "),
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
@@ -42,55 +42,6 @@ func TestRegisterPromptAssetsAndPrepareNPCPrompt(t *testing.T) {
|
||||
if prepared.PromptID != PromptID || prepared.OutputContract.SchemaPath != "dnd_npcs_llm.v1.json" {
|
||||
t.Fatalf("prepared prompt = %#v, want NPC prompt identity and wiring", prepared)
|
||||
}
|
||||
for index, want := range []struct {
|
||||
role string
|
||||
cached bool
|
||||
marker string
|
||||
}{
|
||||
{role: "system", marker: "Dungeons & Dragons gameplay transcripts"},
|
||||
{role: "user", marker: "Transcript units are the only evidence"},
|
||||
{role: "user", cached: true, marker: "most specific supported in-world"},
|
||||
{role: "user", cached: true},
|
||||
{role: "user", marker: "individually identifiable"},
|
||||
{role: "user", cached: true, marker: "observed display name"},
|
||||
{role: "user"},
|
||||
} {
|
||||
if index >= len(prepared.Messages) {
|
||||
t.Fatalf("prepared prompt has %d messages, want at least %d", len(prepared.Messages), index+1)
|
||||
}
|
||||
message := prepared.Messages[index]
|
||||
if message.Role != want.role {
|
||||
t.Errorf("message %d role = %q, want %q", index, message.Role, want.role)
|
||||
}
|
||||
if want.cached {
|
||||
if message.CacheControl == nil || message.CacheControl.Type != scriptorium.CacheControlEphemeral {
|
||||
t.Errorf("message %d cache control = %#v, want ephemeral", index, message.CacheControl)
|
||||
}
|
||||
} else if message.CacheControl != nil {
|
||||
t.Errorf("message %d cache control = %#v, want nil", index, message.CacheControl)
|
||||
}
|
||||
if want.marker != "" && !strings.Contains(message.Content, want.marker) {
|
||||
t.Errorf("message %d content does not contain purpose marker %q", index, want.marker)
|
||||
}
|
||||
}
|
||||
if len(prepared.Messages) != 7 {
|
||||
t.Fatalf("prepared prompt has %d messages, want 7", len(prepared.Messages))
|
||||
}
|
||||
if references := prepared.Messages[3].Content; !strings.Contains(references, "Dana: Mira") || !strings.Contains(references, "Mira: ranger") || !strings.Contains(references, "Greencloak: title") {
|
||||
t.Fatalf("campaign references message = %q, want rendered reference inputs", references)
|
||||
}
|
||||
if transcript := prepared.Messages[6].Content; !strings.Contains(transcript, `{"units":[1]}`) {
|
||||
t.Fatalf("final message = %q, want transcript", transcript)
|
||||
}
|
||||
transcriptMessages := 0
|
||||
for _, message := range prepared.Messages {
|
||||
if strings.Contains(message.Content, `{"units":[1]}`) {
|
||||
transcriptMessages++
|
||||
}
|
||||
}
|
||||
if transcriptMessages != 1 {
|
||||
t.Fatalf("raw transcript rendered in %d messages, want exactly one", transcriptMessages)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPromptMetadataAndDiagnosticsDoNotContainRawAssets(t *testing.T) {
|
||||
@@ -103,7 +54,7 @@ func TestPromptMetadataAndDiagnosticsDoNotContainRawAssets(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, forbidden := range []string{"Include an in-world", "common-dnd-system", "source-unit", "dnd_npcs_llm.v1.json"} {
|
||||
for _, forbidden := range []string{"common-dnd-system", "dnd_npcs_llm.v1.json"} {
|
||||
if strings.Contains(string(payload), forbidden) {
|
||||
t.Fatalf("metadata leaked raw prompt/schema content %q: %s", forbidden, payload)
|
||||
}
|
||||
|
||||
@@ -3,7 +3,6 @@ package scenedescriptions
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -12,7 +11,7 @@ import (
|
||||
"gitea.maximumdirect.net/eric/scriptorium"
|
||||
)
|
||||
|
||||
func TestRegisterPromptAssetsPreparesOrderedSceneDescriptionPrompt(t *testing.T) {
|
||||
func TestRegisterPromptAssetsPreparesSceneDescriptionPrompt(t *testing.T) {
|
||||
registry := llm.NewAssetRegistry()
|
||||
if err := RegisterPromptAssets(registry); err != nil {
|
||||
t.Fatalf("RegisterPromptAssets() error = %v, want nil", err)
|
||||
@@ -32,9 +31,9 @@ func TestRegisterPromptAssetsPreparesOrderedSceneDescriptionPrompt(t *testing.T)
|
||||
PromptID: PromptID, PromptVersion: SchemaVersion, ProfileID: "scene-description-test-profile",
|
||||
Inputs: map[string]scriptorium.ArtifactRef{
|
||||
"transcript": scriptorium.InlineWithURI("file:///session.json", `{"units":[1]}`),
|
||||
"players": scriptorium.Inline("Dana: Mira"),
|
||||
"party": scriptorium.Inline("Mira: ranger"),
|
||||
"glossary": scriptorium.Inline("Greencloak: title"),
|
||||
"players": scriptorium.Inline(" "),
|
||||
"party": scriptorium.Inline(" "),
|
||||
"glossary": scriptorium.Inline(" "),
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
@@ -43,42 +42,6 @@ func TestRegisterPromptAssetsPreparesOrderedSceneDescriptionPrompt(t *testing.T)
|
||||
if prepared.PromptID != PromptID || prepared.OutputContract.SchemaPath != "dnd_scene_descriptions_llm.v1.json" {
|
||||
t.Fatalf("prepared prompt = %#v, want scene-description prompt identity and schema wiring", prepared)
|
||||
}
|
||||
if got, want := messageRoles(prepared.Messages), []string{"system", "user", "user", "user", "user", "user"}; !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("message roles = %#v, want %#v", got, want)
|
||||
}
|
||||
for _, index := range []int{1, 2, 4} {
|
||||
if prepared.Messages[index].CacheControl == nil || prepared.Messages[index].CacheControl.Type != scriptorium.CacheControlEphemeral {
|
||||
t.Fatalf("message %d cache control = %#v, want ephemeral", index, prepared.Messages[index].CacheControl)
|
||||
}
|
||||
}
|
||||
for _, index := range []int{0, 3, 5} {
|
||||
if prepared.Messages[index].CacheControl != nil {
|
||||
t.Fatalf("message %d cache control = %#v, want nil", index, prepared.Messages[index].CacheControl)
|
||||
}
|
||||
}
|
||||
if !strings.Contains(prepared.Messages[0].Content, "Dungeons & Dragons") {
|
||||
t.Fatalf("first message does not use shared D&D system asset: %q", prepared.Messages[0].Content)
|
||||
}
|
||||
for index, want := range []string{"Dana: Mira", "Mira: ranger", "Greencloak: title"} {
|
||||
if !strings.Contains(prepared.Messages[2].Content, want) {
|
||||
t.Fatalf("reference %d not rendered in shared reference message: %q", index, prepared.Messages[2].Content)
|
||||
}
|
||||
}
|
||||
if strings.Contains(prepared.Messages[1].Content, `{"units":[1]}`) || strings.Contains(prepared.Messages[2].Content, `{"units":[1]}`) {
|
||||
t.Fatal("transcript rendered before its final message")
|
||||
}
|
||||
if !strings.Contains(prepared.Messages[5].Content, `{"units":[1]}`) {
|
||||
t.Fatalf("final message does not render transcript: %q", prepared.Messages[5].Content)
|
||||
}
|
||||
transcriptMessages := 0
|
||||
for _, message := range prepared.Messages {
|
||||
if strings.Contains(message.Content, `{"units":[1]}`) {
|
||||
transcriptMessages++
|
||||
}
|
||||
}
|
||||
if transcriptMessages != 1 {
|
||||
t.Fatalf("raw transcript rendered in %d messages, want exactly one", transcriptMessages)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPromptMetadataAndDiagnosticsDoNotContainRawAssets(t *testing.T) {
|
||||
@@ -90,17 +53,9 @@ func TestPromptMetadataAndDiagnosticsDoNotContainRawAssets(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, forbidden := range []string{"Choose exactly one kind", "common-dnd-system", "dnd_scene_descriptions_llm.v1.json"} {
|
||||
for _, forbidden := range []string{"common-dnd-system", "dnd_scene_descriptions_llm.v1.json"} {
|
||||
if strings.Contains(string(payload), forbidden) {
|
||||
t.Fatalf("metadata leaked raw prompt/schema content %q: %s", forbidden, payload)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func messageRoles(messages []scriptorium.RenderedMessage) []string {
|
||||
roles := make([]string, len(messages))
|
||||
for i, message := range messages {
|
||||
roles[i] = string(message.Role)
|
||||
}
|
||||
return roles
|
||||
}
|
||||
|
||||
@@ -11,7 +11,7 @@ import (
|
||||
"gitea.maximumdirect.net/eric/scriptorium"
|
||||
)
|
||||
|
||||
func TestScriptoriumPromptPreparesTranscriptReferencesAndTaskMessages(t *testing.T) {
|
||||
func TestScriptoriumPromptPreparesSpellPrompt(t *testing.T) {
|
||||
transcript := []byte(`{"id":"session-1","segments":[{"id":"u1","text":"Mira casts shield."}]}`)
|
||||
prepared := prepareSpellsPrompt(t, transcript, "Dana: Mira", "Mira: wizard", "Shield: abjuration")
|
||||
|
||||
@@ -21,54 +21,6 @@ func TestScriptoriumPromptPreparesTranscriptReferencesAndTaskMessages(t *testing
|
||||
if prepared.OutputContract.SchemaPath != "dnd_spells_llm.v1.json" {
|
||||
t.Fatalf("schema path = %q, want LLM-only schema", prepared.OutputContract.SchemaPath)
|
||||
}
|
||||
for index, want := range []struct {
|
||||
role string
|
||||
cached bool
|
||||
marker string
|
||||
}{
|
||||
{role: "system", marker: "Dungeons & Dragons gameplay transcripts"},
|
||||
{role: "user", marker: "Transcript units are the only evidence"},
|
||||
{role: "user", cached: true, marker: "most specific supported in-world"},
|
||||
{role: "user", cached: true},
|
||||
{role: "user", cached: true},
|
||||
{role: "user"},
|
||||
{role: "user", marker: "spell-cast artifacts"},
|
||||
{role: "user", cached: true, marker: "source references must collectively support"},
|
||||
{role: "user"},
|
||||
} {
|
||||
if index >= len(prepared.Messages) {
|
||||
t.Fatalf("prepared prompt has %d messages, want at least %d", len(prepared.Messages), index+1)
|
||||
}
|
||||
message := prepared.Messages[index]
|
||||
if message.Role != want.role {
|
||||
t.Errorf("message %d role = %q, want %q", index, message.Role, want.role)
|
||||
}
|
||||
if want.cached {
|
||||
if message.CacheControl == nil || message.CacheControl.Type != scriptorium.CacheControlEphemeral {
|
||||
t.Errorf("message %d cache control = %#v, want ephemeral", index, message.CacheControl)
|
||||
}
|
||||
} else if message.CacheControl != nil {
|
||||
t.Errorf("message %d cache control = %#v, want nil", index, message.CacheControl)
|
||||
}
|
||||
if want.marker != "" && !strings.Contains(message.Content, want.marker) {
|
||||
t.Errorf("message %d content does not contain purpose marker %q", index, want.marker)
|
||||
}
|
||||
}
|
||||
if len(prepared.Messages) != 9 {
|
||||
t.Fatalf("prepared prompt has %d messages, want 9", len(prepared.Messages))
|
||||
}
|
||||
if references := prepared.Messages[3].Content; !strings.Contains(references, "Dana: Mira") || !strings.Contains(references, "Mira: wizard") || !strings.Contains(references, "Shield: abjuration") {
|
||||
t.Fatalf("campaign references message = %q, want rendered reference inputs", references)
|
||||
}
|
||||
if registry := prepared.Messages[4].Content; !strings.Contains(registry, `{"npcs":[]}`) {
|
||||
t.Fatalf("NPC registry message = %q, want registry input", registry)
|
||||
}
|
||||
if catalog := prepared.Messages[5].Content; !strings.Contains(catalog, `{"spell_names":["Cure Wounds"]}`) {
|
||||
t.Fatalf("spell catalog message = %q, want catalog input", catalog)
|
||||
}
|
||||
if final := prepared.Messages[8].Content; !strings.Contains(final, string(transcript)) {
|
||||
t.Fatalf("final message = %q, want transcript", final)
|
||||
}
|
||||
}
|
||||
|
||||
func TestScriptoriumPromptPreparesWithMissingOptionalReferences(t *testing.T) {
|
||||
|
||||
@@ -54,12 +54,12 @@ func TestRegisterPromptAssetsPreparesNormalizationPrompt(t *testing.T) {
|
||||
t.Errorf("message %d role = %q, want %q", index, prepared.Messages[index].Role, role)
|
||||
}
|
||||
}
|
||||
for _, index := range []int{0, 2} {
|
||||
for _, index := range []int{2, 4} {
|
||||
if cache := prepared.Messages[index].CacheControl; cache == nil || cache.Type != scriptorium.CacheControlEphemeral {
|
||||
t.Errorf("message %d cache control = %#v, want ephemeral", index, cache)
|
||||
}
|
||||
}
|
||||
for _, index := range []int{1, 3, 4} {
|
||||
for _, index := range []int{0, 1, 3} {
|
||||
if cache := prepared.Messages[index].CacheControl; cache != nil {
|
||||
t.Errorf("message %d cache control = %#v, want nil", index, cache)
|
||||
}
|
||||
|
||||
166
internal/modules/dnd/register/prompt_cache_test.go
Normal file
166
internal/modules/dnd/register/prompt_cache_test.go
Normal file
@@ -0,0 +1,166 @@
|
||||
package register
|
||||
|
||||
import (
|
||||
"context"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
||||
combatextract "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/combatturns"
|
||||
itemeventextract "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/itemevents"
|
||||
interactionextract "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/npcinteractions"
|
||||
npcextract "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/npcs"
|
||||
scenedescriptionextract "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/scenedescriptions"
|
||||
spellextract "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/spells"
|
||||
"gitea.maximumdirect.net/eric/scriptorium"
|
||||
)
|
||||
|
||||
func TestExtractionPromptsShareRenderedPrefix(t *testing.T) {
|
||||
const (
|
||||
transcriptSentinel = "shared-transcript-sentinel"
|
||||
playersSentinel = "shared-players-sentinel"
|
||||
partySentinel = "shared-party-sentinel"
|
||||
glossarySentinel = "shared-glossary-sentinel"
|
||||
npcSentinel = "npc-registry-sentinel"
|
||||
catalogSentinel = "spell-catalog-sentinel"
|
||||
)
|
||||
registry := llm.NewAssetRegistry()
|
||||
if err := registerPromptAssets(registry); err != nil {
|
||||
t.Fatalf("registerPromptAssets() error = %v", err)
|
||||
}
|
||||
engine := newPromptCacheEngine(t, registry)
|
||||
commonInputs := map[string]scriptorium.ArtifactRef{
|
||||
"transcript": scriptorium.InlineWithURI("file:///session.json", `{"sentinel":"`+transcriptSentinel+`"}`),
|
||||
"players": scriptorium.Inline(playersSentinel),
|
||||
"party": scriptorium.Inline(partySentinel),
|
||||
"glossary": scriptorium.Inline(glossarySentinel),
|
||||
}
|
||||
cases := []struct {
|
||||
name string
|
||||
promptID string
|
||||
promptVersion string
|
||||
inputs map[string]scriptorium.ArtifactRef
|
||||
npcInput bool
|
||||
spellCatalogInput bool
|
||||
}{
|
||||
{name: "npcs", promptID: npcextract.PromptID, promptVersion: npcextract.SchemaVersion, inputs: commonInputs},
|
||||
{name: "item events", promptID: itemeventextract.PromptID, promptVersion: itemeventextract.SchemaVersion, inputs: commonInputs},
|
||||
{name: "scene descriptions", promptID: scenedescriptionextract.PromptID, promptVersion: scenedescriptionextract.SchemaVersion, inputs: commonInputs},
|
||||
{
|
||||
name: "combat turns",
|
||||
promptID: combatextract.PromptID,
|
||||
promptVersion: combatextract.SchemaVersion,
|
||||
inputs: withPromptInputs(commonInputs, map[string]scriptorium.ArtifactRef{
|
||||
"npcs": scriptorium.Inline(`{"sentinel":"` + npcSentinel + `"}`),
|
||||
}),
|
||||
npcInput: true,
|
||||
},
|
||||
{
|
||||
name: "npc interactions",
|
||||
promptID: interactionextract.PromptID,
|
||||
promptVersion: interactionextract.SchemaVersion,
|
||||
inputs: withPromptInputs(commonInputs, map[string]scriptorium.ArtifactRef{
|
||||
"npcs": scriptorium.Inline(`{"sentinel":"` + npcSentinel + `"}`),
|
||||
}),
|
||||
npcInput: true,
|
||||
},
|
||||
{
|
||||
name: "spells",
|
||||
promptID: spellextract.PromptID,
|
||||
promptVersion: spellextract.SchemaVersion,
|
||||
inputs: withPromptInputs(commonInputs, map[string]scriptorium.ArtifactRef{
|
||||
"npcs": scriptorium.Inline(`{"sentinel":"` + npcSentinel + `"}`),
|
||||
"spell_catalog": scriptorium.Inline(`{"sentinel":"` + catalogSentinel + `"}`),
|
||||
}),
|
||||
npcInput: true,
|
||||
spellCatalogInput: true,
|
||||
},
|
||||
}
|
||||
|
||||
var sharedPrefix []scriptorium.RenderedMessage
|
||||
for _, testCase := range cases {
|
||||
t.Run(testCase.name, func(t *testing.T) {
|
||||
prepared, err := engine.Prepare(context.Background(), scriptorium.RunRequest{
|
||||
PromptID: testCase.promptID,
|
||||
PromptVersion: testCase.promptVersion,
|
||||
ProfileID: "prompt-cache-test-profile",
|
||||
Inputs: testCase.inputs,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Prepare() error = %v", err)
|
||||
}
|
||||
transcriptIndex := renderedInputMessageIndex(t, prepared.Messages, transcriptSentinel)
|
||||
prefix := prepared.Messages[:transcriptIndex+1]
|
||||
if len(prefix) != 4 {
|
||||
t.Fatalf("messages through transcript = %d, want 4", len(prefix))
|
||||
}
|
||||
if len(prepared.Messages) <= len(prefix) {
|
||||
t.Fatalf("prepared prompt has %d messages, want lane-specific suffix after transcript", len(prepared.Messages))
|
||||
}
|
||||
if sharedPrefix == nil {
|
||||
sharedPrefix = append([]scriptorium.RenderedMessage(nil), prefix...)
|
||||
} else if !reflect.DeepEqual(prefix, sharedPrefix) {
|
||||
t.Fatalf("rendered prefix = %#v, want %#v", prefix, sharedPrefix)
|
||||
}
|
||||
if testCase.npcInput {
|
||||
assertRenderedInputAfter(t, prepared.Messages, npcSentinel, transcriptIndex)
|
||||
}
|
||||
if testCase.spellCatalogInput {
|
||||
assertRenderedInputAfter(t, prepared.Messages, catalogSentinel, transcriptIndex)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func newPromptCacheEngine(t *testing.T, registry *llm.AssetRegistry) *scriptorium.Engine {
|
||||
t.Helper()
|
||||
options, err := registry.ScriptoriumOptions()
|
||||
if err != nil {
|
||||
t.Fatalf("ScriptoriumOptions() error = %v", err)
|
||||
}
|
||||
options = append(options, scriptorium.WithProfiles(scriptorium.OpenAICompatibleProfile(scriptorium.OpenAICompatibleProfileConfig{
|
||||
ID: "prompt-cache-test-profile", Endpoint: "http://127.0.0.1:1/v1", Model: "prompt-cache-test-model",
|
||||
})))
|
||||
engine, err := scriptorium.NewEngine(scriptorium.Config{Timeout: time.Second}, options...)
|
||||
if err != nil {
|
||||
t.Fatalf("NewEngine() error = %v", err)
|
||||
}
|
||||
return engine
|
||||
}
|
||||
|
||||
func withPromptInputs(inputs, extras map[string]scriptorium.ArtifactRef) map[string]scriptorium.ArtifactRef {
|
||||
merged := make(map[string]scriptorium.ArtifactRef, len(inputs)+len(extras))
|
||||
for name, input := range inputs {
|
||||
merged[name] = input
|
||||
}
|
||||
for name, input := range extras {
|
||||
merged[name] = input
|
||||
}
|
||||
return merged
|
||||
}
|
||||
|
||||
func assertRenderedInputAfter(t *testing.T, messages []scriptorium.RenderedMessage, sentinel string, index int) {
|
||||
t.Helper()
|
||||
if inputIndex := renderedInputMessageIndex(t, messages, sentinel); inputIndex <= index {
|
||||
t.Fatalf("input sentinel %q rendered at message %d, want after transcript message %d", sentinel, inputIndex, index)
|
||||
}
|
||||
}
|
||||
|
||||
func renderedInputMessageIndex(t *testing.T, messages []scriptorium.RenderedMessage, sentinel string) int {
|
||||
t.Helper()
|
||||
index := -1
|
||||
occurrences := 0
|
||||
for messageIndex, message := range messages {
|
||||
count := strings.Count(message.Content, sentinel)
|
||||
if count > 0 {
|
||||
index = messageIndex
|
||||
occurrences += count
|
||||
}
|
||||
}
|
||||
if occurrences != 1 {
|
||||
t.Fatalf("input sentinel %q rendered %d times, want exactly once", sentinel, occurrences)
|
||||
}
|
||||
return index
|
||||
}
|
||||
Reference in New Issue
Block a user