Consolidate D&D prompt ordering tests

This commit is contained in:
2026-07-27 20:14:57 +00:00
parent c0ec068f53
commit d1eaec4dad
9 changed files with 218 additions and 312 deletions

View File

@@ -12,35 +12,43 @@ import (
)
func TestScriptoriumPromptPreparesTranscriptAndTaskMessages(t *testing.T) {
transcript := []byte(`{"id":"session-1","segments":[{"id":"u1","text":"We enter the crypt."}]}`)
prepared := prepareScenesPrompt(t, transcript, "Alice: Aria", "Aria: cleric", "Brightmantle: temple")
transcript := []byte(`{"sentinel":"scene-transcript"}`)
prepared := prepareScenesPrompt(t, transcript, "scene-players", "scene-party", "scene-glossary")
if prepared.PromptID != PromptID {
t.Fatalf("prompt id = %q, want %q", prepared.PromptID, PromptID)
}
transcriptMessages := 0
referenceMessageFound := false
for _, message := range prepared.Messages {
if strings.Contains(message.Content, string(transcript)) {
transcriptMessages++
if message.Role != "user" || message.CacheControl == nil {
t.Fatalf("transcript message did not render as cacheable user message: %#v", message)
}
}
if strings.Contains(message.Content, "Alice: Aria") &&
strings.Contains(message.Content, "Aria: cleric") &&
strings.Contains(message.Content, "Brightmantle: temple") {
referenceMessageFound = true
if message.Role != "user" || message.CacheControl == nil {
t.Fatalf("reference message did not render as cacheable user message: %#v", message)
}
if len(prepared.Messages) != 5 {
t.Fatalf("prepared messages = %d, want 5", len(prepared.Messages))
}
for index, role := range []string{"system", "user", "user", "user", "user"} {
if prepared.Messages[index].Role != role {
t.Errorf("message %d role = %q, want %q", index, prepared.Messages[index].Role, role)
}
}
if transcriptMessages != 1 {
t.Fatalf("raw transcript rendered in %d messages, want exactly one", transcriptMessages)
for _, index := range []int{1, 4} {
if cache := prepared.Messages[index].CacheControl; cache == nil || cache.Type != scriptorium.CacheControlEphemeral {
t.Errorf("message %d cache control = %#v, want ephemeral", index, cache)
}
}
if !referenceMessageFound {
t.Fatalf("reference message did not render all supplied reference material")
for _, index := range []int{0, 2, 3} {
if cache := prepared.Messages[index].CacheControl; cache != nil {
t.Errorf("message %d cache control = %#v, want nil", index, cache)
}
}
if references := prepared.Messages[1].Content; !strings.Contains(references, "scene-players") || !strings.Contains(references, "scene-party") || !strings.Contains(references, "scene-glossary") {
t.Fatalf("reference message = %q, want supplied reference inputs", references)
}
if transcriptMessage := prepared.Messages[4].Content; !strings.Contains(transcriptMessage, "scene-transcript") {
t.Fatalf("final message = %q, want transcript input", transcriptMessage)
}
for index, message := range prepared.Messages {
if index != 1 && (strings.Contains(message.Content, "scene-players") || strings.Contains(message.Content, "scene-party") || strings.Contains(message.Content, "scene-glossary")) {
t.Errorf("message %d unexpectedly rendered reference input", index)
}
if index != 4 && strings.Contains(message.Content, "scene-transcript") {
t.Errorf("message %d unexpectedly rendered transcript input", index)
}
}
}

View File

@@ -50,10 +50,10 @@ func TestScriptoriumPromptPreparesRequiredInputs(t *testing.T) {
PromptID: PromptID, PromptVersion: SchemaVersion, ProfileID: "combat-test-profile",
Inputs: map[string]scriptorium.ArtifactRef{
"transcript": scriptorium.InlineWithURI("file:///session.json", transcript),
"players": scriptorium.Inline("Dana: Mira"),
"party": scriptorium.Inline("Mira: ranger"),
"glossary": scriptorium.Inline("Greencloak: title"),
"npcs": scriptorium.Inline(`{"npcs":[]}`),
"players": scriptorium.Inline(" "),
"party": scriptorium.Inline(" "),
"glossary": scriptorium.Inline(" "),
"npcs": scriptorium.Inline(" "),
},
})
if err != nil {
@@ -62,48 +62,4 @@ func TestScriptoriumPromptPreparesRequiredInputs(t *testing.T) {
if prepared.PromptID != PromptID || prepared.OutputContract.SchemaPath != "dnd_combat_turns_llm.v1.json" {
t.Fatalf("prepared prompt = %#v, want combat prompt identity and schema", prepared)
}
for index, want := range []struct {
role string
cached bool
marker string
}{
{role: "system", marker: "Dungeons & Dragons gameplay transcripts"},
{role: "user", marker: "Transcript units are the only evidence"},
{role: "user", cached: true, marker: "most specific supported in-world"},
{role: "user", cached: true},
{role: "user", cached: true},
{role: "user", marker: "combat-turn artifacts"},
{role: "user", cached: true, marker: "turn_kind"},
{role: "user"},
} {
if index >= len(prepared.Messages) {
t.Fatalf("prepared prompt has %d messages, want at least %d", len(prepared.Messages), index+1)
}
message := prepared.Messages[index]
if message.Role != want.role {
t.Errorf("message %d role = %q, want %q", index, message.Role, want.role)
}
if want.cached {
if message.CacheControl == nil || message.CacheControl.Type != scriptorium.CacheControlEphemeral {
t.Errorf("message %d cache control = %#v, want ephemeral", index, message.CacheControl)
}
} else if message.CacheControl != nil {
t.Errorf("message %d cache control = %#v, want nil", index, message.CacheControl)
}
if want.marker != "" && !strings.Contains(message.Content, want.marker) {
t.Errorf("message %d content does not contain purpose marker %q", index, want.marker)
}
}
if len(prepared.Messages) != 8 {
t.Fatalf("prepared prompt has %d messages, want 8", len(prepared.Messages))
}
if references := prepared.Messages[3].Content; !strings.Contains(references, "Dana: Mira") || !strings.Contains(references, "Mira: ranger") || !strings.Contains(references, "Greencloak: title") {
t.Fatalf("campaign references message = %q, want rendered reference inputs", references)
}
if registry := prepared.Messages[4].Content; !strings.Contains(registry, `{"npcs":[]}`) {
t.Fatalf("NPC registry message = %q, want registry input", registry)
}
if final := prepared.Messages[7].Content; !strings.Contains(final, transcript) {
t.Fatalf("final message = %q, want transcript", final)
}
}

View File

@@ -3,7 +3,6 @@ package itemevents
import (
"context"
"encoding/json"
"io/fs"
"reflect"
"strings"
"testing"
@@ -13,7 +12,7 @@ import (
"gitea.maximumdirect.net/eric/scriptorium"
)
func TestPromptAssetsUseSharedSequenceAndTranscriptLast(t *testing.T) {
func TestPromptAssetsPrepareItemEventPrompt(t *testing.T) {
registry := llm.NewAssetRegistry()
if err := RegisterPromptAssets(registry); err != nil {
t.Fatal(err)
@@ -33,9 +32,9 @@ func TestPromptAssetsUseSharedSequenceAndTranscriptLast(t *testing.T) {
PromptID: PromptID, PromptVersion: SchemaVersion, ProfileID: "item-events-test-profile",
Inputs: map[string]scriptorium.ArtifactRef{
"transcript": scriptorium.InlineWithURI("file:///session.json", `{"segments":[1]}`),
"players": scriptorium.Inline("Dana: Aria"),
"party": scriptorium.Inline("Aria: ranger"),
"glossary": scriptorium.Inline("Moonblade: heirloom"),
"players": scriptorium.Inline(" "),
"party": scriptorium.Inline(" "),
"glossary": scriptorium.Inline(" "),
},
})
if err != nil {
@@ -44,34 +43,6 @@ func TestPromptAssetsUseSharedSequenceAndTranscriptLast(t *testing.T) {
if prepared.PromptID != PromptID || prepared.OutputContract.SchemaPath != "dnd_item_events_llm.v1.json" {
t.Fatalf("prepared prompt = %#v", prepared)
}
want := []struct {
role string
cached bool
marker string
}{
{"system", false, "Dungeons & Dragons gameplay transcripts"},
{"user", false, "Transcript units are the only evidence"},
{"user", true, "most specific supported in-world"},
{"user", true, "Dana: Aria"},
{"user", false, "item and currency events"},
{"user", true, "Ordinary non-depleting use"},
{"user", false, `{"segments":[1]}`},
}
if len(prepared.Messages) != len(want) {
t.Fatalf("prompt messages = %d, want %d", len(prepared.Messages), len(want))
}
for index, expected := range want {
message := prepared.Messages[index]
if message.Role != expected.role || !strings.Contains(message.Content, expected.marker) {
t.Fatalf("message %d = %#v", index, message)
}
if (message.CacheControl != nil) != expected.cached {
t.Fatalf("message %d cache control = %#v", index, message.CacheControl)
}
}
if strings.Contains(prepared.Messages[len(prepared.Messages)-1].Content, "Moonblade: heirloom") {
t.Fatal("transcript message contains optional reference content")
}
}
func TestPromptAssetsDoNotLeakIntoMetadata(t *testing.T) {
@@ -84,7 +55,7 @@ func TestPromptAssetsDoNotLeakIntoMetadata(t *testing.T) {
if err != nil {
t.Fatal(err)
}
for _, forbidden := range []string{"meaningful Dungeons", "common-dnd-system", "start_segment", "dnd_item_events_llm.v1.json"} {
for _, forbidden := range []string{"common-dnd-system", "dnd_item_events_llm.v1.json"} {
if strings.Contains(string(payload), forbidden) {
t.Fatalf("metadata leaked raw asset content %q: %s", forbidden, payload)
}
@@ -102,13 +73,4 @@ func TestPromptManifestReusesOnlySharedAssets(t *testing.T) {
if !reflect.DeepEqual(promptAssetManifest.SharedFiles, want) {
t.Fatalf("shared assets = %#v, want %#v", promptAssetManifest.SharedFiles, want)
}
for _, path := range []string{"assets/prompts/task.md", "assets/prompts/instructions.md"} {
content, err := fs.ReadFile(embeddedAssets, path)
if err != nil {
t.Fatal(err)
}
if strings.Contains(string(content), "Transcript units are the only evidence") || strings.Contains(string(content), "Dungeons & Dragons gameplay transcripts") {
t.Fatalf("module asset %q copied shared prompt text", path)
}
}
}

View File

@@ -51,50 +51,6 @@ func TestRegisterPromptAssetsAndPrepareInteractionPrompt(t *testing.T) {
if prepared.PromptID != PromptID || prepared.OutputContract.SchemaPath != "dnd_npc_interactions_llm.v1.json" {
t.Fatalf("prepared prompt = %#v", prepared)
}
for index, want := range []struct {
role string
cached bool
marker string
}{
{role: "system", marker: "Dungeons & Dragons gameplay transcripts"},
{role: "user", marker: "Transcript units are the only evidence"},
{role: "user", cached: true, marker: "most specific supported in-world"},
{role: "user", cached: true},
{role: "user", cached: true},
{role: "user", marker: "interaction occurrences"},
{role: "user", cached: true, marker: "Use exactly one kind per occurrence"},
{role: "user"},
} {
if index >= len(prepared.Messages) {
t.Fatalf("prepared prompt has %d messages, want at least %d", len(prepared.Messages), index+1)
}
message := prepared.Messages[index]
if message.Role != want.role {
t.Errorf("message %d role = %q, want %q", index, message.Role, want.role)
}
if want.cached {
if message.CacheControl == nil || message.CacheControl.Type != scriptorium.CacheControlEphemeral {
t.Errorf("message %d cache control = %#v, want ephemeral", index, message.CacheControl)
}
} else if message.CacheControl != nil {
t.Errorf("message %d cache control = %#v, want nil", index, message.CacheControl)
}
if want.marker != "" && !strings.Contains(message.Content, want.marker) {
t.Errorf("message %d content does not contain purpose marker %q", index, want.marker)
}
}
if len(prepared.Messages) != 8 {
t.Fatalf("prepared prompt has %d messages, want 8", len(prepared.Messages))
}
if references := prepared.Messages[3].Content; !strings.Contains(references, "Dana: Mira") || !strings.Contains(references, "Mira: ranger") || !strings.Contains(references, "Greencloak: title") {
t.Fatalf("campaign references message = %q, want rendered reference inputs", references)
}
if registry := prepared.Messages[4].Content; !strings.Contains(registry, `{"npcs":[{"name":"Mira Thorn"}]}`) {
t.Fatalf("NPC registry message = %q, want names-only registry input", registry)
}
if final := prepared.Messages[7].Content; !strings.Contains(final, transcript) {
t.Fatalf("final message = %q, want transcript", final)
}
}
func TestPromptMetadataDoesNotExposeAssetContent(t *testing.T) {
@@ -103,7 +59,7 @@ func TestPromptMetadataDoesNotExposeAssetContent(t *testing.T) {
t.Fatalf("scriptoriumPromptMetadata() = %q, %v", hash, err)
}
metadata := newExtractor(t, &fakeInteractionsLLMClient{}).ManifestMetadata()
for _, forbidden := range []string{"combat_opponent", "common-dnd-system", "source_refs", "dnd_npc_interactions_llm.v1.json"} {
for _, forbidden := range []string{"common-dnd-system", "dnd_npc_interactions_llm.v1.json"} {
if strings.Contains(strings.Join(mapValues(metadata), " "), forbidden) {
t.Fatalf("metadata leaked prompt or schema content %q: %#v", forbidden, metadata)
}

View File

@@ -31,9 +31,9 @@ func TestRegisterPromptAssetsAndPrepareNPCPrompt(t *testing.T) {
PromptID: PromptID, PromptVersion: SchemaVersion, ProfileID: "npc-test-profile",
Inputs: map[string]scriptorium.ArtifactRef{
"transcript": scriptorium.InlineWithURI("file:///session.json", `{"units":[1]}`),
"players": scriptorium.Inline("Dana: Mira"),
"party": scriptorium.Inline("Mira: ranger"),
"glossary": scriptorium.Inline("Greencloak: title"),
"players": scriptorium.Inline(" "),
"party": scriptorium.Inline(" "),
"glossary": scriptorium.Inline(" "),
},
})
if err != nil {
@@ -42,55 +42,6 @@ func TestRegisterPromptAssetsAndPrepareNPCPrompt(t *testing.T) {
if prepared.PromptID != PromptID || prepared.OutputContract.SchemaPath != "dnd_npcs_llm.v1.json" {
t.Fatalf("prepared prompt = %#v, want NPC prompt identity and wiring", prepared)
}
for index, want := range []struct {
role string
cached bool
marker string
}{
{role: "system", marker: "Dungeons & Dragons gameplay transcripts"},
{role: "user", marker: "Transcript units are the only evidence"},
{role: "user", cached: true, marker: "most specific supported in-world"},
{role: "user", cached: true},
{role: "user", marker: "individually identifiable"},
{role: "user", cached: true, marker: "observed display name"},
{role: "user"},
} {
if index >= len(prepared.Messages) {
t.Fatalf("prepared prompt has %d messages, want at least %d", len(prepared.Messages), index+1)
}
message := prepared.Messages[index]
if message.Role != want.role {
t.Errorf("message %d role = %q, want %q", index, message.Role, want.role)
}
if want.cached {
if message.CacheControl == nil || message.CacheControl.Type != scriptorium.CacheControlEphemeral {
t.Errorf("message %d cache control = %#v, want ephemeral", index, message.CacheControl)
}
} else if message.CacheControl != nil {
t.Errorf("message %d cache control = %#v, want nil", index, message.CacheControl)
}
if want.marker != "" && !strings.Contains(message.Content, want.marker) {
t.Errorf("message %d content does not contain purpose marker %q", index, want.marker)
}
}
if len(prepared.Messages) != 7 {
t.Fatalf("prepared prompt has %d messages, want 7", len(prepared.Messages))
}
if references := prepared.Messages[3].Content; !strings.Contains(references, "Dana: Mira") || !strings.Contains(references, "Mira: ranger") || !strings.Contains(references, "Greencloak: title") {
t.Fatalf("campaign references message = %q, want rendered reference inputs", references)
}
if transcript := prepared.Messages[6].Content; !strings.Contains(transcript, `{"units":[1]}`) {
t.Fatalf("final message = %q, want transcript", transcript)
}
transcriptMessages := 0
for _, message := range prepared.Messages {
if strings.Contains(message.Content, `{"units":[1]}`) {
transcriptMessages++
}
}
if transcriptMessages != 1 {
t.Fatalf("raw transcript rendered in %d messages, want exactly one", transcriptMessages)
}
}
func TestPromptMetadataAndDiagnosticsDoNotContainRawAssets(t *testing.T) {
@@ -103,7 +54,7 @@ func TestPromptMetadataAndDiagnosticsDoNotContainRawAssets(t *testing.T) {
if err != nil {
t.Fatal(err)
}
for _, forbidden := range []string{"Include an in-world", "common-dnd-system", "source-unit", "dnd_npcs_llm.v1.json"} {
for _, forbidden := range []string{"common-dnd-system", "dnd_npcs_llm.v1.json"} {
if strings.Contains(string(payload), forbidden) {
t.Fatalf("metadata leaked raw prompt/schema content %q: %s", forbidden, payload)
}

View File

@@ -3,7 +3,6 @@ package scenedescriptions
import (
"context"
"encoding/json"
"reflect"
"strings"
"testing"
"time"
@@ -12,7 +11,7 @@ import (
"gitea.maximumdirect.net/eric/scriptorium"
)
func TestRegisterPromptAssetsPreparesOrderedSceneDescriptionPrompt(t *testing.T) {
func TestRegisterPromptAssetsPreparesSceneDescriptionPrompt(t *testing.T) {
registry := llm.NewAssetRegistry()
if err := RegisterPromptAssets(registry); err != nil {
t.Fatalf("RegisterPromptAssets() error = %v, want nil", err)
@@ -32,9 +31,9 @@ func TestRegisterPromptAssetsPreparesOrderedSceneDescriptionPrompt(t *testing.T)
PromptID: PromptID, PromptVersion: SchemaVersion, ProfileID: "scene-description-test-profile",
Inputs: map[string]scriptorium.ArtifactRef{
"transcript": scriptorium.InlineWithURI("file:///session.json", `{"units":[1]}`),
"players": scriptorium.Inline("Dana: Mira"),
"party": scriptorium.Inline("Mira: ranger"),
"glossary": scriptorium.Inline("Greencloak: title"),
"players": scriptorium.Inline(" "),
"party": scriptorium.Inline(" "),
"glossary": scriptorium.Inline(" "),
},
})
if err != nil {
@@ -43,42 +42,6 @@ func TestRegisterPromptAssetsPreparesOrderedSceneDescriptionPrompt(t *testing.T)
if prepared.PromptID != PromptID || prepared.OutputContract.SchemaPath != "dnd_scene_descriptions_llm.v1.json" {
t.Fatalf("prepared prompt = %#v, want scene-description prompt identity and schema wiring", prepared)
}
if got, want := messageRoles(prepared.Messages), []string{"system", "user", "user", "user", "user", "user"}; !reflect.DeepEqual(got, want) {
t.Fatalf("message roles = %#v, want %#v", got, want)
}
for _, index := range []int{1, 2, 4} {
if prepared.Messages[index].CacheControl == nil || prepared.Messages[index].CacheControl.Type != scriptorium.CacheControlEphemeral {
t.Fatalf("message %d cache control = %#v, want ephemeral", index, prepared.Messages[index].CacheControl)
}
}
for _, index := range []int{0, 3, 5} {
if prepared.Messages[index].CacheControl != nil {
t.Fatalf("message %d cache control = %#v, want nil", index, prepared.Messages[index].CacheControl)
}
}
if !strings.Contains(prepared.Messages[0].Content, "Dungeons & Dragons") {
t.Fatalf("first message does not use shared D&D system asset: %q", prepared.Messages[0].Content)
}
for index, want := range []string{"Dana: Mira", "Mira: ranger", "Greencloak: title"} {
if !strings.Contains(prepared.Messages[2].Content, want) {
t.Fatalf("reference %d not rendered in shared reference message: %q", index, prepared.Messages[2].Content)
}
}
if strings.Contains(prepared.Messages[1].Content, `{"units":[1]}`) || strings.Contains(prepared.Messages[2].Content, `{"units":[1]}`) {
t.Fatal("transcript rendered before its final message")
}
if !strings.Contains(prepared.Messages[5].Content, `{"units":[1]}`) {
t.Fatalf("final message does not render transcript: %q", prepared.Messages[5].Content)
}
transcriptMessages := 0
for _, message := range prepared.Messages {
if strings.Contains(message.Content, `{"units":[1]}`) {
transcriptMessages++
}
}
if transcriptMessages != 1 {
t.Fatalf("raw transcript rendered in %d messages, want exactly one", transcriptMessages)
}
}
func TestPromptMetadataAndDiagnosticsDoNotContainRawAssets(t *testing.T) {
@@ -90,17 +53,9 @@ func TestPromptMetadataAndDiagnosticsDoNotContainRawAssets(t *testing.T) {
if err != nil {
t.Fatal(err)
}
for _, forbidden := range []string{"Choose exactly one kind", "common-dnd-system", "dnd_scene_descriptions_llm.v1.json"} {
for _, forbidden := range []string{"common-dnd-system", "dnd_scene_descriptions_llm.v1.json"} {
if strings.Contains(string(payload), forbidden) {
t.Fatalf("metadata leaked raw prompt/schema content %q: %s", forbidden, payload)
}
}
}
func messageRoles(messages []scriptorium.RenderedMessage) []string {
roles := make([]string, len(messages))
for i, message := range messages {
roles[i] = string(message.Role)
}
return roles
}

View File

@@ -11,7 +11,7 @@ import (
"gitea.maximumdirect.net/eric/scriptorium"
)
func TestScriptoriumPromptPreparesTranscriptReferencesAndTaskMessages(t *testing.T) {
func TestScriptoriumPromptPreparesSpellPrompt(t *testing.T) {
transcript := []byte(`{"id":"session-1","segments":[{"id":"u1","text":"Mira casts shield."}]}`)
prepared := prepareSpellsPrompt(t, transcript, "Dana: Mira", "Mira: wizard", "Shield: abjuration")
@@ -21,54 +21,6 @@ func TestScriptoriumPromptPreparesTranscriptReferencesAndTaskMessages(t *testing
if prepared.OutputContract.SchemaPath != "dnd_spells_llm.v1.json" {
t.Fatalf("schema path = %q, want LLM-only schema", prepared.OutputContract.SchemaPath)
}
for index, want := range []struct {
role string
cached bool
marker string
}{
{role: "system", marker: "Dungeons & Dragons gameplay transcripts"},
{role: "user", marker: "Transcript units are the only evidence"},
{role: "user", cached: true, marker: "most specific supported in-world"},
{role: "user", cached: true},
{role: "user", cached: true},
{role: "user"},
{role: "user", marker: "spell-cast artifacts"},
{role: "user", cached: true, marker: "source references must collectively support"},
{role: "user"},
} {
if index >= len(prepared.Messages) {
t.Fatalf("prepared prompt has %d messages, want at least %d", len(prepared.Messages), index+1)
}
message := prepared.Messages[index]
if message.Role != want.role {
t.Errorf("message %d role = %q, want %q", index, message.Role, want.role)
}
if want.cached {
if message.CacheControl == nil || message.CacheControl.Type != scriptorium.CacheControlEphemeral {
t.Errorf("message %d cache control = %#v, want ephemeral", index, message.CacheControl)
}
} else if message.CacheControl != nil {
t.Errorf("message %d cache control = %#v, want nil", index, message.CacheControl)
}
if want.marker != "" && !strings.Contains(message.Content, want.marker) {
t.Errorf("message %d content does not contain purpose marker %q", index, want.marker)
}
}
if len(prepared.Messages) != 9 {
t.Fatalf("prepared prompt has %d messages, want 9", len(prepared.Messages))
}
if references := prepared.Messages[3].Content; !strings.Contains(references, "Dana: Mira") || !strings.Contains(references, "Mira: wizard") || !strings.Contains(references, "Shield: abjuration") {
t.Fatalf("campaign references message = %q, want rendered reference inputs", references)
}
if registry := prepared.Messages[4].Content; !strings.Contains(registry, `{"npcs":[]}`) {
t.Fatalf("NPC registry message = %q, want registry input", registry)
}
if catalog := prepared.Messages[5].Content; !strings.Contains(catalog, `{"spell_names":["Cure Wounds"]}`) {
t.Fatalf("spell catalog message = %q, want catalog input", catalog)
}
if final := prepared.Messages[8].Content; !strings.Contains(final, string(transcript)) {
t.Fatalf("final message = %q, want transcript", final)
}
}
func TestScriptoriumPromptPreparesWithMissingOptionalReferences(t *testing.T) {

View File

@@ -54,12 +54,12 @@ func TestRegisterPromptAssetsPreparesNormalizationPrompt(t *testing.T) {
t.Errorf("message %d role = %q, want %q", index, prepared.Messages[index].Role, role)
}
}
for _, index := range []int{0, 2} {
for _, index := range []int{2, 4} {
if cache := prepared.Messages[index].CacheControl; cache == nil || cache.Type != scriptorium.CacheControlEphemeral {
t.Errorf("message %d cache control = %#v, want ephemeral", index, cache)
}
}
for _, index := range []int{1, 3, 4} {
for _, index := range []int{0, 1, 3} {
if cache := prepared.Messages[index].CacheControl; cache != nil {
t.Errorf("message %d cache control = %#v, want nil", index, cache)
}

View File

@@ -0,0 +1,166 @@
package register
import (
"context"
"reflect"
"strings"
"testing"
"time"
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
combatextract "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/combatturns"
itemeventextract "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/itemevents"
interactionextract "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/npcinteractions"
npcextract "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/npcs"
scenedescriptionextract "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/scenedescriptions"
spellextract "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/spells"
"gitea.maximumdirect.net/eric/scriptorium"
)
func TestExtractionPromptsShareRenderedPrefix(t *testing.T) {
const (
transcriptSentinel = "shared-transcript-sentinel"
playersSentinel = "shared-players-sentinel"
partySentinel = "shared-party-sentinel"
glossarySentinel = "shared-glossary-sentinel"
npcSentinel = "npc-registry-sentinel"
catalogSentinel = "spell-catalog-sentinel"
)
registry := llm.NewAssetRegistry()
if err := registerPromptAssets(registry); err != nil {
t.Fatalf("registerPromptAssets() error = %v", err)
}
engine := newPromptCacheEngine(t, registry)
commonInputs := map[string]scriptorium.ArtifactRef{
"transcript": scriptorium.InlineWithURI("file:///session.json", `{"sentinel":"`+transcriptSentinel+`"}`),
"players": scriptorium.Inline(playersSentinel),
"party": scriptorium.Inline(partySentinel),
"glossary": scriptorium.Inline(glossarySentinel),
}
cases := []struct {
name string
promptID string
promptVersion string
inputs map[string]scriptorium.ArtifactRef
npcInput bool
spellCatalogInput bool
}{
{name: "npcs", promptID: npcextract.PromptID, promptVersion: npcextract.SchemaVersion, inputs: commonInputs},
{name: "item events", promptID: itemeventextract.PromptID, promptVersion: itemeventextract.SchemaVersion, inputs: commonInputs},
{name: "scene descriptions", promptID: scenedescriptionextract.PromptID, promptVersion: scenedescriptionextract.SchemaVersion, inputs: commonInputs},
{
name: "combat turns",
promptID: combatextract.PromptID,
promptVersion: combatextract.SchemaVersion,
inputs: withPromptInputs(commonInputs, map[string]scriptorium.ArtifactRef{
"npcs": scriptorium.Inline(`{"sentinel":"` + npcSentinel + `"}`),
}),
npcInput: true,
},
{
name: "npc interactions",
promptID: interactionextract.PromptID,
promptVersion: interactionextract.SchemaVersion,
inputs: withPromptInputs(commonInputs, map[string]scriptorium.ArtifactRef{
"npcs": scriptorium.Inline(`{"sentinel":"` + npcSentinel + `"}`),
}),
npcInput: true,
},
{
name: "spells",
promptID: spellextract.PromptID,
promptVersion: spellextract.SchemaVersion,
inputs: withPromptInputs(commonInputs, map[string]scriptorium.ArtifactRef{
"npcs": scriptorium.Inline(`{"sentinel":"` + npcSentinel + `"}`),
"spell_catalog": scriptorium.Inline(`{"sentinel":"` + catalogSentinel + `"}`),
}),
npcInput: true,
spellCatalogInput: true,
},
}
var sharedPrefix []scriptorium.RenderedMessage
for _, testCase := range cases {
t.Run(testCase.name, func(t *testing.T) {
prepared, err := engine.Prepare(context.Background(), scriptorium.RunRequest{
PromptID: testCase.promptID,
PromptVersion: testCase.promptVersion,
ProfileID: "prompt-cache-test-profile",
Inputs: testCase.inputs,
})
if err != nil {
t.Fatalf("Prepare() error = %v", err)
}
transcriptIndex := renderedInputMessageIndex(t, prepared.Messages, transcriptSentinel)
prefix := prepared.Messages[:transcriptIndex+1]
if len(prefix) != 4 {
t.Fatalf("messages through transcript = %d, want 4", len(prefix))
}
if len(prepared.Messages) <= len(prefix) {
t.Fatalf("prepared prompt has %d messages, want lane-specific suffix after transcript", len(prepared.Messages))
}
if sharedPrefix == nil {
sharedPrefix = append([]scriptorium.RenderedMessage(nil), prefix...)
} else if !reflect.DeepEqual(prefix, sharedPrefix) {
t.Fatalf("rendered prefix = %#v, want %#v", prefix, sharedPrefix)
}
if testCase.npcInput {
assertRenderedInputAfter(t, prepared.Messages, npcSentinel, transcriptIndex)
}
if testCase.spellCatalogInput {
assertRenderedInputAfter(t, prepared.Messages, catalogSentinel, transcriptIndex)
}
})
}
}
func newPromptCacheEngine(t *testing.T, registry *llm.AssetRegistry) *scriptorium.Engine {
t.Helper()
options, err := registry.ScriptoriumOptions()
if err != nil {
t.Fatalf("ScriptoriumOptions() error = %v", err)
}
options = append(options, scriptorium.WithProfiles(scriptorium.OpenAICompatibleProfile(scriptorium.OpenAICompatibleProfileConfig{
ID: "prompt-cache-test-profile", Endpoint: "http://127.0.0.1:1/v1", Model: "prompt-cache-test-model",
})))
engine, err := scriptorium.NewEngine(scriptorium.Config{Timeout: time.Second}, options...)
if err != nil {
t.Fatalf("NewEngine() error = %v", err)
}
return engine
}
func withPromptInputs(inputs, extras map[string]scriptorium.ArtifactRef) map[string]scriptorium.ArtifactRef {
merged := make(map[string]scriptorium.ArtifactRef, len(inputs)+len(extras))
for name, input := range inputs {
merged[name] = input
}
for name, input := range extras {
merged[name] = input
}
return merged
}
func assertRenderedInputAfter(t *testing.T, messages []scriptorium.RenderedMessage, sentinel string, index int) {
t.Helper()
if inputIndex := renderedInputMessageIndex(t, messages, sentinel); inputIndex <= index {
t.Fatalf("input sentinel %q rendered at message %d, want after transcript message %d", sentinel, inputIndex, index)
}
}
func renderedInputMessageIndex(t *testing.T, messages []scriptorium.RenderedMessage, sentinel string) int {
t.Helper()
index := -1
occurrences := 0
for messageIndex, message := range messages {
count := strings.Count(message.Content, sentinel)
if count > 0 {
index = messageIndex
occurrences += count
}
}
if occurrences != 1 {
t.Fatalf("input sentinel %q rendered %d times, want exactly once", sentinel, occurrences)
}
return index
}