eric/notarius
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Consolidate D&D prompt ordering tests

Browse Source
This commit is contained in:
Eric Rakestraw
2026-07-27 20:14:57 +00:00
parent c0ec068f53
commit d1eaec4dad
9 changed files with 218 additions and 312 deletions
Download Patch File Download Diff File Expand all files Collapse all files

52
internal/modules/dnd/extract/combatturns/scriptorium_assets_test.go
View File

@@ -50,10 +50,10 @@ func TestScriptoriumPromptPreparesRequiredInputs(t *testing.T) {
PromptID: PromptID, PromptVersion: SchemaVersion, ProfileID: "combat-test-profile",
Inputs: map[string]scriptorium.ArtifactRef{
"transcript": scriptorium.InlineWithURI("file:///session.json", transcript),
"players": scriptorium.Inline("Dana: Mira"),
"party": scriptorium.Inline("Mira: ranger"),
"glossary": scriptorium.Inline("Greencloak: title"),
"npcs": scriptorium.Inline(`{"npcs":[]}`),
"players": scriptorium.Inline(" "),
"party": scriptorium.Inline(" "),
"glossary": scriptorium.Inline(" "),
"npcs": scriptorium.Inline(" "),
},
})
if err != nil {
@@ -62,48 +62,4 @@ func TestScriptoriumPromptPreparesRequiredInputs(t *testing.T) {
if prepared.PromptID != PromptID || prepared.OutputContract.SchemaPath != "dnd_combat_turns_llm.v1.json" {
t.Fatalf("prepared prompt = %#v, want combat prompt identity and schema", prepared)
}
for index, want := range []struct {
role string
cached bool
marker string
}{
{role: "system", marker: "Dungeons & Dragons gameplay transcripts"},
{role: "user", marker: "Transcript units are the only evidence"},
{role: "user", cached: true, marker: "most specific supported in-world"},
{role: "user", cached: true},
{role: "user", cached: true},
{role: "user", marker: "combat-turn artifacts"},
{role: "user", cached: true, marker: "turn_kind"},
{role: "user"},
} {
if index >= len(prepared.Messages) {
t.Fatalf("prepared prompt has %d messages, want at least %d", len(prepared.Messages), index+1)
}
message := prepared.Messages[index]
if message.Role != want.role {
t.Errorf("message %d role = %q, want %q", index, message.Role, want.role)
}
if want.cached {
if message.CacheControl == nil || message.CacheControl.Type != scriptorium.CacheControlEphemeral {
t.Errorf("message %d cache control = %#v, want ephemeral", index, message.CacheControl)
}
} else if message.CacheControl != nil {
t.Errorf("message %d cache control = %#v, want nil", index, message.CacheControl)
}
if want.marker != "" && !strings.Contains(message.Content, want.marker) {
t.Errorf("message %d content does not contain purpose marker %q", index, want.marker)
}
}
if len(prepared.Messages) != 8 {
t.Fatalf("prepared prompt has %d messages, want 8", len(prepared.Messages))
}
if references := prepared.Messages[3].Content; !strings.Contains(references, "Dana: Mira") || !strings.Contains(references, "Mira: ranger") || !strings.Contains(references, "Greencloak: title") {
t.Fatalf("campaign references message = %q, want rendered reference inputs", references)
}
if registry := prepared.Messages[4].Content; !strings.Contains(registry, `{"npcs":[]}`) {
t.Fatalf("NPC registry message = %q, want registry input", registry)
}
if final := prepared.Messages[7].Content; !strings.Contains(final, transcript) {
t.Fatalf("final message = %q, want transcript", final)
}
}

48
internal/modules/dnd/extract/itemevents/scriptorium_assets_test.go
View File

@@ -3,7 +3,6 @@ package itemevents
import (
"context"
"encoding/json"
"io/fs"
"reflect"
"strings"
"testing"
@@ -13,7 +12,7 @@ import (
"gitea.maximumdirect.net/eric/scriptorium"
)
func TestPromptAssetsUseSharedSequenceAndTranscriptLast(t *testing.T) {
func TestPromptAssetsPrepareItemEventPrompt(t *testing.T) {
registry := llm.NewAssetRegistry()
if err := RegisterPromptAssets(registry); err != nil {
t.Fatal(err)
@@ -33,9 +32,9 @@ func TestPromptAssetsUseSharedSequenceAndTranscriptLast(t *testing.T) {
PromptID: PromptID, PromptVersion: SchemaVersion, ProfileID: "item-events-test-profile",
Inputs: map[string]scriptorium.ArtifactRef{
"transcript": scriptorium.InlineWithURI("file:///session.json", `{"segments":[1]}`),
"players": scriptorium.Inline("Dana: Aria"),
"party": scriptorium.Inline("Aria: ranger"),
"glossary": scriptorium.Inline("Moonblade: heirloom"),
"players": scriptorium.Inline(" "),
"party": scriptorium.Inline(" "),
"glossary": scriptorium.Inline(" "),
},
})
if err != nil {
@@ -44,34 +43,6 @@ func TestPromptAssetsUseSharedSequenceAndTranscriptLast(t *testing.T) {
if prepared.PromptID != PromptID || prepared.OutputContract.SchemaPath != "dnd_item_events_llm.v1.json" {
t.Fatalf("prepared prompt = %#v", prepared)
}
want := []struct {
role string
cached bool
marker string
}{
{"system", false, "Dungeons & Dragons gameplay transcripts"},
{"user", false, "Transcript units are the only evidence"},
{"user", true, "most specific supported in-world"},
{"user", true, "Dana: Aria"},
{"user", false, "item and currency events"},
{"user", true, "Ordinary non-depleting use"},
{"user", false, `{"segments":[1]}`},
}
if len(prepared.Messages) != len(want) {
t.Fatalf("prompt messages = %d, want %d", len(prepared.Messages), len(want))
}
for index, expected := range want {
message := prepared.Messages[index]
if message.Role != expected.role || !strings.Contains(message.Content, expected.marker) {
t.Fatalf("message %d = %#v", index, message)
}
if (message.CacheControl != nil) != expected.cached {
t.Fatalf("message %d cache control = %#v", index, message.CacheControl)
}
}
if strings.Contains(prepared.Messages[len(prepared.Messages)-1].Content, "Moonblade: heirloom") {
t.Fatal("transcript message contains optional reference content")
}
}
func TestPromptAssetsDoNotLeakIntoMetadata(t *testing.T) {
@@ -84,7 +55,7 @@ func TestPromptAssetsDoNotLeakIntoMetadata(t *testing.T) {
if err != nil {
t.Fatal(err)
}
for _, forbidden := range []string{"meaningful Dungeons", "common-dnd-system", "start_segment", "dnd_item_events_llm.v1.json"} {
for _, forbidden := range []string{"common-dnd-system", "dnd_item_events_llm.v1.json"} {
if strings.Contains(string(payload), forbidden) {
t.Fatalf("metadata leaked raw asset content %q: %s", forbidden, payload)
}
@@ -102,13 +73,4 @@ func TestPromptManifestReusesOnlySharedAssets(t *testing.T) {
if !reflect.DeepEqual(promptAssetManifest.SharedFiles, want) {
t.Fatalf("shared assets = %#v, want %#v", promptAssetManifest.SharedFiles, want)
}
for _, path := range []string{"assets/prompts/task.md", "assets/prompts/instructions.md"} {
content, err := fs.ReadFile(embeddedAssets, path)
if err != nil {
t.Fatal(err)
}
if strings.Contains(string(content), "Transcript units are the only evidence") || strings.Contains(string(content), "Dungeons & Dragons gameplay transcripts") {
t.Fatalf("module asset %q copied shared prompt text", path)
}
}
}

46
internal/modules/dnd/extract/npcinteractions/scriptorium_assets_test.go
View File

@@ -51,50 +51,6 @@ func TestRegisterPromptAssetsAndPrepareInteractionPrompt(t *testing.T) {
if prepared.PromptID != PromptID || prepared.OutputContract.SchemaPath != "dnd_npc_interactions_llm.v1.json" {
t.Fatalf("prepared prompt = %#v", prepared)
}
for index, want := range []struct {
role string
cached bool
marker string
}{
{role: "system", marker: "Dungeons & Dragons gameplay transcripts"},
{role: "user", marker: "Transcript units are the only evidence"},
{role: "user", cached: true, marker: "most specific supported in-world"},
{role: "user", cached: true},
{role: "user", cached: true},
{role: "user", marker: "interaction occurrences"},
{role: "user", cached: true, marker: "Use exactly one kind per occurrence"},
{role: "user"},
} {
if index >= len(prepared.Messages) {
t.Fatalf("prepared prompt has %d messages, want at least %d", len(prepared.Messages), index+1)
}
message := prepared.Messages[index]
if message.Role != want.role {
t.Errorf("message %d role = %q, want %q", index, message.Role, want.role)
}
if want.cached {
if message.CacheControl == nil || message.CacheControl.Type != scriptorium.CacheControlEphemeral {
t.Errorf("message %d cache control = %#v, want ephemeral", index, message.CacheControl)
}
} else if message.CacheControl != nil {
t.Errorf("message %d cache control = %#v, want nil", index, message.CacheControl)
}
if want.marker != "" && !strings.Contains(message.Content, want.marker) {
t.Errorf("message %d content does not contain purpose marker %q", index, want.marker)
}
}
if len(prepared.Messages) != 8 {
t.Fatalf("prepared prompt has %d messages, want 8", len(prepared.Messages))
}
if references := prepared.Messages[3].Content; !strings.Contains(references, "Dana: Mira") || !strings.Contains(references, "Mira: ranger") || !strings.Contains(references, "Greencloak: title") {
t.Fatalf("campaign references message = %q, want rendered reference inputs", references)
}
if registry := prepared.Messages[4].Content; !strings.Contains(registry, `{"npcs":[{"name":"Mira Thorn"}]}`) {
t.Fatalf("NPC registry message = %q, want names-only registry input", registry)
}
if final := prepared.Messages[7].Content; !strings.Contains(final, transcript) {
t.Fatalf("final message = %q, want transcript", final)
}
}
func TestPromptMetadataDoesNotExposeAssetContent(t *testing.T) {
@@ -103,7 +59,7 @@ func TestPromptMetadataDoesNotExposeAssetContent(t *testing.T) {
t.Fatalf("scriptoriumPromptMetadata() = %q, %v", hash, err)
}
metadata := newExtractor(t, &fakeInteractionsLLMClient{}).ManifestMetadata()
for _, forbidden := range []string{"combat_opponent", "common-dnd-system", "source_refs", "dnd_npc_interactions_llm.v1.json"} {
for _, forbidden := range []string{"common-dnd-system", "dnd_npc_interactions_llm.v1.json"} {
if strings.Contains(strings.Join(mapValues(metadata), " "), forbidden) {
t.Fatalf("metadata leaked prompt or schema content %q: %#v", forbidden, metadata)
}

57
internal/modules/dnd/extract/npcs/scriptorium_assets_test.go
View File

@@ -31,9 +31,9 @@ func TestRegisterPromptAssetsAndPrepareNPCPrompt(t *testing.T) {
PromptID: PromptID, PromptVersion: SchemaVersion, ProfileID: "npc-test-profile",
Inputs: map[string]scriptorium.ArtifactRef{
"transcript": scriptorium.InlineWithURI("file:///session.json", `{"units":[1]}`),
"players": scriptorium.Inline("Dana: Mira"),
"party": scriptorium.Inline("Mira: ranger"),
"glossary": scriptorium.Inline("Greencloak: title"),
"players": scriptorium.Inline(" "),
"party": scriptorium.Inline(" "),
"glossary": scriptorium.Inline(" "),
},
})
if err != nil {
@@ -42,55 +42,6 @@ func TestRegisterPromptAssetsAndPrepareNPCPrompt(t *testing.T) {
if prepared.PromptID != PromptID || prepared.OutputContract.SchemaPath != "dnd_npcs_llm.v1.json" {
t.Fatalf("prepared prompt = %#v, want NPC prompt identity and wiring", prepared)
}
for index, want := range []struct {
role string
cached bool
marker string
}{
{role: "system", marker: "Dungeons & Dragons gameplay transcripts"},
{role: "user", marker: "Transcript units are the only evidence"},
{role: "user", cached: true, marker: "most specific supported in-world"},
{role: "user", cached: true},
{role: "user", marker: "individually identifiable"},
{role: "user", cached: true, marker: "observed display name"},
{role: "user"},
} {
if index >= len(prepared.Messages) {
t.Fatalf("prepared prompt has %d messages, want at least %d", len(prepared.Messages), index+1)
}
message := prepared.Messages[index]
if message.Role != want.role {
t.Errorf("message %d role = %q, want %q", index, message.Role, want.role)
}
if want.cached {
if message.CacheControl == nil || message.CacheControl.Type != scriptorium.CacheControlEphemeral {
t.Errorf("message %d cache control = %#v, want ephemeral", index, message.CacheControl)
}
} else if message.CacheControl != nil {
t.Errorf("message %d cache control = %#v, want nil", index, message.CacheControl)
}
if want.marker != "" && !strings.Contains(message.Content, want.marker) {
t.Errorf("message %d content does not contain purpose marker %q", index, want.marker)
}
}
if len(prepared.Messages) != 7 {
t.Fatalf("prepared prompt has %d messages, want 7", len(prepared.Messages))
}
if references := prepared.Messages[3].Content; !strings.Contains(references, "Dana: Mira") || !strings.Contains(references, "Mira: ranger") || !strings.Contains(references, "Greencloak: title") {
t.Fatalf("campaign references message = %q, want rendered reference inputs", references)
}
if transcript := prepared.Messages[6].Content; !strings.Contains(transcript, `{"units":[1]}`) {
t.Fatalf("final message = %q, want transcript", transcript)
}
transcriptMessages := 0
for _, message := range prepared.Messages {
if strings.Contains(message.Content, `{"units":[1]}`) {
transcriptMessages++
}
}
if transcriptMessages != 1 {
t.Fatalf("raw transcript rendered in %d messages, want exactly one", transcriptMessages)
}
}
func TestPromptMetadataAndDiagnosticsDoNotContainRawAssets(t *testing.T) {
@@ -103,7 +54,7 @@ func TestPromptMetadataAndDiagnosticsDoNotContainRawAssets(t *testing.T) {
if err != nil {
t.Fatal(err)
}
for _, forbidden := range []string{"Include an in-world", "common-dnd-system", "source-unit", "dnd_npcs_llm.v1.json"} {
for _, forbidden := range []string{"common-dnd-system", "dnd_npcs_llm.v1.json"} {
if strings.Contains(string(payload), forbidden) {
t.Fatalf("metadata leaked raw prompt/schema content %q: %s", forbidden, payload)
}

55
internal/modules/dnd/extract/scenedescriptions/scriptorium_assets_test.go
View File

@@ -3,7 +3,6 @@ package scenedescriptions
import (
"context"
"encoding/json"
"reflect"
"strings"
"testing"
"time"
@@ -12,7 +11,7 @@ import (
"gitea.maximumdirect.net/eric/scriptorium"
)
func TestRegisterPromptAssetsPreparesOrderedSceneDescriptionPrompt(t *testing.T) {
func TestRegisterPromptAssetsPreparesSceneDescriptionPrompt(t *testing.T) {
registry := llm.NewAssetRegistry()
if err := RegisterPromptAssets(registry); err != nil {
t.Fatalf("RegisterPromptAssets() error = %v, want nil", err)
@@ -32,9 +31,9 @@ func TestRegisterPromptAssetsPreparesOrderedSceneDescriptionPrompt(t *testing.T)
PromptID: PromptID, PromptVersion: SchemaVersion, ProfileID: "scene-description-test-profile",
Inputs: map[string]scriptorium.ArtifactRef{
"transcript": scriptorium.InlineWithURI("file:///session.json", `{"units":[1]}`),
"players": scriptorium.Inline("Dana: Mira"),
"party": scriptorium.Inline("Mira: ranger"),
"glossary": scriptorium.Inline("Greencloak: title"),
"players": scriptorium.Inline(" "),
"party": scriptorium.Inline(" "),
"glossary": scriptorium.Inline(" "),
},
})
if err != nil {
@@ -43,42 +42,6 @@ func TestRegisterPromptAssetsPreparesOrderedSceneDescriptionPrompt(t *testing.T)
if prepared.PromptID != PromptID || prepared.OutputContract.SchemaPath != "dnd_scene_descriptions_llm.v1.json" {
t.Fatalf("prepared prompt = %#v, want scene-description prompt identity and schema wiring", prepared)
}
if got, want := messageRoles(prepared.Messages), []string{"system", "user", "user", "user", "user", "user"}; !reflect.DeepEqual(got, want) {
t.Fatalf("message roles = %#v, want %#v", got, want)
}
for _, index := range []int{1, 2, 4} {
if prepared.Messages[index].CacheControl == nil || prepared.Messages[index].CacheControl.Type != scriptorium.CacheControlEphemeral {
t.Fatalf("message %d cache control = %#v, want ephemeral", index, prepared.Messages[index].CacheControl)
}
}
for _, index := range []int{0, 3, 5} {
if prepared.Messages[index].CacheControl != nil {
t.Fatalf("message %d cache control = %#v, want nil", index, prepared.Messages[index].CacheControl)
}
}
if !strings.Contains(prepared.Messages[0].Content, "Dungeons & Dragons") {
t.Fatalf("first message does not use shared D&D system asset: %q", prepared.Messages[0].Content)
}
for index, want := range []string{"Dana: Mira", "Mira: ranger", "Greencloak: title"} {
if !strings.Contains(prepared.Messages[2].Content, want) {
t.Fatalf("reference %d not rendered in shared reference message: %q", index, prepared.Messages[2].Content)
}
}
if strings.Contains(prepared.Messages[1].Content, `{"units":[1]}`) || strings.Contains(prepared.Messages[2].Content, `{"units":[1]}`) {
t.Fatal("transcript rendered before its final message")
}
if !strings.Contains(prepared.Messages[5].Content, `{"units":[1]}`) {
t.Fatalf("final message does not render transcript: %q", prepared.Messages[5].Content)
}
transcriptMessages := 0
for _, message := range prepared.Messages {
if strings.Contains(message.Content, `{"units":[1]}`) {
transcriptMessages++
}
}
if transcriptMessages != 1 {
t.Fatalf("raw transcript rendered in %d messages, want exactly one", transcriptMessages)
}
}
func TestPromptMetadataAndDiagnosticsDoNotContainRawAssets(t *testing.T) {
@@ -90,17 +53,9 @@ func TestPromptMetadataAndDiagnosticsDoNotContainRawAssets(t *testing.T) {
if err != nil {
t.Fatal(err)
}
for _, forbidden := range []string{"Choose exactly one kind", "common-dnd-system", "dnd_scene_descriptions_llm.v1.json"} {
for _, forbidden := range []string{"common-dnd-system", "dnd_scene_descriptions_llm.v1.json"} {
if strings.Contains(string(payload), forbidden) {
t.Fatalf("metadata leaked raw prompt/schema content %q: %s", forbidden, payload)
}
}
}
func messageRoles(messages []scriptorium.RenderedMessage) []string {
roles := make([]string, len(messages))
for i, message := range messages {
roles[i] = string(message.Role)
}
return roles
}

50
internal/modules/dnd/extract/spells/scriptorium_assets_test.go
View File

@@ -11,7 +11,7 @@ import (
"gitea.maximumdirect.net/eric/scriptorium"
)
func TestScriptoriumPromptPreparesTranscriptReferencesAndTaskMessages(t *testing.T) {
func TestScriptoriumPromptPreparesSpellPrompt(t *testing.T) {
transcript := []byte(`{"id":"session-1","segments":[{"id":"u1","text":"Mira casts shield."}]}`)
prepared := prepareSpellsPrompt(t, transcript, "Dana: Mira", "Mira: wizard", "Shield: abjuration")
@@ -21,54 +21,6 @@ func TestScriptoriumPromptPreparesTranscriptReferencesAndTaskMessages(t *testing
if prepared.OutputContract.SchemaPath != "dnd_spells_llm.v1.json" {
t.Fatalf("schema path = %q, want LLM-only schema", prepared.OutputContract.SchemaPath)
}
for index, want := range []struct {
role string
cached bool
marker string
}{
{role: "system", marker: "Dungeons & Dragons gameplay transcripts"},
{role: "user", marker: "Transcript units are the only evidence"},
{role: "user", cached: true, marker: "most specific supported in-world"},
{role: "user", cached: true},
{role: "user", cached: true},
{role: "user"},
{role: "user", marker: "spell-cast artifacts"},
{role: "user", cached: true, marker: "source references must collectively support"},
{role: "user"},
} {
if index >= len(prepared.Messages) {
t.Fatalf("prepared prompt has %d messages, want at least %d", len(prepared.Messages), index+1)
}
message := prepared.Messages[index]
if message.Role != want.role {
t.Errorf("message %d role = %q, want %q", index, message.Role, want.role)
}
if want.cached {
if message.CacheControl == nil || message.CacheControl.Type != scriptorium.CacheControlEphemeral {
t.Errorf("message %d cache control = %#v, want ephemeral", index, message.CacheControl)
}
} else if message.CacheControl != nil {
t.Errorf("message %d cache control = %#v, want nil", index, message.CacheControl)
}
if want.marker != "" && !strings.Contains(message.Content, want.marker) {
t.Errorf("message %d content does not contain purpose marker %q", index, want.marker)
}
}
if len(prepared.Messages) != 9 {
t.Fatalf("prepared prompt has %d messages, want 9", len(prepared.Messages))
}
if references := prepared.Messages[3].Content; !strings.Contains(references, "Dana: Mira") || !strings.Contains(references, "Mira: wizard") || !strings.Contains(references, "Shield: abjuration") {
t.Fatalf("campaign references message = %q, want rendered reference inputs", references)
}
if registry := prepared.Messages[4].Content; !strings.Contains(registry, `{"npcs":[]}`) {
t.Fatalf("NPC registry message = %q, want registry input", registry)
}
if catalog := prepared.Messages[5].Content; !strings.Contains(catalog, `{"spell_names":["Cure Wounds"]}`) {
t.Fatalf("spell catalog message = %q, want catalog input", catalog)
}
if final := prepared.Messages[8].Content; !strings.Contains(final, string(transcript)) {
t.Fatalf("final message = %q, want transcript", final)
}
}
func TestScriptoriumPromptPreparesWithMissingOptionalReferences(t *testing.T) {