eric/notarius
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Tighten D&D prompt contracts and tests

Browse Source
This commit is contained in:
Eric Rakestraw
2026-08-05 17:38:51 +00:00
parent 4093ff2e8b
commit 299c110267
15 changed files with 78 additions and 331 deletions
Download Patch File Download Diff File Expand all files Collapse all files

34
internal/modules/dnd/extract/combatturns/prompt_assets_test.go
View File

@@ -62,38 +62,4 @@ func TestPromptAssetsPrepareRequiredInputs(t *testing.T) {
if prepared.PromptID != PromptID || prepared.OutputContract.SchemaPath != "dnd_combat_turns_llm.v1.json" {
t.Fatalf("prepared prompt = %#v, want combat prompt identity and schema", prepared)
}
referenceIndex := -1
transcriptIndex := -1
npcIndex := -1
for index, message := range prepared.Messages {
if strings.Contains(message.Content, "combat-player") {
referenceIndex = index
}
if strings.Contains(message.Content, "combat-transcript") {
transcriptIndex = index
}
if strings.Contains(message.Content, "combat-npc") {
npcIndex = index
}
}
instructionsIndex := len(prepared.Messages) - 1
if referenceIndex < 0 || transcriptIndex <= referenceIndex || npcIndex <= transcriptIndex || instructionsIndex <= npcIndex {
t.Fatalf("message order = references %d, transcript %d, NPCs %d, instructions %d; want that order", referenceIndex, transcriptIndex, npcIndex, instructionsIndex)
}
for _, index := range []int{referenceIndex, transcriptIndex, instructionsIndex} {
if cache := prepared.Messages[index].CacheControl; cache == nil || cache.Type != promptkit.CacheControlEphemeral {
t.Fatalf("message %d cache control = %#v, want ephemeral", index, cache)
}
}
for index, message := range prepared.Messages {
if index != referenceIndex && strings.Contains(message.Content, "combat-player") {
t.Errorf("message %d unexpectedly rendered campaign-reference input", index)
}
if index != transcriptIndex && strings.Contains(message.Content, "combat-transcript") {
t.Errorf("message %d unexpectedly rendered transcript input", index)
}
if index != npcIndex && strings.Contains(message.Content, "combat-npc") {
t.Errorf("message %d unexpectedly rendered NPC grounding input", index)
}
}
}

44
internal/modules/dnd/extract/enemyevents/prompt_assets_test.go
View File

@@ -28,33 +28,6 @@ func TestRegisterPromptAssetsAndPrepareEnemyEventPrompt(t *testing.T) {
if prepared.PromptID != PromptID || prepared.OutputContract.SchemaPath != "dnd_enemy_events_llm.v1.json" || prepared.SelectedProfileID != "dnd-extraction" {
t.Fatalf("prepared prompt = %#v", prepared)
}
referenceIndex := renderedMessageIndex(t, prepared.Messages, "enemy-player")
transcriptIndex := renderedMessageIndex(t, prepared.Messages, "enemy-transcript")
if transcriptIndex <= referenceIndex {
t.Fatalf("transcript message index = %d, want after references index %d", transcriptIndex, referenceIndex)
}
groundingIndices := make([]int, 0, 3)
for _, sentinel := range []string{"enemy-npc", "enemy-turn", "enemy-opponent"} {
if index := renderedMessageIndex(t, prepared.Messages, sentinel); index <= transcriptIndex {
t.Fatalf("message containing %q has index %d, want after transcript index %d", sentinel, index, transcriptIndex)
} else {
groundingIndices = append(groundingIndices, index)
}
}
instructionIndex := len(prepared.Messages) - 1
for _, index := range groundingIndices {
if instructionIndex <= index {
t.Fatalf("instruction index = %d, want after grounding index %d", instructionIndex, index)
}
}
if instructionIndex != len(prepared.Messages)-1 {
t.Fatalf("instruction message index = %d, want final message", instructionIndex)
}
for _, index := range []int{referenceIndex, transcriptIndex, instructionIndex} {
if cache := prepared.Messages[index].CacheControl; cache == nil || cache.Type != promptkit.CacheControlEphemeral {
t.Fatalf("message %d cache control = %#v, want ephemeral", index, cache)
}
}
}
func TestEnemyEventPromptRequiresGroundingInputs(t *testing.T) {
@@ -115,20 +88,3 @@ func enemyEventPromptInputs() map[string]promptkit.ArtifactRef {
"npc_interactions": promptkit.Inline(`{"npc_interactions":[{"name":"enemy-opponent","kind":"combat_opponent"}]}`),
}
}
func renderedMessageIndex(t *testing.T, messages []promptkit.RenderedMessage, sentinel string) int {
t.Helper()
index := -1
occurrences := 0
for messageIndex, message := range messages {
count := strings.Count(message.Content, sentinel)
if count > 0 {
index = messageIndex
occurrences += count
}
}
if occurrences != 1 {
t.Fatalf("message sentinel %q rendered %d times, want exactly once", sentinel, occurrences)
}
return index
}

27
internal/modules/dnd/extract/itemevents/prompt_assets_test.go
View File

@@ -42,33 +42,6 @@ func TestPromptAssetsPrepareItemEventPrompt(t *testing.T) {
if prepared.PromptID != PromptID || prepared.OutputContract.SchemaPath != "dnd_item_events_llm.v1.json" {
t.Fatalf("prepared prompt = %#v", prepared)
}
referenceIndex := -1
transcriptIndex := -1
for index, message := range prepared.Messages {
if strings.Contains(message.Content, "item-event-player") {
referenceIndex = index
}
if strings.Contains(message.Content, "item-event-transcript") {
transcriptIndex = index
}
}
instructionsIndex := len(prepared.Messages) - 1
if referenceIndex < 0 || transcriptIndex <= referenceIndex || transcriptIndex >= instructionsIndex {
t.Fatalf("message order = references %d, transcript %d, instructions %d; want that order", referenceIndex, transcriptIndex, instructionsIndex)
}
for _, index := range []int{referenceIndex, transcriptIndex, instructionsIndex} {
if cache := prepared.Messages[index].CacheControl; cache == nil || cache.Type != promptkit.CacheControlEphemeral {
t.Fatalf("message %d cache control = %#v, want ephemeral", index, cache)
}
}
for index, message := range prepared.Messages {
if index != referenceIndex && strings.Contains(message.Content, "item-event-player") {
t.Errorf("message %d unexpectedly rendered campaign-reference input", index)
}
if index != transcriptIndex && strings.Contains(message.Content, "item-event-transcript") {
t.Errorf("message %d unexpectedly rendered transcript input", index)
}
}
}
func TestPromptAssetsDoNotLeakIntoMetadata(t *testing.T) {

36
internal/modules/dnd/extract/locationoccurrences/prompt_assets_test.go
View File

@@ -40,46 +40,12 @@ func TestRegisterPromptAssetsPreparesLocationOccurrencePrompt(t *testing.T) {
t.Fatalf("prepared prompt = %#v", prepared)
}
var registryMessage string
referenceIndex := -1
transcriptIndex := -1
registryIndex := -1
evidenceIndex := -1
for index, message := range prepared.Messages {
if strings.Contains(message.Content, "location-occurrence-player") {
referenceIndex = index
}
if strings.Contains(message.Content, "location-occurrence-transcript") {
transcriptIndex = index
}
for _, message := range prepared.Messages {
if strings.Contains(message.Content, "normalized location registry") {
registryMessage = message.Content
registryIndex = index
}
if strings.Contains(message.Content, "Transcript units are the only evidence") {
evidenceIndex = index
}
}
if !strings.Contains(registryMessage, "location:sha256:test") || !strings.Contains(registryMessage, "location-occurrence-registry") || strings.Contains(registryMessage, "source_refs") {
t.Fatalf("rendered prompt did not preserve source-free registry grounding: %s", registryMessage)
}
instructionsIndex := len(prepared.Messages) - 1
if referenceIndex < 0 || transcriptIndex <= referenceIndex || evidenceIndex <= transcriptIndex || registryIndex <= evidenceIndex || instructionsIndex <= registryIndex {
t.Fatalf("message order = references %d, transcript %d, evidence %d, registry %d, instructions %d; want that order", referenceIndex, transcriptIndex, evidenceIndex, registryIndex, instructionsIndex)
}
for _, index := range []int{referenceIndex, transcriptIndex, instructionsIndex} {
if cache := prepared.Messages[index].CacheControl; cache == nil || cache.Type != promptkit.CacheControlEphemeral {
t.Fatalf("message %d cache control = %#v, want ephemeral", index, cache)
}
}
for index, message := range prepared.Messages {
if index != referenceIndex && strings.Contains(message.Content, "location-occurrence-player") {
t.Errorf("message %d unexpectedly rendered campaign-reference input", index)
}
if index != transcriptIndex && strings.Contains(message.Content, "location-occurrence-transcript") {
t.Errorf("message %d unexpectedly rendered transcript input", index)
}
if index != registryIndex && strings.Contains(message.Content, "location-occurrence-registry") {
t.Errorf("message %d unexpectedly rendered registry grounding input", index)
}
}
}

32
internal/modules/dnd/extract/locations/prompt_assets_test.go
View File

@@ -2,7 +2,6 @@ package locations
import (
"context"
"strings"
"testing"
"time"
@@ -30,34 +29,7 @@ func TestRegisterPromptAssetsPreparesLocationPrompt(t *testing.T) {
if err != nil {
t.Fatalf("Prepare() error = %v", err)
}
if prepared.OutputContract.SchemaPath != "dnd_locations_llm.v1.json" {
t.Fatalf("output contract = %#v", prepared.OutputContract)
}
referenceIndex := -1
transcriptIndex := -1
for index, message := range prepared.Messages {
if strings.Contains(message.Content, "location-player") {
referenceIndex = index
}
if strings.Contains(message.Content, "location-transcript") {
transcriptIndex = index
}
}
instructionsIndex := len(prepared.Messages) - 1
if referenceIndex < 0 || transcriptIndex <= referenceIndex || transcriptIndex >= instructionsIndex {
t.Fatalf("message order = references %d, transcript %d, instructions %d; want that order", referenceIndex, transcriptIndex, instructionsIndex)
}
for _, index := range []int{referenceIndex, transcriptIndex, instructionsIndex} {
if prepared.Messages[index].CacheControl == nil || prepared.Messages[index].CacheControl.Type != promptkit.CacheControlEphemeral {
t.Fatalf("message %d cache control = %#v, want ephemeral", index, prepared.Messages[index].CacheControl)
}
}
for index, message := range prepared.Messages {
if index != referenceIndex && strings.Contains(message.Content, "location-player") {
t.Errorf("message %d unexpectedly rendered campaign-reference input", index)
}
if index != transcriptIndex && strings.Contains(message.Content, "location-transcript") {
t.Errorf("message %d unexpectedly rendered transcript input", index)
}
if prepared.PromptID != PromptID || prepared.OutputContract.SchemaPath != "dnd_locations_llm.v1.json" {
t.Fatalf("prepared prompt = %#v, want location prompt identity and schema wiring", prepared)
}
}

34
internal/modules/dnd/extract/npcinteractions/prompt_assets_test.go
View File

@@ -51,40 +51,6 @@ func TestRegisterPromptAssetsAndPrepareInteractionPrompt(t *testing.T) {
if prepared.PromptID != PromptID || prepared.OutputContract.SchemaPath != "dnd_npc_interactions_llm.v1.json" {
t.Fatalf("prepared prompt = %#v", prepared)
}
referenceIndex := -1
transcriptIndex := -1
npcIndex := -1
for index, message := range prepared.Messages {
if strings.Contains(message.Content, "interaction-player") {
referenceIndex = index
}
if strings.Contains(message.Content, "interaction-transcript") {
transcriptIndex = index
}
if strings.Contains(message.Content, "interaction-npc") {
npcIndex = index
}
}
instructionsIndex := len(prepared.Messages) - 1
if referenceIndex < 0 || transcriptIndex <= referenceIndex || npcIndex <= transcriptIndex || instructionsIndex <= npcIndex {
t.Fatalf("message order = references %d, transcript %d, NPCs %d, instructions %d; want that order", referenceIndex, transcriptIndex, npcIndex, instructionsIndex)
}
for _, index := range []int{referenceIndex, transcriptIndex, instructionsIndex} {
if cache := prepared.Messages[index].CacheControl; cache == nil || cache.Type != promptkit.CacheControlEphemeral {
t.Fatalf("message %d cache control = %#v, want ephemeral", index, cache)
}
}
for index, message := range prepared.Messages {
if index != referenceIndex && strings.Contains(message.Content, "interaction-player") {
t.Errorf("message %d unexpectedly rendered campaign-reference input", index)
}
if index != transcriptIndex && strings.Contains(message.Content, "interaction-transcript") {
t.Errorf("message %d unexpectedly rendered transcript input", index)
}
if index != npcIndex && strings.Contains(message.Content, "interaction-npc") {
t.Errorf("message %d unexpectedly rendered NPC grounding input", index)
}
}
}
func TestPromptMetadataDoesNotExposeAssetContent(t *testing.T) {

27
internal/modules/dnd/extract/npcs/prompt_assets_test.go
View File

@@ -42,33 +42,6 @@ func TestRegisterPromptAssetsAndPrepareNPCPrompt(t *testing.T) {
if prepared.PromptID != PromptID || prepared.OutputContract.SchemaPath != "dnd_npcs_llm.v1.json" {
t.Fatalf("prepared prompt = %#v, want NPC prompt identity and wiring", prepared)
}
referenceIndex := -1
transcriptIndex := -1
for index, message := range prepared.Messages {
if strings.Contains(message.Content, "npc-player") {
referenceIndex = index
}
if strings.Contains(message.Content, "npc-transcript") {
transcriptIndex = index
}
}
instructionsIndex := len(prepared.Messages) - 1
if referenceIndex < 0 || transcriptIndex <= referenceIndex || transcriptIndex >= instructionsIndex {
t.Fatalf("message order = references %d, transcript %d, instructions %d; want that order", referenceIndex, transcriptIndex, instructionsIndex)
}
for _, index := range []int{referenceIndex, transcriptIndex, instructionsIndex} {
if cache := prepared.Messages[index].CacheControl; cache == nil || cache.Type != promptkit.CacheControlEphemeral {
t.Fatalf("message %d cache control = %#v, want ephemeral", index, cache)
}
}
for index, message := range prepared.Messages {
if index != referenceIndex && strings.Contains(message.Content, "npc-player") {
t.Errorf("message %d unexpectedly rendered campaign-reference input", index)
}
if index != transcriptIndex && strings.Contains(message.Content, "npc-transcript") {
t.Errorf("message %d unexpectedly rendered transcript input", index)
}
}
}
func TestPromptMetadataAndDiagnosticsDoNotContainRawAssets(t *testing.T) {

27
internal/modules/dnd/extract/scenedescriptions/prompt_assets_test.go
View File

@@ -42,33 +42,6 @@ func TestRegisterPromptAssetsPreparesSceneDescriptionPrompt(t *testing.T) {
if prepared.PromptID != PromptID || prepared.OutputContract.SchemaPath != "dnd_scene_descriptions_llm.v1.json" {
t.Fatalf("prepared prompt = %#v, want scene-description prompt identity and schema wiring", prepared)
}
referenceIndex := -1
transcriptIndex := -1
for index, message := range prepared.Messages {
if strings.Contains(message.Content, "scene-description-player") {
referenceIndex = index
}
if strings.Contains(message.Content, "scene-description-transcript") {
transcriptIndex = index
}
}
instructionsIndex := len(prepared.Messages) - 1
if referenceIndex < 0 || transcriptIndex <= referenceIndex || transcriptIndex >= instructionsIndex {
t.Fatalf("message order = references %d, transcript %d, instructions %d; want that order", referenceIndex, transcriptIndex, instructionsIndex)
}
for _, index := range []int{referenceIndex, transcriptIndex, instructionsIndex} {
if cache := prepared.Messages[index].CacheControl; cache == nil || cache.Type != promptkit.CacheControlEphemeral {
t.Fatalf("message %d cache control = %#v, want ephemeral", index, cache)
}
}
for index, message := range prepared.Messages {
if index != referenceIndex && strings.Contains(message.Content, "scene-description-player") {
t.Errorf("message %d unexpectedly rendered campaign-reference input", index)
}
if index != transcriptIndex && strings.Contains(message.Content, "scene-description-transcript") {
t.Errorf("message %d unexpectedly rendered transcript input", index)
}
}
}
func TestPromptMetadataAndDiagnosticsDoNotContainRawAssets(t *testing.T) {

29
internal/modules/dnd/extract/spells/prompt_assets_test.go
View File

@@ -21,35 +21,6 @@ func TestPromptAssetsPrepareSpellPrompt(t *testing.T) {
if prepared.OutputContract.SchemaPath != "dnd_spells_llm.v1.json" {
t.Fatalf("schema path = %q, want LLM-only schema", prepared.OutputContract.SchemaPath)
}
indices := map[string]int{
"spell-player": -1,
"spell-transcript": -1,
"spell-npc-sentinel": -1,
"spell-catalog-sentinel": -1,
}
for index, message := range prepared.Messages {
for sentinel := range indices {
if strings.Contains(message.Content, sentinel) {
indices[sentinel] = index
}
}
}
instructionsIndex := len(prepared.Messages) - 1
if indices["spell-player"] < 0 || indices["spell-transcript"] <= indices["spell-player"] || indices["spell-npc-sentinel"] <= indices["spell-transcript"] || indices["spell-catalog-sentinel"] <= indices["spell-npc-sentinel"] || instructionsIndex <= indices["spell-catalog-sentinel"] {
t.Fatalf("message order = %#v with instructions at %d, want references, transcript, NPCs, catalog, instructions", indices, instructionsIndex)
}
for _, index := range []int{indices["spell-player"], indices["spell-transcript"], instructionsIndex} {
if cache := prepared.Messages[index].CacheControl; cache == nil || cache.Type != promptkit.CacheControlEphemeral {
t.Fatalf("message %d cache control = %#v, want ephemeral", index, cache)
}
}
for sentinel, index := range indices {
for messageIndex, message := range prepared.Messages {
if messageIndex != index && strings.Contains(message.Content, sentinel) {
t.Errorf("message %d unexpectedly rendered %q", messageIndex, sentinel)
}
}
}
}
func TestPromptAssetsPrepareWithMissingOptionalReferences(t *testing.T) {