From a8a99c10374c8825271d922e88631b5a9b8b272f Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Mon, 3 Aug 2026 22:57:17 +0000 Subject: [PATCH] Make enemy event prompt tests resilient to refactoring --- .../extract/enemyevents/prompt_assets_test.go | 131 +++++++++--------- .../modules/dnd/register/prompt_cache_test.go | 30 ++-- 2 files changed, 75 insertions(+), 86 deletions(-) diff --git a/internal/modules/dnd/extract/enemyevents/prompt_assets_test.go b/internal/modules/dnd/extract/enemyevents/prompt_assets_test.go index 697926e..23cd73a 100644 --- a/internal/modules/dnd/extract/enemyevents/prompt_assets_test.go +++ b/internal/modules/dnd/extract/enemyevents/prompt_assets_test.go @@ -28,69 +28,52 @@ func TestRegisterPromptAssetsAndPrepareEnemyEventPrompt(t *testing.T) { if prepared.PromptID != PromptID || prepared.OutputContract.SchemaPath != "dnd_enemy_events_llm.v1.json" || prepared.SelectedProfileID != "dnd-extraction" { t.Fatalf("prepared prompt = %#v", prepared) } - transcriptIndex := -1 - for index, message := range prepared.Messages { - if strings.Contains(message.Content, "enemy-transcript") { - transcriptIndex = index - break + transcriptIndex := renderedMessageIndex(t, prepared.Messages, "enemy-transcript") + for _, sentinel := range []string{"enemy-npc", "enemy-turn", "enemy-opponent", "Extract Dungeons & Dragons enemy events"} { + if index := renderedMessageIndex(t, prepared.Messages, sentinel); index <= transcriptIndex { + t.Fatalf("message containing %q has index %d, want after transcript index %d", sentinel, index, transcriptIndex) } } - if transcriptIndex != 3 { - t.Fatalf("transcript message index = %d, want shared-prefix index 3", transcriptIndex) + instructionIndex := renderedMessageIndex(t, prepared.Messages, "Return the `events` array") + if instructionIndex <= transcriptIndex { + t.Fatalf("instruction index = %d, want after transcript index %d", instructionIndex, transcriptIndex) } - for index, role := range []string{"system", "user", "user", "user"} { - if prepared.Messages[index].Role != role { - t.Fatalf("message %d role = %q, want %q", index, prepared.Messages[index].Role, role) - } + if instructionIndex != len(prepared.Messages)-1 { + t.Fatalf("instruction message index = %d, want final message", instructionIndex) } - for _, index := range []int{2, 3} { - if cache := prepared.Messages[index].CacheControl; cache == nil || cache.Type != promptkit.CacheControlEphemeral { - t.Fatalf("message %d cache control = %#v", index, cache) - } - } - for _, index := range []int{0, 1} { - if prepared.Messages[index].CacheControl != nil { - t.Fatalf("message %d cache control = %#v, want nil", index, prepared.Messages[index].CacheControl) - } - } - if !strings.Contains(prepared.Messages[transcriptIndex+3].Content, "enemy-turn") || !strings.Contains(prepared.Messages[transcriptIndex+3].Content, "enemy-opponent") { - t.Fatalf("combat grounding message = %q", prepared.Messages[transcriptIndex+3].Content) + if cache := prepared.Messages[instructionIndex].CacheControl; cache == nil || cache.Type != promptkit.CacheControlEphemeral { + t.Fatalf("final instruction cache control = %#v", cache) } } func TestEnemyEventPromptRequiresGroundingInputs(t *testing.T) { - registry := llm.NewAssetRegistry() - if err := RegisterPromptAssets(registry); err != nil { - t.Fatal(err) - } - options, err := registry.PromptKitOptions() - if err != nil { - t.Fatal(err) - } - options = append(options, promptkit.WithProfiles(promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{ - ID: "dnd-extraction", Endpoint: "http://127.0.0.1:1/v1", Model: "enemy-test-model", - }))) - engine, err := promptkit.NewEngine(promptkit.Config{Timeout: time.Second}, options...) - if err != nil { - t.Fatal(err) - } - _, err = engine.Prepare(context.Background(), promptkit.RunRequest{ - PromptID: PromptID, PromptVersion: SchemaVersion, ProfileID: "dnd-extraction", - Inputs: map[string]promptkit.ArtifactRef{ - "transcript": promptkit.Inline(`{"value":"enemy-transcript"}`), - "players": promptkit.Inline(" "), - "party": promptkit.Inline(" "), - "glossary": promptkit.Inline(" "), - "npcs": promptkit.Inline(`{"npcs":[]}`), - "npc_interactions": promptkit.Inline(`{"npc_interactions":[]}`), - }, - }) - if err == nil || !strings.Contains(err.Error(), "combat_turns") { - t.Fatalf("Prepare() error = %v, want required combat-turn input", err) + engine := newEnemyEventPromptEngine(t) + for _, inputName := range []string{"npcs", "combat_turns", "npc_interactions"} { + t.Run(inputName, func(t *testing.T) { + inputs := enemyEventPromptInputs() + delete(inputs, inputName) + _, err := engine.Prepare(context.Background(), promptkit.RunRequest{ + PromptID: PromptID, PromptVersion: SchemaVersion, Inputs: inputs, + }) + if err == nil || !strings.Contains(err.Error(), inputName) { + t.Fatalf("Prepare() error = %v, want required %q input", err, inputName) + } + }) } } func prepareEnemyEventPrompt(t *testing.T) *promptkit.PreparedRun { + t.Helper() + prepared, err := newEnemyEventPromptEngine(t).Prepare(context.Background(), promptkit.RunRequest{ + PromptID: PromptID, PromptVersion: SchemaVersion, Inputs: enemyEventPromptInputs(), + }) + if err != nil { + t.Fatal(err) + } + return prepared +} + +func newEnemyEventPromptEngine(t *testing.T) *promptkit.Engine { t.Helper() registry := llm.NewAssetRegistry() if err := RegisterPromptAssets(registry); err != nil { @@ -107,20 +90,34 @@ func prepareEnemyEventPrompt(t *testing.T) *promptkit.PreparedRun { if err != nil { t.Fatal(err) } - prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{ - PromptID: PromptID, PromptVersion: SchemaVersion, ProfileID: "dnd-extraction", - Inputs: map[string]promptkit.ArtifactRef{ - "transcript": promptkit.Inline(`{"value":"enemy-transcript"}`), - "players": promptkit.Inline("enemy-player"), - "party": promptkit.Inline("enemy-party"), - "glossary": promptkit.Inline("enemy-glossary"), - "npcs": promptkit.Inline(`{"npcs":[{"name":"enemy-npc"}]}`), - "combat_turns": promptkit.Inline(`{"combat_turns":[{"actor":"enemy-turn","turn_kind":"turn"}]}`), - "npc_interactions": promptkit.Inline(`{"npc_interactions":[{"name":"enemy-opponent","kind":"combat_opponent"}]}`), - }, - }) - if err != nil { - t.Fatal(err) - } - return prepared + return engine +} + +func enemyEventPromptInputs() map[string]promptkit.ArtifactRef { + return map[string]promptkit.ArtifactRef{ + "transcript": promptkit.Inline(`{"value":"enemy-transcript"}`), + "players": promptkit.Inline("enemy-player"), + "party": promptkit.Inline("enemy-party"), + "glossary": promptkit.Inline("enemy-glossary"), + "npcs": promptkit.Inline(`{"npcs":[{"name":"enemy-npc"}]}`), + "combat_turns": promptkit.Inline(`{"combat_turns":[{"actor":"enemy-turn","turn_kind":"turn"}]}`), + "npc_interactions": promptkit.Inline(`{"npc_interactions":[{"name":"enemy-opponent","kind":"combat_opponent"}]}`), + } +} + +func renderedMessageIndex(t *testing.T, messages []promptkit.RenderedMessage, sentinel string) int { + t.Helper() + index := -1 + occurrences := 0 + for messageIndex, message := range messages { + count := strings.Count(message.Content, sentinel) + if count > 0 { + index = messageIndex + occurrences += count + } + } + if occurrences != 1 { + t.Fatalf("message sentinel %q rendered %d times, want exactly once", sentinel, occurrences) + } + return index } diff --git a/internal/modules/dnd/register/prompt_cache_test.go b/internal/modules/dnd/register/prompt_cache_test.go index bc69b39..7aaeb1c 100644 --- a/internal/modules/dnd/register/prompt_cache_test.go +++ b/internal/modules/dnd/register/prompt_cache_test.go @@ -39,12 +39,11 @@ func TestExtractionPromptsShareRenderedPrefix(t *testing.T) { "glossary": promptkit.Inline(glossarySentinel), } cases := []struct { - name string - promptID string - promptVersion string - inputs map[string]promptkit.ArtifactRef - npcInput bool - spellCatalogInput bool + name string + promptID string + promptVersion string + inputs map[string]promptkit.ArtifactRef + inputSentinels []string }{ {name: "npcs", promptID: npcextract.PromptID, promptVersion: npcextract.SchemaVersion, inputs: commonInputs}, {name: "item events", promptID: itemeventextract.PromptID, promptVersion: itemeventextract.SchemaVersion, inputs: commonInputs}, @@ -56,7 +55,7 @@ func TestExtractionPromptsShareRenderedPrefix(t *testing.T) { inputs: withPromptInputs(commonInputs, map[string]promptkit.ArtifactRef{ "npcs": promptkit.Inline(`{"sentinel":"` + npcSentinel + `"}`), }), - npcInput: true, + inputSentinels: []string{npcSentinel}, }, { name: "enemy events", @@ -67,7 +66,7 @@ func TestExtractionPromptsShareRenderedPrefix(t *testing.T) { "combat_turns": promptkit.Inline(`{"sentinel":"combat-turns-sentinel"}`), "npc_interactions": promptkit.Inline(`{"sentinel":"npc-interactions-sentinel"}`), }), - npcInput: true, + inputSentinels: []string{npcSentinel, "combat-turns-sentinel", "npc-interactions-sentinel"}, }, { name: "npc interactions", @@ -76,7 +75,7 @@ func TestExtractionPromptsShareRenderedPrefix(t *testing.T) { inputs: withPromptInputs(commonInputs, map[string]promptkit.ArtifactRef{ "npcs": promptkit.Inline(`{"sentinel":"` + npcSentinel + `"}`), }), - npcInput: true, + inputSentinels: []string{npcSentinel}, }, { name: "spells", @@ -86,8 +85,7 @@ func TestExtractionPromptsShareRenderedPrefix(t *testing.T) { "npcs": promptkit.Inline(`{"sentinel":"` + npcSentinel + `"}`), "spell_catalog": promptkit.Inline(`{"sentinel":"` + catalogSentinel + `"}`), }), - npcInput: true, - spellCatalogInput: true, + inputSentinels: []string{npcSentinel, catalogSentinel}, }, } @@ -105,9 +103,6 @@ func TestExtractionPromptsShareRenderedPrefix(t *testing.T) { } transcriptIndex := renderedInputMessageIndex(t, prepared.Messages, transcriptSentinel) prefix := prepared.Messages[:transcriptIndex+1] - if len(prefix) != 4 { - t.Fatalf("messages through transcript = %d, want 4", len(prefix)) - } if len(prepared.Messages) <= len(prefix) { t.Fatalf("prepared prompt has %d messages, want lane-specific suffix after transcript", len(prepared.Messages)) } @@ -116,11 +111,8 @@ func TestExtractionPromptsShareRenderedPrefix(t *testing.T) { } else if !reflect.DeepEqual(prefix, sharedPrefix) { t.Fatalf("rendered prefix = %#v, want %#v", prefix, sharedPrefix) } - if testCase.npcInput { - assertRenderedInputAfter(t, prepared.Messages, npcSentinel, transcriptIndex) - } - if testCase.spellCatalogInput { - assertRenderedInputAfter(t, prepared.Messages, catalogSentinel, transcriptIndex) + for _, sentinel := range testCase.inputSentinels { + assertRenderedInputAfter(t, prepared.Messages, sentinel, transcriptIndex) } }) }