Tighten D&D prompt contracts and tests
This commit is contained in:
@@ -20,7 +20,7 @@ import (
|
||||
"gitea.maximumdirect.net/eric/promptkit"
|
||||
)
|
||||
|
||||
func TestExtractionPromptsShareRenderedPrefix(t *testing.T) {
|
||||
func TestExtractionPromptComposition(t *testing.T) {
|
||||
const (
|
||||
transcriptSentinel = "shared-transcript-sentinel"
|
||||
playersSentinel = "shared-players-sentinel"
|
||||
@@ -28,6 +28,7 @@ func TestExtractionPromptsShareRenderedPrefix(t *testing.T) {
|
||||
glossarySentinel = "shared-glossary-sentinel"
|
||||
npcSentinel = "npc-registry-sentinel"
|
||||
catalogSentinel = "spell-catalog-sentinel"
|
||||
evidenceSentinel = "Transcript units are the only evidence"
|
||||
)
|
||||
registry := llm.NewAssetRegistry()
|
||||
if err := registerPromptAssets(registry); err != nil {
|
||||
@@ -41,15 +42,15 @@ func TestExtractionPromptsShareRenderedPrefix(t *testing.T) {
|
||||
"glossary": promptkit.Inline(glossarySentinel),
|
||||
}
|
||||
cases := []struct {
|
||||
name string
|
||||
promptID string
|
||||
promptVersion string
|
||||
inputs map[string]promptkit.ArtifactRef
|
||||
inputSentinels []string
|
||||
name string
|
||||
promptID string
|
||||
promptVersion string
|
||||
inputs map[string]promptkit.ArtifactRef
|
||||
suffixGroups [][]string
|
||||
}{
|
||||
{name: "npcs", promptID: npcextract.PromptID, promptVersion: npcextract.SchemaVersion, inputs: commonInputs},
|
||||
{name: "locations", promptID: locationextract.PromptID, promptVersion: locationextract.SchemaVersion, inputs: commonInputs},
|
||||
{name: "item events", promptID: itemeventextract.PromptID, promptVersion: itemeventextract.SchemaVersion, inputs: commonInputs},
|
||||
{name: "npcs", promptID: npcextract.PromptID, promptVersion: npcextract.SchemaVersion, inputs: commonInputs, suffixGroups: [][]string{{evidenceSentinel}}},
|
||||
{name: "locations", promptID: locationextract.PromptID, promptVersion: locationextract.SchemaVersion, inputs: commonInputs, suffixGroups: [][]string{{evidenceSentinel}}},
|
||||
{name: "item events", promptID: itemeventextract.PromptID, promptVersion: itemeventextract.SchemaVersion, inputs: commonInputs, suffixGroups: [][]string{{evidenceSentinel}}},
|
||||
{name: "scene descriptions", promptID: scenedescriptionextract.PromptID, promptVersion: scenedescriptionextract.SchemaVersion, inputs: commonInputs},
|
||||
{
|
||||
name: "combat turns",
|
||||
@@ -58,7 +59,7 @@ func TestExtractionPromptsShareRenderedPrefix(t *testing.T) {
|
||||
inputs: withPromptInputs(commonInputs, map[string]promptkit.ArtifactRef{
|
||||
"npcs": promptkit.Inline(`{"sentinel":"` + npcSentinel + `"}`),
|
||||
}),
|
||||
inputSentinels: []string{npcSentinel},
|
||||
suffixGroups: [][]string{{evidenceSentinel}, {npcSentinel}},
|
||||
},
|
||||
{
|
||||
name: "enemy events",
|
||||
@@ -69,7 +70,11 @@ func TestExtractionPromptsShareRenderedPrefix(t *testing.T) {
|
||||
"combat_turns": promptkit.Inline(`{"sentinel":"combat-turns-sentinel"}`),
|
||||
"npc_interactions": promptkit.Inline(`{"sentinel":"npc-interactions-sentinel"}`),
|
||||
}),
|
||||
inputSentinels: []string{npcSentinel, "combat-turns-sentinel", "npc-interactions-sentinel"},
|
||||
suffixGroups: [][]string{
|
||||
{evidenceSentinel},
|
||||
{npcSentinel},
|
||||
{"combat-turns-sentinel", "npc-interactions-sentinel"},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "npc interactions",
|
||||
@@ -78,7 +83,7 @@ func TestExtractionPromptsShareRenderedPrefix(t *testing.T) {
|
||||
inputs: withPromptInputs(commonInputs, map[string]promptkit.ArtifactRef{
|
||||
"npcs": promptkit.Inline(`{"sentinel":"` + npcSentinel + `"}`),
|
||||
}),
|
||||
inputSentinels: []string{npcSentinel},
|
||||
suffixGroups: [][]string{{evidenceSentinel}, {npcSentinel}},
|
||||
},
|
||||
{
|
||||
name: "location occurrences",
|
||||
@@ -87,7 +92,7 @@ func TestExtractionPromptsShareRenderedPrefix(t *testing.T) {
|
||||
inputs: withPromptInputs(commonInputs, map[string]promptkit.ArtifactRef{
|
||||
"locations": promptkit.Inline(`{"sentinel":"location-registry-sentinel"}`),
|
||||
}),
|
||||
inputSentinels: []string{"location-registry-sentinel"},
|
||||
suffixGroups: [][]string{{evidenceSentinel}, {"location-registry-sentinel"}},
|
||||
},
|
||||
{
|
||||
name: "spells",
|
||||
@@ -97,7 +102,7 @@ func TestExtractionPromptsShareRenderedPrefix(t *testing.T) {
|
||||
"npcs": promptkit.Inline(`{"sentinel":"` + npcSentinel + `"}`),
|
||||
"spell_catalog": promptkit.Inline(`{"sentinel":"` + catalogSentinel + `"}`),
|
||||
}),
|
||||
inputSentinels: []string{npcSentinel, catalogSentinel},
|
||||
suffixGroups: [][]string{{evidenceSentinel}, {npcSentinel}, {catalogSentinel}},
|
||||
},
|
||||
}
|
||||
|
||||
@@ -114,6 +119,12 @@ func TestExtractionPromptsShareRenderedPrefix(t *testing.T) {
|
||||
t.Fatalf("Prepare() error = %v", err)
|
||||
}
|
||||
transcriptIndex := renderedInputMessageIndex(t, prepared.Messages, transcriptSentinel)
|
||||
referenceIndex := renderedInputMessageIndex(t, prepared.Messages, playersSentinel)
|
||||
if referenceIndex >= transcriptIndex {
|
||||
t.Fatalf("campaign references rendered at message %d, want before transcript message %d", referenceIndex, transcriptIndex)
|
||||
}
|
||||
assertEphemeralCache(t, prepared.Messages, referenceIndex)
|
||||
assertEphemeralCache(t, prepared.Messages, transcriptIndex)
|
||||
prefix := prepared.Messages[:transcriptIndex+1]
|
||||
if len(prepared.Messages) <= len(prefix) {
|
||||
t.Fatalf("prepared prompt has %d messages, want lane-specific suffix after transcript", len(prepared.Messages))
|
||||
@@ -123,9 +134,27 @@ func TestExtractionPromptsShareRenderedPrefix(t *testing.T) {
|
||||
} else if !reflect.DeepEqual(prefix, sharedPrefix) {
|
||||
t.Fatalf("rendered prefix = %#v, want %#v", prefix, sharedPrefix)
|
||||
}
|
||||
for _, sentinel := range testCase.inputSentinels {
|
||||
assertRenderedInputAfter(t, prepared.Messages, sentinel, transcriptIndex)
|
||||
previousIndex := transcriptIndex
|
||||
for _, group := range testCase.suffixGroups {
|
||||
groupIndex := -1
|
||||
for _, sentinel := range group {
|
||||
inputIndex := renderedInputMessageIndex(t, prepared.Messages, sentinel)
|
||||
if groupIndex < 0 {
|
||||
groupIndex = inputIndex
|
||||
} else if inputIndex != groupIndex {
|
||||
t.Fatalf("input sentinel %q rendered at message %d, want grouped at message %d", sentinel, inputIndex, groupIndex)
|
||||
}
|
||||
}
|
||||
if groupIndex <= previousIndex {
|
||||
t.Fatalf("prompt suffix group rendered at message %d, want after message %d", groupIndex, previousIndex)
|
||||
}
|
||||
previousIndex = groupIndex
|
||||
}
|
||||
instructionIndex := len(prepared.Messages) - 1
|
||||
if instructionIndex <= previousIndex {
|
||||
t.Fatalf("instructions rendered at message %d, want after lane input message %d", instructionIndex, previousIndex)
|
||||
}
|
||||
assertEphemeralCache(t, prepared.Messages, instructionIndex)
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -157,10 +186,10 @@ func withPromptInputs(inputs, extras map[string]promptkit.ArtifactRef) map[strin
|
||||
return merged
|
||||
}
|
||||
|
||||
func assertRenderedInputAfter(t *testing.T, messages []promptkit.RenderedMessage, sentinel string, index int) {
|
||||
func assertEphemeralCache(t *testing.T, messages []promptkit.RenderedMessage, index int) {
|
||||
t.Helper()
|
||||
if inputIndex := renderedInputMessageIndex(t, messages, sentinel); inputIndex <= index {
|
||||
t.Fatalf("input sentinel %q rendered at message %d, want after transcript message %d", sentinel, inputIndex, index)
|
||||
if cache := messages[index].CacheControl; cache == nil || cache.Type != promptkit.CacheControlEphemeral {
|
||||
t.Fatalf("message %d cache control = %#v, want ephemeral", index, cache)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user