diff --git a/internal/modules/dnd/chunk/scenes/scriptorium_assets_test.go b/internal/modules/dnd/chunk/scenes/scriptorium_assets_test.go index 583ca24..355ad88 100644 --- a/internal/modules/dnd/chunk/scenes/scriptorium_assets_test.go +++ b/internal/modules/dnd/chunk/scenes/scriptorium_assets_test.go @@ -12,35 +12,43 @@ import ( ) func TestScriptoriumPromptPreparesTranscriptAndTaskMessages(t *testing.T) { - transcript := []byte(`{"id":"session-1","segments":[{"id":"u1","text":"We enter the crypt."}]}`) - prepared := prepareScenesPrompt(t, transcript, "Alice: Aria", "Aria: cleric", "Brightmantle: temple") + transcript := []byte(`{"sentinel":"scene-transcript"}`) + prepared := prepareScenesPrompt(t, transcript, "scene-players", "scene-party", "scene-glossary") if prepared.PromptID != PromptID { t.Fatalf("prompt id = %q, want %q", prepared.PromptID, PromptID) } - transcriptMessages := 0 - referenceMessageFound := false - for _, message := range prepared.Messages { - if strings.Contains(message.Content, string(transcript)) { - transcriptMessages++ - if message.Role != "user" || message.CacheControl == nil { - t.Fatalf("transcript message did not render as cacheable user message: %#v", message) - } - } - if strings.Contains(message.Content, "Alice: Aria") && - strings.Contains(message.Content, "Aria: cleric") && - strings.Contains(message.Content, "Brightmantle: temple") { - referenceMessageFound = true - if message.Role != "user" || message.CacheControl == nil { - t.Fatalf("reference message did not render as cacheable user message: %#v", message) - } + if len(prepared.Messages) != 5 { + t.Fatalf("prepared messages = %d, want 5", len(prepared.Messages)) + } + for index, role := range []string{"system", "user", "user", "user", "user"} { + if prepared.Messages[index].Role != role { + t.Errorf("message %d role = %q, want %q", index, prepared.Messages[index].Role, role) } } - if transcriptMessages != 1 { - t.Fatalf("raw transcript rendered in %d messages, want exactly one", transcriptMessages) + for _, index := range []int{1, 4} { + if cache := prepared.Messages[index].CacheControl; cache == nil || cache.Type != scriptorium.CacheControlEphemeral { + t.Errorf("message %d cache control = %#v, want ephemeral", index, cache) + } } - if !referenceMessageFound { - t.Fatalf("reference message did not render all supplied reference material") + for _, index := range []int{0, 2, 3} { + if cache := prepared.Messages[index].CacheControl; cache != nil { + t.Errorf("message %d cache control = %#v, want nil", index, cache) + } + } + if references := prepared.Messages[1].Content; !strings.Contains(references, "scene-players") || !strings.Contains(references, "scene-party") || !strings.Contains(references, "scene-glossary") { + t.Fatalf("reference message = %q, want supplied reference inputs", references) + } + if transcriptMessage := prepared.Messages[4].Content; !strings.Contains(transcriptMessage, "scene-transcript") { + t.Fatalf("final message = %q, want transcript input", transcriptMessage) + } + for index, message := range prepared.Messages { + if index != 1 && (strings.Contains(message.Content, "scene-players") || strings.Contains(message.Content, "scene-party") || strings.Contains(message.Content, "scene-glossary")) { + t.Errorf("message %d unexpectedly rendered reference input", index) + } + if index != 4 && strings.Contains(message.Content, "scene-transcript") { + t.Errorf("message %d unexpectedly rendered transcript input", index) + } } } diff --git a/internal/modules/dnd/extract/combatturns/scriptorium_assets_test.go b/internal/modules/dnd/extract/combatturns/scriptorium_assets_test.go index 291f1d8..0b2f5f3 100644 --- a/internal/modules/dnd/extract/combatturns/scriptorium_assets_test.go +++ b/internal/modules/dnd/extract/combatturns/scriptorium_assets_test.go @@ -50,10 +50,10 @@ func TestScriptoriumPromptPreparesRequiredInputs(t *testing.T) { PromptID: PromptID, PromptVersion: SchemaVersion, ProfileID: "combat-test-profile", Inputs: map[string]scriptorium.ArtifactRef{ "transcript": scriptorium.InlineWithURI("file:///session.json", transcript), - "players": scriptorium.Inline("Dana: Mira"), - "party": scriptorium.Inline("Mira: ranger"), - "glossary": scriptorium.Inline("Greencloak: title"), - "npcs": scriptorium.Inline(`{"npcs":[]}`), + "players": scriptorium.Inline(" "), + "party": scriptorium.Inline(" "), + "glossary": scriptorium.Inline(" "), + "npcs": scriptorium.Inline(" "), }, }) if err != nil { @@ -62,48 +62,4 @@ func TestScriptoriumPromptPreparesRequiredInputs(t *testing.T) { if prepared.PromptID != PromptID || prepared.OutputContract.SchemaPath != "dnd_combat_turns_llm.v1.json" { t.Fatalf("prepared prompt = %#v, want combat prompt identity and schema", prepared) } - for index, want := range []struct { - role string - cached bool - marker string - }{ - {role: "system", marker: "Dungeons & Dragons gameplay transcripts"}, - {role: "user", marker: "Transcript units are the only evidence"}, - {role: "user", cached: true, marker: "most specific supported in-world"}, - {role: "user", cached: true}, - {role: "user", cached: true}, - {role: "user", marker: "combat-turn artifacts"}, - {role: "user", cached: true, marker: "turn_kind"}, - {role: "user"}, - } { - if index >= len(prepared.Messages) { - t.Fatalf("prepared prompt has %d messages, want at least %d", len(prepared.Messages), index+1) - } - message := prepared.Messages[index] - if message.Role != want.role { - t.Errorf("message %d role = %q, want %q", index, message.Role, want.role) - } - if want.cached { - if message.CacheControl == nil || message.CacheControl.Type != scriptorium.CacheControlEphemeral { - t.Errorf("message %d cache control = %#v, want ephemeral", index, message.CacheControl) - } - } else if message.CacheControl != nil { - t.Errorf("message %d cache control = %#v, want nil", index, message.CacheControl) - } - if want.marker != "" && !strings.Contains(message.Content, want.marker) { - t.Errorf("message %d content does not contain purpose marker %q", index, want.marker) - } - } - if len(prepared.Messages) != 8 { - t.Fatalf("prepared prompt has %d messages, want 8", len(prepared.Messages)) - } - if references := prepared.Messages[3].Content; !strings.Contains(references, "Dana: Mira") || !strings.Contains(references, "Mira: ranger") || !strings.Contains(references, "Greencloak: title") { - t.Fatalf("campaign references message = %q, want rendered reference inputs", references) - } - if registry := prepared.Messages[4].Content; !strings.Contains(registry, `{"npcs":[]}`) { - t.Fatalf("NPC registry message = %q, want registry input", registry) - } - if final := prepared.Messages[7].Content; !strings.Contains(final, transcript) { - t.Fatalf("final message = %q, want transcript", final) - } } diff --git a/internal/modules/dnd/extract/itemevents/scriptorium_assets_test.go b/internal/modules/dnd/extract/itemevents/scriptorium_assets_test.go index c8f8867..23106ff 100644 --- a/internal/modules/dnd/extract/itemevents/scriptorium_assets_test.go +++ b/internal/modules/dnd/extract/itemevents/scriptorium_assets_test.go @@ -3,7 +3,6 @@ package itemevents import ( "context" "encoding/json" - "io/fs" "reflect" "strings" "testing" @@ -13,7 +12,7 @@ import ( "gitea.maximumdirect.net/eric/scriptorium" ) -func TestPromptAssetsUseSharedSequenceAndTranscriptLast(t *testing.T) { +func TestPromptAssetsPrepareItemEventPrompt(t *testing.T) { registry := llm.NewAssetRegistry() if err := RegisterPromptAssets(registry); err != nil { t.Fatal(err) @@ -33,9 +32,9 @@ func TestPromptAssetsUseSharedSequenceAndTranscriptLast(t *testing.T) { PromptID: PromptID, PromptVersion: SchemaVersion, ProfileID: "item-events-test-profile", Inputs: map[string]scriptorium.ArtifactRef{ "transcript": scriptorium.InlineWithURI("file:///session.json", `{"segments":[1]}`), - "players": scriptorium.Inline("Dana: Aria"), - "party": scriptorium.Inline("Aria: ranger"), - "glossary": scriptorium.Inline("Moonblade: heirloom"), + "players": scriptorium.Inline(" "), + "party": scriptorium.Inline(" "), + "glossary": scriptorium.Inline(" "), }, }) if err != nil { @@ -44,34 +43,6 @@ func TestPromptAssetsUseSharedSequenceAndTranscriptLast(t *testing.T) { if prepared.PromptID != PromptID || prepared.OutputContract.SchemaPath != "dnd_item_events_llm.v1.json" { t.Fatalf("prepared prompt = %#v", prepared) } - want := []struct { - role string - cached bool - marker string - }{ - {"system", false, "Dungeons & Dragons gameplay transcripts"}, - {"user", false, "Transcript units are the only evidence"}, - {"user", true, "most specific supported in-world"}, - {"user", true, "Dana: Aria"}, - {"user", false, "item and currency events"}, - {"user", true, "Ordinary non-depleting use"}, - {"user", false, `{"segments":[1]}`}, - } - if len(prepared.Messages) != len(want) { - t.Fatalf("prompt messages = %d, want %d", len(prepared.Messages), len(want)) - } - for index, expected := range want { - message := prepared.Messages[index] - if message.Role != expected.role || !strings.Contains(message.Content, expected.marker) { - t.Fatalf("message %d = %#v", index, message) - } - if (message.CacheControl != nil) != expected.cached { - t.Fatalf("message %d cache control = %#v", index, message.CacheControl) - } - } - if strings.Contains(prepared.Messages[len(prepared.Messages)-1].Content, "Moonblade: heirloom") { - t.Fatal("transcript message contains optional reference content") - } } func TestPromptAssetsDoNotLeakIntoMetadata(t *testing.T) { @@ -84,7 +55,7 @@ func TestPromptAssetsDoNotLeakIntoMetadata(t *testing.T) { if err != nil { t.Fatal(err) } - for _, forbidden := range []string{"meaningful Dungeons", "common-dnd-system", "start_segment", "dnd_item_events_llm.v1.json"} { + for _, forbidden := range []string{"common-dnd-system", "dnd_item_events_llm.v1.json"} { if strings.Contains(string(payload), forbidden) { t.Fatalf("metadata leaked raw asset content %q: %s", forbidden, payload) } @@ -102,13 +73,4 @@ func TestPromptManifestReusesOnlySharedAssets(t *testing.T) { if !reflect.DeepEqual(promptAssetManifest.SharedFiles, want) { t.Fatalf("shared assets = %#v, want %#v", promptAssetManifest.SharedFiles, want) } - for _, path := range []string{"assets/prompts/task.md", "assets/prompts/instructions.md"} { - content, err := fs.ReadFile(embeddedAssets, path) - if err != nil { - t.Fatal(err) - } - if strings.Contains(string(content), "Transcript units are the only evidence") || strings.Contains(string(content), "Dungeons & Dragons gameplay transcripts") { - t.Fatalf("module asset %q copied shared prompt text", path) - } - } } diff --git a/internal/modules/dnd/extract/npcinteractions/scriptorium_assets_test.go b/internal/modules/dnd/extract/npcinteractions/scriptorium_assets_test.go index 1d5af4e..e8bbb9e 100644 --- a/internal/modules/dnd/extract/npcinteractions/scriptorium_assets_test.go +++ b/internal/modules/dnd/extract/npcinteractions/scriptorium_assets_test.go @@ -51,50 +51,6 @@ func TestRegisterPromptAssetsAndPrepareInteractionPrompt(t *testing.T) { if prepared.PromptID != PromptID || prepared.OutputContract.SchemaPath != "dnd_npc_interactions_llm.v1.json" { t.Fatalf("prepared prompt = %#v", prepared) } - for index, want := range []struct { - role string - cached bool - marker string - }{ - {role: "system", marker: "Dungeons & Dragons gameplay transcripts"}, - {role: "user", marker: "Transcript units are the only evidence"}, - {role: "user", cached: true, marker: "most specific supported in-world"}, - {role: "user", cached: true}, - {role: "user", cached: true}, - {role: "user", marker: "interaction occurrences"}, - {role: "user", cached: true, marker: "Use exactly one kind per occurrence"}, - {role: "user"}, - } { - if index >= len(prepared.Messages) { - t.Fatalf("prepared prompt has %d messages, want at least %d", len(prepared.Messages), index+1) - } - message := prepared.Messages[index] - if message.Role != want.role { - t.Errorf("message %d role = %q, want %q", index, message.Role, want.role) - } - if want.cached { - if message.CacheControl == nil || message.CacheControl.Type != scriptorium.CacheControlEphemeral { - t.Errorf("message %d cache control = %#v, want ephemeral", index, message.CacheControl) - } - } else if message.CacheControl != nil { - t.Errorf("message %d cache control = %#v, want nil", index, message.CacheControl) - } - if want.marker != "" && !strings.Contains(message.Content, want.marker) { - t.Errorf("message %d content does not contain purpose marker %q", index, want.marker) - } - } - if len(prepared.Messages) != 8 { - t.Fatalf("prepared prompt has %d messages, want 8", len(prepared.Messages)) - } - if references := prepared.Messages[3].Content; !strings.Contains(references, "Dana: Mira") || !strings.Contains(references, "Mira: ranger") || !strings.Contains(references, "Greencloak: title") { - t.Fatalf("campaign references message = %q, want rendered reference inputs", references) - } - if registry := prepared.Messages[4].Content; !strings.Contains(registry, `{"npcs":[{"name":"Mira Thorn"}]}`) { - t.Fatalf("NPC registry message = %q, want names-only registry input", registry) - } - if final := prepared.Messages[7].Content; !strings.Contains(final, transcript) { - t.Fatalf("final message = %q, want transcript", final) - } } func TestPromptMetadataDoesNotExposeAssetContent(t *testing.T) { @@ -103,7 +59,7 @@ func TestPromptMetadataDoesNotExposeAssetContent(t *testing.T) { t.Fatalf("scriptoriumPromptMetadata() = %q, %v", hash, err) } metadata := newExtractor(t, &fakeInteractionsLLMClient{}).ManifestMetadata() - for _, forbidden := range []string{"combat_opponent", "common-dnd-system", "source_refs", "dnd_npc_interactions_llm.v1.json"} { + for _, forbidden := range []string{"common-dnd-system", "dnd_npc_interactions_llm.v1.json"} { if strings.Contains(strings.Join(mapValues(metadata), " "), forbidden) { t.Fatalf("metadata leaked prompt or schema content %q: %#v", forbidden, metadata) } diff --git a/internal/modules/dnd/extract/npcs/scriptorium_assets_test.go b/internal/modules/dnd/extract/npcs/scriptorium_assets_test.go index 8dc28e6..a618b36 100644 --- a/internal/modules/dnd/extract/npcs/scriptorium_assets_test.go +++ b/internal/modules/dnd/extract/npcs/scriptorium_assets_test.go @@ -31,9 +31,9 @@ func TestRegisterPromptAssetsAndPrepareNPCPrompt(t *testing.T) { PromptID: PromptID, PromptVersion: SchemaVersion, ProfileID: "npc-test-profile", Inputs: map[string]scriptorium.ArtifactRef{ "transcript": scriptorium.InlineWithURI("file:///session.json", `{"units":[1]}`), - "players": scriptorium.Inline("Dana: Mira"), - "party": scriptorium.Inline("Mira: ranger"), - "glossary": scriptorium.Inline("Greencloak: title"), + "players": scriptorium.Inline(" "), + "party": scriptorium.Inline(" "), + "glossary": scriptorium.Inline(" "), }, }) if err != nil { @@ -42,55 +42,6 @@ func TestRegisterPromptAssetsAndPrepareNPCPrompt(t *testing.T) { if prepared.PromptID != PromptID || prepared.OutputContract.SchemaPath != "dnd_npcs_llm.v1.json" { t.Fatalf("prepared prompt = %#v, want NPC prompt identity and wiring", prepared) } - for index, want := range []struct { - role string - cached bool - marker string - }{ - {role: "system", marker: "Dungeons & Dragons gameplay transcripts"}, - {role: "user", marker: "Transcript units are the only evidence"}, - {role: "user", cached: true, marker: "most specific supported in-world"}, - {role: "user", cached: true}, - {role: "user", marker: "individually identifiable"}, - {role: "user", cached: true, marker: "observed display name"}, - {role: "user"}, - } { - if index >= len(prepared.Messages) { - t.Fatalf("prepared prompt has %d messages, want at least %d", len(prepared.Messages), index+1) - } - message := prepared.Messages[index] - if message.Role != want.role { - t.Errorf("message %d role = %q, want %q", index, message.Role, want.role) - } - if want.cached { - if message.CacheControl == nil || message.CacheControl.Type != scriptorium.CacheControlEphemeral { - t.Errorf("message %d cache control = %#v, want ephemeral", index, message.CacheControl) - } - } else if message.CacheControl != nil { - t.Errorf("message %d cache control = %#v, want nil", index, message.CacheControl) - } - if want.marker != "" && !strings.Contains(message.Content, want.marker) { - t.Errorf("message %d content does not contain purpose marker %q", index, want.marker) - } - } - if len(prepared.Messages) != 7 { - t.Fatalf("prepared prompt has %d messages, want 7", len(prepared.Messages)) - } - if references := prepared.Messages[3].Content; !strings.Contains(references, "Dana: Mira") || !strings.Contains(references, "Mira: ranger") || !strings.Contains(references, "Greencloak: title") { - t.Fatalf("campaign references message = %q, want rendered reference inputs", references) - } - if transcript := prepared.Messages[6].Content; !strings.Contains(transcript, `{"units":[1]}`) { - t.Fatalf("final message = %q, want transcript", transcript) - } - transcriptMessages := 0 - for _, message := range prepared.Messages { - if strings.Contains(message.Content, `{"units":[1]}`) { - transcriptMessages++ - } - } - if transcriptMessages != 1 { - t.Fatalf("raw transcript rendered in %d messages, want exactly one", transcriptMessages) - } } func TestPromptMetadataAndDiagnosticsDoNotContainRawAssets(t *testing.T) { @@ -103,7 +54,7 @@ func TestPromptMetadataAndDiagnosticsDoNotContainRawAssets(t *testing.T) { if err != nil { t.Fatal(err) } - for _, forbidden := range []string{"Include an in-world", "common-dnd-system", "source-unit", "dnd_npcs_llm.v1.json"} { + for _, forbidden := range []string{"common-dnd-system", "dnd_npcs_llm.v1.json"} { if strings.Contains(string(payload), forbidden) { t.Fatalf("metadata leaked raw prompt/schema content %q: %s", forbidden, payload) } diff --git a/internal/modules/dnd/extract/scenedescriptions/scriptorium_assets_test.go b/internal/modules/dnd/extract/scenedescriptions/scriptorium_assets_test.go index c833bc4..11c53fd 100644 --- a/internal/modules/dnd/extract/scenedescriptions/scriptorium_assets_test.go +++ b/internal/modules/dnd/extract/scenedescriptions/scriptorium_assets_test.go @@ -3,7 +3,6 @@ package scenedescriptions import ( "context" "encoding/json" - "reflect" "strings" "testing" "time" @@ -12,7 +11,7 @@ import ( "gitea.maximumdirect.net/eric/scriptorium" ) -func TestRegisterPromptAssetsPreparesOrderedSceneDescriptionPrompt(t *testing.T) { +func TestRegisterPromptAssetsPreparesSceneDescriptionPrompt(t *testing.T) { registry := llm.NewAssetRegistry() if err := RegisterPromptAssets(registry); err != nil { t.Fatalf("RegisterPromptAssets() error = %v, want nil", err) @@ -32,9 +31,9 @@ func TestRegisterPromptAssetsPreparesOrderedSceneDescriptionPrompt(t *testing.T) PromptID: PromptID, PromptVersion: SchemaVersion, ProfileID: "scene-description-test-profile", Inputs: map[string]scriptorium.ArtifactRef{ "transcript": scriptorium.InlineWithURI("file:///session.json", `{"units":[1]}`), - "players": scriptorium.Inline("Dana: Mira"), - "party": scriptorium.Inline("Mira: ranger"), - "glossary": scriptorium.Inline("Greencloak: title"), + "players": scriptorium.Inline(" "), + "party": scriptorium.Inline(" "), + "glossary": scriptorium.Inline(" "), }, }) if err != nil { @@ -43,42 +42,6 @@ func TestRegisterPromptAssetsPreparesOrderedSceneDescriptionPrompt(t *testing.T) if prepared.PromptID != PromptID || prepared.OutputContract.SchemaPath != "dnd_scene_descriptions_llm.v1.json" { t.Fatalf("prepared prompt = %#v, want scene-description prompt identity and schema wiring", prepared) } - if got, want := messageRoles(prepared.Messages), []string{"system", "user", "user", "user", "user", "user"}; !reflect.DeepEqual(got, want) { - t.Fatalf("message roles = %#v, want %#v", got, want) - } - for _, index := range []int{1, 2, 4} { - if prepared.Messages[index].CacheControl == nil || prepared.Messages[index].CacheControl.Type != scriptorium.CacheControlEphemeral { - t.Fatalf("message %d cache control = %#v, want ephemeral", index, prepared.Messages[index].CacheControl) - } - } - for _, index := range []int{0, 3, 5} { - if prepared.Messages[index].CacheControl != nil { - t.Fatalf("message %d cache control = %#v, want nil", index, prepared.Messages[index].CacheControl) - } - } - if !strings.Contains(prepared.Messages[0].Content, "Dungeons & Dragons") { - t.Fatalf("first message does not use shared D&D system asset: %q", prepared.Messages[0].Content) - } - for index, want := range []string{"Dana: Mira", "Mira: ranger", "Greencloak: title"} { - if !strings.Contains(prepared.Messages[2].Content, want) { - t.Fatalf("reference %d not rendered in shared reference message: %q", index, prepared.Messages[2].Content) - } - } - if strings.Contains(prepared.Messages[1].Content, `{"units":[1]}`) || strings.Contains(prepared.Messages[2].Content, `{"units":[1]}`) { - t.Fatal("transcript rendered before its final message") - } - if !strings.Contains(prepared.Messages[5].Content, `{"units":[1]}`) { - t.Fatalf("final message does not render transcript: %q", prepared.Messages[5].Content) - } - transcriptMessages := 0 - for _, message := range prepared.Messages { - if strings.Contains(message.Content, `{"units":[1]}`) { - transcriptMessages++ - } - } - if transcriptMessages != 1 { - t.Fatalf("raw transcript rendered in %d messages, want exactly one", transcriptMessages) - } } func TestPromptMetadataAndDiagnosticsDoNotContainRawAssets(t *testing.T) { @@ -90,17 +53,9 @@ func TestPromptMetadataAndDiagnosticsDoNotContainRawAssets(t *testing.T) { if err != nil { t.Fatal(err) } - for _, forbidden := range []string{"Choose exactly one kind", "common-dnd-system", "dnd_scene_descriptions_llm.v1.json"} { + for _, forbidden := range []string{"common-dnd-system", "dnd_scene_descriptions_llm.v1.json"} { if strings.Contains(string(payload), forbidden) { t.Fatalf("metadata leaked raw prompt/schema content %q: %s", forbidden, payload) } } } - -func messageRoles(messages []scriptorium.RenderedMessage) []string { - roles := make([]string, len(messages)) - for i, message := range messages { - roles[i] = string(message.Role) - } - return roles -} diff --git a/internal/modules/dnd/extract/spells/scriptorium_assets_test.go b/internal/modules/dnd/extract/spells/scriptorium_assets_test.go index d313e64..c7ed766 100644 --- a/internal/modules/dnd/extract/spells/scriptorium_assets_test.go +++ b/internal/modules/dnd/extract/spells/scriptorium_assets_test.go @@ -11,7 +11,7 @@ import ( "gitea.maximumdirect.net/eric/scriptorium" ) -func TestScriptoriumPromptPreparesTranscriptReferencesAndTaskMessages(t *testing.T) { +func TestScriptoriumPromptPreparesSpellPrompt(t *testing.T) { transcript := []byte(`{"id":"session-1","segments":[{"id":"u1","text":"Mira casts shield."}]}`) prepared := prepareSpellsPrompt(t, transcript, "Dana: Mira", "Mira: wizard", "Shield: abjuration") @@ -21,54 +21,6 @@ func TestScriptoriumPromptPreparesTranscriptReferencesAndTaskMessages(t *testing if prepared.OutputContract.SchemaPath != "dnd_spells_llm.v1.json" { t.Fatalf("schema path = %q, want LLM-only schema", prepared.OutputContract.SchemaPath) } - for index, want := range []struct { - role string - cached bool - marker string - }{ - {role: "system", marker: "Dungeons & Dragons gameplay transcripts"}, - {role: "user", marker: "Transcript units are the only evidence"}, - {role: "user", cached: true, marker: "most specific supported in-world"}, - {role: "user", cached: true}, - {role: "user", cached: true}, - {role: "user"}, - {role: "user", marker: "spell-cast artifacts"}, - {role: "user", cached: true, marker: "source references must collectively support"}, - {role: "user"}, - } { - if index >= len(prepared.Messages) { - t.Fatalf("prepared prompt has %d messages, want at least %d", len(prepared.Messages), index+1) - } - message := prepared.Messages[index] - if message.Role != want.role { - t.Errorf("message %d role = %q, want %q", index, message.Role, want.role) - } - if want.cached { - if message.CacheControl == nil || message.CacheControl.Type != scriptorium.CacheControlEphemeral { - t.Errorf("message %d cache control = %#v, want ephemeral", index, message.CacheControl) - } - } else if message.CacheControl != nil { - t.Errorf("message %d cache control = %#v, want nil", index, message.CacheControl) - } - if want.marker != "" && !strings.Contains(message.Content, want.marker) { - t.Errorf("message %d content does not contain purpose marker %q", index, want.marker) - } - } - if len(prepared.Messages) != 9 { - t.Fatalf("prepared prompt has %d messages, want 9", len(prepared.Messages)) - } - if references := prepared.Messages[3].Content; !strings.Contains(references, "Dana: Mira") || !strings.Contains(references, "Mira: wizard") || !strings.Contains(references, "Shield: abjuration") { - t.Fatalf("campaign references message = %q, want rendered reference inputs", references) - } - if registry := prepared.Messages[4].Content; !strings.Contains(registry, `{"npcs":[]}`) { - t.Fatalf("NPC registry message = %q, want registry input", registry) - } - if catalog := prepared.Messages[5].Content; !strings.Contains(catalog, `{"spell_names":["Cure Wounds"]}`) { - t.Fatalf("spell catalog message = %q, want catalog input", catalog) - } - if final := prepared.Messages[8].Content; !strings.Contains(final, string(transcript)) { - t.Fatalf("final message = %q, want transcript", final) - } } func TestScriptoriumPromptPreparesWithMissingOptionalReferences(t *testing.T) { diff --git a/internal/modules/dnd/normalize/npcs/scriptorium_assets_test.go b/internal/modules/dnd/normalize/npcs/scriptorium_assets_test.go index 70f45bd..634a1ad 100644 --- a/internal/modules/dnd/normalize/npcs/scriptorium_assets_test.go +++ b/internal/modules/dnd/normalize/npcs/scriptorium_assets_test.go @@ -54,12 +54,12 @@ func TestRegisterPromptAssetsPreparesNormalizationPrompt(t *testing.T) { t.Errorf("message %d role = %q, want %q", index, prepared.Messages[index].Role, role) } } - for _, index := range []int{0, 2} { + for _, index := range []int{2, 4} { if cache := prepared.Messages[index].CacheControl; cache == nil || cache.Type != scriptorium.CacheControlEphemeral { t.Errorf("message %d cache control = %#v, want ephemeral", index, cache) } } - for _, index := range []int{1, 3, 4} { + for _, index := range []int{0, 1, 3} { if cache := prepared.Messages[index].CacheControl; cache != nil { t.Errorf("message %d cache control = %#v, want nil", index, cache) } diff --git a/internal/modules/dnd/register/prompt_cache_test.go b/internal/modules/dnd/register/prompt_cache_test.go new file mode 100644 index 0000000..22f5ea7 --- /dev/null +++ b/internal/modules/dnd/register/prompt_cache_test.go @@ -0,0 +1,166 @@ +package register + +import ( + "context" + "reflect" + "strings" + "testing" + "time" + + "gitea.maximumdirect.net/eric/notarius/internal/framework/llm" + combatextract "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/combatturns" + itemeventextract "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/itemevents" + interactionextract "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/npcinteractions" + npcextract "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/npcs" + scenedescriptionextract "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/scenedescriptions" + spellextract "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/spells" + "gitea.maximumdirect.net/eric/scriptorium" +) + +func TestExtractionPromptsShareRenderedPrefix(t *testing.T) { + const ( + transcriptSentinel = "shared-transcript-sentinel" + playersSentinel = "shared-players-sentinel" + partySentinel = "shared-party-sentinel" + glossarySentinel = "shared-glossary-sentinel" + npcSentinel = "npc-registry-sentinel" + catalogSentinel = "spell-catalog-sentinel" + ) + registry := llm.NewAssetRegistry() + if err := registerPromptAssets(registry); err != nil { + t.Fatalf("registerPromptAssets() error = %v", err) + } + engine := newPromptCacheEngine(t, registry) + commonInputs := map[string]scriptorium.ArtifactRef{ + "transcript": scriptorium.InlineWithURI("file:///session.json", `{"sentinel":"`+transcriptSentinel+`"}`), + "players": scriptorium.Inline(playersSentinel), + "party": scriptorium.Inline(partySentinel), + "glossary": scriptorium.Inline(glossarySentinel), + } + cases := []struct { + name string + promptID string + promptVersion string + inputs map[string]scriptorium.ArtifactRef + npcInput bool + spellCatalogInput bool + }{ + {name: "npcs", promptID: npcextract.PromptID, promptVersion: npcextract.SchemaVersion, inputs: commonInputs}, + {name: "item events", promptID: itemeventextract.PromptID, promptVersion: itemeventextract.SchemaVersion, inputs: commonInputs}, + {name: "scene descriptions", promptID: scenedescriptionextract.PromptID, promptVersion: scenedescriptionextract.SchemaVersion, inputs: commonInputs}, + { + name: "combat turns", + promptID: combatextract.PromptID, + promptVersion: combatextract.SchemaVersion, + inputs: withPromptInputs(commonInputs, map[string]scriptorium.ArtifactRef{ + "npcs": scriptorium.Inline(`{"sentinel":"` + npcSentinel + `"}`), + }), + npcInput: true, + }, + { + name: "npc interactions", + promptID: interactionextract.PromptID, + promptVersion: interactionextract.SchemaVersion, + inputs: withPromptInputs(commonInputs, map[string]scriptorium.ArtifactRef{ + "npcs": scriptorium.Inline(`{"sentinel":"` + npcSentinel + `"}`), + }), + npcInput: true, + }, + { + name: "spells", + promptID: spellextract.PromptID, + promptVersion: spellextract.SchemaVersion, + inputs: withPromptInputs(commonInputs, map[string]scriptorium.ArtifactRef{ + "npcs": scriptorium.Inline(`{"sentinel":"` + npcSentinel + `"}`), + "spell_catalog": scriptorium.Inline(`{"sentinel":"` + catalogSentinel + `"}`), + }), + npcInput: true, + spellCatalogInput: true, + }, + } + + var sharedPrefix []scriptorium.RenderedMessage + for _, testCase := range cases { + t.Run(testCase.name, func(t *testing.T) { + prepared, err := engine.Prepare(context.Background(), scriptorium.RunRequest{ + PromptID: testCase.promptID, + PromptVersion: testCase.promptVersion, + ProfileID: "prompt-cache-test-profile", + Inputs: testCase.inputs, + }) + if err != nil { + t.Fatalf("Prepare() error = %v", err) + } + transcriptIndex := renderedInputMessageIndex(t, prepared.Messages, transcriptSentinel) + prefix := prepared.Messages[:transcriptIndex+1] + if len(prefix) != 4 { + t.Fatalf("messages through transcript = %d, want 4", len(prefix)) + } + if len(prepared.Messages) <= len(prefix) { + t.Fatalf("prepared prompt has %d messages, want lane-specific suffix after transcript", len(prepared.Messages)) + } + if sharedPrefix == nil { + sharedPrefix = append([]scriptorium.RenderedMessage(nil), prefix...) + } else if !reflect.DeepEqual(prefix, sharedPrefix) { + t.Fatalf("rendered prefix = %#v, want %#v", prefix, sharedPrefix) + } + if testCase.npcInput { + assertRenderedInputAfter(t, prepared.Messages, npcSentinel, transcriptIndex) + } + if testCase.spellCatalogInput { + assertRenderedInputAfter(t, prepared.Messages, catalogSentinel, transcriptIndex) + } + }) + } +} + +func newPromptCacheEngine(t *testing.T, registry *llm.AssetRegistry) *scriptorium.Engine { + t.Helper() + options, err := registry.ScriptoriumOptions() + if err != nil { + t.Fatalf("ScriptoriumOptions() error = %v", err) + } + options = append(options, scriptorium.WithProfiles(scriptorium.OpenAICompatibleProfile(scriptorium.OpenAICompatibleProfileConfig{ + ID: "prompt-cache-test-profile", Endpoint: "http://127.0.0.1:1/v1", Model: "prompt-cache-test-model", + }))) + engine, err := scriptorium.NewEngine(scriptorium.Config{Timeout: time.Second}, options...) + if err != nil { + t.Fatalf("NewEngine() error = %v", err) + } + return engine +} + +func withPromptInputs(inputs, extras map[string]scriptorium.ArtifactRef) map[string]scriptorium.ArtifactRef { + merged := make(map[string]scriptorium.ArtifactRef, len(inputs)+len(extras)) + for name, input := range inputs { + merged[name] = input + } + for name, input := range extras { + merged[name] = input + } + return merged +} + +func assertRenderedInputAfter(t *testing.T, messages []scriptorium.RenderedMessage, sentinel string, index int) { + t.Helper() + if inputIndex := renderedInputMessageIndex(t, messages, sentinel); inputIndex <= index { + t.Fatalf("input sentinel %q rendered at message %d, want after transcript message %d", sentinel, inputIndex, index) + } +} + +func renderedInputMessageIndex(t *testing.T, messages []scriptorium.RenderedMessage, sentinel string) int { + t.Helper() + index := -1 + occurrences := 0 + for messageIndex, message := range messages { + count := strings.Count(message.Content, sentinel) + if count > 0 { + index = messageIndex + occurrences += count + } + } + if occurrences != 1 { + t.Fatalf("input sentinel %q rendered %d times, want exactly once", sentinel, occurrences) + } + return index +}