From 447c4f73f9becd2daa08ef4e8201e0eef415c59a Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Tue, 21 Jul 2026 14:04:52 +0000 Subject: [PATCH] Protect deterministic D&D prompt inputs --- .../combatturns/scriptorium_assets_test.go | 50 +++++++ .../extract/npcs/scriptorium_assets_test.go | 25 +++- .../extract/spells/scriptorium_assets_test.go | 44 +++--- .../modules/dnd/shared/prompt_inputs_test.go | 131 +++++++++++++----- 4 files changed, 183 insertions(+), 67 deletions(-) diff --git a/internal/modules/dnd/extract/combatturns/scriptorium_assets_test.go b/internal/modules/dnd/extract/combatturns/scriptorium_assets_test.go index 4b94af2..a7e1843 100644 --- a/internal/modules/dnd/extract/combatturns/scriptorium_assets_test.go +++ b/internal/modules/dnd/extract/combatturns/scriptorium_assets_test.go @@ -1,11 +1,14 @@ package combatturns import ( + "context" "io/fs" "strings" "testing" + "time" "gitea.maximumdirect.net/eric/notarius/internal/framework/llm" + "gitea.maximumdirect.net/eric/scriptorium" ) func TestRegisterPromptAssetsAndPrepareCombatPrompt(t *testing.T) { @@ -42,3 +45,50 @@ func TestRegisterPromptAssetsAndPrepareCombatPrompt(t *testing.T) { t.Fatalf("scriptoriumPromptMetadata() = %q, %v; want digest", hash, err) } } + +func TestScriptoriumPromptPreparesRequiredInputs(t *testing.T) { + registry := llm.NewAssetRegistry() + if err := RegisterPromptAssets(registry); err != nil { + t.Fatalf("RegisterPromptAssets() error = %v, want nil", err) + } + options, err := registry.ScriptoriumOptions() + if err != nil { + t.Fatalf("ScriptoriumOptions() error = %v, want nil", err) + } + options = append(options, scriptorium.WithProfiles(scriptorium.OpenAICompatibleProfile(scriptorium.OpenAICompatibleProfileConfig{ + ID: "combat-test-profile", Endpoint: "http://127.0.0.1:1/v1", Model: "combat-test-model", + }))) + engine, err := scriptorium.NewEngine(scriptorium.Config{Timeout: time.Second}, options...) + if err != nil { + t.Fatalf("NewEngine() error = %v, want nil", err) + } + transcript := `{"units":[1]}` + prepared, err := engine.Prepare(context.Background(), scriptorium.RunRequest{ + PromptID: PromptID, PromptVersion: SchemaVersion, ProfileID: "combat-test-profile", + Inputs: map[string]scriptorium.ArtifactRef{ + "transcript": scriptorium.InlineWithURI("file:///session.json", transcript), + "players": scriptorium.Inline("Dana: Mira"), + "party": scriptorium.Inline("Mira: ranger"), + "glossary": scriptorium.Inline("Greencloak: title"), + "npcs": scriptorium.Inline(`{"npcs":[]}`), + }, + }) + if err != nil { + t.Fatalf("Prepare() error = %v, want nil", err) + } + if prepared.PromptID != PromptID || prepared.OutputContract.SchemaPath != "dnd_combat_turns_llm.v1.json" { + t.Fatalf("prepared prompt = %#v, want combat prompt identity and schema", prepared) + } + for _, want := range []string{transcript, "Dana: Mira", "Mira: ranger", "Greencloak: title", `{"npcs":[]}`} { + found := false + for _, message := range prepared.Messages { + if strings.Contains(message.Content, want) { + found = true + break + } + } + if !found { + t.Fatalf("prepared prompt did not render required input %q", want) + } + } +} diff --git a/internal/modules/dnd/extract/npcs/scriptorium_assets_test.go b/internal/modules/dnd/extract/npcs/scriptorium_assets_test.go index 2ecdabb..97a3203 100644 --- a/internal/modules/dnd/extract/npcs/scriptorium_assets_test.go +++ b/internal/modules/dnd/extract/npcs/scriptorium_assets_test.go @@ -39,14 +39,29 @@ func TestRegisterPromptAssetsAndPrepareNPCPrompt(t *testing.T) { if err != nil { t.Fatalf("Prepare() error = %v, want nil", err) } - if prepared.PromptID != PromptID || prepared.OutputContract.SchemaPath != "dnd_npcs_llm.v1.json" || len(prepared.Messages) != 5 { + if prepared.PromptID != PromptID || prepared.OutputContract.SchemaPath != "dnd_npcs_llm.v1.json" { t.Fatalf("prepared prompt = %#v, want NPC prompt identity and wiring", prepared) } - if !strings.Contains(prepared.Messages[1].Content, `{"units":[1]}`) || !strings.Contains(prepared.Messages[2].Content, "Dana: Mira") || !strings.Contains(prepared.Messages[2].Content, "Mira: ranger") { - t.Fatalf("prepared prompt inputs do not include transcript/references") + for _, want := range []string{`{"units":[1]}`, "Dana: Mira", "Mira: ranger", "Greencloak: title"} { + found := false + for _, message := range prepared.Messages { + if strings.Contains(message.Content, want) { + found = true + break + } + } + if !found { + t.Fatalf("prepared prompt did not render required input %q", want) + } } - if strings.Contains(prepared.Messages[3].Content, `{"units":[1]}`) || strings.Contains(prepared.Messages[4].Content, `{"units":[1]}`) { - t.Fatal("task or instruction prompt leaked raw transcript") + transcriptMessages := 0 + for _, message := range prepared.Messages { + if strings.Contains(message.Content, `{"units":[1]}`) { + transcriptMessages++ + } + } + if transcriptMessages != 1 { + t.Fatalf("raw transcript rendered in %d messages, want exactly one", transcriptMessages) } } diff --git a/internal/modules/dnd/extract/spells/scriptorium_assets_test.go b/internal/modules/dnd/extract/spells/scriptorium_assets_test.go index 3c4299d..13da182 100644 --- a/internal/modules/dnd/extract/spells/scriptorium_assets_test.go +++ b/internal/modules/dnd/extract/spells/scriptorium_assets_test.go @@ -21,32 +21,24 @@ func TestScriptoriumPromptPreparesTranscriptReferencesAndTaskMessages(t *testing if prepared.OutputContract.SchemaPath != "dnd_spells_llm.v1.json" { t.Fatalf("schema path = %q, want LLM-only schema", prepared.OutputContract.SchemaPath) } - if got := len(prepared.Messages); got != 7 { - t.Fatalf("message count = %d, want 7", got) - } - if !strings.Contains(prepared.Messages[1].Content, string(transcript)) { - t.Fatalf("transcript message did not include source input") - } - if prepared.Messages[1].CacheControl == nil || prepared.Messages[2].CacheControl == nil { - t.Fatalf("expected transcript and reference messages to be cacheable: %#v", prepared.Messages) - } - if !strings.Contains(prepared.Messages[2].Content, "Dana: Mira") { - t.Fatalf("reference message missing player content") - } - if !strings.Contains(prepared.Messages[2].Content, "Mira: wizard") { - t.Fatalf("reference message missing party content") - } - if !strings.Contains(prepared.Messages[2].Content, "Shield: abjuration") { - t.Fatalf("reference message missing glossary content") - } - if !strings.Contains(prepared.Messages[4].Content, `{"spell_names":["Cure Wounds"]}`) { - t.Fatalf("catalog message missing canonical spell-name input: %s", prepared.Messages[4].Content) - } - if !strings.Contains(prepared.Messages[3].Content, `{"npcs":[]}`) { - t.Fatalf("NPC registry message missing empty registry input: %s", prepared.Messages[3].Content) - } - if strings.Contains(prepared.Messages[5].Content, string(transcript)) { - t.Fatalf("task message leaked transcript bytes") + for _, want := range []string{ + string(transcript), + "Dana: Mira", + "Mira: wizard", + "Shield: abjuration", + `{"spell_names":["Cure Wounds"]}`, + `{"npcs":[]}`, + } { + found := false + for _, message := range prepared.Messages { + if strings.Contains(message.Content, want) { + found = true + break + } + } + if !found { + t.Fatalf("prepared prompt did not render required input %q", want) + } } } diff --git a/internal/modules/dnd/shared/prompt_inputs_test.go b/internal/modules/dnd/shared/prompt_inputs_test.go index 77b62ef..9c328d9 100644 --- a/internal/modules/dnd/shared/prompt_inputs_test.go +++ b/internal/modules/dnd/shared/prompt_inputs_test.go @@ -1,57 +1,107 @@ package shared import ( + "reflect" "strings" "testing" "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" ) -func TestPromptInputsBuildExpectedInputs(t *testing.T) { +func TestPromptInputsIdentity(t *testing.T) { source := contracts.NewLLMInputMaterial("source", "application/json", []byte("source text"), "sha256:source", "file:///source.json") - references := contracts.ReferenceSet{Slots: map[string]contracts.ResolvedReferenceSlot{ + baseReferences := contracts.ReferenceSet{Slots: map[string]contracts.ResolvedReferenceSlot{ "players": slotWithContent("players", "Alice: Aria"), "party": slotWithContent("party", "Aria: cleric"), "glossary": slotWithContent("glossary", "Brightmantle: temple"), }} - - inputs := PromptInputs(source, references) - for _, name := range []string{"transcript", "players", "party", "glossary"} { - if _, ok := inputs[name]; !ok { - t.Fatalf("PromptInputs() missing %q: %#v", name, inputs) - } - } - if _, ok := inputs["roster"]; ok { - t.Fatalf("PromptInputs() included roster input: %#v", inputs) - } - if got := inputs["transcript"].Name; got != "transcript" { - t.Fatalf("transcript name = %q, want transcript", got) - } - if got := string(inputs["transcript"].Content); got != "source text" { - t.Fatalf("transcript content = %q, want source text", got) - } - if got := string(inputs["players"].Content); got != "Alice: Aria" { - t.Fatalf("players content = %q, want player reference", got) - } - if got := string(inputs["party"].Content); got != "Aria: cleric" { - t.Fatalf("party content = %q, want party reference", got) - } - if got := string(inputs["glossary"].Content); got != "Brightmantle: temple" { - t.Fatalf("glossary content = %q, want glossary reference", got) - } -} - -func TestPromptInputsUseRosterWhenPartyIsEmpty(t *testing.T) { - inputs := PromptInputs(contracts.LLMInputMaterial{}, contracts.ReferenceSet{Slots: map[string]contracts.ResolvedReferenceSlot{ + orderedReferences := contracts.ReferenceSet{Slots: map[string]contracts.ResolvedReferenceSlot{ + "players": {Items: []contracts.ReferenceItem{ + referenceItem("players", "file:///b.txt", "sha256:bbb", "second"), + referenceItem("players", "file:///a.txt", "sha256:aaa", "first"), + }}, + }} + reversedReferences := contracts.ReferenceSet{Slots: map[string]contracts.ResolvedReferenceSlot{ + "players": {Items: []contracts.ReferenceItem{ + referenceItem("players", "file:///a.txt", "sha256:aaa", "first"), + referenceItem("players", "file:///b.txt", "sha256:bbb", "second"), + }}, + }} + rosterReferences := contracts.ReferenceSet{Slots: map[string]contracts.ResolvedReferenceSlot{ "party": {}, "roster": slotWithContent("roster", "Legacy roster text"), - }}) + }} + explicitPartyReferences := contracts.ReferenceSet{Slots: map[string]contracts.ResolvedReferenceSlot{ + "party": slotWithContent("party", "Current party text"), + "roster": slotWithContent("roster", "Legacy roster text"), + }} - if got := string(inputs["party"].Content); got != "Legacy roster text" { - t.Fatalf("party content = %q, want roster fallback content", got) + tests := []struct { + name string + references contracts.ReferenceSet + other *contracts.ReferenceSet + want map[string]string + }{ + { + name: "identical materials produce deeply equal inputs", + references: baseReferences, + other: &baseReferences, + want: map[string]string{ + "transcript": "source text", + "players": "Alice: Aria", + "party": "Aria: cleric", + "glossary": "Brightmantle: temple", + }, + }, + { + name: "reference insertion order does not change bytes", + references: orderedReferences, + other: &reversedReferences, + want: map[string]string{"players": "Reference 1\nOrigin-URI: file:///a.txt\nDigest: sha256:aaa\n\nfirst\n\nReference 2\nOrigin-URI: file:///b.txt\nDigest: sha256:bbb\n\nsecond"}, + }, + { + name: "roster becomes canonical party input", + references: rosterReferences, + want: map[string]string{"party": "Legacy roster text"}, + }, + { + name: "explicit party wins over roster", + references: explicitPartyReferences, + want: map[string]string{"party": "Current party text"}, + }, + { + name: "missing optional slots use placeholders", + references: contracts.ReferenceSet{}, + want: map[string]string{ + "players": " ", + "party": " ", + "glossary": " ", + }, + }, } - if _, ok := inputs["roster"]; ok { - t.Fatalf("PromptInputs() included roster input: %#v", inputs) + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + inputs := PromptInputs(source, test.references) + if test.other != nil { + other := PromptInputs(source, *test.other) + if !reflect.DeepEqual(inputs, other) { + t.Fatalf("PromptInputs() differs for equivalent references:\nfirst=%#v\nsecond=%#v", inputs, other) + } + } + for name, want := range test.want { + got, ok := inputs[name] + if !ok { + t.Fatalf("PromptInputs() missing %q: %#v", name, inputs) + } + if got := string(got.Content); got != want { + t.Fatalf("%s content = %q, want %q", name, got, want) + } + } + if _, ok := inputs["roster"]; ok { + t.Fatalf("PromptInputs() included roster input: %#v", inputs) + } + }) } } @@ -149,3 +199,12 @@ func slotWithContent(name string, content string) contracts.ResolvedReferenceSlo }}, } } + +func referenceItem(slotName, uri, digest, content string) contracts.ReferenceItem { + return contracts.ReferenceItem{ + SlotName: slotName, + Content: []byte(content), + Digest: digest, + Origin: contracts.ReferenceOrigin{URI: uri}, + } +}