eric/notarius
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Adjust the prompt FS layout and test strategy

Browse Source
This commit is contained in:
Eric Rakestraw
2026-07-06 11:26:21 -05:00
parent 8cafa64174
commit 47cf7e76ec
20 changed files with 266 additions and 111 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
internal/modules/chunk/dnd/scenes/assets/prompts/dnd.scenes.yaml
View File

@@ -5,7 +5,10 @@ inputs:
- name: transcript
required: true
content_type: application/json
- name: roster
- name: players
required: false
content_type: text/plain
- name: party
required: false
content_type: text/plain
- name: glossary
@@ -13,13 +16,13 @@ inputs:
content_type: text/plain
messages:
- role: system
content_file: ./common-dnd-system.md
content_file: ./sharedassets/common-dnd-system.md
- role: user
content_file: ./common-dnd-transcript.md
content_file: ./sharedassets/common-dnd-transcript.md
cache_control:
type: ephemeral
- role: user
content_file: ./common-dnd-references.md
content_file: ./sharedassets/common-dnd-references.md
cache_control:
type: ephemeral
- role: user

12
internal/modules/chunk/dnd/scenes/chunker.go
View File

@@ -35,9 +35,19 @@ var referenceSlots = []contracts.ReferenceSlot{
Description: "Optional campaign glossary reference material used only for scene disambiguation.",
AcceptedMediaTypes: append([]string(nil), acceptedReferenceMediaTypes...),
},
{
Name: "party",
Description: "Optional party roster reference material used only for scene disambiguation.",
AcceptedMediaTypes: append([]string(nil), acceptedReferenceMediaTypes...),
},
{
Name: "players",
Description: "Optional player list reference material used only for scene disambiguation.",
AcceptedMediaTypes: append([]string(nil), acceptedReferenceMediaTypes...),
},
{
Name: "roster",
Description: "Optional campaign roster or player-character reference material used only for scene disambiguation.",
Description: "Deprecated alias for party roster reference material used only for scene disambiguation.",
AcceptedMediaTypes: append([]string(nil), acceptedReferenceMediaTypes...),
},
}

61
internal/modules/chunk/dnd/scenes/chunker_test.go
View File

@@ -81,9 +81,19 @@ func wantReferenceSlots() []contracts.ReferenceSlot {
Description: "Optional campaign glossary reference material used only for scene disambiguation.",
AcceptedMediaTypes: append([]string(nil), accepted...),
},
{
Name: "party",
Description: "Optional party roster reference material used only for scene disambiguation.",
AcceptedMediaTypes: append([]string(nil), accepted...),
},
{
Name: "players",
Description: "Optional player list reference material used only for scene disambiguation.",
AcceptedMediaTypes: append([]string(nil), accepted...),
},
{
Name: "roster",
Description: "Optional campaign roster or player-character reference material used only for scene disambiguation.",
Description: "Deprecated alias for party roster reference material used only for scene disambiguation.",
AcceptedMediaTypes: append([]string(nil), accepted...),
},
}
@@ -146,8 +156,11 @@ func TestChunkReturnsSceneChunksFromStructuredOutput(t *testing.T) {
if got := string(transcript.Content); got != sceneTranscriptJSON {
t.Fatalf("transcript content = %q, want original source input", got)
}
if got := string(req.Inputs["roster"].Content); got != " " {
t.Fatalf("roster input = %q, want empty reference placeholder", got)
if got := string(req.Inputs["players"].Content); got != " " {
t.Fatalf("players input = %q, want empty reference placeholder", got)
}
if got := string(req.Inputs["party"].Content); got != " " {
t.Fatalf("party input = %q, want empty reference placeholder", got)
}
if got := string(req.Inputs["glossary"].Content); got != " " {
t.Fatalf("glossary input = %q, want empty reference placeholder", got)
@@ -204,10 +217,16 @@ func TestChunkPassesReferencesAsPromptInputs(t *testing.T) {
req := chunkRequestWithClient(client)
req.References = contracts.ReferenceSet{
Slots: map[string]contracts.ResolvedReferenceSlot{
"roster": {
Slot: contracts.ReferenceSlot{Name: "roster"},
"players": {
Slot: contracts.ReferenceSlot{Name: "players"},
Items: []contracts.ReferenceItem{
{SlotName: "roster", Content: []byte("Aria: cleric")},
{SlotName: "players", Content: []byte("Alice: Aria")},
},
},
"party": {
Slot: contracts.ReferenceSlot{Name: "party"},
Items: []contracts.ReferenceItem{
{SlotName: "party", Content: []byte("Aria: cleric")},
},
},
"glossary": {
@@ -223,8 +242,11 @@ func TestChunkPassesReferencesAsPromptInputs(t *testing.T) {
t.Fatalf("Chunk() error = %v, want nil", err)
}
request := client.requests[0]
if got := string(request.Inputs["roster"].Content); got != "Aria: cleric" {
t.Fatalf("roster input = %q, want reference content", got)
if got := string(request.Inputs["players"].Content); got != "Alice: Aria" {
t.Fatalf("players input = %q, want reference content", got)
}
if got := string(request.Inputs["party"].Content); got != "Aria: cleric" {
t.Fatalf("party input = %q, want reference content", got)
}
if got := string(request.Inputs["glossary"].Content); got != "Brightmantle: local temple" {
t.Fatalf("glossary input = %q, want reference content", got)
@@ -234,6 +256,29 @@ func TestChunkPassesReferencesAsPromptInputs(t *testing.T) {
}
}
func TestPromptInputsMapLegacyRosterReferenceToParty(t *testing.T) {
inputs := promptInputs(contracts.ChunkRequest{
SourceInput: sceneSourceInput(),
References: contracts.ReferenceSet{
Slots: map[string]contracts.ResolvedReferenceSlot{
"roster": {
Slot: contracts.ReferenceSlot{Name: "roster"},
Items: []contracts.ReferenceItem{
{SlotName: "roster", Content: []byte("Legacy roster text")},
},
},
},
},
})
if got := string(inputs["party"].Content); got != "Legacy roster text" {
t.Fatalf("party input = %q, want legacy roster content", got)
}
if _, ok := inputs["roster"]; ok {
t.Fatalf("roster prompt input was present; want only party input")
}
}
func TestChunkRejectsWhitespaceOnlyBoundaryCaveats(t *testing.T) {
client := &fakeScenesLLMClient{
response: chunkResponse{

7
internal/modules/chunk/dnd/scenes/scriptorium_assets.go
View File

@@ -29,9 +29,14 @@ func RegisterPromptAssets(registry *llm.AssetRegistry) error {
}
func promptInputs(req contracts.ChunkRequest) contracts.LLMInputSet {
partySlot := req.References.Slots["party"]
if len(partySlot.Items) == 0 {
partySlot = req.References.Slots["roster"]
}
return contracts.LLMInputSet{
"transcript": transcriptPromptInput(req.SourceInput),
"roster": referencePromptMaterial("roster", req.References.Slots["roster"]),
"players": referencePromptMaterial("players", req.References.Slots["players"]),
"party": referencePromptMaterial("party", partySlot),
"glossary": referencePromptMaterial("glossary", req.References.Slots["glossary"]),
}
}

30
internal/modules/chunk/dnd/scenes/scriptorium_assets_test.go
View File

@@ -14,7 +14,7 @@ import (
func TestScriptoriumPromptPreparesTranscriptAndTaskMessages(t *testing.T) {
transcript := []byte(`{"id":"session-1","segments":[{"id":"u1","text":"We enter the crypt."}]}`)
prepared := prepareScenesPrompt(t, transcript, "Aria: cleric", "Brightmantle: temple")
prepared := prepareScenesPrompt(t, transcript, "Alice: Aria", "Aria: cleric", "Brightmantle: temple")
if prepared.PromptID != PromptID {
t.Fatalf("prompt id = %q, want %q", prepared.PromptID, PromptID)
@@ -25,21 +25,20 @@ func TestScriptoriumPromptPreparesTranscriptAndTaskMessages(t *testing.T) {
if prepared.Messages[1].Role != "user" || prepared.Messages[1].CacheControl == nil {
t.Fatalf("transcript message did not render as cacheable user message: %#v", prepared.Messages[1])
}
wantTranscript := "A transcript of a Dungeons & Dragons gameplay session is provided below.\n\n" + string(transcript) + "\n"
if prepared.Messages[1].Content != wantTranscript {
t.Fatalf("transcript message = %q, want byte-identical shared transcript body", prepared.Messages[1].Content)
if !strings.Contains(prepared.Messages[1].Content, string(transcript)) {
t.Fatalf("transcript message did not include source input")
}
if prepared.Messages[2].CacheControl == nil {
t.Fatalf("reference message did not render as cacheable user message: %#v", prepared.Messages[2])
}
if !strings.Contains(prepared.Messages[2].Content, "Roster reference:\nAria: cleric") {
t.Fatalf("reference message missing roster content: %q", prepared.Messages[2].Content)
if !strings.Contains(prepared.Messages[2].Content, "Alice: Aria") {
t.Fatalf("reference message missing player content")
}
if !strings.Contains(prepared.Messages[2].Content, "Glossary reference:\nBrightmantle: temple") {
t.Fatalf("reference message missing glossary content: %q", prepared.Messages[2].Content)
if !strings.Contains(prepared.Messages[2].Content, "Aria: cleric") {
t.Fatalf("reference message missing party content")
}
if !strings.Contains(prepared.Messages[3].Content, "Divide the provided transcript") {
t.Fatalf("task message missing scene task text: %q", prepared.Messages[3].Content)
if !strings.Contains(prepared.Messages[2].Content, "Brightmantle: temple") {
t.Fatalf("reference message missing glossary content")
}
if strings.Contains(prepared.Messages[3].Content, string(transcript)) {
t.Fatalf("task message leaked transcript bytes")
@@ -48,7 +47,7 @@ func TestScriptoriumPromptPreparesTranscriptAndTaskMessages(t *testing.T) {
func TestScriptoriumPromptDiagnosticsOmitRawMaterials(t *testing.T) {
transcript := []byte(`{"secret":"source text"}`)
prepared := prepareScenesPrompt(t, transcript, "private roster note", "private glossary note")
prepared := prepareScenesPrompt(t, transcript, "private player note", "private party note", "private glossary note")
metadata := New().ManifestMetadata()
payload, err := json.Marshal(map[string]any{
@@ -70,9 +69,9 @@ func TestScriptoriumPromptDiagnosticsOmitRawMaterials(t *testing.T) {
diagnostics := string(payload)
for _, forbidden := range []string{
"source text",
"private roster note",
"private player note",
"private party note",
"private glossary note",
"Divide the provided transcript",
`"properties"`,
"start_unit_id",
} {
@@ -88,7 +87,7 @@ func TestScriptoriumPromptDiagnosticsOmitRawMaterials(t *testing.T) {
}
}
func prepareScenesPrompt(t *testing.T, transcript []byte, roster string, glossary string) *scriptorium.PreparedRun {
func prepareScenesPrompt(t *testing.T, transcript []byte, players string, party string, glossary string) *scriptorium.PreparedRun {
t.Helper()
registry := llm.NewAssetRegistry()
if err := sharedassets.Register(registry); err != nil {
@@ -104,7 +103,8 @@ func prepareScenesPrompt(t *testing.T, transcript []byte, roster string, glossar
ProfileID: "scene-test-profile",
Inputs: map[string]scriptorium.ArtifactRef{
"transcript": scriptorium.InlineWithURI("file:///session.json", string(transcript)),
"roster": scriptorium.Inline(roster),
"players": scriptorium.Inline(players),
"party": scriptorium.Inline(party),
"glossary": scriptorium.Inline(glossary),
},
})