eric/notarius
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Adjust the prompt FS layout and test strategy

Browse Source
This commit is contained in:
Eric Rakestraw
2026-07-06 11:26:21 -05:00
parent 8cafa64174
commit 47cf7e76ec
20 changed files with 266 additions and 111 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
internal/modules/extract/dnd/spells/assets/prompts/dnd.spells.yaml
View File

@@ -5,7 +5,10 @@ inputs:
- name: transcript
required: true
content_type: application/json
- name: roster
- name: players
required: false
content_type: text/plain
- name: party
required: false
content_type: text/plain
- name: glossary
@@ -13,13 +16,13 @@ inputs:
content_type: text/plain
messages:
- role: system
content_file: ./common-dnd-system.md
content_file: ./sharedassets/common-dnd-system.md
- role: user
content_file: ./common-dnd-transcript.md
content_file: ./sharedassets/common-dnd-transcript.md
cache_control:
type: ephemeral
- role: user
content_file: ./common-dnd-references.md
content_file: ./sharedassets/common-dnd-references.md
cache_control:
type: ephemeral
- role: user

4
internal/modules/extract/dnd/spells/assets/prompts/instructions.md
View File

@@ -4,8 +4,8 @@ Return only D&D spell-cast artifacts. For each spell cast, identify the in-world
caster, spell name, effect, narrative description, and source references using
source_id, start_unit_id, and end_unit_id.
Use roster and glossary reference material only to clarify source text. Do not
return spells, casters, or effects that are mentioned only in reference
Use player, party, and glossary reference material only to clarify source text.
Do not return spells, casters, or effects that are mentioned only in reference
material.
Return exactly one JSON object and no explanatory text.

12
internal/modules/extract/dnd/spells/extractor.go
View File

@@ -39,9 +39,19 @@ var referenceSlots = []contracts.ReferenceSlot{
Description: "Optional campaign glossary reference material used only for disambiguation.",
AcceptedMediaTypes: append([]string(nil), acceptedReferenceMediaTypes...),
},
{
Name: "party",
Description: "Optional party roster reference material used only for disambiguation.",
AcceptedMediaTypes: append([]string(nil), acceptedReferenceMediaTypes...),
},
{
Name: "players",
Description: "Optional player list reference material used only for disambiguation.",
AcceptedMediaTypes: append([]string(nil), acceptedReferenceMediaTypes...),
},
{
Name: "roster",
Description: "Optional campaign roster or player-character reference material used only for disambiguation.",
Description: "Deprecated alias for party roster reference material used only for disambiguation.",
AcceptedMediaTypes: append([]string(nil), acceptedReferenceMediaTypes...),
},
}

42
internal/modules/extract/dnd/spells/extractor_test.go
View File

@@ -109,10 +109,16 @@ func TestExtractPassesReferencesAsPromptInputs(t *testing.T) {
req := extractionRequestWithClient(client)
req.References = contracts.ReferenceSet{
Slots: map[string]contracts.ResolvedReferenceSlot{
"roster": {
Slot: contracts.ReferenceSlot{Name: "roster"},
"players": {
Slot: contracts.ReferenceSlot{Name: "players"},
Items: []contracts.ReferenceItem{
{SlotName: "roster", Content: []byte("Aria Brightmantle: party cleric")},
{SlotName: "players", Content: []byte("Alice: Aria Brightmantle")},
},
},
"party": {
Slot: contracts.ReferenceSlot{Name: "party"},
Items: []contracts.ReferenceItem{
{SlotName: "party", Content: []byte("Aria Brightmantle: party cleric")},
},
},
"glossary": {
@@ -135,8 +141,11 @@ func TestExtractPassesReferencesAsPromptInputs(t *testing.T) {
if request.PromptID != PromptID || request.PromptVersion != SchemaVersion {
t.Fatalf("prompt = %q/%q, want %q/%q", request.PromptID, request.PromptVersion, PromptID, SchemaVersion)
}
if got := string(request.Inputs["roster"].Content); got != "Aria Brightmantle: party cleric" {
t.Fatalf("roster input = %q, want reference content", got)
if got := string(request.Inputs["players"].Content); got != "Alice: Aria Brightmantle" {
t.Fatalf("players input = %q, want reference content", got)
}
if got := string(request.Inputs["party"].Content); got != "Aria Brightmantle: party cleric" {
t.Fatalf("party input = %q, want reference content", got)
}
if got := string(request.Inputs["glossary"].Content); got != "Brightmantle: local temple name" {
t.Fatalf("glossary input = %q, want reference content", got)
@@ -146,6 +155,29 @@ func TestExtractPassesReferencesAsPromptInputs(t *testing.T) {
}
}
func TestPromptInputsMapLegacyRosterReferenceToParty(t *testing.T) {
inputs := promptInputs(contracts.ExtractionRequest{
SourceInput: spellSourceInput(),
References: contracts.ReferenceSet{
Slots: map[string]contracts.ResolvedReferenceSlot{
"roster": {
Slot: contracts.ReferenceSlot{Name: "roster"},
Items: []contracts.ReferenceItem{
{SlotName: "roster", Content: []byte("Legacy roster text")},
},
},
},
},
})
if got := string(inputs["party"].Content); got != "Legacy roster text" {
t.Fatalf("party input = %q, want legacy roster content", got)
}
if _, ok := inputs["roster"]; ok {
t.Fatalf("roster prompt input was present; want only party input")
}
}
func TestExtractReturnsNoCandidatesForEmptyResponse(t *testing.T) {
client := &fakeSpellsLLMClient{response: extractionResponse{SpellCasts: []spellCastResponse{}}}

12
internal/modules/extract/dnd/spells/registry_test.go
View File

@@ -43,9 +43,19 @@ func TestModuleSpec(t *testing.T) {
Description: "Optional campaign glossary reference material used only for disambiguation.",
AcceptedMediaTypes: []string{"application/json", "application/x-yaml", "application/yaml", "text/markdown", "text/plain"},
},
{
Name: "party",
Description: "Optional party roster reference material used only for disambiguation.",
AcceptedMediaTypes: []string{"application/json", "application/x-yaml", "application/yaml", "text/markdown", "text/plain"},
},
{
Name: "players",
Description: "Optional player list reference material used only for disambiguation.",
AcceptedMediaTypes: []string{"application/json", "application/x-yaml", "application/yaml", "text/markdown", "text/plain"},
},
{
Name: "roster",
Description: "Optional campaign roster or player-character reference material used only for disambiguation.",
Description: "Deprecated alias for party roster reference material used only for disambiguation.",
AcceptedMediaTypes: []string{"application/json", "application/x-yaml", "application/yaml", "text/markdown", "text/plain"},
},
},

34
internal/modules/extract/dnd/spells/runner_test.go
View File

@@ -110,7 +110,7 @@ func TestRunnerProcessesSeriatimInputWithDNDSpellsExtractor(t *testing.T) {
}
}
func TestRunnerPassesRosterAndGlossaryReferencesToDNDSpellsPrompt(t *testing.T) {
func TestRunnerPassesPartyAndGlossaryReferencesToDNDSpellsPrompt(t *testing.T) {
raw := readDNDSpellsFixture(t)
expectedDoc := parseDNDSpellsFixture(t, raw)
resolved := resolveDNDSpellsPipeline(t)
@@ -147,7 +147,7 @@ func TestRunnerPassesRosterAndGlossaryReferencesToDNDSpellsPrompt(t *testing.T)
t.Fatalf("len(Approved) = %d, want 1", len(output.Approved))
}
if len(output.Manifest.References) != 2 {
t.Fatalf("manifest references = %#v, want roster and glossary provenance", output.Manifest.References)
t.Fatalf("manifest references = %#v, want party and glossary provenance", output.Manifest.References)
}
if len(llmClient.requests) != 1 {
t.Fatalf("LLM calls = %d, want 1", len(llmClient.requests))
@@ -156,15 +156,15 @@ func TestRunnerPassesRosterAndGlossaryReferencesToDNDSpellsPrompt(t *testing.T)
if request.PromptID != PromptID || request.PromptVersion != SchemaVersion {
t.Fatalf("prompt = %q/%q, want %q/%q", request.PromptID, request.PromptVersion, PromptID, SchemaVersion)
}
if got := string(request.Inputs["roster"].Content); got != "Aria: party cleric\nBorin: fighter" {
t.Fatalf("roster input = %q, want reference text", got)
if got := string(request.Inputs["party"].Content); got != "Aria: party cleric\nBorin: fighter" {
t.Fatalf("party input = %q, want reference text", got)
}
if got := string(request.Inputs["glossary"].Content); got != "Fire Bolt: evocation cantrip" {
t.Fatalf("glossary input = %q, want reference text", got)
}
}
func TestRunnerDoesNotExtractSpellMentionedOnlyInRoster(t *testing.T) {
func TestRunnerDoesNotExtractSpellMentionedOnlyInPartyReference(t *testing.T) {
raw := readDNDSpellsFixture(t)
resolved := resolveDNDSpellsPipeline(t)
resolved.ResolvedPipeline.ArtifactLanes[0].ExtractReferences.ReferenceSet = dndSpellsReferenceSet(
@@ -185,7 +185,7 @@ func TestRunnerDoesNotExtractSpellMentionedOnlyInRoster(t *testing.T) {
}
if len(output.Approved) != 0 {
t.Fatalf("approved artifacts = %#v, want no roster-only spell casts", output.Approved)
t.Fatalf("approved artifacts = %#v, want no party-reference-only spell casts", output.Approved)
}
if len(llmClient.requests) != 1 {
t.Fatalf("LLM calls = %d, want 1", len(llmClient.requests))
@@ -194,8 +194,8 @@ func TestRunnerDoesNotExtractSpellMentionedOnlyInRoster(t *testing.T) {
if request.PromptID != PromptID || request.PromptVersion != SchemaVersion {
t.Fatalf("prompt = %q/%q, want %q/%q", request.PromptID, request.PromptVersion, PromptID, SchemaVersion)
}
if got := string(request.Inputs["roster"].Content); !strings.Contains(got, "Lightning Bolt") {
t.Fatalf("roster input = %q, want roster-only spell in reference input", got)
if got := string(request.Inputs["party"].Content); !strings.Contains(got, "Lightning Bolt") {
t.Fatalf("party input = %q, want party-reference-only spell in reference input", got)
}
if output.Manifest.ValidationStatus != "approved" {
t.Fatalf("ValidationStatus = %q, want approved empty extraction", output.Manifest.ValidationStatus)
@@ -247,19 +247,19 @@ func TestRunnerRejectsDNDSpellCastWithInvalidSourceRef(t *testing.T) {
}
}
func dndSpellsReferenceSet(roster string, glossary string) contracts.ReferenceSet {
func dndSpellsReferenceSet(party string, glossary string) contracts.ReferenceSet {
slots := make(map[string]contracts.ResolvedReferenceSlot)
if strings.TrimSpace(roster) != "" {
slots["roster"] = contracts.ResolvedReferenceSlot{
Slot: contracts.ReferenceSlot{Name: "roster"},
if strings.TrimSpace(party) != "" {
slots["party"] = contracts.ResolvedReferenceSlot{
Slot: contracts.ReferenceSlot{Name: "party"},
Items: []contracts.ReferenceItem{
{
SlotName: "roster",
SlotName: "party",
MediaType: "text/plain; charset=utf-8",
Content: []byte(roster),
Digest: "sha256:roster",
Origin: contracts.ReferenceOrigin{Type: "file", URI: "file:///tmp/roster.txt"},
SizeBytes: int64(len(roster)),
Content: []byte(party),
Digest: "sha256:party",
Origin: contracts.ReferenceOrigin{Type: "file", URI: "file:///tmp/party.txt"},
SizeBytes: int64(len(party)),
BindingSource: contracts.ReferenceBindingSourceConfig,
},
},

7
internal/modules/extract/dnd/spells/scriptorium_assets.go
View File

@@ -29,9 +29,14 @@ func RegisterPromptAssets(registry *llm.AssetRegistry) error {
}
func promptInputs(req contracts.ExtractionRequest) contracts.LLMInputSet {
partySlot := req.References.Slots["party"]
if len(partySlot.Items) == 0 {
partySlot = req.References.Slots["roster"]
}
return contracts.LLMInputSet{
"transcript": transcriptPromptInput(req.SourceInput),
"roster": referencePromptMaterial("roster", req.References.Slots["roster"]),
"players": referencePromptMaterial("players", req.References.Slots["players"]),
"party": referencePromptMaterial("party", partySlot),
"glossary": referencePromptMaterial("glossary", req.References.Slots["glossary"]),
}
}

39
internal/modules/extract/dnd/spells/scriptorium_assets_test.go
View File

@@ -15,7 +15,7 @@ import (
func TestScriptoriumPromptPreparesTranscriptReferencesAndTaskMessages(t *testing.T) {
transcript := []byte(`{"id":"session-1","segments":[{"id":"u1","text":"Mira casts shield."}]}`)
prepared := prepareSpellsPrompt(t, transcript, "Mira: wizard", "Shield: abjuration")
prepared := prepareSpellsPrompt(t, transcript, "Dana: Mira", "Mira: wizard", "Shield: abjuration")
if prepared.PromptID != PromptID {
t.Fatalf("prompt id = %q, want %q", prepared.PromptID, PromptID)
@@ -23,21 +23,20 @@ func TestScriptoriumPromptPreparesTranscriptReferencesAndTaskMessages(t *testing
if got := len(prepared.Messages); got != 5 {
t.Fatalf("message count = %d, want 5", got)
}
wantTranscript := "A transcript of a Dungeons & Dragons gameplay session is provided below.\n\n" + string(transcript) + "\n"
if prepared.Messages[1].Content != wantTranscript {
t.Fatalf("transcript message = %q, want byte-identical shared transcript body", prepared.Messages[1].Content)
if !strings.Contains(prepared.Messages[1].Content, string(transcript)) {
t.Fatalf("transcript message did not include source input")
}
if prepared.Messages[1].CacheControl == nil || prepared.Messages[2].CacheControl == nil {
t.Fatalf("expected transcript and reference messages to be cacheable: %#v", prepared.Messages)
}
if !strings.Contains(prepared.Messages[2].Content, "Roster reference:\nMira: wizard") {
t.Fatalf("reference message missing roster content: %q", prepared.Messages[2].Content)
if !strings.Contains(prepared.Messages[2].Content, "Dana: Mira") {
t.Fatalf("reference message missing player content")
}
if !strings.Contains(prepared.Messages[2].Content, "Glossary reference:\nShield: abjuration") {
t.Fatalf("reference message missing glossary content: %q", prepared.Messages[2].Content)
if !strings.Contains(prepared.Messages[2].Content, "Mira: wizard") {
t.Fatalf("reference message missing party content")
}
if !strings.Contains(prepared.Messages[3].Content, "Extract Dungeons & Dragons spell-cast artifacts") {
t.Fatalf("task message missing spell task text: %q", prepared.Messages[3].Content)
if !strings.Contains(prepared.Messages[2].Content, "Shield: abjuration") {
t.Fatalf("reference message missing glossary content")
}
if strings.Contains(prepared.Messages[3].Content, string(transcript)) {
t.Fatalf("task message leaked transcript bytes")
@@ -46,13 +45,10 @@ func TestScriptoriumPromptPreparesTranscriptReferencesAndTaskMessages(t *testing
func TestScriptoriumPromptPreparesWithMissingOptionalReferences(t *testing.T) {
transcript := []byte(`{"id":"session-1","segments":[]}`)
prepared := prepareSpellsPrompt(t, transcript, " ", " ")
prepared := prepareSpellsPrompt(t, transcript, " ", " ", " ")
if !strings.Contains(prepared.Messages[2].Content, "Roster reference:\n ") {
t.Fatalf("reference message did not include empty roster input: %q", prepared.Messages[2].Content)
}
if !strings.Contains(prepared.Messages[2].Content, "Glossary reference:\n ") {
t.Fatalf("reference message did not include empty glossary input: %q", prepared.Messages[2].Content)
if !strings.Contains(prepared.Messages[2].Content, " ") {
t.Fatalf("reference message did not include empty optional reference placeholders")
}
}
@@ -101,8 +97,8 @@ func TestSingleReferencePromptInputKeepsContentOnly(t *testing.T) {
func TestScriptoriumPromptDiagnosticsOmitRawMaterials(t *testing.T) {
transcript := []byte(`{"secret":"source text"}`)
reference := "private roster note"
prepared := prepareSpellsPrompt(t, transcript, reference, " ")
reference := "private party note"
prepared := prepareSpellsPrompt(t, transcript, "private player note", reference, " ")
metadata := New().ManifestMetadata()
payload, err := json.Marshal(map[string]any{
@@ -124,8 +120,8 @@ func TestScriptoriumPromptDiagnosticsOmitRawMaterials(t *testing.T) {
diagnostics := string(payload)
for _, forbidden := range []string{
"source text",
"private player note",
reference,
"Extract Dungeons & Dragons spell-cast artifacts",
`"properties"`,
"spell_casts",
} {
@@ -141,7 +137,7 @@ func TestScriptoriumPromptDiagnosticsOmitRawMaterials(t *testing.T) {
}
}
func prepareSpellsPrompt(t *testing.T, transcript []byte, roster string, glossary string) *scriptorium.PreparedRun {
func prepareSpellsPrompt(t *testing.T, transcript []byte, players string, party string, glossary string) *scriptorium.PreparedRun {
t.Helper()
registry := llm.NewAssetRegistry()
if err := sharedassets.Register(registry); err != nil {
@@ -169,7 +165,8 @@ func prepareSpellsPrompt(t *testing.T, transcript []byte, roster string, glossar
ProfileID: "spell-test-profile",
Inputs: map[string]scriptorium.ArtifactRef{
"transcript": scriptorium.InlineWithURI("file:///session.json", string(transcript)),
"roster": scriptorium.Inline(roster),
"players": scriptorium.Inline(players),
"party": scriptorium.Inline(party),
"glossary": scriptorium.Inline(glossary),
},
})