Use references in D&D spell extraction
This commit is contained in:
@@ -110,6 +110,96 @@ func TestRunnerProcessesSeriatimInputWithDNDSpellsExtractor(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerPassesRosterAndGlossaryReferencesToDNDSpellsPrompt(t *testing.T) {
|
||||
raw := readDNDSpellsFixture(t)
|
||||
expectedDoc := parseDNDSpellsFixture(t, raw)
|
||||
resolved := resolveDNDSpellsPipeline(t)
|
||||
resolved.ResolvedPipeline.ArtifactLanes[0].ReferenceSet = dndSpellsReferenceSet(
|
||||
"Aria: party cleric\nBorin: fighter",
|
||||
"Fire Bolt: evocation cantrip",
|
||||
)
|
||||
llmClient := &fakeSpellsLLMClient{
|
||||
response: extractionResponse{
|
||||
SpellCasts: []spellCastResponse{
|
||||
{
|
||||
Caster: "Borin",
|
||||
Spell: "Fire Bolt",
|
||||
Effect: "Scorches the wight.",
|
||||
NarrativeDescription: "Borin hurls fire at the wight.",
|
||||
SourceRefs: []source.SourceRef{
|
||||
{SourceID: expectedDoc.ID, StartUnitID: "seg-003", EndUnitID: "seg-003"},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
output, err := pipeline.New(dndSpellsRunnerRegistries(t)).Run(context.Background(), pipeline.RunInput{
|
||||
Pipeline: resolved.ResolvedPipeline,
|
||||
RawInput: raw,
|
||||
LLMClient: llmClient,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v, want nil", err)
|
||||
}
|
||||
|
||||
if len(output.Approved) != 1 {
|
||||
t.Fatalf("len(Approved) = %d, want 1", len(output.Approved))
|
||||
}
|
||||
if len(output.Manifest.References) != 2 {
|
||||
t.Fatalf("manifest references = %#v, want roster and glossary provenance", output.Manifest.References)
|
||||
}
|
||||
if len(llmClient.requests) != 1 {
|
||||
t.Fatalf("LLM calls = %d, want 1", len(llmClient.requests))
|
||||
}
|
||||
user := llmClient.requests[0].Messages[1].Content
|
||||
for _, want := range []string{
|
||||
"Roster reference material:",
|
||||
"Aria: party cleric",
|
||||
"Glossary reference material:",
|
||||
"Fire Bolt: evocation cantrip",
|
||||
} {
|
||||
if !strings.Contains(user, want) {
|
||||
t.Fatalf("user prompt = %q, want substring %q", user, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerDoesNotExtractSpellMentionedOnlyInRoster(t *testing.T) {
|
||||
raw := readDNDSpellsFixture(t)
|
||||
resolved := resolveDNDSpellsPipeline(t)
|
||||
resolved.ResolvedPipeline.ArtifactLanes[0].ReferenceSet = dndSpellsReferenceSet(
|
||||
"Mira: wizard who can cast Lightning Bolt",
|
||||
"",
|
||||
)
|
||||
llmClient := &fakeSpellsLLMClient{
|
||||
response: extractionResponse{SpellCasts: []spellCastResponse{}},
|
||||
}
|
||||
|
||||
output, err := pipeline.New(dndSpellsRunnerRegistries(t)).Run(context.Background(), pipeline.RunInput{
|
||||
Pipeline: resolved.ResolvedPipeline,
|
||||
RawInput: raw,
|
||||
LLMClient: llmClient,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v, want nil", err)
|
||||
}
|
||||
|
||||
if len(output.Approved) != 0 {
|
||||
t.Fatalf("approved artifacts = %#v, want no roster-only spell casts", output.Approved)
|
||||
}
|
||||
if len(llmClient.requests) != 1 {
|
||||
t.Fatalf("LLM calls = %d, want 1", len(llmClient.requests))
|
||||
}
|
||||
user := llmClient.requests[0].Messages[1].Content
|
||||
if !strings.Contains(user, "Lightning Bolt") {
|
||||
t.Fatalf("user prompt = %q, want roster-only spell in reference section", user)
|
||||
}
|
||||
if output.Manifest.ValidationStatus != "approved" {
|
||||
t.Fatalf("ValidationStatus = %q, want approved empty extraction", output.Manifest.ValidationStatus)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerRejectsDNDSpellCastWithInvalidSourceRef(t *testing.T) {
|
||||
raw := readDNDSpellsFixture(t)
|
||||
resolved := resolveDNDSpellsPipeline(t)
|
||||
@@ -155,6 +245,43 @@ func TestRunnerRejectsDNDSpellCastWithInvalidSourceRef(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func dndSpellsReferenceSet(roster string, glossary string) contracts.ReferenceSet {
|
||||
slots := make(map[string]contracts.ResolvedReferenceSlot)
|
||||
if strings.TrimSpace(roster) != "" {
|
||||
slots["roster"] = contracts.ResolvedReferenceSlot{
|
||||
Slot: contracts.ReferenceSlot{Name: "roster"},
|
||||
Items: []contracts.ReferenceItem{
|
||||
{
|
||||
SlotName: "roster",
|
||||
MediaType: "text/plain; charset=utf-8",
|
||||
Content: []byte(roster),
|
||||
Digest: "sha256:roster",
|
||||
Origin: contracts.ReferenceOrigin{Type: "file", URI: "file:///tmp/roster.txt"},
|
||||
SizeBytes: int64(len(roster)),
|
||||
BindingSource: contracts.ReferenceBindingSourceConfig,
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
if strings.TrimSpace(glossary) != "" {
|
||||
slots["glossary"] = contracts.ResolvedReferenceSlot{
|
||||
Slot: contracts.ReferenceSlot{Name: "glossary"},
|
||||
Items: []contracts.ReferenceItem{
|
||||
{
|
||||
SlotName: "glossary",
|
||||
MediaType: "text/plain; charset=utf-8",
|
||||
Content: []byte(glossary),
|
||||
Digest: "sha256:glossary",
|
||||
Origin: contracts.ReferenceOrigin{Type: "file", URI: "file:///tmp/glossary.txt"},
|
||||
SizeBytes: int64(len(glossary)),
|
||||
BindingSource: contracts.ReferenceBindingSourceConfig,
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
return contracts.ReferenceSet{Slots: slots}
|
||||
}
|
||||
|
||||
func TestRunnerFailsWhenDNDSpellsExtractorReturnsMalformedOutput(t *testing.T) {
|
||||
raw := readDNDSpellsFixture(t)
|
||||
resolved := resolveDNDSpellsPipeline(t)
|
||||
|
||||
Reference in New Issue
Block a user