Add NPC registry grounding for spell extraction

This commit is contained in:
2026-07-21 03:12:03 +00:00
parent fb043325e1
commit 20cfbfd311
26 changed files with 901 additions and 58 deletions

View File

@@ -17,6 +17,9 @@ inputs:
- name: glossary
required: false
content_type: text/plain
- name: npcs
required: false
content_type: application/json
messages:
- role: system
content_file: ./sharedassets/common-dnd-system.md
@@ -30,6 +33,8 @@ messages:
type: ephemeral
- role: user
content_file: ./catalog.md
- role: user
content_file: ./npc_registry.md
- role: user
content_file: ./task.md
- role: user

View File

@@ -0,0 +1,11 @@
The optional canonical NPC registry for this extraction is provided below as
durable JSON. Use it only to prefer exact canonical NPC names and recognize
their aliases when the transcript identifies a caster.
Registry entries are grounding material, not evidence that a spell was cast.
Do not extract a spell, caster, effect, or source reference from the registry.
NPC source references describe registry provenance and may belong to another
session; they are never spell evidence. Preserve the existing player and party
policy for identifying PCs from the transcript.
{{ input "npcs" }}

View File

@@ -4,6 +4,7 @@ import (
"bytes"
"context"
"fmt"
"sort"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
@@ -34,12 +35,19 @@ var referenceSlotDescriptions = shared.ReferenceSlotDescriptions{
func referenceSlots() []contracts.ReferenceSlot {
slots := shared.ReferenceSlots(referenceSlotDescriptions)
return append(slots, contracts.ReferenceSlot{
slots = append(slots, contracts.ReferenceSlot{
Name: spellcatalog.SpellCatalogReferenceSlot,
Description: "Optional canonical spell-name catalog used for extraction grounding.",
AcceptedMediaTypes: []string{"application/json"},
MaxBytes: 1048576,
}, contracts.ReferenceSlot{
Name: NPCRegistryReferenceSlot,
Description: "Optional normalized NPC registry used for canonical caster-name grounding.",
AcceptedMediaTypes: []string{"application/json"},
MaxBytes: NPCRegistryMaxBytes,
})
sort.Slice(slots, func(i, j int) bool { return slots[i].Name < slots[j].Name })
return slots
}
var _ contracts.Extractor[dnd.SpellList] = (*Extractor)(nil)
@@ -51,6 +59,7 @@ type Extractor struct {
llm contracts.StructuredLLMClient
effectiveCatalog spellcatalog.EffectiveCatalog
catalogPromptInput contracts.LLMInputMaterial
npcRegistry npcRegistryPromptInput
promptSHA string
responseSchemaSHA string
}
@@ -74,6 +83,10 @@ func New(llmClient contracts.StructuredLLMClient, _ Options, references ...contr
if err != nil {
return nil, extractorErrorf("prepare spell catalog prompt input: %w", err)
}
npcRegistry, err := resolveNPCRegistry(referenceSet)
if err != nil {
return nil, extractorErrorf("prepare NPC registry prompt input: %w", err)
}
promptSHA, err := scriptoriumPromptMetadata()
if err != nil {
return nil, extractorErrorf("load prompt metadata: %w", err)
@@ -86,6 +99,7 @@ func New(llmClient contracts.StructuredLLMClient, _ Options, references ...contr
llm: llmClient,
effectiveCatalog: effectiveCatalog,
catalogPromptInput: catalogPromptInput,
npcRegistry: npcRegistry,
promptSHA: promptSHA,
responseSchemaSHA: responseSchema.SHA256,
}, nil
@@ -113,6 +127,10 @@ func (e *Extractor) ManifestMetadata() map[string]any {
"response_schema_version": SchemaVersion,
"response_schema_sha256": e.responseSchemaSHA,
}
if e.npcRegistry.bound {
metadata["npc_registry_digest"] = e.npcRegistry.digest
metadata["npc_count"] = e.npcRegistry.count
}
return metadata
}
@@ -120,11 +138,15 @@ func (e *Extractor) CheckpointFingerprints() []pipeline.CheckpointFingerprint {
if e == nil {
return nil
}
return []pipeline.CheckpointFingerprint{
fingerprints := []pipeline.CheckpointFingerprint{
{Name: "effective_catalog", Value: e.effectiveCatalog.Digest()},
{Name: "prompt", Value: e.promptSHA},
{Name: "response_schema", Value: e.responseSchemaSHA},
}
if e.npcRegistry.bound {
fingerprints = append(fingerprints, pipeline.CheckpointFingerprint{Name: "npc_registry", Value: e.npcRegistry.digest})
}
return fingerprints
}
func (e *Extractor) Extract(ctx context.Context, req contracts.TypedExtractionRequest) (contracts.TypedExtractionResult[dnd.SpellList], error) {
@@ -157,6 +179,7 @@ func (e *Extractor) Extract(ctx context.Context, req contracts.TypedExtractionRe
var response extractionResponse
inputs := shared.PromptInputs(sourceInput, req.References)
inputs[spellcatalog.SpellCatalogReferenceSlot] = e.catalogPromptInput.Clone()
inputs[NPCRegistryReferenceSlot] = e.npcRegistry.input.Clone()
if _, err := e.llm.CompleteStructured(ctx, contracts.StructuredCompletionRequest{
StageName: Key,
PromptID: PromptID,

View File

@@ -0,0 +1,89 @@
package spells
import (
"crypto/sha256"
"encoding/hex"
"fmt"
"mime"
"strings"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
npccodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/npcs"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/npcs/identity"
)
const (
NPCRegistryReferenceSlot = "npcs"
NPCRegistryMaxBytes = 1048576
)
type npcRegistryPromptInput struct {
input contracts.LLMInputMaterial
digest string
count int
bound bool
}
func resolveNPCRegistry(references contracts.ReferenceSet) (npcRegistryPromptInput, error) {
slot, ok := references.Slots[NPCRegistryReferenceSlot]
if !ok {
return npcRegistryPromptInput{
input: contracts.NewLLMInputMaterial(
NPCRegistryReferenceSlot,
"application/json",
[]byte(`{"npcs":[]}`),
"",
"",
),
}, nil
}
if len(slot.Items) != 1 {
return npcRegistryPromptInput{}, fmt.Errorf("reference slot %q must contain exactly one item", NPCRegistryReferenceSlot)
}
item := slot.Items[0]
mediaType, _, err := mime.ParseMediaType(item.MediaType)
if err != nil {
return npcRegistryPromptInput{}, fmt.Errorf("reference slot %q item media type %q is invalid: %w", NPCRegistryReferenceSlot, item.MediaType, err)
}
if !strings.EqualFold(mediaType, "application/json") {
return npcRegistryPromptInput{}, fmt.Errorf("reference slot %q item media type %q must be application/json", NPCRegistryReferenceSlot, item.MediaType)
}
if len(item.Content) > NPCRegistryMaxBytes {
return npcRegistryPromptInput{}, fmt.Errorf("reference slot %q item is %d bytes, limit %d", NPCRegistryReferenceSlot, len(item.Content), NPCRegistryMaxBytes)
}
codec := npccodec.New()
value, err := codec.Decode(item.Content)
if err != nil {
return npcRegistryPromptInput{}, fmt.Errorf("decode NPC registry: %w", err)
}
if issues := identity.ValidateList(value); len(issues) > 0 {
return npcRegistryPromptInput{}, fmt.Errorf("validate NPC registry identity: %s", formatNPCIdentityIssues(issues))
}
content, err := codec.Encode(value)
if err != nil {
return npcRegistryPromptInput{}, fmt.Errorf("encode canonical NPC registry: %w", err)
}
digest := semanticNPCRegistryDigest(content)
return npcRegistryPromptInput{
input: contracts.NewLLMInputMaterial(NPCRegistryReferenceSlot, "application/json", content, digest, ""),
digest: digest,
count: len(value.NPCs),
bound: true,
}, nil
}
func semanticNPCRegistryDigest(content []byte) string {
sum := sha256.Sum256(content)
return "sha256:" + hex.EncodeToString(sum[:])
}
func formatNPCIdentityIssues(issues []identity.Issue) string {
parts := make([]string, len(issues))
for index, issue := range issues {
parts[index] = fmt.Sprintf("%s at record %d", issue.Code, issue.RecordIndex)
}
return strings.Join(parts, ", ")
}

View File

@@ -0,0 +1,236 @@
package spells
import (
"bytes"
"context"
"encoding/json"
"strings"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
npccodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/npcs"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/npcs/identity"
)
func TestResolveNPCRegistryUsesExactEmptyPromptWhenUnbound(t *testing.T) {
resolved, err := resolveNPCRegistry(contracts.ReferenceSet{})
if err != nil {
t.Fatalf("resolveNPCRegistry() error = %v, want nil", err)
}
if resolved.bound || resolved.digest != "" || resolved.count != 0 {
t.Fatalf("resolved unbound registry = %#v, want no semantic metadata", resolved)
}
if resolved.input.Name != NPCRegistryReferenceSlot || resolved.input.MediaType != "application/json" || resolved.input.Digest != "" || resolved.input.OriginURI != "" {
t.Fatalf("unbound prompt input metadata = %#v, want name/media type only", resolved.input)
}
if got := string(resolved.input.Content); got != `{"npcs":[]}` {
t.Fatalf("unbound prompt input = %q, want exact empty registry", got)
}
if resolved.input.SizeBytes != int64(len(`{"npcs":[]}`)) {
t.Fatalf("unbound prompt input size = %d, want %d", resolved.input.SizeBytes, len(`{"npcs":[]}`))
}
if metadata := newExtractor(t, &fakeSpellsLLMClient{}).ManifestMetadata(); metadata["npc_registry_digest"] != nil || metadata["npc_count"] != nil {
t.Fatalf("unbound extractor metadata = %#v, want no NPC registry fields", metadata)
}
fingerprints := newExtractor(t, &fakeSpellsLLMClient{}).CheckpointFingerprints()
for _, fingerprint := range fingerprints {
if fingerprint.Name == "npc_registry" {
t.Fatalf("unbound checkpoint fingerprints = %#v, want no NPC registry fingerprint", fingerprints)
}
}
}
func TestResolveNPCRegistryCanonicalizesContentAndUsesSemanticDigest(t *testing.T) {
value := validNPCRegistryList()
canonical := encodeNPCRegistry(t, value)
raw := append([]byte(" \n"), canonical...)
raw = append(raw, []byte("\n ")...)
resolved, err := resolveNPCRegistry(npcRegistryReference(raw, "file:///another-session/npcs.json"))
if err != nil {
t.Fatalf("resolveNPCRegistry() error = %v, want nil", err)
}
if !resolved.bound || resolved.count != len(value.NPCs) {
t.Fatalf("resolved registry = %#v, want bound registry with %d NPC", resolved, len(value.NPCs))
}
if !bytes.Equal(resolved.input.Content, canonical) {
t.Fatalf("canonical prompt input = %s, want %s", resolved.input.Content, canonical)
}
if resolved.input.Digest != semanticNPCRegistryDigest(canonical) || resolved.digest != resolved.input.Digest {
t.Fatalf("semantic digest = %q/%q, want %q", resolved.input.Digest, resolved.digest, semanticNPCRegistryDigest(canonical))
}
if resolved.input.OriginURI != "" {
t.Fatalf("prompt input origin = %q, want no provenance path", resolved.input.OriginURI)
}
resolved.input.Content[0] = 'X'
again, err := resolveNPCRegistry(npcRegistryReference(raw, "file:///another-session/npcs.json"))
if err != nil {
t.Fatalf("second resolveNPCRegistry() error = %v, want nil", err)
}
if !bytes.Equal(again.input.Content, canonical) {
t.Fatalf("canonical content changed after caller mutation = %s, want %s", again.input.Content, canonical)
}
}
func TestResolveNPCRegistryRejectsInvalidBoundaryValues(t *testing.T) {
valid := validNPCRegistryList()
second := validNPCRegistryList().NPCs[0]
second.ID = identity.DeriveID("Captain Vale")
second.Name = "Captain Vale"
second.Aliases = []string{"The Greencloak"}
valueWithAliasCollision := dnd.NPCList{NPCs: []dnd.NPC{valid.NPCs[0], second}}
invalidID := valid
invalidID.NPCs[0].ID = "not-an-npc-id"
tests := []struct {
name string
reference contracts.ReferenceSet
wantError string
}{
{name: "zero items", reference: contracts.ReferenceSet{Slots: map[string]contracts.ResolvedReferenceSlot{NPCRegistryReferenceSlot: {Items: []contracts.ReferenceItem{}}}}, wantError: "exactly one"},
{name: "multiple", reference: contracts.ReferenceSet{Slots: map[string]contracts.ResolvedReferenceSlot{NPCRegistryReferenceSlot: {Items: []contracts.ReferenceItem{{Content: []byte(`{"npcs":[]}`)}, {Content: []byte(`{"npcs":[]}`)}}}}}, wantError: "exactly one"},
{name: "wrong media type", reference: npcRegistryReferenceWithMedia([]byte(`{"npcs":[]}`), "text/plain"), wantError: "must be application/json"},
{name: "malformed JSON", reference: npcRegistryReference([]byte(`{"npcs":[`), "file:///private.json"), wantError: "decode NPC registry"},
{name: "unknown field", reference: npcRegistryReference([]byte(`{"npcs":[],"unexpected":true}`), "file:///private.json"), wantError: "unknown field"},
{name: "invalid ID", reference: npcRegistryReference(marshalNPCRegistry(t, invalidID), "file:///private.json"), wantError: "decode NPC registry"},
{name: "alias collision", reference: npcRegistryReference(encodeNPCRegistry(t, valueWithAliasCollision), "file:///private.json"), wantError: string(identity.IssueAliasOwnershipCollision)},
{name: "byte limit", reference: npcRegistryReference(bytes.Repeat([]byte("x"), NPCRegistryMaxBytes+1), "file:///private.json"), wantError: "limit"},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
_, err := resolveNPCRegistry(test.reference)
if err == nil || !strings.Contains(err.Error(), test.wantError) {
t.Fatalf("resolveNPCRegistry() error = %v, want %q", err, test.wantError)
}
if strings.Contains(err.Error(), "Mira Thorn") || strings.Contains(err.Error(), "The Greencloak") || strings.Contains(err.Error(), "private.json") {
t.Fatalf("error leaked registry content or provenance: %v", err)
}
})
}
}
func TestNPCRegistryFingerprintIsSemanticAndDefensive(t *testing.T) {
value := validNPCRegistryList()
canonical := encodeNPCRegistry(t, value)
pretty, err := json.MarshalIndent(value, "", " ")
if err != nil {
t.Fatalf("MarshalIndent() error = %v", err)
}
first := newExtractor(t, &fakeSpellsLLMClient{}, npcRegistryReference(canonical, "file:///one.json"))
second := newExtractor(t, &fakeSpellsLLMClient{}, npcRegistryReference(pretty, "file:///two.json"))
firstFingerprints := checkpointFingerprintMap(first.CheckpointFingerprints())
secondFingerprints := checkpointFingerprintMap(second.CheckpointFingerprints())
if firstFingerprints["npc_registry"] == "" || firstFingerprints["npc_registry"] != secondFingerprints["npc_registry"] {
t.Fatalf("semantic NPC fingerprints = %#v and %#v, want same npc_registry value", firstFingerprints, secondFingerprints)
}
returned := first.CheckpointFingerprints()
returned[0].Name = "caller-mutated"
if first.CheckpointFingerprints()[0].Name == "caller-mutated" {
t.Fatal("CheckpointFingerprints() returned caller-mutable slice state")
}
changed := validNPCRegistryList()
changed.NPCs[0].Description = "A different description."
changedFingerprint := checkpointFingerprintMap(newExtractor(t, &fakeSpellsLLMClient{}, npcRegistryReference(encodeNPCRegistry(t, changed), "file:///three.json")).CheckpointFingerprints())
if changedFingerprint["npc_registry"] == firstFingerprints["npc_registry"] {
t.Fatalf("semantic NPC fingerprint did not change: %#v", changedFingerprint)
}
metadata := first.ManifestMetadata()
encoded, err := json.Marshal(map[string]any{"metadata": metadata, "fingerprints": firstFingerprints})
if err != nil {
t.Fatalf("marshal metadata: %v", err)
}
for _, forbidden := range []string{"Mira Thorn", "The Greencloak", "another-session", "one.json"} {
if strings.Contains(string(encoded), forbidden) {
t.Fatalf("metadata or fingerprints leaked %q: %s", forbidden, encoded)
}
}
if metadata["npc_registry_digest"] != firstFingerprints["npc_registry"] || metadata["npc_count"] != 1 {
t.Fatalf("NPC registry metadata = %#v, want digest and count only", metadata)
}
}
func TestExtractPassesCanonicalNPCRegistryToLLMWithoutProvenance(t *testing.T) {
client := &fakeSpellsLLMClient{response: extractionResponse{SpellCasts: []spellCastResponse{}}}
canonical := encodeNPCRegistry(t, validNPCRegistryList())
extractor := newExtractor(t, client, npcRegistryReference(append([]byte("\n"), canonical...), "file:///npc-session.json"))
if _, err := extractor.Extract(context.Background(), extractionRequest()); err != nil {
t.Fatalf("Extract() error = %v, want nil", err)
}
input := client.requests[0].Inputs[NPCRegistryReferenceSlot]
if input.Name != NPCRegistryReferenceSlot || input.MediaType != "application/json" || input.Digest != semanticNPCRegistryDigest(canonical) || input.OriginURI != "" {
t.Fatalf("NPC prompt input metadata = %#v, want semantic metadata without provenance", input)
}
if !bytes.Equal(input.Content, canonical) {
t.Fatalf("NPC prompt input = %s, want canonical JSON %s", input.Content, canonical)
}
}
func validNPCRegistryList() dnd.NPCList {
return dnd.NPCList{NPCs: []dnd.NPC{{
ID: identity.DeriveID("Mira Thorn"),
Name: "Mira Thorn",
Aliases: []string{"The Greencloak"},
Description: "A guarded ranger who watches the northern road.",
Relationships: []dnd.NPCRelationship{{
Target: "Captain Vale", Relationship: "reports to",
}},
SourceRefs: []source.SourceRef{{SourceID: "npc-session", StartUnitID: 41, EndUnitID: 43}},
}}}
}
func encodeNPCRegistry(t *testing.T, value dnd.NPCList) []byte {
t.Helper()
content, err := npccodec.New().Encode(value)
if err != nil {
t.Fatalf("encode NPC registry: %v", err)
}
return content
}
func marshalNPCRegistry(t *testing.T, value dnd.NPCList) []byte {
t.Helper()
content, err := json.Marshal(value)
if err != nil {
t.Fatalf("marshal NPC registry: %v", err)
}
return content
}
func npcRegistryReference(content []byte, origin string) contracts.ReferenceSet {
references := npcRegistryReferenceWithMedia(content, "application/json; charset=utf-8")
item := references.Slots[NPCRegistryReferenceSlot].Items[0]
item.Origin.URI = origin
slot := references.Slots[NPCRegistryReferenceSlot]
slot.Items[0] = item
references.Slots[NPCRegistryReferenceSlot] = slot
return references
}
func npcRegistryReferenceWithMedia(content []byte, mediaType string) contracts.ReferenceSet {
return contracts.ReferenceSet{Slots: map[string]contracts.ResolvedReferenceSlot{
NPCRegistryReferenceSlot: {
Slot: contracts.ReferenceSlot{Name: NPCRegistryReferenceSlot, AcceptedMediaTypes: []string{"application/json"}, MaxBytes: NPCRegistryMaxBytes},
Items: []contracts.ReferenceItem{{
SlotName: NPCRegistryReferenceSlot,
MediaType: mediaType,
Content: append([]byte(nil), content...),
Origin: contracts.ReferenceOrigin{Type: "file", URI: "file:///npc-registry.json"},
}},
},
}}
}
func checkpointFingerprintMap(values []pipeline.CheckpointFingerprint) map[string]string {
result := make(map[string]string, len(values))
for _, value := range values {
result[value.Name] = value.Value
}
return result
}

View File

@@ -39,6 +39,12 @@ func TestModuleSpec(t *testing.T) {
Description: "Optional campaign glossary reference material used only for disambiguation.",
AcceptedMediaTypes: []string{"application/json", "application/x-yaml", "application/yaml", "text/markdown", "text/plain"},
},
{
Name: NPCRegistryReferenceSlot,
Description: "Optional normalized NPC registry used for canonical caster-name grounding.",
AcceptedMediaTypes: []string{"application/json"},
MaxBytes: NPCRegistryMaxBytes,
},
{
Name: "party",
Description: "Optional party roster reference material used only for disambiguation.",

View File

@@ -15,6 +15,7 @@ func RegisterPromptAssets(registry *llm.AssetRegistry) error {
promptFS, err := shared.ModulePromptFS("dnd.spells", embeddedAssets, []promptfs.ModulePromptFile{
{Name: "dnd.spells.yaml", Path: "assets/prompts/dnd.spells.yaml"},
{Name: "catalog.md", Path: "assets/prompts/catalog.md"},
{Name: "npc_registry.md", Path: "assets/prompts/npc_registry.md"},
{Name: "task.md", Path: "assets/prompts/task.md"},
{Name: "instructions.md", Path: "assets/prompts/instructions.md"},
})
@@ -32,6 +33,7 @@ func scriptoriumPromptMetadata() (string, error) {
parts := append([]llm.AssetHashPart{
{FS: embeddedAssets, Path: "assets/prompts/dnd.spells.yaml"},
{FS: embeddedAssets, Path: "assets/prompts/catalog.md"},
{FS: embeddedAssets, Path: "assets/prompts/npc_registry.md"},
{FS: embeddedAssets, Path: "assets/prompts/task.md"},
{FS: embeddedAssets, Path: "assets/prompts/instructions.md"},
}, append(shared.CommonHashParts(), shared.ReferenceHashParts()...)...)

View File

@@ -21,8 +21,8 @@ func TestScriptoriumPromptPreparesTranscriptReferencesAndTaskMessages(t *testing
if prepared.OutputContract.SchemaPath != "dnd_spells_llm.v1.json" {
t.Fatalf("schema path = %q, want LLM-only schema", prepared.OutputContract.SchemaPath)
}
if got := len(prepared.Messages); got != 6 {
t.Fatalf("message count = %d, want 6", got)
if got := len(prepared.Messages); got != 7 {
t.Fatalf("message count = %d, want 7", got)
}
if !strings.Contains(prepared.Messages[1].Content, string(transcript)) {
t.Fatalf("transcript message did not include source input")
@@ -42,7 +42,10 @@ func TestScriptoriumPromptPreparesTranscriptReferencesAndTaskMessages(t *testing
if !strings.Contains(prepared.Messages[3].Content, `{"spell_names":["Cure Wounds"]}`) {
t.Fatalf("catalog message missing canonical spell-name input: %s", prepared.Messages[3].Content)
}
if strings.Contains(prepared.Messages[4].Content, string(transcript)) {
if !strings.Contains(prepared.Messages[4].Content, `{"npcs":[]}`) {
t.Fatalf("NPC registry message missing empty registry input: %s", prepared.Messages[4].Content)
}
if strings.Contains(prepared.Messages[5].Content, string(transcript)) {
t.Fatalf("task message leaked transcript bytes")
}
}
@@ -129,6 +132,7 @@ func prepareSpellsPrompt(t *testing.T, transcript []byte, players string, party
Inputs: map[string]scriptorium.ArtifactRef{
"transcript": scriptorium.InlineWithURI("file:///session.json", string(transcript)),
"spell_catalog": scriptorium.Inline(`{"spell_names":["Cure Wounds"]}`),
"npcs": scriptorium.Inline(`{"npcs":[]}`),
"players": scriptorium.Inline(players),
"party": scriptorium.Inline(party),
"glossary": scriptorium.Inline(glossary),