Add grounded NPC interaction extractor
This commit is contained in:
6
internal/modules/dnd/extract/npcinteractions/assets.go
Normal file
6
internal/modules/dnd/extract/npcinteractions/assets.go
Normal file
@@ -0,0 +1,6 @@
|
||||
package npcinteractions
|
||||
|
||||
import "embed"
|
||||
|
||||
//go:embed assets/schemas/dnd_npc_interactions_llm.v1.json assets/prompts/*.yaml assets/prompts/*.md
|
||||
var embeddedAssets embed.FS
|
||||
@@ -0,0 +1,49 @@
|
||||
id: dnd.npc_interactions
|
||||
version: "v1"
|
||||
default_profile: gemini-2-flash
|
||||
inputs:
|
||||
- name: transcript
|
||||
required: true
|
||||
content_type: application/json
|
||||
- name: players
|
||||
required: false
|
||||
content_type: text/plain
|
||||
- name: party
|
||||
required: false
|
||||
content_type: text/plain
|
||||
- name: glossary
|
||||
required: false
|
||||
content_type: text/plain
|
||||
- name: npcs
|
||||
required: true
|
||||
content_type: application/json
|
||||
messages:
|
||||
- role: system
|
||||
content_file: ./sharedassets/common-dnd-system.md
|
||||
- role: user
|
||||
content_file: ./sharedassets/common-dnd-extraction-evidence.md
|
||||
- role: user
|
||||
content_file: ./sharedassets/common-dnd-identity.md
|
||||
cache_control:
|
||||
type: ephemeral
|
||||
- role: user
|
||||
content_file: ./sharedassets/common-dnd-references.md
|
||||
cache_control:
|
||||
type: ephemeral
|
||||
- role: user
|
||||
content_file: ./sharedassets/common-dnd-npcs.md
|
||||
cache_control:
|
||||
type: ephemeral
|
||||
- role: user
|
||||
content_file: ./task.md
|
||||
- role: user
|
||||
content_file: ./instructions.md
|
||||
cache_control:
|
||||
type: ephemeral
|
||||
- role: user
|
||||
content_file: ./sharedassets/common-dnd-transcript.md
|
||||
output:
|
||||
format: json
|
||||
validation_mode: json_schema
|
||||
schema_path: dnd_npc_interactions_llm.v1.json
|
||||
repair_attempts: 0
|
||||
@@ -0,0 +1,18 @@
|
||||
Return the interactions array even when no interaction is established. Every
|
||||
record must contain name, kind, and source_refs. Cite transcript ranges that
|
||||
support both the NPC identity and the interaction kind.
|
||||
|
||||
Use exactly one kind per occurrence:
|
||||
|
||||
- mentioned: the NPC is referred to but is not established as present or communicating;
|
||||
- noncombat_presence: the NPC is present and relevant but does not meaningfully participate in dialogue or combat;
|
||||
- dialogue: the NPC speaks, responds, or is directly engaged in a meaningful non-combat exchange;
|
||||
- combat_ally: the NPC actively participates in combat on the party's side;
|
||||
- combat_opponent: the NPC actively participates in combat against the party; or
|
||||
- other: the transcript clearly establishes a direct NPC occurrence that fits none of the preceding kinds.
|
||||
|
||||
When activities overlap, active combat participation outranks dialogue,
|
||||
presence, and mention; dialogue outranks noncombat presence and mention; and
|
||||
noncombat presence outranks mention. Other is only for directly evidenced
|
||||
activity outside those categories. Split an occurrence rather than assigning
|
||||
both combat alignments.
|
||||
@@ -0,0 +1,16 @@
|
||||
Extract Dungeons & Dragons NPC interaction occurrences from the supplied
|
||||
transcript.
|
||||
|
||||
Include an occurrence only when the transcript establishes one supplied NPC,
|
||||
one interaction kind, and a coherent passage supporting both. Use only names
|
||||
from the supplied NPC registry. The registry helps ground identity but never
|
||||
proves that an interaction occurred.
|
||||
|
||||
Do not summarize, infer relationships, sentiment, factions, motives, aliases,
|
||||
or persistent state. Do not identify player characters, anonymous groups, or
|
||||
invented NPCs. Return an empty interactions array when no supplied NPC has an
|
||||
evidenced interaction in this transcript passage.
|
||||
|
||||
Keep occurrences within this transcript chunk. Split records when an NPC's
|
||||
interaction kind changes, when combat alignment changes, or when an NPC is
|
||||
first mentioned and later becomes present.
|
||||
@@ -0,0 +1,41 @@
|
||||
{
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"$id": "notarius.dnd.npc_interactions.llm",
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["interactions"],
|
||||
"properties": {
|
||||
"interactions": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["name", "kind", "source_refs"],
|
||||
"properties": {
|
||||
"name": {
|
||||
"type": "string"
|
||||
},
|
||||
"kind": {
|
||||
"type": "string"
|
||||
},
|
||||
"source_refs": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["start_unit_id", "end_unit_id"],
|
||||
"properties": {
|
||||
"start_unit_id": {
|
||||
"type": "integer"
|
||||
},
|
||||
"end_unit_id": {
|
||||
"type": "integer"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
113
internal/modules/dnd/extract/npcinteractions/canonicalize.go
Normal file
113
internal/modules/dnd/extract/npcinteractions/canonicalize.go
Normal file
@@ -0,0 +1,113 @@
|
||||
package npcinteractions
|
||||
|
||||
import (
|
||||
"sort"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
||||
)
|
||||
|
||||
func canonicalizeResponse(response *extractionResponse, doc *source.SourceDocument) {
|
||||
if response == nil {
|
||||
return
|
||||
}
|
||||
for index := range response.Interactions {
|
||||
canonicalizeInteraction(&response.Interactions[index])
|
||||
}
|
||||
sort.SliceStable(response.Interactions, func(i, j int) bool {
|
||||
left, leftOK := earliestSourcePosition(doc, response.Interactions[i])
|
||||
right, rightOK := earliestSourcePosition(doc, response.Interactions[j])
|
||||
if leftOK != rightOK {
|
||||
return leftOK
|
||||
}
|
||||
if !leftOK {
|
||||
return false
|
||||
}
|
||||
return left < right
|
||||
})
|
||||
}
|
||||
|
||||
func canonicalizeInteraction(interaction *interactionResponse) {
|
||||
if interaction == nil {
|
||||
return
|
||||
}
|
||||
sort.SliceStable(interaction.SourceRefs, func(i, j int) bool {
|
||||
left := interaction.SourceRefs[i]
|
||||
right := interaction.SourceRefs[j]
|
||||
if unitSortValue(left.StartUnitID) != unitSortValue(right.StartUnitID) {
|
||||
return unitSortValue(left.StartUnitID) < unitSortValue(right.StartUnitID)
|
||||
}
|
||||
return unitSortValue(left.EndUnitID) < unitSortValue(right.EndUnitID)
|
||||
})
|
||||
interaction.SourceRefs = dedupeSourceRefs(interaction.SourceRefs)
|
||||
}
|
||||
|
||||
func dedupeSourceRefs(refs []interactionSourceRefResponse) []interactionSourceRefResponse {
|
||||
if len(refs) < 2 {
|
||||
return refs
|
||||
}
|
||||
out := refs[:0]
|
||||
var previous interactionSourceRefResponse
|
||||
for index, ref := range refs {
|
||||
if index > 0 && previous == ref {
|
||||
continue
|
||||
}
|
||||
out = append(out, ref)
|
||||
previous = ref
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func earliestSourcePosition(doc *source.SourceDocument, interaction interactionResponse) (int, bool) {
|
||||
if doc == nil {
|
||||
return 0, false
|
||||
}
|
||||
found := false
|
||||
earliest := 0
|
||||
for _, ref := range interaction.SourceRefs {
|
||||
candidate := source.SourceRef{SourceID: doc.ID, StartUnitID: ref.StartUnitID, EndUnitID: ref.EndUnitID}
|
||||
if err := source.ValidateRef(doc, candidate); err != nil {
|
||||
continue
|
||||
}
|
||||
index, ok := source.UnitIndex(doc, candidate.StartUnitID)
|
||||
if !ok || (found && index >= earliest) {
|
||||
continue
|
||||
}
|
||||
earliest = index
|
||||
found = true
|
||||
}
|
||||
return earliest, found
|
||||
}
|
||||
|
||||
func unitSortValue(value int) int {
|
||||
if value <= 0 {
|
||||
return int(^uint(0) >> 1)
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
func canonicalInteractionList(response extractionResponse, sourceID string) dnd.NPCInteractionList {
|
||||
interactions := make([]dnd.NPCInteraction, len(response.Interactions))
|
||||
for index, interaction := range response.Interactions {
|
||||
interactions[index] = dnd.NPCInteraction{
|
||||
Name: interaction.Name,
|
||||
Kind: dnd.NPCInteractionKind(interaction.Kind),
|
||||
SourceRefs: canonicalSourceRefs(interaction.SourceRefs, sourceID),
|
||||
}
|
||||
}
|
||||
if response.Interactions == nil {
|
||||
interactions = nil
|
||||
}
|
||||
return dnd.NPCInteractionList{Interactions: interactions}
|
||||
}
|
||||
|
||||
func canonicalSourceRefs(refs []interactionSourceRefResponse, sourceID string) []source.SourceRef {
|
||||
if refs == nil {
|
||||
return nil
|
||||
}
|
||||
out := make([]source.SourceRef, len(refs))
|
||||
for index, ref := range refs {
|
||||
out[index] = source.SourceRef{SourceID: sourceID, StartUnitID: ref.StartUnitID, EndUnitID: ref.EndUnitID}
|
||||
}
|
||||
return out
|
||||
}
|
||||
225
internal/modules/dnd/extract/npcinteractions/extractor.go
Normal file
225
internal/modules/dnd/extract/npcinteractions/extractor.go
Normal file
@@ -0,0 +1,225 @@
|
||||
package npcinteractions
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sort"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
||||
npcregistry "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/npcs/registry"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared"
|
||||
)
|
||||
|
||||
const (
|
||||
Key = "dnd/npc-interactions"
|
||||
mappingPolicy = "dnd.npc_interactions.extract_mapping.v1"
|
||||
)
|
||||
|
||||
const (
|
||||
NPCRegistryReferenceSlot = npcregistry.ReferenceSlot
|
||||
NPCRegistryMaxBytes = npcregistry.MaxBytes
|
||||
)
|
||||
|
||||
var requiredCapabilities = []string{
|
||||
"chunks",
|
||||
"source.transcript",
|
||||
}
|
||||
|
||||
var providedCapabilities = []string{
|
||||
"dnd.npc_interactions",
|
||||
}
|
||||
|
||||
var referenceSlotDescriptions = shared.ReferenceSlotDescriptions{
|
||||
Glossary: "Optional campaign glossary reference material used only for interaction disambiguation.",
|
||||
Party: "Optional party roster reference material used only for interaction disambiguation.",
|
||||
Players: "Optional player list reference material used only for interaction disambiguation.",
|
||||
Roster: "Deprecated alias for party roster reference material used only for interaction disambiguation.",
|
||||
}
|
||||
|
||||
func referenceSlots() []contracts.ReferenceSlot {
|
||||
slots := shared.ReferenceSlots(referenceSlotDescriptions)
|
||||
slots = append(slots, contracts.ReferenceSlot{
|
||||
Name: NPCRegistryReferenceSlot,
|
||||
Description: "Required normalized NPC registry used only for interaction identity grounding, never as interaction evidence.",
|
||||
Required: true,
|
||||
AcceptedMediaTypes: []string{"application/json"},
|
||||
AcceptedArtifactKinds: []contracts.ArtifactKind{dnd.NPCListKind},
|
||||
MaxBytes: NPCRegistryMaxBytes,
|
||||
})
|
||||
sort.Slice(slots, func(i, j int) bool { return slots[i].Name < slots[j].Name })
|
||||
return slots
|
||||
}
|
||||
|
||||
var _ contracts.Extractor[dnd.NPCInteractionList] = (*Extractor)(nil)
|
||||
var _ contracts.ManifestMetadataProvider = (*Extractor)(nil)
|
||||
var _ pipeline.CheckpointFingerprintProvider = (*Extractor)(nil)
|
||||
|
||||
type Options struct{}
|
||||
|
||||
type Extractor struct {
|
||||
llm contracts.StructuredLLMClient
|
||||
npcResolver *npcregistry.Resolver
|
||||
promptSHA string
|
||||
responseSchemaSHA string
|
||||
}
|
||||
|
||||
func New(llmClient contracts.StructuredLLMClient, _ Options, references ...contracts.ReferenceSet) (*Extractor, error) {
|
||||
if llmClient == nil {
|
||||
return nil, extractorErrorf("LLM client must not be nil")
|
||||
}
|
||||
if len(references) > 1 {
|
||||
return nil, extractorErrorf("at most one reference set may be supplied")
|
||||
}
|
||||
var referenceSet contracts.ReferenceSet
|
||||
if len(references) == 1 {
|
||||
referenceSet = references[0]
|
||||
}
|
||||
npcResolver, err := npcregistry.NewResolver(referenceSet)
|
||||
if err != nil {
|
||||
return nil, extractorErrorf("prepare NPC registry prompt input: %w", err)
|
||||
}
|
||||
promptSHA, err := scriptoriumPromptMetadata()
|
||||
if err != nil {
|
||||
return nil, extractorErrorf("load prompt metadata: %w", err)
|
||||
}
|
||||
responseSchema, err := loadResponseSchema()
|
||||
if err != nil {
|
||||
return nil, extractorErrorf("load response schema: %w", err)
|
||||
}
|
||||
return &Extractor{
|
||||
llm: llmClient,
|
||||
npcResolver: npcResolver,
|
||||
promptSHA: promptSHA,
|
||||
responseSchemaSHA: responseSchema.SHA256,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (e *Extractor) Key() string { return Key }
|
||||
|
||||
func (e *Extractor) ReferenceSlots() []contracts.ReferenceSlot { return referenceSlots() }
|
||||
|
||||
func (e *Extractor) ManifestMetadata() map[string]any {
|
||||
if e == nil {
|
||||
return nil
|
||||
}
|
||||
metadata := map[string]any{
|
||||
"prompt_id": PromptID,
|
||||
"prompt_version": SchemaVersion,
|
||||
"prompt_sha256": e.promptSHA,
|
||||
"mapping_policy": mappingPolicy,
|
||||
"response_schema_key": string(ResponseSchemaKey),
|
||||
"response_schema_id": ResponseSchemaID,
|
||||
"response_schema_name": ResponseSchemaName,
|
||||
"response_schema_version": SchemaVersion,
|
||||
"response_schema_sha256": e.responseSchemaSHA,
|
||||
}
|
||||
seeded := e.npcResolver.Seeded()
|
||||
if seeded.Bound() {
|
||||
metadata["npc_registry_digest"] = seeded.Digest()
|
||||
metadata["npc_count"] = seeded.Count()
|
||||
}
|
||||
return metadata
|
||||
}
|
||||
|
||||
func (e *Extractor) CheckpointFingerprints() []pipeline.CheckpointFingerprint {
|
||||
if e == nil {
|
||||
return nil
|
||||
}
|
||||
seeded := e.npcResolver.Seeded()
|
||||
return []pipeline.CheckpointFingerprint{
|
||||
{Name: "prompt", Value: e.promptSHA},
|
||||
{Name: "response_schema", Value: e.responseSchemaSHA},
|
||||
{Name: "mapping_policy", Value: mappingPolicy},
|
||||
{Name: "npc_registry", Value: seeded.ProjectionDigest()},
|
||||
}
|
||||
}
|
||||
|
||||
func (e *Extractor) Extract(ctx context.Context, req contracts.TypedExtractionRequest) (contracts.TypedExtractionResult[dnd.NPCInteractionList], error) {
|
||||
if e == nil {
|
||||
return contracts.TypedExtractionResult[dnd.NPCInteractionList]{}, extractorErrorf("extractor must not be nil")
|
||||
}
|
||||
if e.llm == nil {
|
||||
return contracts.TypedExtractionResult[dnd.NPCInteractionList]{}, extractorErrorf("LLM client must not be nil")
|
||||
}
|
||||
if ctx == nil {
|
||||
return contracts.TypedExtractionResult[dnd.NPCInteractionList]{}, extractorErrorf("context must not be nil")
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return contracts.TypedExtractionResult[dnd.NPCInteractionList]{}, extractorErrorf("context error before extraction: %w", err)
|
||||
}
|
||||
if req.Source == nil {
|
||||
return contracts.TypedExtractionResult[dnd.NPCInteractionList]{}, extractorErrorf("source must not be nil")
|
||||
}
|
||||
if req.Chunk == nil {
|
||||
return contracts.TypedExtractionResult[dnd.NPCInteractionList]{}, extractorErrorf("chunk must not be nil")
|
||||
}
|
||||
if len(req.Chunk.Units) == 0 {
|
||||
return contracts.TypedExtractionResult[dnd.NPCInteractionList]{}, extractorErrorf("chunk %q units must not be empty", req.Chunk.ID)
|
||||
}
|
||||
sourceInput, err := shared.ChunkPromptMaterial(req)
|
||||
if err != nil {
|
||||
return contracts.TypedExtractionResult[dnd.NPCInteractionList]{}, extractorErrorf("%w", err)
|
||||
}
|
||||
npcRegistry, err := e.npcResolver.Resolve(req.References)
|
||||
if err != nil {
|
||||
return contracts.TypedExtractionResult[dnd.NPCInteractionList]{}, extractorErrorf("resolve NPC registry: %w", err)
|
||||
}
|
||||
if !npcRegistry.Bound() {
|
||||
return contracts.TypedExtractionResult[dnd.NPCInteractionList]{}, extractorErrorf("NPC registry reference is required")
|
||||
}
|
||||
|
||||
var response extractionResponse
|
||||
inputs := shared.PromptInputs(sourceInput, req.References)
|
||||
inputs[NPCRegistryReferenceSlot] = npcRegistry.PromptInput()
|
||||
if _, err := e.llm.CompleteStructured(ctx, contracts.StructuredCompletionRequest{
|
||||
StageName: Key,
|
||||
PromptID: PromptID,
|
||||
PromptVersion: SchemaVersion,
|
||||
ProfileID: req.LLMProfile,
|
||||
SessionID: req.SessionID,
|
||||
Inputs: inputs,
|
||||
}, &response); err != nil {
|
||||
return contracts.TypedExtractionResult[dnd.NPCInteractionList]{}, extractorErrorf("complete structured output: %w", err)
|
||||
}
|
||||
canonicalizeResponse(&response, req.Source)
|
||||
return contracts.TypedExtractionResult[dnd.NPCInteractionList]{Value: canonicalInteractionList(response, req.Source.ID)}, nil
|
||||
}
|
||||
|
||||
func ModuleSpec() pipeline.ModuleSpec {
|
||||
return pipeline.ModuleSpec{
|
||||
Key: Key,
|
||||
Stage: pipeline.StageExtract,
|
||||
Requires: append([]string(nil), requiredCapabilities...),
|
||||
Provides: append([]string(nil), providedCapabilities...),
|
||||
ArtifactKind: dnd.NPCInteractionListKind,
|
||||
ReferenceSlots: referenceSlots(),
|
||||
}
|
||||
}
|
||||
|
||||
func Register(registry *pipeline.ExtractorRegistry) error {
|
||||
return pipeline.RegisterExtractorBuilder(registry, ModuleSpec(), validateOptions, func(request pipeline.BuildRequest) (contracts.Extractor[dnd.NPCInteractionList], error) {
|
||||
options, err := DecodeOptions(request.Options)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return New(request.Dependencies.LLM, options, request.References)
|
||||
})
|
||||
}
|
||||
|
||||
func validateOptions(options map[string]any) error {
|
||||
_, err := DecodeOptions(options)
|
||||
return err
|
||||
}
|
||||
|
||||
func DecodeOptions(options map[string]any) (Options, error) {
|
||||
if err := pipeline.RejectUnknownOptions(options); err != nil {
|
||||
return Options{}, extractorErrorf("%w", err)
|
||||
}
|
||||
return Options{}, nil
|
||||
}
|
||||
|
||||
func extractorErrorf(format string, args ...any) error {
|
||||
return fmt.Errorf("dnd NPC interactions extractor: "+format, args...)
|
||||
}
|
||||
370
internal/modules/dnd/extract/npcinteractions/extractor_test.go
Normal file
370
internal/modules/dnd/extract/npcinteractions/extractor_test.go
Normal file
@@ -0,0 +1,370 @@
|
||||
package npcinteractions
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
||||
npccodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/npcs"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/npcs/identity"
|
||||
)
|
||||
|
||||
func TestExtractMapsEveryKindAndOrdersBySourcePosition(t *testing.T) {
|
||||
client := &fakeInteractionsLLMClient{response: extractionResponse{Interactions: []interactionResponse{
|
||||
{Name: "Other", Kind: "other", SourceRefs: interactionRefs(30, 30)},
|
||||
{Name: "Opponent", Kind: "combat_opponent", SourceRefs: interactionRefs(20, 20)},
|
||||
{Name: "Ally", Kind: "combat_ally", SourceRefs: interactionRefs(5, 5)},
|
||||
{Name: "Speaker", Kind: "dialogue", SourceRefs: append(interactionRefs(2, 2), interactionRefs(2, 2)...)},
|
||||
{Name: "Present", Kind: "noncombat_presence", SourceRefs: interactionRefs(7, 7)},
|
||||
{Name: "Mentioned", Kind: "mentioned", SourceRefs: interactionRefs(10, 10)},
|
||||
{Name: "Invalid", Kind: "unsupported", SourceRefs: interactionRefs(0, 0)},
|
||||
}}}
|
||||
references := requiredRegistryReferences(t, "Mentioned", "Speaker", "Present", "Ally", "Opponent", "Other", "Invalid")
|
||||
req := extractionRequest()
|
||||
req.References = references
|
||||
|
||||
result, err := newExtractor(t, client, references).Extract(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("Extract() error = %v", err)
|
||||
}
|
||||
if got := interactionNames(result.Value); !reflect.DeepEqual(got, []string{"Mentioned", "Speaker", "Present", "Ally", "Opponent", "Other", "Invalid"}) {
|
||||
t.Fatalf("interaction order = %#v", got)
|
||||
}
|
||||
if got := interactionKinds(result.Value); !reflect.DeepEqual(got, []dnd.NPCInteractionKind{
|
||||
dnd.NPCInteractionKindMentioned,
|
||||
dnd.NPCInteractionKindDialogue,
|
||||
dnd.NPCInteractionKindNoncombatPresence,
|
||||
dnd.NPCInteractionKindCombatAlly,
|
||||
dnd.NPCInteractionKindCombatOpponent,
|
||||
dnd.NPCInteractionKindOther,
|
||||
"unsupported",
|
||||
}) {
|
||||
t.Fatalf("interaction kinds = %#v", got)
|
||||
}
|
||||
if refs := result.Value.Interactions[1].SourceRefs; !reflect.DeepEqual(refs, []source.SourceRef{{SourceID: "session-alpha", StartUnitID: 2, EndUnitID: 2}}) {
|
||||
t.Fatalf("canonical source refs = %#v", refs)
|
||||
}
|
||||
if invalid := result.Value.Interactions[6]; invalid.Name != "Invalid" || invalid.Kind != "unsupported" || !reflect.DeepEqual(invalid.SourceRefs, []source.SourceRef{{SourceID: "session-alpha"}}) {
|
||||
t.Fatalf("invalid candidate = %#v, want preserved values with current source identity", invalid)
|
||||
}
|
||||
if len(client.requests) != 1 {
|
||||
t.Fatalf("LLM calls = %d, want 1", len(client.requests))
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewRequiresLLMAndRejectsAmbiguousReferenceSets(t *testing.T) {
|
||||
if _, err := New(nil, Options{}); err == nil || !strings.Contains(err.Error(), "LLM client") {
|
||||
t.Fatalf("New(nil) error = %v", err)
|
||||
}
|
||||
if _, err := New(&fakeInteractionsLLMClient{}, Options{}, contracts.ReferenceSet{}, contracts.ReferenceSet{}); err == nil || !strings.Contains(err.Error(), "at most one reference set") {
|
||||
t.Fatalf("New() error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractUsesNamesOnlyRegistryAndCurrentTranscriptEvidence(t *testing.T) {
|
||||
client := &fakeInteractionsLLMClient{response: extractionResponse{Interactions: []interactionResponse{{
|
||||
Name: "Mira Thorn", Kind: "dialogue", SourceRefs: interactionRefs(10, 10),
|
||||
}}}}
|
||||
references := requiredRegistryReferences(t, "Mira Thorn", "Hooded Guard")
|
||||
req := extractionRequest()
|
||||
req.References = references
|
||||
if _, err := newExtractor(t, client, references).Extract(context.Background(), req); err != nil {
|
||||
t.Fatalf("Extract() error = %v", err)
|
||||
}
|
||||
request := client.requests[0]
|
||||
registry := request.Inputs[NPCRegistryReferenceSlot]
|
||||
if registry.Name != NPCRegistryReferenceSlot || registry.MediaType != "application/json" || string(registry.Content) != `{"npcs":[{"name":"Mira Thorn"},{"name":"Hooded Guard"}]}` {
|
||||
t.Fatalf("registry prompt input = %#v, want exact names-only projection", registry)
|
||||
}
|
||||
for _, forbidden := range []string{"npc:sha256:", "other-session", "start_unit_id"} {
|
||||
if strings.Contains(string(registry.Content), forbidden) {
|
||||
t.Fatalf("registry prompt input leaked %q: %s", forbidden, registry.Content)
|
||||
}
|
||||
}
|
||||
if strings.Contains(string(request.Inputs["transcript"].Content), "other-session") {
|
||||
t.Fatal("transcript input contains registry evidence")
|
||||
}
|
||||
metadata, err := json.Marshal(newExtractor(t, &fakeInteractionsLLMClient{}, references).ManifestMetadata())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if strings.Contains(string(metadata), "Mira Thorn") || strings.Contains(string(metadata), "other-session") {
|
||||
t.Fatalf("manifest metadata leaked registry content: %s", metadata)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractRequiresBoundRegistryBeforeLLMCall(t *testing.T) {
|
||||
client := &fakeInteractionsLLMClient{}
|
||||
if _, err := newExtractor(t, client).Extract(context.Background(), extractionRequest()); err == nil || !strings.Contains(err.Error(), "NPC registry reference is required") {
|
||||
t.Fatalf("Extract() error = %v, want required registry failure", err)
|
||||
}
|
||||
if len(client.requests) != 0 {
|
||||
t.Fatalf("LLM calls = %d, want no call", len(client.requests))
|
||||
}
|
||||
|
||||
malformed := contracts.ReferenceSet{Slots: map[string]contracts.ResolvedReferenceSlot{
|
||||
NPCRegistryReferenceSlot: {
|
||||
Slot: contracts.ReferenceSlot{Name: NPCRegistryReferenceSlot},
|
||||
Items: []contracts.ReferenceItem{{SlotName: NPCRegistryReferenceSlot, MediaType: "application/json", Content: []byte(`{"secret":"private source"}`)}},
|
||||
},
|
||||
}}
|
||||
if _, err := New(client, Options{}, malformed); err == nil || !strings.Contains(err.Error(), "prepare NPC registry") || strings.Contains(err.Error(), "private source") {
|
||||
t.Fatalf("New() error = %v, want content-safe malformed registry error", err)
|
||||
}
|
||||
if len(client.requests) != 0 {
|
||||
t.Fatalf("LLM calls = %d, want no call", len(client.requests))
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractResolvesGeneratedRegistryAtOperationTime(t *testing.T) {
|
||||
client := &fakeInteractionsLLMClient{response: extractionResponse{Interactions: []interactionResponse{}}}
|
||||
references := requiredRegistryReferences(t, "Mira Thorn")
|
||||
req := extractionRequest()
|
||||
req.References = references
|
||||
extractor := newExtractor(t, client)
|
||||
if _, err := extractor.Extract(context.Background(), req); err != nil {
|
||||
t.Fatalf("Extract() error = %v", err)
|
||||
}
|
||||
if input := client.requests[0].Inputs[NPCRegistryReferenceSlot]; string(input.Content) != `{"npcs":[{"name":"Mira Thorn"}]}` || input.OriginURI != "" {
|
||||
t.Fatalf("generated registry prompt input = %#v", input)
|
||||
}
|
||||
metadata := extractor.ManifestMetadata()
|
||||
if _, ok := metadata["npc_registry_digest"]; ok {
|
||||
t.Fatalf("operation-time registry leaked into static metadata: %#v", metadata)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractAcceptsEmptyBoundRegistryAndEmptyResponse(t *testing.T) {
|
||||
content, err := npccodec.New().Encode(dnd.NPCList{NPCs: []dnd.NPC{}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
references := contracts.ReferenceSet{Slots: map[string]contracts.ResolvedReferenceSlot{
|
||||
NPCRegistryReferenceSlot: {
|
||||
Slot: contracts.ReferenceSlot{Name: NPCRegistryReferenceSlot},
|
||||
Items: []contracts.ReferenceItem{{SlotName: NPCRegistryReferenceSlot, MediaType: npccodec.MediaType, Content: content}},
|
||||
},
|
||||
}}
|
||||
client := &fakeInteractionsLLMClient{response: extractionResponse{Interactions: []interactionResponse{}}}
|
||||
req := extractionRequest()
|
||||
req.References = references
|
||||
result, err := newExtractor(t, client, references).Extract(context.Background(), req)
|
||||
if err != nil || result.Value.Interactions == nil || len(result.Value.Interactions) != 0 {
|
||||
t.Fatalf("Extract() = %#v, %v; want present empty interactions", result, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractRejectsInvalidRequestsAndProviderFailures(t *testing.T) {
|
||||
references := requiredRegistryReferences(t, "Mira Thorn")
|
||||
valid := extractionRequest()
|
||||
valid.References = references
|
||||
canceled, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
extractor := newExtractor(t, &fakeInteractionsLLMClient{}, references)
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
ctx context.Context
|
||||
req contracts.TypedExtractionRequest
|
||||
want string
|
||||
}{
|
||||
{"nil context", nil, valid, "context"},
|
||||
{"canceled context", canceled, valid, "context"},
|
||||
{"nil source", context.Background(), contracts.TypedExtractionRequest{Chunk: valid.Chunk, References: references}, "source"},
|
||||
{"nil chunk", context.Background(), contracts.TypedExtractionRequest{Source: valid.Source, References: references}, "chunk"},
|
||||
{"empty chunk", context.Background(), emptyChunkRequest(valid), "units"},
|
||||
{"source input mismatch", context.Background(), mismatchedSourceInputRequest(valid), "must match chunk"},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
if _, err := extractor.Extract(test.ctx, test.req); err == nil || !strings.Contains(err.Error(), test.want) {
|
||||
t.Fatalf("Extract() error = %v, want %q", err, test.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
if _, err := newExtractor(t, &fakeInteractionsLLMClient{err: errors.New("provider unavailable")}, references).Extract(context.Background(), valid); err == nil || !strings.Contains(err.Error(), "provider unavailable") {
|
||||
t.Fatalf("provider error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestModuleSpecRegistrationMetadataAndFingerprints(t *testing.T) {
|
||||
got := ModuleSpec()
|
||||
if got.Key != Key || got.Stage != pipeline.StageExtract || got.ArtifactKind != dnd.NPCInteractionListKind || !reflect.DeepEqual(got.Requires, []string{"chunks", "source.transcript"}) || !reflect.DeepEqual(got.Provides, []string{"dnd.npc_interactions"}) {
|
||||
t.Fatalf("ModuleSpec() = %#v", got)
|
||||
}
|
||||
var registrySlot contracts.ReferenceSlot
|
||||
for _, slot := range got.ReferenceSlots {
|
||||
if slot.Name == NPCRegistryReferenceSlot {
|
||||
registrySlot = slot
|
||||
}
|
||||
}
|
||||
if !registrySlot.Required || !reflect.DeepEqual(registrySlot.AcceptedArtifactKinds, []contracts.ArtifactKind{dnd.NPCListKind}) || registrySlot.MaxBytes != NPCRegistryMaxBytes {
|
||||
t.Fatalf("NPC registry slot = %#v", registrySlot)
|
||||
}
|
||||
got.ReferenceSlots[0].AcceptedMediaTypes[0] = "changed"
|
||||
if ModuleSpec().ReferenceSlots[0].AcceptedMediaTypes[0] == "changed" {
|
||||
t.Fatal("ModuleSpec() returned mutable reference slots")
|
||||
}
|
||||
extractorSlots := newExtractor(t, &fakeInteractionsLLMClient{}).ReferenceSlots()
|
||||
extractorSlots[0].AcceptedMediaTypes[0] = "changed"
|
||||
if newExtractor(t, &fakeInteractionsLLMClient{}).ReferenceSlots()[0].AcceptedMediaTypes[0] == "changed" {
|
||||
t.Fatal("ReferenceSlots() returned mutable reference slots")
|
||||
}
|
||||
registry := pipeline.NewExtractorRegistry()
|
||||
if err := Register(registry); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, ok := registry.Spec(Key); !ok {
|
||||
t.Fatalf("registry missing %q", Key)
|
||||
}
|
||||
if _, err := DecodeOptions(map[string]any{"unexpected": true}); err == nil {
|
||||
t.Fatal("DecodeOptions() accepted unknown option")
|
||||
}
|
||||
|
||||
references := requiredRegistryReferences(t, "Mira Thorn")
|
||||
extractor := newExtractor(t, &fakeInteractionsLLMClient{}, references)
|
||||
metadata := extractor.ManifestMetadata()
|
||||
for key, want := range map[string]string{
|
||||
"prompt_id": PromptID, "prompt_version": SchemaVersion, "mapping_policy": mappingPolicy,
|
||||
"response_schema_key": string(ResponseSchemaKey), "response_schema_id": ResponseSchemaID,
|
||||
"response_schema_name": ResponseSchemaName, "response_schema_version": SchemaVersion,
|
||||
} {
|
||||
if metadata[key] != want {
|
||||
t.Fatalf("metadata[%q] = %#v, want %q", key, metadata[key], want)
|
||||
}
|
||||
}
|
||||
for _, key := range []string{"prompt_sha256", "response_schema_sha256", "npc_registry_digest"} {
|
||||
if value, ok := metadata[key].(string); !ok || !strings.HasPrefix(value, "sha256:") {
|
||||
t.Fatalf("metadata[%q] = %#v, want digest", key, metadata[key])
|
||||
}
|
||||
}
|
||||
wantFingerprints := map[string]struct{}{"prompt": {}, "response_schema": {}, "mapping_policy": {}, "npc_registry": {}}
|
||||
for _, fingerprint := range extractor.CheckpointFingerprints() {
|
||||
delete(wantFingerprints, fingerprint.Name)
|
||||
}
|
||||
if len(wantFingerprints) != 0 {
|
||||
t.Fatalf("missing fingerprints = %#v", wantFingerprints)
|
||||
}
|
||||
}
|
||||
|
||||
func extractionRequest() contracts.TypedExtractionRequest {
|
||||
doc := sourceDocument()
|
||||
chunk := &source.Chunk{
|
||||
ID: "session-alpha:chunk:0", SourceID: doc.ID, Index: 0,
|
||||
Ref: source.SourceRef{SourceID: doc.ID, StartUnitID: 10, EndUnitID: 30},
|
||||
Content: []byte(`{"units":[10,2,7,5,20,30]}`), MediaType: "application/json",
|
||||
Units: append([]source.SourceUnit(nil), doc.Units...),
|
||||
}
|
||||
return contracts.TypedExtractionRequest{
|
||||
Source: doc, Chunk: chunk,
|
||||
SourceInput: contracts.NewLLMInputMaterial("source", chunk.MediaType, chunk.Content, "sha256:chunk", "file:///session-alpha.json"),
|
||||
SessionID: "session-123", LLMProfile: "profile-npc-interactions",
|
||||
}
|
||||
}
|
||||
|
||||
func sourceDocument() *source.SourceDocument {
|
||||
return &source.SourceDocument{
|
||||
ID: "session-alpha", Kind: "transcript", Format: "application/vnd.seriatim.minimal+json", Digest: "sha256:source",
|
||||
Units: []source.SourceUnit{
|
||||
{ID: 10, Kind: "transcript_segment", Text: "Mira Thorn is mentioned."},
|
||||
{ID: 2, Kind: "transcript_segment", Text: "Mira Thorn speaks to the party."},
|
||||
{ID: 7, Kind: "transcript_segment", Text: "Mira Thorn watches nearby."},
|
||||
{ID: 5, Kind: "transcript_segment", Text: "Mira Thorn joins the party in combat."},
|
||||
{ID: 20, Kind: "transcript_segment", Text: "Mira Thorn attacks the party."},
|
||||
{ID: 30, Kind: "transcript_segment", Text: "Mira Thorn performs a ritual."},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func requiredRegistryReferences(t *testing.T, names ...string) contracts.ReferenceSet {
|
||||
t.Helper()
|
||||
npcs := make([]dnd.NPC, len(names))
|
||||
for index, name := range names {
|
||||
npcs[index] = dnd.NPC{
|
||||
ID: identity.DeriveID(name), Name: name,
|
||||
SourceRefs: []source.SourceRef{{SourceID: "other-session", StartUnitID: index + 1, EndUnitID: index + 1}},
|
||||
}
|
||||
}
|
||||
content, err := npccodec.New().Encode(dnd.NPCList{NPCs: npcs})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return contracts.ReferenceSet{Slots: map[string]contracts.ResolvedReferenceSlot{
|
||||
NPCRegistryReferenceSlot: {
|
||||
Slot: contracts.ReferenceSlot{Name: NPCRegistryReferenceSlot},
|
||||
Items: []contracts.ReferenceItem{{SlotName: NPCRegistryReferenceSlot, MediaType: npccodec.MediaType, Content: content, Origin: contracts.ReferenceOrigin{Type: "generated"}}},
|
||||
},
|
||||
}}
|
||||
}
|
||||
|
||||
func interactionRefs(start, end int) []interactionSourceRefResponse {
|
||||
return []interactionSourceRefResponse{{StartUnitID: start, EndUnitID: end}}
|
||||
}
|
||||
|
||||
func interactionNames(value dnd.NPCInteractionList) []string {
|
||||
names := make([]string, len(value.Interactions))
|
||||
for index, interaction := range value.Interactions {
|
||||
names[index] = interaction.Name
|
||||
}
|
||||
return names
|
||||
}
|
||||
|
||||
func interactionKinds(value dnd.NPCInteractionList) []dnd.NPCInteractionKind {
|
||||
kinds := make([]dnd.NPCInteractionKind, len(value.Interactions))
|
||||
for index, interaction := range value.Interactions {
|
||||
kinds[index] = interaction.Kind
|
||||
}
|
||||
return kinds
|
||||
}
|
||||
|
||||
func emptyChunkRequest(req contracts.TypedExtractionRequest) contracts.TypedExtractionRequest {
|
||||
req.Chunk = &source.Chunk{ID: req.Chunk.ID, SourceID: req.Chunk.SourceID, Index: req.Chunk.Index}
|
||||
return req
|
||||
}
|
||||
|
||||
func mismatchedSourceInputRequest(req contracts.TypedExtractionRequest) contracts.TypedExtractionRequest {
|
||||
req.SourceInput = contracts.NewLLMInputMaterial("source", "application/json", []byte(`{"different":true}`), "sha256:other", "")
|
||||
return req
|
||||
}
|
||||
|
||||
func newExtractor(t *testing.T, client contracts.StructuredLLMClient, references ...contracts.ReferenceSet) *Extractor {
|
||||
t.Helper()
|
||||
extractor, err := New(client, Options{}, references...)
|
||||
if err != nil {
|
||||
t.Fatalf("New() error = %v", err)
|
||||
}
|
||||
return extractor
|
||||
}
|
||||
|
||||
type fakeInteractionsLLMClient struct {
|
||||
response extractionResponse
|
||||
err error
|
||||
requests []contracts.StructuredCompletionRequest
|
||||
}
|
||||
|
||||
func (client *fakeInteractionsLLMClient) CompleteStructured(_ context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
|
||||
client.requests = append(client.requests, cloneStructuredCompletionRequest(req))
|
||||
if client.err != nil {
|
||||
return contracts.StructuredCompletionResponse{}, client.err
|
||||
}
|
||||
target, ok := out.(*extractionResponse)
|
||||
if !ok {
|
||||
return contracts.StructuredCompletionResponse{}, errors.New("unexpected output target")
|
||||
}
|
||||
*target = client.response
|
||||
content, err := json.Marshal(client.response)
|
||||
if err != nil {
|
||||
return contracts.StructuredCompletionResponse{}, err
|
||||
}
|
||||
return contracts.StructuredCompletionResponse{Content: content}, nil
|
||||
}
|
||||
|
||||
func cloneStructuredCompletionRequest(req contracts.StructuredCompletionRequest) contracts.StructuredCompletionRequest {
|
||||
req.Inputs = req.Inputs.Clone()
|
||||
return req
|
||||
}
|
||||
16
internal/modules/dnd/extract/npcinteractions/model.go
Normal file
16
internal/modules/dnd/extract/npcinteractions/model.go
Normal file
@@ -0,0 +1,16 @@
|
||||
package npcinteractions
|
||||
|
||||
type extractionResponse struct {
|
||||
Interactions []interactionResponse `json:"interactions"`
|
||||
}
|
||||
|
||||
type interactionResponse struct {
|
||||
Name string `json:"name"`
|
||||
Kind string `json:"kind"`
|
||||
SourceRefs []interactionSourceRefResponse `json:"source_refs"`
|
||||
}
|
||||
|
||||
type interactionSourceRefResponse struct {
|
||||
StartUnitID int `json:"start_unit_id"`
|
||||
EndUnitID int `json:"end_unit_id"`
|
||||
}
|
||||
18
internal/modules/dnd/extract/npcinteractions/model_test.go
Normal file
18
internal/modules/dnd/extract/npcinteractions/model_test.go
Normal file
@@ -0,0 +1,18 @@
|
||||
package npcinteractions
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestExtractionResponsePreservesValidatorOwnedSemantics(t *testing.T) {
|
||||
content := []byte(`{"interactions":[{"name":"","kind":"unsupported","source_refs":[{"start_unit_id":0,"end_unit_id":-1}]}]}`)
|
||||
var response extractionResponse
|
||||
if err := json.Unmarshal(content, &response); err != nil {
|
||||
t.Fatalf("json.Unmarshal() error = %v", err)
|
||||
}
|
||||
interaction := response.Interactions[0]
|
||||
if interaction.Name != "" || interaction.Kind != "unsupported" || interaction.SourceRefs[0] != (interactionSourceRefResponse{StartUnitID: 0, EndUnitID: -1}) {
|
||||
t.Fatalf("decoded response = %#v", interaction)
|
||||
}
|
||||
}
|
||||
21
internal/modules/dnd/extract/npcinteractions/schema.go
Normal file
21
internal/modules/dnd/extract/npcinteractions/schema.go
Normal file
@@ -0,0 +1,21 @@
|
||||
package npcinteractions
|
||||
|
||||
import "gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
||||
|
||||
const (
|
||||
PromptID = "dnd.npc_interactions"
|
||||
ResponseSchemaKey = llm.ResponseSchemaKey("dnd_npc_interactions_llm")
|
||||
ResponseSchemaID = "notarius.dnd.npc_interactions.llm"
|
||||
ResponseSchemaName = "notarius_dnd_npc_interactions_llm_v1"
|
||||
SchemaVersion = "v1"
|
||||
)
|
||||
|
||||
func loadResponseSchema() (llm.ResponseSchema, error) {
|
||||
return llm.LoadResponseSchema(embeddedAssets, llm.ResponseSchemaDefinition{
|
||||
Key: ResponseSchemaKey,
|
||||
ID: ResponseSchemaID,
|
||||
Version: SchemaVersion,
|
||||
Name: ResponseSchemaName,
|
||||
AssetPath: "assets/schemas/dnd_npc_interactions_llm.v1.json",
|
||||
})
|
||||
}
|
||||
106
internal/modules/dnd/extract/npcinteractions/schema_test.go
Normal file
106
internal/modules/dnd/extract/npcinteractions/schema_test.go
Normal file
@@ -0,0 +1,106 @@
|
||||
package npcinteractions
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/santhosh-tekuri/jsonschema/v6"
|
||||
)
|
||||
|
||||
func TestResponseSchemaOwnsOnlyPrivateStructuralContract(t *testing.T) {
|
||||
schema, err := loadResponseSchema()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if schema.Key != ResponseSchemaKey || schema.ID != ResponseSchemaID || schema.Name != ResponseSchemaName || schema.Version != SchemaVersion || !strings.HasPrefix(schema.SHA256, "sha256:") || !json.Valid(schema.JSONSchema) {
|
||||
t.Fatalf("schema = %#v", schema)
|
||||
}
|
||||
valid := validInteractionResponse()
|
||||
content, err := json.Marshal(valid)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := validateJSONSchema(content, schema.JSONSchema); err != nil {
|
||||
t.Fatalf("valid response rejected: %v", err)
|
||||
}
|
||||
|
||||
semanticCandidate := validInteractionResponse()
|
||||
interaction := semanticCandidate["interactions"].([]any)[0].(map[string]any)
|
||||
interaction["name"] = ""
|
||||
interaction["kind"] = "unsupported"
|
||||
ref := interaction["source_refs"].([]any)[0].(map[string]any)
|
||||
ref["start_unit_id"] = 0
|
||||
ref["end_unit_id"] = -1
|
||||
content, err = json.Marshal(semanticCandidate)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := validateJSONSchema(content, schema.JSONSchema); err != nil {
|
||||
t.Fatalf("schema rejected validator-owned semantics: %v", err)
|
||||
}
|
||||
|
||||
for _, mutate := range []func(map[string]any){
|
||||
func(record map[string]any) { delete(record, "name") },
|
||||
func(record map[string]any) { record["kind"] = 1 },
|
||||
func(record map[string]any) { record["unexpected"] = true },
|
||||
func(record map[string]any) {
|
||||
record["source_refs"].([]any)[0].(map[string]any)["source_id"] = "assigned later"
|
||||
},
|
||||
} {
|
||||
candidate := validInteractionResponse()
|
||||
mutate(candidate["interactions"].([]any)[0].(map[string]any))
|
||||
content, err := json.Marshal(candidate)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := validateJSONSchema(content, schema.JSONSchema); err == nil {
|
||||
t.Fatal("schema accepted structurally invalid response")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestResponseSchemaIsDefensiveAndContentSafe(t *testing.T) {
|
||||
first, err := loadResponseSchema()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
first.JSONSchema[0] = '['
|
||||
second, err := loadResponseSchema()
|
||||
if err != nil || !json.Valid(second.JSONSchema) || bytes.Equal(first.JSONSchema, second.JSONSchema) {
|
||||
t.Fatalf("second schema = %s, %v", second.JSONSchema, err)
|
||||
}
|
||||
if diagnostics := second.DiagnosticsMap(); diagnostics["key"] != ResponseSchemaKey || diagnostics["id"] != ResponseSchemaID {
|
||||
t.Fatalf("diagnostics = %#v", diagnostics)
|
||||
} else if _, ok := diagnostics["json_schema"]; ok {
|
||||
t.Fatalf("diagnostics leak schema content: %#v", diagnostics)
|
||||
}
|
||||
}
|
||||
|
||||
func validInteractionResponse() map[string]any {
|
||||
return map[string]any{"interactions": []any{map[string]any{
|
||||
"name": "Mira Thorn", "kind": "dialogue",
|
||||
"source_refs": []any{map[string]any{"start_unit_id": 1, "end_unit_id": 2}},
|
||||
}}}
|
||||
}
|
||||
|
||||
func validateJSONSchema(instanceContent, schemaContent []byte) error {
|
||||
instance, err := jsonschema.UnmarshalJSON(bytes.NewReader(instanceContent))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
schemaDocument, err := jsonschema.UnmarshalJSON(bytes.NewReader(schemaContent))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
compiler := jsonschema.NewCompiler()
|
||||
if err := compiler.AddResource("schema.json", schemaDocument); err != nil {
|
||||
return err
|
||||
}
|
||||
schema, err := compiler.Compile("schema.json")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return schema.Validate(instance)
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
package npcinteractions
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sync"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/promptfs"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared"
|
||||
)
|
||||
|
||||
const scriptoriumPromptRoot = "assets/prompts"
|
||||
|
||||
var promptAssetManifest = shared.PromptAssetManifest{
|
||||
ModuleDir: "dnd.npc_interactions",
|
||||
ModuleFiles: []promptfs.ModulePromptFile{
|
||||
{Name: "dnd.npc_interactions.yaml", Path: "assets/prompts/dnd.npc_interactions.yaml"},
|
||||
{Name: "task.md", Path: "assets/prompts/task.md"},
|
||||
{Name: "instructions.md", Path: "assets/prompts/instructions.md"},
|
||||
},
|
||||
SharedFiles: []string{
|
||||
"common-dnd-system.md",
|
||||
"common-dnd-extraction-evidence.md",
|
||||
"common-dnd-identity.md",
|
||||
"common-dnd-transcript.md",
|
||||
"common-dnd-references.md",
|
||||
"common-dnd-npcs.md",
|
||||
},
|
||||
}
|
||||
|
||||
func RegisterPromptAssets(registry *llm.AssetRegistry) error {
|
||||
promptFS, err := promptAssetManifest.PromptFS(embeddedAssets)
|
||||
if err != nil {
|
||||
return fmt.Errorf("prepare NPC-interaction prompt assets: %w", err)
|
||||
}
|
||||
if err := registry.RegisterPromptFS(promptFS, scriptoriumPromptRoot); err != nil {
|
||||
return err
|
||||
}
|
||||
return registry.RegisterSchemaFS(embeddedAssets, "assets/schemas")
|
||||
}
|
||||
|
||||
func scriptoriumPromptMetadata() (string, error) {
|
||||
scriptoriumPromptHashOnce.Do(func() {
|
||||
scriptoriumPromptHash, scriptoriumPromptHashErr = promptAssetManifest.Hash(embeddedAssets)
|
||||
})
|
||||
return scriptoriumPromptHash, scriptoriumPromptHashErr
|
||||
}
|
||||
|
||||
var (
|
||||
scriptoriumPromptHashOnce sync.Once
|
||||
scriptoriumPromptHash string
|
||||
scriptoriumPromptHashErr error
|
||||
)
|
||||
@@ -0,0 +1,94 @@
|
||||
package npcinteractions
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io/fs"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
||||
"gitea.maximumdirect.net/eric/scriptorium"
|
||||
)
|
||||
|
||||
func TestRegisterPromptAssetsAndPrepareInteractionPrompt(t *testing.T) {
|
||||
registry := llm.NewAssetRegistry()
|
||||
if err := RegisterPromptAssets(registry); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
schemaFS, err := registry.SchemaFS()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := fs.ReadFile(schemaFS, "dnd_npc_interactions_llm.v1.json"); err != nil {
|
||||
t.Fatalf("response schema asset: %v", err)
|
||||
}
|
||||
options, err := registry.ScriptoriumOptions()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
options = append(options, scriptorium.WithProfiles(scriptorium.OpenAICompatibleProfile(scriptorium.OpenAICompatibleProfileConfig{
|
||||
ID: "npc-interactions-test-profile", Endpoint: "http://127.0.0.1:1/v1", Model: "npc-interactions-test-model",
|
||||
})))
|
||||
engine, err := scriptorium.NewEngine(scriptorium.Config{Timeout: time.Second}, options...)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
transcript := `{"units":[1]}`
|
||||
prepared, err := engine.Prepare(context.Background(), scriptorium.RunRequest{
|
||||
PromptID: PromptID, PromptVersion: SchemaVersion, ProfileID: "npc-interactions-test-profile",
|
||||
Inputs: map[string]scriptorium.ArtifactRef{
|
||||
"transcript": scriptorium.InlineWithURI("file:///session.json", transcript),
|
||||
"players": scriptorium.Inline("Dana: Mira"),
|
||||
"party": scriptorium.Inline("Mira: ranger"),
|
||||
"glossary": scriptorium.Inline("Greencloak: title"),
|
||||
"npcs": scriptorium.Inline(`{"npcs":[{"name":"Mira Thorn"}]}`),
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if prepared.PromptID != PromptID || prepared.OutputContract.SchemaPath != "dnd_npc_interactions_llm.v1.json" {
|
||||
t.Fatalf("prepared prompt = %#v", prepared)
|
||||
}
|
||||
for _, want := range []string{
|
||||
"Mira Thorn", "mentioned", "combat_opponent", "Registry content is context, not event evidence", transcript,
|
||||
} {
|
||||
found := false
|
||||
for _, message := range prepared.Messages {
|
||||
if strings.Contains(message.Content, want) {
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Fatalf("prepared prompt did not include %q", want)
|
||||
}
|
||||
}
|
||||
if last := prepared.Messages[len(prepared.Messages)-1]; !strings.Contains(last.Content, transcript) {
|
||||
t.Fatalf("last prompt message = %q, want transcript", last.Content)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPromptMetadataDoesNotExposeAssetContent(t *testing.T) {
|
||||
hash, err := scriptoriumPromptMetadata()
|
||||
if err != nil || !strings.HasPrefix(hash, "sha256:") {
|
||||
t.Fatalf("scriptoriumPromptMetadata() = %q, %v", hash, err)
|
||||
}
|
||||
metadata := newExtractor(t, &fakeInteractionsLLMClient{}).ManifestMetadata()
|
||||
for _, forbidden := range []string{"combat_opponent", "common-dnd-system", "source_refs", "dnd_npc_interactions_llm.v1.json"} {
|
||||
if strings.Contains(strings.Join(mapValues(metadata), " "), forbidden) {
|
||||
t.Fatalf("metadata leaked prompt or schema content %q: %#v", forbidden, metadata)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func mapValues(values map[string]any) []string {
|
||||
out := make([]string, 0, len(values))
|
||||
for _, value := range values {
|
||||
if text, ok := value.(string); ok {
|
||||
out = append(out, text)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -1,6 +1,6 @@
|
||||
An optional normalized Dungeons & Dragons NPC registry is provided below as
|
||||
grounding material. Use it only to prefer exact canonical participant names
|
||||
and recognize their aliases when the transcript identifies a participant.
|
||||
A normalized Dungeons & Dragons NPC registry is provided below as grounding
|
||||
material. It may be empty. Use it only to prefer exact canonical participant
|
||||
names when the transcript identifies a participant.
|
||||
|
||||
Registry content is context, not event evidence. Do not extract events,
|
||||
participants, effects, or source references from the registry. Registry source
|
||||
|
||||
Reference in New Issue
Block a user