From 2b9d2eaeaa1cce269f0530316a70ecc7a630d2c0 Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Thu, 23 Jul 2026 13:26:27 +0000 Subject: [PATCH] Add grounded NPC interaction extractor --- .../dnd/extract/npcinteractions/assets.go | 6 + .../assets/prompts/dnd.npc_interactions.yaml | 49 +++ .../assets/prompts/instructions.md | 18 + .../npcinteractions/assets/prompts/task.md | 16 + .../schemas/dnd_npc_interactions_llm.v1.json | 41 ++ .../extract/npcinteractions/canonicalize.go | 113 ++++++ .../dnd/extract/npcinteractions/extractor.go | 225 +++++++++++ .../extract/npcinteractions/extractor_test.go | 370 ++++++++++++++++++ .../dnd/extract/npcinteractions/model.go | 16 + .../dnd/extract/npcinteractions/model_test.go | 18 + .../dnd/extract/npcinteractions/schema.go | 21 + .../extract/npcinteractions/schema_test.go | 106 +++++ .../npcinteractions/scriptorium_assets.go | 53 +++ .../scriptorium_assets_test.go | 94 +++++ .../shared/assets/prompts/common-dnd-npcs.md | 6 +- 15 files changed, 1149 insertions(+), 3 deletions(-) create mode 100644 internal/modules/dnd/extract/npcinteractions/assets.go create mode 100644 internal/modules/dnd/extract/npcinteractions/assets/prompts/dnd.npc_interactions.yaml create mode 100644 internal/modules/dnd/extract/npcinteractions/assets/prompts/instructions.md create mode 100644 internal/modules/dnd/extract/npcinteractions/assets/prompts/task.md create mode 100644 internal/modules/dnd/extract/npcinteractions/assets/schemas/dnd_npc_interactions_llm.v1.json create mode 100644 internal/modules/dnd/extract/npcinteractions/canonicalize.go create mode 100644 internal/modules/dnd/extract/npcinteractions/extractor.go create mode 100644 internal/modules/dnd/extract/npcinteractions/extractor_test.go create mode 100644 internal/modules/dnd/extract/npcinteractions/model.go create mode 100644 internal/modules/dnd/extract/npcinteractions/model_test.go create mode 100644 internal/modules/dnd/extract/npcinteractions/schema.go create mode 100644 internal/modules/dnd/extract/npcinteractions/schema_test.go create mode 100644 internal/modules/dnd/extract/npcinteractions/scriptorium_assets.go create mode 100644 internal/modules/dnd/extract/npcinteractions/scriptorium_assets_test.go diff --git a/internal/modules/dnd/extract/npcinteractions/assets.go b/internal/modules/dnd/extract/npcinteractions/assets.go new file mode 100644 index 0000000..dfde6ee --- /dev/null +++ b/internal/modules/dnd/extract/npcinteractions/assets.go @@ -0,0 +1,6 @@ +package npcinteractions + +import "embed" + +//go:embed assets/schemas/dnd_npc_interactions_llm.v1.json assets/prompts/*.yaml assets/prompts/*.md +var embeddedAssets embed.FS diff --git a/internal/modules/dnd/extract/npcinteractions/assets/prompts/dnd.npc_interactions.yaml b/internal/modules/dnd/extract/npcinteractions/assets/prompts/dnd.npc_interactions.yaml new file mode 100644 index 0000000..bd8ec3b --- /dev/null +++ b/internal/modules/dnd/extract/npcinteractions/assets/prompts/dnd.npc_interactions.yaml @@ -0,0 +1,49 @@ +id: dnd.npc_interactions +version: "v1" +default_profile: gemini-2-flash +inputs: + - name: transcript + required: true + content_type: application/json + - name: players + required: false + content_type: text/plain + - name: party + required: false + content_type: text/plain + - name: glossary + required: false + content_type: text/plain + - name: npcs + required: true + content_type: application/json +messages: + - role: system + content_file: ./sharedassets/common-dnd-system.md + - role: user + content_file: ./sharedassets/common-dnd-extraction-evidence.md + - role: user + content_file: ./sharedassets/common-dnd-identity.md + cache_control: + type: ephemeral + - role: user + content_file: ./sharedassets/common-dnd-references.md + cache_control: + type: ephemeral + - role: user + content_file: ./sharedassets/common-dnd-npcs.md + cache_control: + type: ephemeral + - role: user + content_file: ./task.md + - role: user + content_file: ./instructions.md + cache_control: + type: ephemeral + - role: user + content_file: ./sharedassets/common-dnd-transcript.md +output: + format: json + validation_mode: json_schema + schema_path: dnd_npc_interactions_llm.v1.json + repair_attempts: 0 diff --git a/internal/modules/dnd/extract/npcinteractions/assets/prompts/instructions.md b/internal/modules/dnd/extract/npcinteractions/assets/prompts/instructions.md new file mode 100644 index 0000000..b5b90be --- /dev/null +++ b/internal/modules/dnd/extract/npcinteractions/assets/prompts/instructions.md @@ -0,0 +1,18 @@ +Return the interactions array even when no interaction is established. Every +record must contain name, kind, and source_refs. Cite transcript ranges that +support both the NPC identity and the interaction kind. + +Use exactly one kind per occurrence: + +- mentioned: the NPC is referred to but is not established as present or communicating; +- noncombat_presence: the NPC is present and relevant but does not meaningfully participate in dialogue or combat; +- dialogue: the NPC speaks, responds, or is directly engaged in a meaningful non-combat exchange; +- combat_ally: the NPC actively participates in combat on the party's side; +- combat_opponent: the NPC actively participates in combat against the party; or +- other: the transcript clearly establishes a direct NPC occurrence that fits none of the preceding kinds. + +When activities overlap, active combat participation outranks dialogue, +presence, and mention; dialogue outranks noncombat presence and mention; and +noncombat presence outranks mention. Other is only for directly evidenced +activity outside those categories. Split an occurrence rather than assigning +both combat alignments. diff --git a/internal/modules/dnd/extract/npcinteractions/assets/prompts/task.md b/internal/modules/dnd/extract/npcinteractions/assets/prompts/task.md new file mode 100644 index 0000000..0feda98 --- /dev/null +++ b/internal/modules/dnd/extract/npcinteractions/assets/prompts/task.md @@ -0,0 +1,16 @@ +Extract Dungeons & Dragons NPC interaction occurrences from the supplied +transcript. + +Include an occurrence only when the transcript establishes one supplied NPC, +one interaction kind, and a coherent passage supporting both. Use only names +from the supplied NPC registry. The registry helps ground identity but never +proves that an interaction occurred. + +Do not summarize, infer relationships, sentiment, factions, motives, aliases, +or persistent state. Do not identify player characters, anonymous groups, or +invented NPCs. Return an empty interactions array when no supplied NPC has an +evidenced interaction in this transcript passage. + +Keep occurrences within this transcript chunk. Split records when an NPC's +interaction kind changes, when combat alignment changes, or when an NPC is +first mentioned and later becomes present. diff --git a/internal/modules/dnd/extract/npcinteractions/assets/schemas/dnd_npc_interactions_llm.v1.json b/internal/modules/dnd/extract/npcinteractions/assets/schemas/dnd_npc_interactions_llm.v1.json new file mode 100644 index 0000000..1b78e5e --- /dev/null +++ b/internal/modules/dnd/extract/npcinteractions/assets/schemas/dnd_npc_interactions_llm.v1.json @@ -0,0 +1,41 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "notarius.dnd.npc_interactions.llm", + "type": "object", + "additionalProperties": false, + "required": ["interactions"], + "properties": { + "interactions": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": ["name", "kind", "source_refs"], + "properties": { + "name": { + "type": "string" + }, + "kind": { + "type": "string" + }, + "source_refs": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": ["start_unit_id", "end_unit_id"], + "properties": { + "start_unit_id": { + "type": "integer" + }, + "end_unit_id": { + "type": "integer" + } + } + } + } + } + } + } + } +} diff --git a/internal/modules/dnd/extract/npcinteractions/canonicalize.go b/internal/modules/dnd/extract/npcinteractions/canonicalize.go new file mode 100644 index 0000000..d1ae7f1 --- /dev/null +++ b/internal/modules/dnd/extract/npcinteractions/canonicalize.go @@ -0,0 +1,113 @@ +package npcinteractions + +import ( + "sort" + + "gitea.maximumdirect.net/eric/notarius/internal/core/source" + "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd" +) + +func canonicalizeResponse(response *extractionResponse, doc *source.SourceDocument) { + if response == nil { + return + } + for index := range response.Interactions { + canonicalizeInteraction(&response.Interactions[index]) + } + sort.SliceStable(response.Interactions, func(i, j int) bool { + left, leftOK := earliestSourcePosition(doc, response.Interactions[i]) + right, rightOK := earliestSourcePosition(doc, response.Interactions[j]) + if leftOK != rightOK { + return leftOK + } + if !leftOK { + return false + } + return left < right + }) +} + +func canonicalizeInteraction(interaction *interactionResponse) { + if interaction == nil { + return + } + sort.SliceStable(interaction.SourceRefs, func(i, j int) bool { + left := interaction.SourceRefs[i] + right := interaction.SourceRefs[j] + if unitSortValue(left.StartUnitID) != unitSortValue(right.StartUnitID) { + return unitSortValue(left.StartUnitID) < unitSortValue(right.StartUnitID) + } + return unitSortValue(left.EndUnitID) < unitSortValue(right.EndUnitID) + }) + interaction.SourceRefs = dedupeSourceRefs(interaction.SourceRefs) +} + +func dedupeSourceRefs(refs []interactionSourceRefResponse) []interactionSourceRefResponse { + if len(refs) < 2 { + return refs + } + out := refs[:0] + var previous interactionSourceRefResponse + for index, ref := range refs { + if index > 0 && previous == ref { + continue + } + out = append(out, ref) + previous = ref + } + return out +} + +func earliestSourcePosition(doc *source.SourceDocument, interaction interactionResponse) (int, bool) { + if doc == nil { + return 0, false + } + found := false + earliest := 0 + for _, ref := range interaction.SourceRefs { + candidate := source.SourceRef{SourceID: doc.ID, StartUnitID: ref.StartUnitID, EndUnitID: ref.EndUnitID} + if err := source.ValidateRef(doc, candidate); err != nil { + continue + } + index, ok := source.UnitIndex(doc, candidate.StartUnitID) + if !ok || (found && index >= earliest) { + continue + } + earliest = index + found = true + } + return earliest, found +} + +func unitSortValue(value int) int { + if value <= 0 { + return int(^uint(0) >> 1) + } + return value +} + +func canonicalInteractionList(response extractionResponse, sourceID string) dnd.NPCInteractionList { + interactions := make([]dnd.NPCInteraction, len(response.Interactions)) + for index, interaction := range response.Interactions { + interactions[index] = dnd.NPCInteraction{ + Name: interaction.Name, + Kind: dnd.NPCInteractionKind(interaction.Kind), + SourceRefs: canonicalSourceRefs(interaction.SourceRefs, sourceID), + } + } + if response.Interactions == nil { + interactions = nil + } + return dnd.NPCInteractionList{Interactions: interactions} +} + +func canonicalSourceRefs(refs []interactionSourceRefResponse, sourceID string) []source.SourceRef { + if refs == nil { + return nil + } + out := make([]source.SourceRef, len(refs)) + for index, ref := range refs { + out[index] = source.SourceRef{SourceID: sourceID, StartUnitID: ref.StartUnitID, EndUnitID: ref.EndUnitID} + } + return out +} diff --git a/internal/modules/dnd/extract/npcinteractions/extractor.go b/internal/modules/dnd/extract/npcinteractions/extractor.go new file mode 100644 index 0000000..8bdcacd --- /dev/null +++ b/internal/modules/dnd/extract/npcinteractions/extractor.go @@ -0,0 +1,225 @@ +package npcinteractions + +import ( + "context" + "fmt" + "sort" + + "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" + "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" + "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd" + npcregistry "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/npcs/registry" + "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared" +) + +const ( + Key = "dnd/npc-interactions" + mappingPolicy = "dnd.npc_interactions.extract_mapping.v1" +) + +const ( + NPCRegistryReferenceSlot = npcregistry.ReferenceSlot + NPCRegistryMaxBytes = npcregistry.MaxBytes +) + +var requiredCapabilities = []string{ + "chunks", + "source.transcript", +} + +var providedCapabilities = []string{ + "dnd.npc_interactions", +} + +var referenceSlotDescriptions = shared.ReferenceSlotDescriptions{ + Glossary: "Optional campaign glossary reference material used only for interaction disambiguation.", + Party: "Optional party roster reference material used only for interaction disambiguation.", + Players: "Optional player list reference material used only for interaction disambiguation.", + Roster: "Deprecated alias for party roster reference material used only for interaction disambiguation.", +} + +func referenceSlots() []contracts.ReferenceSlot { + slots := shared.ReferenceSlots(referenceSlotDescriptions) + slots = append(slots, contracts.ReferenceSlot{ + Name: NPCRegistryReferenceSlot, + Description: "Required normalized NPC registry used only for interaction identity grounding, never as interaction evidence.", + Required: true, + AcceptedMediaTypes: []string{"application/json"}, + AcceptedArtifactKinds: []contracts.ArtifactKind{dnd.NPCListKind}, + MaxBytes: NPCRegistryMaxBytes, + }) + sort.Slice(slots, func(i, j int) bool { return slots[i].Name < slots[j].Name }) + return slots +} + +var _ contracts.Extractor[dnd.NPCInteractionList] = (*Extractor)(nil) +var _ contracts.ManifestMetadataProvider = (*Extractor)(nil) +var _ pipeline.CheckpointFingerprintProvider = (*Extractor)(nil) + +type Options struct{} + +type Extractor struct { + llm contracts.StructuredLLMClient + npcResolver *npcregistry.Resolver + promptSHA string + responseSchemaSHA string +} + +func New(llmClient contracts.StructuredLLMClient, _ Options, references ...contracts.ReferenceSet) (*Extractor, error) { + if llmClient == nil { + return nil, extractorErrorf("LLM client must not be nil") + } + if len(references) > 1 { + return nil, extractorErrorf("at most one reference set may be supplied") + } + var referenceSet contracts.ReferenceSet + if len(references) == 1 { + referenceSet = references[0] + } + npcResolver, err := npcregistry.NewResolver(referenceSet) + if err != nil { + return nil, extractorErrorf("prepare NPC registry prompt input: %w", err) + } + promptSHA, err := scriptoriumPromptMetadata() + if err != nil { + return nil, extractorErrorf("load prompt metadata: %w", err) + } + responseSchema, err := loadResponseSchema() + if err != nil { + return nil, extractorErrorf("load response schema: %w", err) + } + return &Extractor{ + llm: llmClient, + npcResolver: npcResolver, + promptSHA: promptSHA, + responseSchemaSHA: responseSchema.SHA256, + }, nil +} + +func (e *Extractor) Key() string { return Key } + +func (e *Extractor) ReferenceSlots() []contracts.ReferenceSlot { return referenceSlots() } + +func (e *Extractor) ManifestMetadata() map[string]any { + if e == nil { + return nil + } + metadata := map[string]any{ + "prompt_id": PromptID, + "prompt_version": SchemaVersion, + "prompt_sha256": e.promptSHA, + "mapping_policy": mappingPolicy, + "response_schema_key": string(ResponseSchemaKey), + "response_schema_id": ResponseSchemaID, + "response_schema_name": ResponseSchemaName, + "response_schema_version": SchemaVersion, + "response_schema_sha256": e.responseSchemaSHA, + } + seeded := e.npcResolver.Seeded() + if seeded.Bound() { + metadata["npc_registry_digest"] = seeded.Digest() + metadata["npc_count"] = seeded.Count() + } + return metadata +} + +func (e *Extractor) CheckpointFingerprints() []pipeline.CheckpointFingerprint { + if e == nil { + return nil + } + seeded := e.npcResolver.Seeded() + return []pipeline.CheckpointFingerprint{ + {Name: "prompt", Value: e.promptSHA}, + {Name: "response_schema", Value: e.responseSchemaSHA}, + {Name: "mapping_policy", Value: mappingPolicy}, + {Name: "npc_registry", Value: seeded.ProjectionDigest()}, + } +} + +func (e *Extractor) Extract(ctx context.Context, req contracts.TypedExtractionRequest) (contracts.TypedExtractionResult[dnd.NPCInteractionList], error) { + if e == nil { + return contracts.TypedExtractionResult[dnd.NPCInteractionList]{}, extractorErrorf("extractor must not be nil") + } + if e.llm == nil { + return contracts.TypedExtractionResult[dnd.NPCInteractionList]{}, extractorErrorf("LLM client must not be nil") + } + if ctx == nil { + return contracts.TypedExtractionResult[dnd.NPCInteractionList]{}, extractorErrorf("context must not be nil") + } + if err := ctx.Err(); err != nil { + return contracts.TypedExtractionResult[dnd.NPCInteractionList]{}, extractorErrorf("context error before extraction: %w", err) + } + if req.Source == nil { + return contracts.TypedExtractionResult[dnd.NPCInteractionList]{}, extractorErrorf("source must not be nil") + } + if req.Chunk == nil { + return contracts.TypedExtractionResult[dnd.NPCInteractionList]{}, extractorErrorf("chunk must not be nil") + } + if len(req.Chunk.Units) == 0 { + return contracts.TypedExtractionResult[dnd.NPCInteractionList]{}, extractorErrorf("chunk %q units must not be empty", req.Chunk.ID) + } + sourceInput, err := shared.ChunkPromptMaterial(req) + if err != nil { + return contracts.TypedExtractionResult[dnd.NPCInteractionList]{}, extractorErrorf("%w", err) + } + npcRegistry, err := e.npcResolver.Resolve(req.References) + if err != nil { + return contracts.TypedExtractionResult[dnd.NPCInteractionList]{}, extractorErrorf("resolve NPC registry: %w", err) + } + if !npcRegistry.Bound() { + return contracts.TypedExtractionResult[dnd.NPCInteractionList]{}, extractorErrorf("NPC registry reference is required") + } + + var response extractionResponse + inputs := shared.PromptInputs(sourceInput, req.References) + inputs[NPCRegistryReferenceSlot] = npcRegistry.PromptInput() + if _, err := e.llm.CompleteStructured(ctx, contracts.StructuredCompletionRequest{ + StageName: Key, + PromptID: PromptID, + PromptVersion: SchemaVersion, + ProfileID: req.LLMProfile, + SessionID: req.SessionID, + Inputs: inputs, + }, &response); err != nil { + return contracts.TypedExtractionResult[dnd.NPCInteractionList]{}, extractorErrorf("complete structured output: %w", err) + } + canonicalizeResponse(&response, req.Source) + return contracts.TypedExtractionResult[dnd.NPCInteractionList]{Value: canonicalInteractionList(response, req.Source.ID)}, nil +} + +func ModuleSpec() pipeline.ModuleSpec { + return pipeline.ModuleSpec{ + Key: Key, + Stage: pipeline.StageExtract, + Requires: append([]string(nil), requiredCapabilities...), + Provides: append([]string(nil), providedCapabilities...), + ArtifactKind: dnd.NPCInteractionListKind, + ReferenceSlots: referenceSlots(), + } +} + +func Register(registry *pipeline.ExtractorRegistry) error { + return pipeline.RegisterExtractorBuilder(registry, ModuleSpec(), validateOptions, func(request pipeline.BuildRequest) (contracts.Extractor[dnd.NPCInteractionList], error) { + options, err := DecodeOptions(request.Options) + if err != nil { + return nil, err + } + return New(request.Dependencies.LLM, options, request.References) + }) +} + +func validateOptions(options map[string]any) error { + _, err := DecodeOptions(options) + return err +} + +func DecodeOptions(options map[string]any) (Options, error) { + if err := pipeline.RejectUnknownOptions(options); err != nil { + return Options{}, extractorErrorf("%w", err) + } + return Options{}, nil +} + +func extractorErrorf(format string, args ...any) error { + return fmt.Errorf("dnd NPC interactions extractor: "+format, args...) +} diff --git a/internal/modules/dnd/extract/npcinteractions/extractor_test.go b/internal/modules/dnd/extract/npcinteractions/extractor_test.go new file mode 100644 index 0000000..b06c2fc --- /dev/null +++ b/internal/modules/dnd/extract/npcinteractions/extractor_test.go @@ -0,0 +1,370 @@ +package npcinteractions + +import ( + "context" + "encoding/json" + "errors" + "reflect" + "strings" + "testing" + + "gitea.maximumdirect.net/eric/notarius/internal/core/source" + "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" + "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" + "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd" + npccodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/npcs" + "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/npcs/identity" +) + +func TestExtractMapsEveryKindAndOrdersBySourcePosition(t *testing.T) { + client := &fakeInteractionsLLMClient{response: extractionResponse{Interactions: []interactionResponse{ + {Name: "Other", Kind: "other", SourceRefs: interactionRefs(30, 30)}, + {Name: "Opponent", Kind: "combat_opponent", SourceRefs: interactionRefs(20, 20)}, + {Name: "Ally", Kind: "combat_ally", SourceRefs: interactionRefs(5, 5)}, + {Name: "Speaker", Kind: "dialogue", SourceRefs: append(interactionRefs(2, 2), interactionRefs(2, 2)...)}, + {Name: "Present", Kind: "noncombat_presence", SourceRefs: interactionRefs(7, 7)}, + {Name: "Mentioned", Kind: "mentioned", SourceRefs: interactionRefs(10, 10)}, + {Name: "Invalid", Kind: "unsupported", SourceRefs: interactionRefs(0, 0)}, + }}} + references := requiredRegistryReferences(t, "Mentioned", "Speaker", "Present", "Ally", "Opponent", "Other", "Invalid") + req := extractionRequest() + req.References = references + + result, err := newExtractor(t, client, references).Extract(context.Background(), req) + if err != nil { + t.Fatalf("Extract() error = %v", err) + } + if got := interactionNames(result.Value); !reflect.DeepEqual(got, []string{"Mentioned", "Speaker", "Present", "Ally", "Opponent", "Other", "Invalid"}) { + t.Fatalf("interaction order = %#v", got) + } + if got := interactionKinds(result.Value); !reflect.DeepEqual(got, []dnd.NPCInteractionKind{ + dnd.NPCInteractionKindMentioned, + dnd.NPCInteractionKindDialogue, + dnd.NPCInteractionKindNoncombatPresence, + dnd.NPCInteractionKindCombatAlly, + dnd.NPCInteractionKindCombatOpponent, + dnd.NPCInteractionKindOther, + "unsupported", + }) { + t.Fatalf("interaction kinds = %#v", got) + } + if refs := result.Value.Interactions[1].SourceRefs; !reflect.DeepEqual(refs, []source.SourceRef{{SourceID: "session-alpha", StartUnitID: 2, EndUnitID: 2}}) { + t.Fatalf("canonical source refs = %#v", refs) + } + if invalid := result.Value.Interactions[6]; invalid.Name != "Invalid" || invalid.Kind != "unsupported" || !reflect.DeepEqual(invalid.SourceRefs, []source.SourceRef{{SourceID: "session-alpha"}}) { + t.Fatalf("invalid candidate = %#v, want preserved values with current source identity", invalid) + } + if len(client.requests) != 1 { + t.Fatalf("LLM calls = %d, want 1", len(client.requests)) + } +} + +func TestNewRequiresLLMAndRejectsAmbiguousReferenceSets(t *testing.T) { + if _, err := New(nil, Options{}); err == nil || !strings.Contains(err.Error(), "LLM client") { + t.Fatalf("New(nil) error = %v", err) + } + if _, err := New(&fakeInteractionsLLMClient{}, Options{}, contracts.ReferenceSet{}, contracts.ReferenceSet{}); err == nil || !strings.Contains(err.Error(), "at most one reference set") { + t.Fatalf("New() error = %v", err) + } +} + +func TestExtractUsesNamesOnlyRegistryAndCurrentTranscriptEvidence(t *testing.T) { + client := &fakeInteractionsLLMClient{response: extractionResponse{Interactions: []interactionResponse{{ + Name: "Mira Thorn", Kind: "dialogue", SourceRefs: interactionRefs(10, 10), + }}}} + references := requiredRegistryReferences(t, "Mira Thorn", "Hooded Guard") + req := extractionRequest() + req.References = references + if _, err := newExtractor(t, client, references).Extract(context.Background(), req); err != nil { + t.Fatalf("Extract() error = %v", err) + } + request := client.requests[0] + registry := request.Inputs[NPCRegistryReferenceSlot] + if registry.Name != NPCRegistryReferenceSlot || registry.MediaType != "application/json" || string(registry.Content) != `{"npcs":[{"name":"Mira Thorn"},{"name":"Hooded Guard"}]}` { + t.Fatalf("registry prompt input = %#v, want exact names-only projection", registry) + } + for _, forbidden := range []string{"npc:sha256:", "other-session", "start_unit_id"} { + if strings.Contains(string(registry.Content), forbidden) { + t.Fatalf("registry prompt input leaked %q: %s", forbidden, registry.Content) + } + } + if strings.Contains(string(request.Inputs["transcript"].Content), "other-session") { + t.Fatal("transcript input contains registry evidence") + } + metadata, err := json.Marshal(newExtractor(t, &fakeInteractionsLLMClient{}, references).ManifestMetadata()) + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(metadata), "Mira Thorn") || strings.Contains(string(metadata), "other-session") { + t.Fatalf("manifest metadata leaked registry content: %s", metadata) + } +} + +func TestExtractRequiresBoundRegistryBeforeLLMCall(t *testing.T) { + client := &fakeInteractionsLLMClient{} + if _, err := newExtractor(t, client).Extract(context.Background(), extractionRequest()); err == nil || !strings.Contains(err.Error(), "NPC registry reference is required") { + t.Fatalf("Extract() error = %v, want required registry failure", err) + } + if len(client.requests) != 0 { + t.Fatalf("LLM calls = %d, want no call", len(client.requests)) + } + + malformed := contracts.ReferenceSet{Slots: map[string]contracts.ResolvedReferenceSlot{ + NPCRegistryReferenceSlot: { + Slot: contracts.ReferenceSlot{Name: NPCRegistryReferenceSlot}, + Items: []contracts.ReferenceItem{{SlotName: NPCRegistryReferenceSlot, MediaType: "application/json", Content: []byte(`{"secret":"private source"}`)}}, + }, + }} + if _, err := New(client, Options{}, malformed); err == nil || !strings.Contains(err.Error(), "prepare NPC registry") || strings.Contains(err.Error(), "private source") { + t.Fatalf("New() error = %v, want content-safe malformed registry error", err) + } + if len(client.requests) != 0 { + t.Fatalf("LLM calls = %d, want no call", len(client.requests)) + } +} + +func TestExtractResolvesGeneratedRegistryAtOperationTime(t *testing.T) { + client := &fakeInteractionsLLMClient{response: extractionResponse{Interactions: []interactionResponse{}}} + references := requiredRegistryReferences(t, "Mira Thorn") + req := extractionRequest() + req.References = references + extractor := newExtractor(t, client) + if _, err := extractor.Extract(context.Background(), req); err != nil { + t.Fatalf("Extract() error = %v", err) + } + if input := client.requests[0].Inputs[NPCRegistryReferenceSlot]; string(input.Content) != `{"npcs":[{"name":"Mira Thorn"}]}` || input.OriginURI != "" { + t.Fatalf("generated registry prompt input = %#v", input) + } + metadata := extractor.ManifestMetadata() + if _, ok := metadata["npc_registry_digest"]; ok { + t.Fatalf("operation-time registry leaked into static metadata: %#v", metadata) + } +} + +func TestExtractAcceptsEmptyBoundRegistryAndEmptyResponse(t *testing.T) { + content, err := npccodec.New().Encode(dnd.NPCList{NPCs: []dnd.NPC{}}) + if err != nil { + t.Fatal(err) + } + references := contracts.ReferenceSet{Slots: map[string]contracts.ResolvedReferenceSlot{ + NPCRegistryReferenceSlot: { + Slot: contracts.ReferenceSlot{Name: NPCRegistryReferenceSlot}, + Items: []contracts.ReferenceItem{{SlotName: NPCRegistryReferenceSlot, MediaType: npccodec.MediaType, Content: content}}, + }, + }} + client := &fakeInteractionsLLMClient{response: extractionResponse{Interactions: []interactionResponse{}}} + req := extractionRequest() + req.References = references + result, err := newExtractor(t, client, references).Extract(context.Background(), req) + if err != nil || result.Value.Interactions == nil || len(result.Value.Interactions) != 0 { + t.Fatalf("Extract() = %#v, %v; want present empty interactions", result, err) + } +} + +func TestExtractRejectsInvalidRequestsAndProviderFailures(t *testing.T) { + references := requiredRegistryReferences(t, "Mira Thorn") + valid := extractionRequest() + valid.References = references + canceled, cancel := context.WithCancel(context.Background()) + cancel() + extractor := newExtractor(t, &fakeInteractionsLLMClient{}, references) + for _, test := range []struct { + name string + ctx context.Context + req contracts.TypedExtractionRequest + want string + }{ + {"nil context", nil, valid, "context"}, + {"canceled context", canceled, valid, "context"}, + {"nil source", context.Background(), contracts.TypedExtractionRequest{Chunk: valid.Chunk, References: references}, "source"}, + {"nil chunk", context.Background(), contracts.TypedExtractionRequest{Source: valid.Source, References: references}, "chunk"}, + {"empty chunk", context.Background(), emptyChunkRequest(valid), "units"}, + {"source input mismatch", context.Background(), mismatchedSourceInputRequest(valid), "must match chunk"}, + } { + t.Run(test.name, func(t *testing.T) { + if _, err := extractor.Extract(test.ctx, test.req); err == nil || !strings.Contains(err.Error(), test.want) { + t.Fatalf("Extract() error = %v, want %q", err, test.want) + } + }) + } + if _, err := newExtractor(t, &fakeInteractionsLLMClient{err: errors.New("provider unavailable")}, references).Extract(context.Background(), valid); err == nil || !strings.Contains(err.Error(), "provider unavailable") { + t.Fatalf("provider error = %v", err) + } +} + +func TestModuleSpecRegistrationMetadataAndFingerprints(t *testing.T) { + got := ModuleSpec() + if got.Key != Key || got.Stage != pipeline.StageExtract || got.ArtifactKind != dnd.NPCInteractionListKind || !reflect.DeepEqual(got.Requires, []string{"chunks", "source.transcript"}) || !reflect.DeepEqual(got.Provides, []string{"dnd.npc_interactions"}) { + t.Fatalf("ModuleSpec() = %#v", got) + } + var registrySlot contracts.ReferenceSlot + for _, slot := range got.ReferenceSlots { + if slot.Name == NPCRegistryReferenceSlot { + registrySlot = slot + } + } + if !registrySlot.Required || !reflect.DeepEqual(registrySlot.AcceptedArtifactKinds, []contracts.ArtifactKind{dnd.NPCListKind}) || registrySlot.MaxBytes != NPCRegistryMaxBytes { + t.Fatalf("NPC registry slot = %#v", registrySlot) + } + got.ReferenceSlots[0].AcceptedMediaTypes[0] = "changed" + if ModuleSpec().ReferenceSlots[0].AcceptedMediaTypes[0] == "changed" { + t.Fatal("ModuleSpec() returned mutable reference slots") + } + extractorSlots := newExtractor(t, &fakeInteractionsLLMClient{}).ReferenceSlots() + extractorSlots[0].AcceptedMediaTypes[0] = "changed" + if newExtractor(t, &fakeInteractionsLLMClient{}).ReferenceSlots()[0].AcceptedMediaTypes[0] == "changed" { + t.Fatal("ReferenceSlots() returned mutable reference slots") + } + registry := pipeline.NewExtractorRegistry() + if err := Register(registry); err != nil { + t.Fatal(err) + } + if _, ok := registry.Spec(Key); !ok { + t.Fatalf("registry missing %q", Key) + } + if _, err := DecodeOptions(map[string]any{"unexpected": true}); err == nil { + t.Fatal("DecodeOptions() accepted unknown option") + } + + references := requiredRegistryReferences(t, "Mira Thorn") + extractor := newExtractor(t, &fakeInteractionsLLMClient{}, references) + metadata := extractor.ManifestMetadata() + for key, want := range map[string]string{ + "prompt_id": PromptID, "prompt_version": SchemaVersion, "mapping_policy": mappingPolicy, + "response_schema_key": string(ResponseSchemaKey), "response_schema_id": ResponseSchemaID, + "response_schema_name": ResponseSchemaName, "response_schema_version": SchemaVersion, + } { + if metadata[key] != want { + t.Fatalf("metadata[%q] = %#v, want %q", key, metadata[key], want) + } + } + for _, key := range []string{"prompt_sha256", "response_schema_sha256", "npc_registry_digest"} { + if value, ok := metadata[key].(string); !ok || !strings.HasPrefix(value, "sha256:") { + t.Fatalf("metadata[%q] = %#v, want digest", key, metadata[key]) + } + } + wantFingerprints := map[string]struct{}{"prompt": {}, "response_schema": {}, "mapping_policy": {}, "npc_registry": {}} + for _, fingerprint := range extractor.CheckpointFingerprints() { + delete(wantFingerprints, fingerprint.Name) + } + if len(wantFingerprints) != 0 { + t.Fatalf("missing fingerprints = %#v", wantFingerprints) + } +} + +func extractionRequest() contracts.TypedExtractionRequest { + doc := sourceDocument() + chunk := &source.Chunk{ + ID: "session-alpha:chunk:0", SourceID: doc.ID, Index: 0, + Ref: source.SourceRef{SourceID: doc.ID, StartUnitID: 10, EndUnitID: 30}, + Content: []byte(`{"units":[10,2,7,5,20,30]}`), MediaType: "application/json", + Units: append([]source.SourceUnit(nil), doc.Units...), + } + return contracts.TypedExtractionRequest{ + Source: doc, Chunk: chunk, + SourceInput: contracts.NewLLMInputMaterial("source", chunk.MediaType, chunk.Content, "sha256:chunk", "file:///session-alpha.json"), + SessionID: "session-123", LLMProfile: "profile-npc-interactions", + } +} + +func sourceDocument() *source.SourceDocument { + return &source.SourceDocument{ + ID: "session-alpha", Kind: "transcript", Format: "application/vnd.seriatim.minimal+json", Digest: "sha256:source", + Units: []source.SourceUnit{ + {ID: 10, Kind: "transcript_segment", Text: "Mira Thorn is mentioned."}, + {ID: 2, Kind: "transcript_segment", Text: "Mira Thorn speaks to the party."}, + {ID: 7, Kind: "transcript_segment", Text: "Mira Thorn watches nearby."}, + {ID: 5, Kind: "transcript_segment", Text: "Mira Thorn joins the party in combat."}, + {ID: 20, Kind: "transcript_segment", Text: "Mira Thorn attacks the party."}, + {ID: 30, Kind: "transcript_segment", Text: "Mira Thorn performs a ritual."}, + }, + } +} + +func requiredRegistryReferences(t *testing.T, names ...string) contracts.ReferenceSet { + t.Helper() + npcs := make([]dnd.NPC, len(names)) + for index, name := range names { + npcs[index] = dnd.NPC{ + ID: identity.DeriveID(name), Name: name, + SourceRefs: []source.SourceRef{{SourceID: "other-session", StartUnitID: index + 1, EndUnitID: index + 1}}, + } + } + content, err := npccodec.New().Encode(dnd.NPCList{NPCs: npcs}) + if err != nil { + t.Fatal(err) + } + return contracts.ReferenceSet{Slots: map[string]contracts.ResolvedReferenceSlot{ + NPCRegistryReferenceSlot: { + Slot: contracts.ReferenceSlot{Name: NPCRegistryReferenceSlot}, + Items: []contracts.ReferenceItem{{SlotName: NPCRegistryReferenceSlot, MediaType: npccodec.MediaType, Content: content, Origin: contracts.ReferenceOrigin{Type: "generated"}}}, + }, + }} +} + +func interactionRefs(start, end int) []interactionSourceRefResponse { + return []interactionSourceRefResponse{{StartUnitID: start, EndUnitID: end}} +} + +func interactionNames(value dnd.NPCInteractionList) []string { + names := make([]string, len(value.Interactions)) + for index, interaction := range value.Interactions { + names[index] = interaction.Name + } + return names +} + +func interactionKinds(value dnd.NPCInteractionList) []dnd.NPCInteractionKind { + kinds := make([]dnd.NPCInteractionKind, len(value.Interactions)) + for index, interaction := range value.Interactions { + kinds[index] = interaction.Kind + } + return kinds +} + +func emptyChunkRequest(req contracts.TypedExtractionRequest) contracts.TypedExtractionRequest { + req.Chunk = &source.Chunk{ID: req.Chunk.ID, SourceID: req.Chunk.SourceID, Index: req.Chunk.Index} + return req +} + +func mismatchedSourceInputRequest(req contracts.TypedExtractionRequest) contracts.TypedExtractionRequest { + req.SourceInput = contracts.NewLLMInputMaterial("source", "application/json", []byte(`{"different":true}`), "sha256:other", "") + return req +} + +func newExtractor(t *testing.T, client contracts.StructuredLLMClient, references ...contracts.ReferenceSet) *Extractor { + t.Helper() + extractor, err := New(client, Options{}, references...) + if err != nil { + t.Fatalf("New() error = %v", err) + } + return extractor +} + +type fakeInteractionsLLMClient struct { + response extractionResponse + err error + requests []contracts.StructuredCompletionRequest +} + +func (client *fakeInteractionsLLMClient) CompleteStructured(_ context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) { + client.requests = append(client.requests, cloneStructuredCompletionRequest(req)) + if client.err != nil { + return contracts.StructuredCompletionResponse{}, client.err + } + target, ok := out.(*extractionResponse) + if !ok { + return contracts.StructuredCompletionResponse{}, errors.New("unexpected output target") + } + *target = client.response + content, err := json.Marshal(client.response) + if err != nil { + return contracts.StructuredCompletionResponse{}, err + } + return contracts.StructuredCompletionResponse{Content: content}, nil +} + +func cloneStructuredCompletionRequest(req contracts.StructuredCompletionRequest) contracts.StructuredCompletionRequest { + req.Inputs = req.Inputs.Clone() + return req +} diff --git a/internal/modules/dnd/extract/npcinteractions/model.go b/internal/modules/dnd/extract/npcinteractions/model.go new file mode 100644 index 0000000..2e33b86 --- /dev/null +++ b/internal/modules/dnd/extract/npcinteractions/model.go @@ -0,0 +1,16 @@ +package npcinteractions + +type extractionResponse struct { + Interactions []interactionResponse `json:"interactions"` +} + +type interactionResponse struct { + Name string `json:"name"` + Kind string `json:"kind"` + SourceRefs []interactionSourceRefResponse `json:"source_refs"` +} + +type interactionSourceRefResponse struct { + StartUnitID int `json:"start_unit_id"` + EndUnitID int `json:"end_unit_id"` +} diff --git a/internal/modules/dnd/extract/npcinteractions/model_test.go b/internal/modules/dnd/extract/npcinteractions/model_test.go new file mode 100644 index 0000000..3968fae --- /dev/null +++ b/internal/modules/dnd/extract/npcinteractions/model_test.go @@ -0,0 +1,18 @@ +package npcinteractions + +import ( + "encoding/json" + "testing" +) + +func TestExtractionResponsePreservesValidatorOwnedSemantics(t *testing.T) { + content := []byte(`{"interactions":[{"name":"","kind":"unsupported","source_refs":[{"start_unit_id":0,"end_unit_id":-1}]}]}`) + var response extractionResponse + if err := json.Unmarshal(content, &response); err != nil { + t.Fatalf("json.Unmarshal() error = %v", err) + } + interaction := response.Interactions[0] + if interaction.Name != "" || interaction.Kind != "unsupported" || interaction.SourceRefs[0] != (interactionSourceRefResponse{StartUnitID: 0, EndUnitID: -1}) { + t.Fatalf("decoded response = %#v", interaction) + } +} diff --git a/internal/modules/dnd/extract/npcinteractions/schema.go b/internal/modules/dnd/extract/npcinteractions/schema.go new file mode 100644 index 0000000..dd50fb8 --- /dev/null +++ b/internal/modules/dnd/extract/npcinteractions/schema.go @@ -0,0 +1,21 @@ +package npcinteractions + +import "gitea.maximumdirect.net/eric/notarius/internal/framework/llm" + +const ( + PromptID = "dnd.npc_interactions" + ResponseSchemaKey = llm.ResponseSchemaKey("dnd_npc_interactions_llm") + ResponseSchemaID = "notarius.dnd.npc_interactions.llm" + ResponseSchemaName = "notarius_dnd_npc_interactions_llm_v1" + SchemaVersion = "v1" +) + +func loadResponseSchema() (llm.ResponseSchema, error) { + return llm.LoadResponseSchema(embeddedAssets, llm.ResponseSchemaDefinition{ + Key: ResponseSchemaKey, + ID: ResponseSchemaID, + Version: SchemaVersion, + Name: ResponseSchemaName, + AssetPath: "assets/schemas/dnd_npc_interactions_llm.v1.json", + }) +} diff --git a/internal/modules/dnd/extract/npcinteractions/schema_test.go b/internal/modules/dnd/extract/npcinteractions/schema_test.go new file mode 100644 index 0000000..e524555 --- /dev/null +++ b/internal/modules/dnd/extract/npcinteractions/schema_test.go @@ -0,0 +1,106 @@ +package npcinteractions + +import ( + "bytes" + "encoding/json" + "strings" + "testing" + + "github.com/santhosh-tekuri/jsonschema/v6" +) + +func TestResponseSchemaOwnsOnlyPrivateStructuralContract(t *testing.T) { + schema, err := loadResponseSchema() + if err != nil { + t.Fatal(err) + } + if schema.Key != ResponseSchemaKey || schema.ID != ResponseSchemaID || schema.Name != ResponseSchemaName || schema.Version != SchemaVersion || !strings.HasPrefix(schema.SHA256, "sha256:") || !json.Valid(schema.JSONSchema) { + t.Fatalf("schema = %#v", schema) + } + valid := validInteractionResponse() + content, err := json.Marshal(valid) + if err != nil { + t.Fatal(err) + } + if err := validateJSONSchema(content, schema.JSONSchema); err != nil { + t.Fatalf("valid response rejected: %v", err) + } + + semanticCandidate := validInteractionResponse() + interaction := semanticCandidate["interactions"].([]any)[0].(map[string]any) + interaction["name"] = "" + interaction["kind"] = "unsupported" + ref := interaction["source_refs"].([]any)[0].(map[string]any) + ref["start_unit_id"] = 0 + ref["end_unit_id"] = -1 + content, err = json.Marshal(semanticCandidate) + if err != nil { + t.Fatal(err) + } + if err := validateJSONSchema(content, schema.JSONSchema); err != nil { + t.Fatalf("schema rejected validator-owned semantics: %v", err) + } + + for _, mutate := range []func(map[string]any){ + func(record map[string]any) { delete(record, "name") }, + func(record map[string]any) { record["kind"] = 1 }, + func(record map[string]any) { record["unexpected"] = true }, + func(record map[string]any) { + record["source_refs"].([]any)[0].(map[string]any)["source_id"] = "assigned later" + }, + } { + candidate := validInteractionResponse() + mutate(candidate["interactions"].([]any)[0].(map[string]any)) + content, err := json.Marshal(candidate) + if err != nil { + t.Fatal(err) + } + if err := validateJSONSchema(content, schema.JSONSchema); err == nil { + t.Fatal("schema accepted structurally invalid response") + } + } +} + +func TestResponseSchemaIsDefensiveAndContentSafe(t *testing.T) { + first, err := loadResponseSchema() + if err != nil { + t.Fatal(err) + } + first.JSONSchema[0] = '[' + second, err := loadResponseSchema() + if err != nil || !json.Valid(second.JSONSchema) || bytes.Equal(first.JSONSchema, second.JSONSchema) { + t.Fatalf("second schema = %s, %v", second.JSONSchema, err) + } + if diagnostics := second.DiagnosticsMap(); diagnostics["key"] != ResponseSchemaKey || diagnostics["id"] != ResponseSchemaID { + t.Fatalf("diagnostics = %#v", diagnostics) + } else if _, ok := diagnostics["json_schema"]; ok { + t.Fatalf("diagnostics leak schema content: %#v", diagnostics) + } +} + +func validInteractionResponse() map[string]any { + return map[string]any{"interactions": []any{map[string]any{ + "name": "Mira Thorn", "kind": "dialogue", + "source_refs": []any{map[string]any{"start_unit_id": 1, "end_unit_id": 2}}, + }}} +} + +func validateJSONSchema(instanceContent, schemaContent []byte) error { + instance, err := jsonschema.UnmarshalJSON(bytes.NewReader(instanceContent)) + if err != nil { + return err + } + schemaDocument, err := jsonschema.UnmarshalJSON(bytes.NewReader(schemaContent)) + if err != nil { + return err + } + compiler := jsonschema.NewCompiler() + if err := compiler.AddResource("schema.json", schemaDocument); err != nil { + return err + } + schema, err := compiler.Compile("schema.json") + if err != nil { + return err + } + return schema.Validate(instance) +} diff --git a/internal/modules/dnd/extract/npcinteractions/scriptorium_assets.go b/internal/modules/dnd/extract/npcinteractions/scriptorium_assets.go new file mode 100644 index 0000000..821b8b4 --- /dev/null +++ b/internal/modules/dnd/extract/npcinteractions/scriptorium_assets.go @@ -0,0 +1,53 @@ +package npcinteractions + +import ( + "fmt" + "sync" + + "gitea.maximumdirect.net/eric/notarius/internal/framework/llm" + "gitea.maximumdirect.net/eric/notarius/internal/framework/promptfs" + "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared" +) + +const scriptoriumPromptRoot = "assets/prompts" + +var promptAssetManifest = shared.PromptAssetManifest{ + ModuleDir: "dnd.npc_interactions", + ModuleFiles: []promptfs.ModulePromptFile{ + {Name: "dnd.npc_interactions.yaml", Path: "assets/prompts/dnd.npc_interactions.yaml"}, + {Name: "task.md", Path: "assets/prompts/task.md"}, + {Name: "instructions.md", Path: "assets/prompts/instructions.md"}, + }, + SharedFiles: []string{ + "common-dnd-system.md", + "common-dnd-extraction-evidence.md", + "common-dnd-identity.md", + "common-dnd-transcript.md", + "common-dnd-references.md", + "common-dnd-npcs.md", + }, +} + +func RegisterPromptAssets(registry *llm.AssetRegistry) error { + promptFS, err := promptAssetManifest.PromptFS(embeddedAssets) + if err != nil { + return fmt.Errorf("prepare NPC-interaction prompt assets: %w", err) + } + if err := registry.RegisterPromptFS(promptFS, scriptoriumPromptRoot); err != nil { + return err + } + return registry.RegisterSchemaFS(embeddedAssets, "assets/schemas") +} + +func scriptoriumPromptMetadata() (string, error) { + scriptoriumPromptHashOnce.Do(func() { + scriptoriumPromptHash, scriptoriumPromptHashErr = promptAssetManifest.Hash(embeddedAssets) + }) + return scriptoriumPromptHash, scriptoriumPromptHashErr +} + +var ( + scriptoriumPromptHashOnce sync.Once + scriptoriumPromptHash string + scriptoriumPromptHashErr error +) diff --git a/internal/modules/dnd/extract/npcinteractions/scriptorium_assets_test.go b/internal/modules/dnd/extract/npcinteractions/scriptorium_assets_test.go new file mode 100644 index 0000000..135a78e --- /dev/null +++ b/internal/modules/dnd/extract/npcinteractions/scriptorium_assets_test.go @@ -0,0 +1,94 @@ +package npcinteractions + +import ( + "context" + "io/fs" + "strings" + "testing" + "time" + + "gitea.maximumdirect.net/eric/notarius/internal/framework/llm" + "gitea.maximumdirect.net/eric/scriptorium" +) + +func TestRegisterPromptAssetsAndPrepareInteractionPrompt(t *testing.T) { + registry := llm.NewAssetRegistry() + if err := RegisterPromptAssets(registry); err != nil { + t.Fatal(err) + } + schemaFS, err := registry.SchemaFS() + if err != nil { + t.Fatal(err) + } + if _, err := fs.ReadFile(schemaFS, "dnd_npc_interactions_llm.v1.json"); err != nil { + t.Fatalf("response schema asset: %v", err) + } + options, err := registry.ScriptoriumOptions() + if err != nil { + t.Fatal(err) + } + options = append(options, scriptorium.WithProfiles(scriptorium.OpenAICompatibleProfile(scriptorium.OpenAICompatibleProfileConfig{ + ID: "npc-interactions-test-profile", Endpoint: "http://127.0.0.1:1/v1", Model: "npc-interactions-test-model", + }))) + engine, err := scriptorium.NewEngine(scriptorium.Config{Timeout: time.Second}, options...) + if err != nil { + t.Fatal(err) + } + transcript := `{"units":[1]}` + prepared, err := engine.Prepare(context.Background(), scriptorium.RunRequest{ + PromptID: PromptID, PromptVersion: SchemaVersion, ProfileID: "npc-interactions-test-profile", + Inputs: map[string]scriptorium.ArtifactRef{ + "transcript": scriptorium.InlineWithURI("file:///session.json", transcript), + "players": scriptorium.Inline("Dana: Mira"), + "party": scriptorium.Inline("Mira: ranger"), + "glossary": scriptorium.Inline("Greencloak: title"), + "npcs": scriptorium.Inline(`{"npcs":[{"name":"Mira Thorn"}]}`), + }, + }) + if err != nil { + t.Fatal(err) + } + if prepared.PromptID != PromptID || prepared.OutputContract.SchemaPath != "dnd_npc_interactions_llm.v1.json" { + t.Fatalf("prepared prompt = %#v", prepared) + } + for _, want := range []string{ + "Mira Thorn", "mentioned", "combat_opponent", "Registry content is context, not event evidence", transcript, + } { + found := false + for _, message := range prepared.Messages { + if strings.Contains(message.Content, want) { + found = true + break + } + } + if !found { + t.Fatalf("prepared prompt did not include %q", want) + } + } + if last := prepared.Messages[len(prepared.Messages)-1]; !strings.Contains(last.Content, transcript) { + t.Fatalf("last prompt message = %q, want transcript", last.Content) + } +} + +func TestPromptMetadataDoesNotExposeAssetContent(t *testing.T) { + hash, err := scriptoriumPromptMetadata() + if err != nil || !strings.HasPrefix(hash, "sha256:") { + t.Fatalf("scriptoriumPromptMetadata() = %q, %v", hash, err) + } + metadata := newExtractor(t, &fakeInteractionsLLMClient{}).ManifestMetadata() + for _, forbidden := range []string{"combat_opponent", "common-dnd-system", "source_refs", "dnd_npc_interactions_llm.v1.json"} { + if strings.Contains(strings.Join(mapValues(metadata), " "), forbidden) { + t.Fatalf("metadata leaked prompt or schema content %q: %#v", forbidden, metadata) + } + } +} + +func mapValues(values map[string]any) []string { + out := make([]string, 0, len(values)) + for _, value := range values { + if text, ok := value.(string); ok { + out = append(out, text) + } + } + return out +} diff --git a/internal/modules/dnd/shared/assets/prompts/common-dnd-npcs.md b/internal/modules/dnd/shared/assets/prompts/common-dnd-npcs.md index 403f9db..317f9cb 100644 --- a/internal/modules/dnd/shared/assets/prompts/common-dnd-npcs.md +++ b/internal/modules/dnd/shared/assets/prompts/common-dnd-npcs.md @@ -1,6 +1,6 @@ -An optional normalized Dungeons & Dragons NPC registry is provided below as -grounding material. Use it only to prefer exact canonical participant names -and recognize their aliases when the transcript identifies a participant. +A normalized Dungeons & Dragons NPC registry is provided below as grounding +material. It may be empty. Use it only to prefer exact canonical participant +names when the transcript identifies a participant. Registry content is context, not event evidence. Do not extract events, participants, effects, or source references from the registry. Registry source