Add D&D item event extractor
This commit is contained in:
@@ -103,6 +103,7 @@ func TestCodecRejectsStrictJSONAndApprovedBoundaries(t *testing.T) {
|
|||||||
{"party transfer", strings.Replace(validJSON, `"kind":"acquired","to":"party"`, `"kind":"transferred","from":"party","to":"Borin"`, 1), "holders are incompatible"},
|
{"party transfer", strings.Replace(validJSON, `"kind":"acquired","to":"party"`, `"kind":"transferred","from":"party","to":"Borin"`, 1), "holders are incompatible"},
|
||||||
{"self transfer", strings.Replace(validJSON, `"kind":"acquired","to":"party"`, `"kind":"transferred","from":"Aria","to":"aria"`, 1), "holders are incompatible"},
|
{"self transfer", strings.Replace(validJSON, `"kind":"acquired","to":"party"`, `"kind":"transferred","from":"Aria","to":"aria"`, 1), "holders are incompatible"},
|
||||||
{"missing source refs", strings.Replace(validJSON, `,"source_refs":[{"source_id":"session","start_unit_id":1,"end_unit_id":1}]`, "", 1), "source_refs must contain"},
|
{"missing source refs", strings.Replace(validJSON, `,"source_refs":[{"source_id":"session","start_unit_id":1,"end_unit_id":1}]`, "", 1), "source_refs must contain"},
|
||||||
|
{"empty source refs", strings.Replace(validJSON, `[{"source_id":"session","start_unit_id":1,"end_unit_id":1}]`, `[]`, 1), "source_refs must contain"},
|
||||||
}
|
}
|
||||||
for _, test := range tests {
|
for _, test := range tests {
|
||||||
t.Run(test.name, func(t *testing.T) {
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
|||||||
6
internal/modules/dnd/extract/itemevents/assets.go
Normal file
6
internal/modules/dnd/extract/itemevents/assets.go
Normal file
@@ -0,0 +1,6 @@
|
|||||||
|
package itemevents
|
||||||
|
|
||||||
|
import "embed"
|
||||||
|
|
||||||
|
//go:embed assets/schemas/dnd_item_events_llm.v1.json assets/prompts/*.yaml assets/prompts/*.md
|
||||||
|
var embeddedAssets embed.FS
|
||||||
@@ -0,0 +1,42 @@
|
|||||||
|
id: dnd.item_events
|
||||||
|
version: "v1"
|
||||||
|
default_profile: gemini-2-flash
|
||||||
|
inputs:
|
||||||
|
- name: transcript
|
||||||
|
required: true
|
||||||
|
content_type: application/json
|
||||||
|
- name: players
|
||||||
|
required: false
|
||||||
|
content_type: text/plain
|
||||||
|
- name: party
|
||||||
|
required: false
|
||||||
|
content_type: text/plain
|
||||||
|
- name: glossary
|
||||||
|
required: false
|
||||||
|
content_type: text/plain
|
||||||
|
messages:
|
||||||
|
- role: system
|
||||||
|
content_file: ./sharedassets/common-dnd-system.md
|
||||||
|
- role: user
|
||||||
|
content_file: ./sharedassets/common-dnd-extraction-evidence.md
|
||||||
|
- role: user
|
||||||
|
content_file: ./sharedassets/common-dnd-identity.md
|
||||||
|
cache_control:
|
||||||
|
type: ephemeral
|
||||||
|
- role: user
|
||||||
|
content_file: ./sharedassets/common-dnd-references.md
|
||||||
|
cache_control:
|
||||||
|
type: ephemeral
|
||||||
|
- role: user
|
||||||
|
content_file: ./task.md
|
||||||
|
- role: user
|
||||||
|
content_file: ./instructions.md
|
||||||
|
cache_control:
|
||||||
|
type: ephemeral
|
||||||
|
- role: user
|
||||||
|
content_file: ./sharedassets/common-dnd-transcript.md
|
||||||
|
output:
|
||||||
|
format: json
|
||||||
|
validation_mode: json_schema
|
||||||
|
schema_path: dnd_item_events_llm.v1.json
|
||||||
|
repair_attempts: 0
|
||||||
@@ -0,0 +1,22 @@
|
|||||||
|
Return one event only when the transcript establishes a meaningful item or
|
||||||
|
currency occurrence. Use a concise observed item name and preserve the stated
|
||||||
|
currency denomination; include quantity only when the transcript explicitly
|
||||||
|
states it.
|
||||||
|
|
||||||
|
Use `discovered` when the party learns of or encounters an item without
|
||||||
|
establishing possession. Use `acquired` when the party or a party member gains
|
||||||
|
possession. Use `lost` when party possession ends through a gift, sale, payment,
|
||||||
|
theft, abandonment, or destruction not caused by intended use. Use `consumed`
|
||||||
|
when intended use depletes an item or currency. Use `transferred` only when
|
||||||
|
possession moves between two distinct named party members.
|
||||||
|
|
||||||
|
For `discovered`, omit both holders. For `acquired`, provide only `to`; for
|
||||||
|
`lost` and `consumed`, provide only `from`; and for `transferred`, provide both
|
||||||
|
`from` and `to`. Use `party` only for collective or unresolved party possession,
|
||||||
|
never for either side of a transfer. Do not emit a transfer for a gift, sale, or
|
||||||
|
payment outside the party.
|
||||||
|
|
||||||
|
Ordinary non-depleting use is not an event. Do not infer acquisition from a
|
||||||
|
discovery, or discovery from an acquisition: emit both only when each is
|
||||||
|
independently established. Every event needs at least one narrow transcript
|
||||||
|
range. Return no lore, inventory totals, aliases, or unstated holder details.
|
||||||
@@ -0,0 +1,7 @@
|
|||||||
|
Extract meaningful Dungeons & Dragons item and currency events established by
|
||||||
|
the transcript. Record only discoveries and changes in party possession, with
|
||||||
|
the transcript ranges that support each event.
|
||||||
|
|
||||||
|
This is an event history, not an inventory or ledger. Do not calculate balances,
|
||||||
|
resolve item identity across records, or infer ownership that the transcript
|
||||||
|
does not establish.
|
||||||
@@ -0,0 +1,36 @@
|
|||||||
|
{
|
||||||
|
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||||
|
"$id": "notarius.dnd.item_events.llm",
|
||||||
|
"type": "object",
|
||||||
|
"additionalProperties": false,
|
||||||
|
"required": ["events"],
|
||||||
|
"properties": {
|
||||||
|
"events": {
|
||||||
|
"type": "array",
|
||||||
|
"items": {
|
||||||
|
"type": "object",
|
||||||
|
"additionalProperties": false,
|
||||||
|
"required": ["name", "kind", "source_refs"],
|
||||||
|
"properties": {
|
||||||
|
"name": {"type": "string"},
|
||||||
|
"kind": {"type": "string"},
|
||||||
|
"quantity": {"type": "integer"},
|
||||||
|
"from": {"type": "string"},
|
||||||
|
"to": {"type": "string"},
|
||||||
|
"source_refs": {
|
||||||
|
"type": "array",
|
||||||
|
"items": {
|
||||||
|
"type": "object",
|
||||||
|
"additionalProperties": false,
|
||||||
|
"required": ["start_segment", "end_segment"],
|
||||||
|
"properties": {
|
||||||
|
"start_segment": {"type": "integer"},
|
||||||
|
"end_segment": {"type": "integer"}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
99
internal/modules/dnd/extract/itemevents/canonicalize.go
Normal file
99
internal/modules/dnd/extract/itemevents/canonicalize.go
Normal file
@@ -0,0 +1,99 @@
|
|||||||
|
package itemevents
|
||||||
|
|
||||||
|
import (
|
||||||
|
"sort"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared"
|
||||||
|
)
|
||||||
|
|
||||||
|
type orderedItemEventResponse struct {
|
||||||
|
value itemEventResponse
|
||||||
|
earliest int
|
||||||
|
hasEvidence bool
|
||||||
|
}
|
||||||
|
|
||||||
|
func canonicalizeResponse(response *extractionResponse, order shared.SourceRefOrder, sourceID string) {
|
||||||
|
if response == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
ordered := make([]orderedItemEventResponse, len(response.Events))
|
||||||
|
for index := range response.Events {
|
||||||
|
earliest, hasEvidence := canonicalizeItemEvent(&response.Events[index], order, sourceID)
|
||||||
|
ordered[index] = orderedItemEventResponse{
|
||||||
|
value: response.Events[index],
|
||||||
|
earliest: earliest,
|
||||||
|
hasEvidence: hasEvidence,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sort.SliceStable(ordered, func(left, right int) bool {
|
||||||
|
if ordered[left].hasEvidence != ordered[right].hasEvidence {
|
||||||
|
return ordered[left].hasEvidence
|
||||||
|
}
|
||||||
|
if !ordered[left].hasEvidence {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
return ordered[left].earliest < ordered[right].earliest
|
||||||
|
})
|
||||||
|
for index := range ordered {
|
||||||
|
response.Events[index] = ordered[index].value
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func canonicalizeItemEvent(event *itemEventResponse, order shared.SourceRefOrder, sourceID string) (int, bool) {
|
||||||
|
if event == nil {
|
||||||
|
return 0, false
|
||||||
|
}
|
||||||
|
refs := order.Canonicalize(itemEventSourceRefs(event.SourceRefs, sourceID))
|
||||||
|
event.SourceRefs = itemEventResponseRefs(refs)
|
||||||
|
return order.EarliestValid(refs)
|
||||||
|
}
|
||||||
|
|
||||||
|
func canonicalItemEventList(response extractionResponse, sourceID string) dnd.ItemEventList {
|
||||||
|
if response.Events == nil {
|
||||||
|
return dnd.ItemEventList{}
|
||||||
|
}
|
||||||
|
events := make([]dnd.ItemEvent, len(response.Events))
|
||||||
|
for index, event := range response.Events {
|
||||||
|
events[index] = dnd.ItemEvent{
|
||||||
|
Name: event.Name,
|
||||||
|
Kind: dnd.ItemEventKind(event.Kind),
|
||||||
|
Quantity: cloneQuantity(event.Quantity),
|
||||||
|
From: event.From,
|
||||||
|
To: event.To,
|
||||||
|
SourceRefs: itemEventSourceRefs(event.SourceRefs, sourceID),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return dnd.ItemEventList{Events: events}
|
||||||
|
}
|
||||||
|
|
||||||
|
func itemEventSourceRefs(refs []itemEventSourceRefResponse, sourceID string) []source.SourceRef {
|
||||||
|
if refs == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
values := make([]source.SourceRef, len(refs))
|
||||||
|
for index, ref := range refs {
|
||||||
|
values[index] = source.SourceRef{SourceID: sourceID, StartUnitID: ref.StartSegment, EndUnitID: ref.EndSegment}
|
||||||
|
}
|
||||||
|
return values
|
||||||
|
}
|
||||||
|
|
||||||
|
func itemEventResponseRefs(refs []source.SourceRef) []itemEventSourceRefResponse {
|
||||||
|
if refs == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
values := make([]itemEventSourceRefResponse, len(refs))
|
||||||
|
for index, ref := range refs {
|
||||||
|
values[index] = itemEventSourceRefResponse{StartSegment: ref.StartUnitID, EndSegment: ref.EndUnitID}
|
||||||
|
}
|
||||||
|
return values
|
||||||
|
}
|
||||||
|
|
||||||
|
func cloneQuantity(value *int) *int {
|
||||||
|
if value == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
quantity := *value
|
||||||
|
return &quantity
|
||||||
|
}
|
||||||
163
internal/modules/dnd/extract/itemevents/extractor.go
Normal file
163
internal/modules/dnd/extract/itemevents/extractor.go
Normal file
@@ -0,0 +1,163 @@
|
|||||||
|
// Package itemevents extracts source-grounded D&D item events.
|
||||||
|
package itemevents
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared"
|
||||||
|
)
|
||||||
|
|
||||||
|
const Key = "dnd/item-events"
|
||||||
|
|
||||||
|
const mappingPolicy = "dnd.item_events.extract_mapping.v1"
|
||||||
|
|
||||||
|
var requiredCapabilities = []string{
|
||||||
|
"chunks",
|
||||||
|
"source.transcript",
|
||||||
|
}
|
||||||
|
|
||||||
|
var providedCapabilities = []string{
|
||||||
|
"dnd.item_events",
|
||||||
|
}
|
||||||
|
|
||||||
|
var referenceSlotDescriptions = shared.ReferenceSlotDescriptions{
|
||||||
|
Glossary: "Optional campaign glossary reference material used only for item and holder disambiguation.",
|
||||||
|
Party: "Optional party roster reference material used only for item and holder disambiguation.",
|
||||||
|
Players: "Optional player list reference material used only for item and holder disambiguation.",
|
||||||
|
Roster: "Deprecated alias for party roster reference material used only for item and holder disambiguation.",
|
||||||
|
}
|
||||||
|
|
||||||
|
func referenceSlots() []contracts.ReferenceSlot {
|
||||||
|
return shared.ReferenceSlots(referenceSlotDescriptions)
|
||||||
|
}
|
||||||
|
|
||||||
|
var _ contracts.Extractor[dnd.ItemEventList] = (*Extractor)(nil)
|
||||||
|
var _ contracts.ManifestMetadataProvider = (*Extractor)(nil)
|
||||||
|
var _ pipeline.CheckpointFingerprintProvider = (*Extractor)(nil)
|
||||||
|
|
||||||
|
type Options struct{}
|
||||||
|
|
||||||
|
type Extractor struct {
|
||||||
|
llm contracts.StructuredLLMClient
|
||||||
|
promptSHA string
|
||||||
|
responseSchemaSHA string
|
||||||
|
}
|
||||||
|
|
||||||
|
func New(llmClient contracts.StructuredLLMClient, _ Options, references ...contracts.ReferenceSet) (*Extractor, error) {
|
||||||
|
if llmClient == nil {
|
||||||
|
return nil, extractorErrorf("LLM client must not be nil")
|
||||||
|
}
|
||||||
|
if len(references) > 1 {
|
||||||
|
return nil, extractorErrorf("at most one reference set may be supplied")
|
||||||
|
}
|
||||||
|
promptSHA, err := scriptoriumPromptMetadata()
|
||||||
|
if err != nil {
|
||||||
|
return nil, extractorErrorf("load prompt metadata: %w", err)
|
||||||
|
}
|
||||||
|
responseSchema, err := loadResponseSchema()
|
||||||
|
if err != nil {
|
||||||
|
return nil, extractorErrorf("load response schema: %w", err)
|
||||||
|
}
|
||||||
|
return &Extractor{llm: llmClient, promptSHA: promptSHA, responseSchemaSHA: responseSchema.SHA256}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *Extractor) Key() string { return Key }
|
||||||
|
|
||||||
|
func (e *Extractor) ReferenceSlots() []contracts.ReferenceSlot { return referenceSlots() }
|
||||||
|
|
||||||
|
func (e *Extractor) ManifestMetadata() map[string]any {
|
||||||
|
if e == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return map[string]any{
|
||||||
|
"prompt_id": PromptID,
|
||||||
|
"prompt_version": SchemaVersion,
|
||||||
|
"prompt_sha256": e.promptSHA,
|
||||||
|
"response_schema_key": string(ResponseSchemaKey),
|
||||||
|
"response_schema_id": ResponseSchemaID,
|
||||||
|
"response_schema_name": ResponseSchemaName,
|
||||||
|
"response_schema_version": SchemaVersion,
|
||||||
|
"response_schema_sha256": e.responseSchemaSHA,
|
||||||
|
"mapping_policy": mappingPolicy,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *Extractor) CheckpointFingerprints() []pipeline.CheckpointFingerprint {
|
||||||
|
if e == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return []pipeline.CheckpointFingerprint{
|
||||||
|
{Name: "prompt", Value: e.promptSHA},
|
||||||
|
{Name: "response_schema", Value: e.responseSchemaSHA},
|
||||||
|
{Name: "mapping_policy", Value: mappingPolicy},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *Extractor) Extract(ctx context.Context, req contracts.TypedExtractionRequest) (contracts.TypedExtractionResult[dnd.ItemEventList], error) {
|
||||||
|
if e == nil {
|
||||||
|
return contracts.TypedExtractionResult[dnd.ItemEventList]{}, extractorErrorf("extractor must not be nil")
|
||||||
|
}
|
||||||
|
if e.llm == nil {
|
||||||
|
return contracts.TypedExtractionResult[dnd.ItemEventList]{}, extractorErrorf("LLM client must not be nil")
|
||||||
|
}
|
||||||
|
sourceInput, err := shared.PrepareChunkExtraction(ctx, req)
|
||||||
|
if err != nil {
|
||||||
|
return contracts.TypedExtractionResult[dnd.ItemEventList]{}, extractorErrorf("%w", err)
|
||||||
|
}
|
||||||
|
order := shared.NewSourceRefOrder(req.Source)
|
||||||
|
|
||||||
|
var response extractionResponse
|
||||||
|
if _, err := e.llm.CompleteStructured(ctx, contracts.StructuredCompletionRequest{
|
||||||
|
StageName: Key,
|
||||||
|
PromptID: PromptID,
|
||||||
|
PromptVersion: SchemaVersion,
|
||||||
|
ProfileID: req.LLMProfile,
|
||||||
|
SessionID: req.SessionID,
|
||||||
|
Inputs: shared.PromptInputs(sourceInput, req.References),
|
||||||
|
}, &response); err != nil {
|
||||||
|
return contracts.TypedExtractionResult[dnd.ItemEventList]{}, extractorErrorf("complete structured output: %w", err)
|
||||||
|
}
|
||||||
|
canonicalizeResponse(&response, order, req.Source.ID)
|
||||||
|
return contracts.TypedExtractionResult[dnd.ItemEventList]{Value: canonicalItemEventList(response, req.Source.ID)}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func ModuleSpec() pipeline.ModuleSpec {
|
||||||
|
return pipeline.ModuleSpec{
|
||||||
|
Key: Key,
|
||||||
|
Stage: pipeline.StageExtract,
|
||||||
|
Requires: append([]string(nil), requiredCapabilities...),
|
||||||
|
Provides: append([]string(nil), providedCapabilities...),
|
||||||
|
ArtifactKind: dnd.ItemEventListKind,
|
||||||
|
ReferenceSlots: referenceSlots(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func Register(registry *pipeline.ExtractorRegistry) error {
|
||||||
|
return pipeline.RegisterExtractorBuilder(registry, ModuleSpec(), validateOptions, func(request pipeline.BuildRequest) (contracts.Extractor[dnd.ItemEventList], error) {
|
||||||
|
options, err := DecodeOptions(request.Options)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return New(request.Dependencies.LLM, options, request.References)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func validateOptions(options map[string]any) error {
|
||||||
|
_, err := DecodeOptions(options)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
func DecodeOptions(options map[string]any) (Options, error) {
|
||||||
|
if err := pipeline.RejectUnknownOptions(options); err != nil {
|
||||||
|
return Options{}, extractorErrorf("%w", err)
|
||||||
|
}
|
||||||
|
return Options{}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func extractorErrorf(format string, args ...any) error {
|
||||||
|
return fmt.Errorf("dnd item events extractor: "+format, args...)
|
||||||
|
}
|
||||||
147
internal/modules/dnd/extract/itemevents/extractor_test.go
Normal file
147
internal/modules/dnd/extract/itemevents/extractor_test.go
Normal file
@@ -0,0 +1,147 @@
|
|||||||
|
package itemevents
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"reflect"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestExtractMapsAllEventKindsAndCanonicalizesEvidence(t *testing.T) {
|
||||||
|
quantity := 20
|
||||||
|
client := &fakeItemEventsLLMClient{response: extractionResponse{Events: []itemEventResponse{
|
||||||
|
{Name: "Torch", Kind: "lost", From: "party", SourceRefs: responseRefs(5, 5)},
|
||||||
|
{Name: "Moonblade", Kind: "transferred", From: "Aria", To: "Borin", SourceRefs: responseRefs(3, 3)},
|
||||||
|
{Name: "Hidden Cache", Kind: "discovered", SourceRefs: []itemEventSourceRefResponse{{StartSegment: 1, EndSegment: 1}, {StartSegment: 1, EndSegment: 1}}},
|
||||||
|
{Name: "Gold Pieces", Kind: "acquired", Quantity: &quantity, To: "party", SourceRefs: responseRefs(2, 2)},
|
||||||
|
{Name: "Healing Potion", Kind: "consumed", From: "party", SourceRefs: responseRefs(4, 4)},
|
||||||
|
}}}
|
||||||
|
|
||||||
|
result, err := newExtractor(t, client).Extract(context.Background(), extractionRequest())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Extract() error = %v", err)
|
||||||
|
}
|
||||||
|
if got := []dnd.ItemEventKind{result.Value.Events[0].Kind, result.Value.Events[1].Kind, result.Value.Events[2].Kind, result.Value.Events[3].Kind, result.Value.Events[4].Kind}; !reflect.DeepEqual(got, []dnd.ItemEventKind{
|
||||||
|
dnd.ItemEventKindDiscovered, dnd.ItemEventKindAcquired, dnd.ItemEventKindTransferred, dnd.ItemEventKindConsumed, dnd.ItemEventKindLost,
|
||||||
|
}) {
|
||||||
|
t.Fatalf("event kinds = %#v, want source order", got)
|
||||||
|
}
|
||||||
|
acquired := result.Value.Events[1]
|
||||||
|
if acquired.Name != "Gold Pieces" || acquired.To != "party" || acquired.Quantity == nil || *acquired.Quantity != 20 {
|
||||||
|
t.Fatalf("acquired event = %#v", acquired)
|
||||||
|
}
|
||||||
|
if result.Value.Events[0].Quantity != nil || result.Value.Events[0].From != "" || result.Value.Events[0].To != "" {
|
||||||
|
t.Fatalf("discovered event = %#v, want omitted optional fields", result.Value.Events[0])
|
||||||
|
}
|
||||||
|
if refs := result.Value.Events[0].SourceRefs; len(refs) != 1 || refs[0] != (source.SourceRef{SourceID: "session-alpha", StartUnitID: 1, EndUnitID: 1}) {
|
||||||
|
t.Fatalf("source refs = %#v, want source ID attachment and deduplication", refs)
|
||||||
|
}
|
||||||
|
if acquired.Quantity == client.response.Events[3].Quantity {
|
||||||
|
t.Fatal("mapped quantity aliases private response")
|
||||||
|
}
|
||||||
|
if len(client.requests) != 1 {
|
||||||
|
t.Fatalf("LLM calls = %d, want 1", len(client.requests))
|
||||||
|
}
|
||||||
|
request := client.requests[0]
|
||||||
|
if request.StageName != Key || request.PromptID != PromptID || request.PromptVersion != SchemaVersion || request.ProfileID != "profile-item-events" || request.SessionID != "session-123" {
|
||||||
|
t.Fatalf("LLM request identity = %#v", request)
|
||||||
|
}
|
||||||
|
if transcript := request.Inputs["transcript"]; string(transcript.Content) != string(extractionRequest().Chunk.Content) || transcript.Name != "transcript" {
|
||||||
|
t.Fatalf("transcript input = %#v", transcript)
|
||||||
|
}
|
||||||
|
for _, name := range []string{"players", "party", "glossary"} {
|
||||||
|
if input := request.Inputs[name]; string(input.Content) != " " {
|
||||||
|
t.Fatalf("absent %s input = %#v, want retained empty prompt material", name, input)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExtractPreservesInvalidCandidatesAndEmptyResults(t *testing.T) {
|
||||||
|
client := &fakeItemEventsLLMClient{content: []byte(`{"events":[{"name":"","kind":"transferred","quantity":0,"from":"party","to":"Party","source_refs":[{"start_segment":99,"end_segment":-1}]}]}`)}
|
||||||
|
result, err := newExtractor(t, client).Extract(context.Background(), extractionRequest())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Extract() error = %v", err)
|
||||||
|
}
|
||||||
|
event := result.Value.Events[0]
|
||||||
|
if event.Name != "" || event.Kind != "transferred" || event.Quantity == nil || *event.Quantity != 0 || event.From != "party" || event.To != "Party" {
|
||||||
|
t.Fatalf("invalid candidate = %#v, want values preserved", event)
|
||||||
|
}
|
||||||
|
if refs := event.SourceRefs; len(refs) != 1 || refs[0] != (source.SourceRef{SourceID: "session-alpha", StartUnitID: 99, EndUnitID: -1}) {
|
||||||
|
t.Fatalf("source refs = %#v, want invalid range preserved", refs)
|
||||||
|
}
|
||||||
|
|
||||||
|
empty, err := newExtractor(t, &fakeItemEventsLLMClient{response: extractionResponse{Events: []itemEventResponse{}}}).Extract(context.Background(), extractionRequest())
|
||||||
|
if err != nil || empty.Value.Events == nil || len(empty.Value.Events) != 0 {
|
||||||
|
t.Fatalf("empty provider result = %#v, %v; want valid empty list", empty.Value, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExtractUsesSourceDocumentOrderForCandidates(t *testing.T) {
|
||||||
|
client := &fakeItemEventsLLMClient{response: extractionResponse{Events: []itemEventResponse{
|
||||||
|
{Name: "Later", Kind: "discovered", SourceRefs: responseRefs(10, 10)},
|
||||||
|
{Name: "Earlier", Kind: "discovered", SourceRefs: responseRefs(100, 100)},
|
||||||
|
}}}
|
||||||
|
req := extractionRequest()
|
||||||
|
req.Source.Units = []source.SourceUnit{{ID: 100}, {ID: 10}}
|
||||||
|
req.Chunk.Units = append([]source.SourceUnit(nil), req.Source.Units...)
|
||||||
|
req.Chunk.Ref = source.SourceRef{SourceID: req.Source.ID, StartUnitID: 100, EndUnitID: 10}
|
||||||
|
result, err := newExtractor(t, client).Extract(context.Background(), req)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got := []string{result.Value.Events[0].Name, result.Value.Events[1].Name}; !reflect.DeepEqual(got, []string{"Earlier", "Later"}) {
|
||||||
|
t.Fatalf("event order = %#v, want source document order", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExtractUsesOnlySupportedPromptInputs(t *testing.T) {
|
||||||
|
client := &fakeItemEventsLLMClient{response: extractionResponse{Events: []itemEventResponse{}}}
|
||||||
|
req := extractionRequest()
|
||||||
|
req.References = contracts.ReferenceSet{Slots: map[string]contracts.ResolvedReferenceSlot{
|
||||||
|
"players": {Slot: contracts.ReferenceSlot{Name: "players"}, Items: []contracts.ReferenceItem{{SlotName: "players", Content: []byte("Dana: Aria")}}},
|
||||||
|
"party": {Slot: contracts.ReferenceSlot{Name: "party"}, Items: []contracts.ReferenceItem{{SlotName: "party", Content: []byte("Aria: ranger")}}},
|
||||||
|
"glossary": {Slot: contracts.ReferenceSlot{Name: "glossary"}, Items: []contracts.ReferenceItem{{SlotName: "glossary", Content: []byte("Moonblade: heirloom")}}},
|
||||||
|
"npcs": {Slot: contracts.ReferenceSlot{Name: "npcs"}, Items: []contracts.ReferenceItem{{SlotName: "npcs", Content: []byte("must not be used")}}},
|
||||||
|
}}
|
||||||
|
if _, err := newExtractor(t, client).Extract(context.Background(), req); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
inputs := client.requests[0].Inputs
|
||||||
|
if string(inputs["players"].Content) != "Dana: Aria" || string(inputs["party"].Content) != "Aria: ranger" || string(inputs["glossary"].Content) != "Moonblade: heirloom" {
|
||||||
|
t.Fatalf("reference inputs = %#v", inputs)
|
||||||
|
}
|
||||||
|
if _, ok := inputs["npcs"]; ok {
|
||||||
|
t.Fatalf("unexpected generated-lane input: %#v", inputs)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExtractRetainsLocalErrorContext(t *testing.T) {
|
||||||
|
request := extractionRequest()
|
||||||
|
extractor := newExtractor(t, &fakeItemEventsLLMClient{})
|
||||||
|
var nilExtractor *Extractor
|
||||||
|
request.SourceInput = contracts.NewLLMInputMaterial("source", "application/json", []byte(`{"different":true}`), "sha256:other", "file:///other.json")
|
||||||
|
for _, test := range []struct {
|
||||||
|
name string
|
||||||
|
extractor *Extractor
|
||||||
|
req contracts.TypedExtractionRequest
|
||||||
|
want string
|
||||||
|
}{
|
||||||
|
{"nil extractor", nilExtractor, extractionRequest(), "extractor"},
|
||||||
|
{"nil client", &Extractor{}, extractionRequest(), "LLM client"},
|
||||||
|
{"preflight", extractor, request, "must match chunk"},
|
||||||
|
} {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
if _, err := test.extractor.Extract(context.Background(), test.req); err == nil || !strings.Contains(err.Error(), "dnd item events") || !strings.Contains(err.Error(), test.want) {
|
||||||
|
t.Fatalf("Extract() error = %v", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
if _, err := newExtractor(t, &fakeItemEventsLLMClient{err: errors.New("provider unavailable")}).Extract(context.Background(), extractionRequest()); err == nil || !strings.Contains(err.Error(), "provider unavailable") {
|
||||||
|
t.Fatalf("provider error = %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
19
internal/modules/dnd/extract/itemevents/model.go
Normal file
19
internal/modules/dnd/extract/itemevents/model.go
Normal file
@@ -0,0 +1,19 @@
|
|||||||
|
package itemevents
|
||||||
|
|
||||||
|
type extractionResponse struct {
|
||||||
|
Events []itemEventResponse `json:"events"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type itemEventResponse struct {
|
||||||
|
Name string `json:"name"`
|
||||||
|
Kind string `json:"kind"`
|
||||||
|
Quantity *int `json:"quantity,omitempty"`
|
||||||
|
From string `json:"from,omitempty"`
|
||||||
|
To string `json:"to,omitempty"`
|
||||||
|
SourceRefs []itemEventSourceRefResponse `json:"source_refs"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type itemEventSourceRefResponse struct {
|
||||||
|
StartSegment int `json:"start_segment"`
|
||||||
|
EndSegment int `json:"end_segment"`
|
||||||
|
}
|
||||||
65
internal/modules/dnd/extract/itemevents/registry_test.go
Normal file
65
internal/modules/dnd/extract/itemevents/registry_test.go
Normal file
@@ -0,0 +1,65 @@
|
|||||||
|
package itemevents
|
||||||
|
|
||||||
|
import (
|
||||||
|
"reflect"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestConstructorSpecOptionsAndMetadata(t *testing.T) {
|
||||||
|
if _, err := New(nil, Options{}); err == nil || !strings.Contains(err.Error(), "LLM client") {
|
||||||
|
t.Fatalf("New(nil) error = %v", err)
|
||||||
|
}
|
||||||
|
if _, err := New(&fakeItemEventsLLMClient{}, Options{}, contracts.ReferenceSet{}, contracts.ReferenceSet{}); err == nil || !strings.Contains(err.Error(), "at most one reference set") {
|
||||||
|
t.Fatalf("New() error = %v", err)
|
||||||
|
}
|
||||||
|
want := pipeline.ModuleSpec{
|
||||||
|
Key: Key, Stage: pipeline.StageExtract, Requires: []string{"chunks", "source.transcript"}, Provides: []string{"dnd.item_events"}, ArtifactKind: dnd.ItemEventListKind,
|
||||||
|
ReferenceSlots: []contracts.ReferenceSlot{
|
||||||
|
{Name: "glossary", Description: referenceSlotDescriptions.Glossary, AcceptedMediaTypes: []string{"application/json", "application/x-yaml", "application/yaml", "text/markdown", "text/plain"}},
|
||||||
|
{Name: "party", Description: referenceSlotDescriptions.Party, AcceptedMediaTypes: []string{"application/json", "application/x-yaml", "application/yaml", "text/markdown", "text/plain"}},
|
||||||
|
{Name: "players", Description: referenceSlotDescriptions.Players, AcceptedMediaTypes: []string{"application/json", "application/x-yaml", "application/yaml", "text/markdown", "text/plain"}},
|
||||||
|
{Name: "roster", Description: referenceSlotDescriptions.Roster, AcceptedMediaTypes: []string{"application/json", "application/x-yaml", "application/yaml", "text/markdown", "text/plain"}},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
if got := ModuleSpec(); !reflect.DeepEqual(got, want) {
|
||||||
|
t.Fatalf("ModuleSpec() = %#v, want %#v", got, want)
|
||||||
|
}
|
||||||
|
if _, err := DecodeOptions(map[string]any{"unexpected": true}); err == nil || !strings.Contains(err.Error(), "unknown option") {
|
||||||
|
t.Fatalf("DecodeOptions() error = %v", err)
|
||||||
|
}
|
||||||
|
registry := pipeline.NewExtractorRegistry()
|
||||||
|
if err := Register(registry); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got, ok := registry.Spec(Key); !ok || !reflect.DeepEqual(got, want) {
|
||||||
|
t.Fatalf("registry spec = %#v, %t", got, ok)
|
||||||
|
}
|
||||||
|
|
||||||
|
extractor := newExtractor(t, &fakeItemEventsLLMClient{})
|
||||||
|
metadata := extractor.ManifestMetadata()
|
||||||
|
for key, want := range map[string]string{
|
||||||
|
"prompt_id": PromptID, "prompt_version": SchemaVersion, "response_schema_key": string(ResponseSchemaKey), "response_schema_id": ResponseSchemaID,
|
||||||
|
"response_schema_name": ResponseSchemaName, "response_schema_version": SchemaVersion, "mapping_policy": mappingPolicy,
|
||||||
|
} {
|
||||||
|
if metadata[key] != want {
|
||||||
|
t.Fatalf("metadata[%q] = %#v, want %q", key, metadata[key], want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, key := range []string{"prompt_sha256", "response_schema_sha256"} {
|
||||||
|
if value, ok := metadata[key].(string); !ok || !strings.HasPrefix(value, "sha256:") {
|
||||||
|
t.Fatalf("metadata[%q] = %#v", key, metadata[key])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if got := extractor.CheckpointFingerprints(); !reflect.DeepEqual(got, []pipeline.CheckpointFingerprint{
|
||||||
|
{Name: "prompt", Value: metadata["prompt_sha256"].(string)},
|
||||||
|
{Name: "response_schema", Value: metadata["response_schema_sha256"].(string)},
|
||||||
|
{Name: "mapping_policy", Value: mappingPolicy},
|
||||||
|
}) {
|
||||||
|
t.Fatalf("CheckpointFingerprints() = %#v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
21
internal/modules/dnd/extract/itemevents/schema.go
Normal file
21
internal/modules/dnd/extract/itemevents/schema.go
Normal file
@@ -0,0 +1,21 @@
|
|||||||
|
package itemevents
|
||||||
|
|
||||||
|
import "gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
||||||
|
|
||||||
|
const (
|
||||||
|
PromptID = "dnd.item_events"
|
||||||
|
ResponseSchemaKey = llm.ResponseSchemaKey("dnd_item_events_llm")
|
||||||
|
ResponseSchemaID = "notarius.dnd.item_events.llm"
|
||||||
|
ResponseSchemaName = "notarius_dnd_item_events_llm_v1"
|
||||||
|
SchemaVersion = "v1"
|
||||||
|
)
|
||||||
|
|
||||||
|
func loadResponseSchema() (llm.ResponseSchema, error) {
|
||||||
|
return llm.LoadResponseSchema(embeddedAssets, llm.ResponseSchemaDefinition{
|
||||||
|
Key: ResponseSchemaKey,
|
||||||
|
ID: ResponseSchemaID,
|
||||||
|
Version: SchemaVersion,
|
||||||
|
Name: ResponseSchemaName,
|
||||||
|
AssetPath: "assets/schemas/dnd_item_events_llm.v1.json",
|
||||||
|
})
|
||||||
|
}
|
||||||
80
internal/modules/dnd/extract/itemevents/schema_test.go
Normal file
80
internal/modules/dnd/extract/itemevents/schema_test.go
Normal file
@@ -0,0 +1,80 @@
|
|||||||
|
package itemevents
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"encoding/json"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/santhosh-tekuri/jsonschema/v6"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestResponseSchemaIsStrictlyStructuralAndPrivate(t *testing.T) {
|
||||||
|
schema, err := loadResponseSchema()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if schema.Key != ResponseSchemaKey || schema.ID != ResponseSchemaID || schema.Name != ResponseSchemaName || schema.Version != SchemaVersion || !strings.HasPrefix(schema.SHA256, "sha256:") || !json.Valid(schema.JSONSchema) {
|
||||||
|
t.Fatalf("schema = %#v", schema)
|
||||||
|
}
|
||||||
|
valid := map[string]any{"events": []any{map[string]any{
|
||||||
|
"name": "", "kind": "unsupported", "quantity": 0, "from": "party", "to": "Party",
|
||||||
|
"source_refs": []any{map[string]any{"start_segment": 0, "end_segment": -1}},
|
||||||
|
}}}
|
||||||
|
content, err := json.Marshal(valid)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := validateJSONSchema(content, schema.JSONSchema); err != nil {
|
||||||
|
t.Fatalf("private schema rejected validator-owned values: %v", err)
|
||||||
|
}
|
||||||
|
for _, test := range []struct {
|
||||||
|
name string
|
||||||
|
value map[string]any
|
||||||
|
}{
|
||||||
|
{"missing events", map[string]any{}},
|
||||||
|
{"missing event name", map[string]any{"events": []any{map[string]any{"kind": "acquired", "source_refs": []any{}}}}},
|
||||||
|
{"unknown event field", map[string]any{"events": []any{map[string]any{"name": "Ring", "kind": "acquired", "source_refs": []any{}, "extra": true}}}},
|
||||||
|
{"unknown reference field", map[string]any{"events": []any{map[string]any{"name": "Ring", "kind": "acquired", "source_refs": []any{map[string]any{"start_segment": 1, "end_segment": 1, "extra": true}}}}}},
|
||||||
|
{"noninteger range", map[string]any{"events": []any{map[string]any{"name": "Ring", "kind": "acquired", "source_refs": []any{map[string]any{"start_segment": 1.5, "end_segment": 1}}}}}},
|
||||||
|
} {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
content, err := json.Marshal(test.value)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := validateJSONSchema(content, schema.JSONSchema); err == nil {
|
||||||
|
t.Fatal("private schema accepted invalid structure")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
first := schema.JSONSchema
|
||||||
|
first[0] = '['
|
||||||
|
second, err := loadResponseSchema()
|
||||||
|
if err != nil || !json.Valid(second.JSONSchema) || bytes.Equal(first, second.JSONSchema) {
|
||||||
|
t.Fatalf("defensive schema copy = %s, %v", second.JSONSchema, err)
|
||||||
|
}
|
||||||
|
if _, ok := second.DiagnosticsMap()["json_schema"]; ok {
|
||||||
|
t.Fatalf("schema diagnostics leaked content: %#v", second.DiagnosticsMap())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func validateJSONSchema(instanceContent, schemaContent []byte) error {
|
||||||
|
instance, err := jsonschema.UnmarshalJSON(bytes.NewReader(instanceContent))
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
document, err := jsonschema.UnmarshalJSON(bytes.NewReader(schemaContent))
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
compiler := jsonschema.NewCompiler()
|
||||||
|
if err := compiler.AddResource("schema.json", document); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
compiled, err := compiler.Compile("schema.json")
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return compiled.Validate(instance)
|
||||||
|
}
|
||||||
@@ -0,0 +1,52 @@
|
|||||||
|
package itemevents
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"sync"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/promptfs"
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared"
|
||||||
|
)
|
||||||
|
|
||||||
|
const scriptoriumPromptRoot = "assets/prompts"
|
||||||
|
|
||||||
|
var promptAssetManifest = shared.PromptAssetManifest{
|
||||||
|
ModuleDir: "dnd.item_events",
|
||||||
|
ModuleFiles: []promptfs.ModulePromptFile{
|
||||||
|
{Name: "dnd.item_events.yaml", Path: "assets/prompts/dnd.item_events.yaml"},
|
||||||
|
{Name: "task.md", Path: "assets/prompts/task.md"},
|
||||||
|
{Name: "instructions.md", Path: "assets/prompts/instructions.md"},
|
||||||
|
},
|
||||||
|
SharedFiles: []string{
|
||||||
|
"common-dnd-system.md",
|
||||||
|
"common-dnd-extraction-evidence.md",
|
||||||
|
"common-dnd-identity.md",
|
||||||
|
"common-dnd-references.md",
|
||||||
|
"common-dnd-transcript.md",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
func RegisterPromptAssets(registry *llm.AssetRegistry) error {
|
||||||
|
promptFS, err := promptAssetManifest.PromptFS(embeddedAssets)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("prepare item event prompt assets: %w", err)
|
||||||
|
}
|
||||||
|
if err := registry.RegisterPromptFS(promptFS, scriptoriumPromptRoot); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return registry.RegisterSchemaFS(embeddedAssets, "assets/schemas")
|
||||||
|
}
|
||||||
|
|
||||||
|
func scriptoriumPromptMetadata() (string, error) {
|
||||||
|
scriptoriumPromptHashOnce.Do(func() {
|
||||||
|
scriptoriumPromptHash, scriptoriumPromptHashErr = promptAssetManifest.Hash(embeddedAssets)
|
||||||
|
})
|
||||||
|
return scriptoriumPromptHash, scriptoriumPromptHashErr
|
||||||
|
}
|
||||||
|
|
||||||
|
var (
|
||||||
|
scriptoriumPromptHashOnce sync.Once
|
||||||
|
scriptoriumPromptHash string
|
||||||
|
scriptoriumPromptHashErr error
|
||||||
|
)
|
||||||
@@ -0,0 +1,114 @@
|
|||||||
|
package itemevents
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"io/fs"
|
||||||
|
"reflect"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
||||||
|
"gitea.maximumdirect.net/eric/scriptorium"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestPromptAssetsUseSharedSequenceAndTranscriptLast(t *testing.T) {
|
||||||
|
registry := llm.NewAssetRegistry()
|
||||||
|
if err := RegisterPromptAssets(registry); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
options, err := registry.ScriptoriumOptions()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
options = append(options, scriptorium.WithProfiles(scriptorium.OpenAICompatibleProfile(scriptorium.OpenAICompatibleProfileConfig{
|
||||||
|
ID: "item-events-test-profile", Endpoint: "http://127.0.0.1:1/v1", Model: "item-events-test-model",
|
||||||
|
})))
|
||||||
|
engine, err := scriptorium.NewEngine(scriptorium.Config{Timeout: time.Second}, options...)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
prepared, err := engine.Prepare(context.Background(), scriptorium.RunRequest{
|
||||||
|
PromptID: PromptID, PromptVersion: SchemaVersion, ProfileID: "item-events-test-profile",
|
||||||
|
Inputs: map[string]scriptorium.ArtifactRef{
|
||||||
|
"transcript": scriptorium.InlineWithURI("file:///session.json", `{"segments":[1]}`),
|
||||||
|
"players": scriptorium.Inline("Dana: Aria"),
|
||||||
|
"party": scriptorium.Inline("Aria: ranger"),
|
||||||
|
"glossary": scriptorium.Inline("Moonblade: heirloom"),
|
||||||
|
},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if prepared.PromptID != PromptID || prepared.OutputContract.SchemaPath != "dnd_item_events_llm.v1.json" {
|
||||||
|
t.Fatalf("prepared prompt = %#v", prepared)
|
||||||
|
}
|
||||||
|
want := []struct {
|
||||||
|
role string
|
||||||
|
cached bool
|
||||||
|
marker string
|
||||||
|
}{
|
||||||
|
{"system", false, "Dungeons & Dragons gameplay transcripts"},
|
||||||
|
{"user", false, "Transcript units are the only evidence"},
|
||||||
|
{"user", true, "most specific supported in-world"},
|
||||||
|
{"user", true, "Dana: Aria"},
|
||||||
|
{"user", false, "item and currency events"},
|
||||||
|
{"user", true, "Ordinary non-depleting use"},
|
||||||
|
{"user", false, `{"segments":[1]}`},
|
||||||
|
}
|
||||||
|
if len(prepared.Messages) != len(want) {
|
||||||
|
t.Fatalf("prompt messages = %d, want %d", len(prepared.Messages), len(want))
|
||||||
|
}
|
||||||
|
for index, expected := range want {
|
||||||
|
message := prepared.Messages[index]
|
||||||
|
if message.Role != expected.role || !strings.Contains(message.Content, expected.marker) {
|
||||||
|
t.Fatalf("message %d = %#v", index, message)
|
||||||
|
}
|
||||||
|
if (message.CacheControl != nil) != expected.cached {
|
||||||
|
t.Fatalf("message %d cache control = %#v", index, message.CacheControl)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if strings.Contains(prepared.Messages[len(prepared.Messages)-1].Content, "Moonblade: heirloom") {
|
||||||
|
t.Fatal("transcript message contains optional reference content")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPromptAssetsDoNotLeakIntoMetadata(t *testing.T) {
|
||||||
|
hash, err := scriptoriumPromptMetadata()
|
||||||
|
if err != nil || !strings.HasPrefix(hash, "sha256:") {
|
||||||
|
t.Fatalf("scriptoriumPromptMetadata() = %q, %v", hash, err)
|
||||||
|
}
|
||||||
|
metadata := newExtractor(t, &fakeItemEventsLLMClient{}).ManifestMetadata()
|
||||||
|
payload, err := json.Marshal(metadata)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, forbidden := range []string{"meaningful Dungeons", "common-dnd-system", "start_segment", "dnd_item_events_llm.v1.json"} {
|
||||||
|
if strings.Contains(string(payload), forbidden) {
|
||||||
|
t.Fatalf("metadata leaked raw asset content %q: %s", forbidden, payload)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPromptManifestReusesOnlySharedAssets(t *testing.T) {
|
||||||
|
want := []string{
|
||||||
|
"common-dnd-system.md",
|
||||||
|
"common-dnd-extraction-evidence.md",
|
||||||
|
"common-dnd-identity.md",
|
||||||
|
"common-dnd-references.md",
|
||||||
|
"common-dnd-transcript.md",
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(promptAssetManifest.SharedFiles, want) {
|
||||||
|
t.Fatalf("shared assets = %#v, want %#v", promptAssetManifest.SharedFiles, want)
|
||||||
|
}
|
||||||
|
for _, path := range []string{"assets/prompts/task.md", "assets/prompts/instructions.md"} {
|
||||||
|
content, err := fs.ReadFile(embeddedAssets, path)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if strings.Contains(string(content), "Transcript units are the only evidence") || strings.Contains(string(content), "Dungeons & Dragons gameplay transcripts") {
|
||||||
|
t.Fatalf("module asset %q copied shared prompt text", path)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
92
internal/modules/dnd/extract/itemevents/test_helpers_test.go
Normal file
92
internal/modules/dnd/extract/itemevents/test_helpers_test.go
Normal file
@@ -0,0 +1,92 @@
|
|||||||
|
package itemevents
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||||
|
)
|
||||||
|
|
||||||
|
func extractionRequest() contracts.TypedExtractionRequest {
|
||||||
|
doc := sourceDocument()
|
||||||
|
chunk := &source.Chunk{
|
||||||
|
ID: "session-alpha:chunk:0",
|
||||||
|
SourceID: doc.ID,
|
||||||
|
Index: 0,
|
||||||
|
Ref: source.SourceRef{SourceID: doc.ID, StartUnitID: 1, EndUnitID: 5},
|
||||||
|
Content: []byte(`{"segments":[1,2,3,4,5]}`),
|
||||||
|
MediaType: "application/json",
|
||||||
|
Units: append([]source.SourceUnit(nil), doc.Units...),
|
||||||
|
}
|
||||||
|
return contracts.TypedExtractionRequest{
|
||||||
|
Source: doc,
|
||||||
|
Chunk: chunk,
|
||||||
|
SourceInput: contracts.NewLLMInputMaterial("source", chunk.MediaType, chunk.Content, "sha256:chunk", "file:///session-alpha.json"),
|
||||||
|
SessionID: "session-123",
|
||||||
|
LLMProfile: "profile-item-events",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func sourceDocument() *source.SourceDocument {
|
||||||
|
return &source.SourceDocument{
|
||||||
|
ID: "session-alpha", Kind: "transcript", Format: "application/vnd.seriatim.minimal+json", Digest: "sha256:test",
|
||||||
|
Units: []source.SourceUnit{
|
||||||
|
{ID: 1, Kind: "transcript_segment", Text: "The party discovers a hidden cache."},
|
||||||
|
{ID: 2, Kind: "transcript_segment", Text: "They acquire 20 gold pieces."},
|
||||||
|
{ID: 3, Kind: "transcript_segment", Text: "Aria gives Borin the moonblade."},
|
||||||
|
{ID: 4, Kind: "transcript_segment", Text: "The party consumes a healing potion."},
|
||||||
|
{ID: 5, Kind: "transcript_segment", Text: "The party loses a torch."},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func responseRefs(start, end int) []itemEventSourceRefResponse {
|
||||||
|
return []itemEventSourceRefResponse{{StartSegment: start, EndSegment: end}}
|
||||||
|
}
|
||||||
|
|
||||||
|
func newExtractor(t *testing.T, client contracts.StructuredLLMClient, references ...contracts.ReferenceSet) *Extractor {
|
||||||
|
t.Helper()
|
||||||
|
extractor, err := New(client, Options{}, references...)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("New() error = %v", err)
|
||||||
|
}
|
||||||
|
return extractor
|
||||||
|
}
|
||||||
|
|
||||||
|
func cloneStructuredCompletionRequest(req contracts.StructuredCompletionRequest) contracts.StructuredCompletionRequest {
|
||||||
|
req.Inputs = req.Inputs.Clone()
|
||||||
|
return req
|
||||||
|
}
|
||||||
|
|
||||||
|
type fakeItemEventsLLMClient struct {
|
||||||
|
response extractionResponse
|
||||||
|
content []byte
|
||||||
|
err error
|
||||||
|
requests []contracts.StructuredCompletionRequest
|
||||||
|
}
|
||||||
|
|
||||||
|
func (client *fakeItemEventsLLMClient) CompleteStructured(_ context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
|
||||||
|
client.requests = append(client.requests, cloneStructuredCompletionRequest(req))
|
||||||
|
if client.err != nil {
|
||||||
|
return contracts.StructuredCompletionResponse{}, client.err
|
||||||
|
}
|
||||||
|
target, ok := out.(*extractionResponse)
|
||||||
|
if !ok {
|
||||||
|
return contracts.StructuredCompletionResponse{}, errors.New("unexpected output target")
|
||||||
|
}
|
||||||
|
content := append([]byte(nil), client.content...)
|
||||||
|
if len(content) == 0 {
|
||||||
|
var err error
|
||||||
|
content, err = json.Marshal(client.response)
|
||||||
|
if err != nil {
|
||||||
|
return contracts.StructuredCompletionResponse{}, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(content, target); err != nil {
|
||||||
|
return contracts.StructuredCompletionResponse{}, err
|
||||||
|
}
|
||||||
|
return contracts.StructuredCompletionResponse{Content: content}, nil
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user