From f51b22bea7d4dc352f9938e4627e3c7c05bb06fd Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Sat, 25 Jul 2026 21:49:24 +0000 Subject: [PATCH] Add D&D item event extractor --- .../dnd/codec/itemevents/codec_test.go | 1 + .../modules/dnd/extract/itemevents/assets.go | 6 + .../assets/prompts/dnd.item_events.yaml | 42 +++++ .../itemevents/assets/prompts/instructions.md | 22 +++ .../extract/itemevents/assets/prompts/task.md | 7 + .../schemas/dnd_item_events_llm.v1.json | 36 ++++ .../dnd/extract/itemevents/canonicalize.go | 99 +++++++++++ .../dnd/extract/itemevents/extractor.go | 163 ++++++++++++++++++ .../dnd/extract/itemevents/extractor_test.go | 147 ++++++++++++++++ .../modules/dnd/extract/itemevents/model.go | 19 ++ .../dnd/extract/itemevents/registry_test.go | 65 +++++++ .../modules/dnd/extract/itemevents/schema.go | 21 +++ .../dnd/extract/itemevents/schema_test.go | 80 +++++++++ .../extract/itemevents/scriptorium_assets.go | 52 ++++++ .../itemevents/scriptorium_assets_test.go | 114 ++++++++++++ .../extract/itemevents/test_helpers_test.go | 92 ++++++++++ 16 files changed, 966 insertions(+) create mode 100644 internal/modules/dnd/extract/itemevents/assets.go create mode 100644 internal/modules/dnd/extract/itemevents/assets/prompts/dnd.item_events.yaml create mode 100644 internal/modules/dnd/extract/itemevents/assets/prompts/instructions.md create mode 100644 internal/modules/dnd/extract/itemevents/assets/prompts/task.md create mode 100644 internal/modules/dnd/extract/itemevents/assets/schemas/dnd_item_events_llm.v1.json create mode 100644 internal/modules/dnd/extract/itemevents/canonicalize.go create mode 100644 internal/modules/dnd/extract/itemevents/extractor.go create mode 100644 internal/modules/dnd/extract/itemevents/extractor_test.go create mode 100644 internal/modules/dnd/extract/itemevents/model.go create mode 100644 internal/modules/dnd/extract/itemevents/registry_test.go create mode 100644 internal/modules/dnd/extract/itemevents/schema.go create mode 100644 internal/modules/dnd/extract/itemevents/schema_test.go create mode 100644 internal/modules/dnd/extract/itemevents/scriptorium_assets.go create mode 100644 internal/modules/dnd/extract/itemevents/scriptorium_assets_test.go create mode 100644 internal/modules/dnd/extract/itemevents/test_helpers_test.go diff --git a/internal/modules/dnd/codec/itemevents/codec_test.go b/internal/modules/dnd/codec/itemevents/codec_test.go index db69582..7066247 100644 --- a/internal/modules/dnd/codec/itemevents/codec_test.go +++ b/internal/modules/dnd/codec/itemevents/codec_test.go @@ -103,6 +103,7 @@ func TestCodecRejectsStrictJSONAndApprovedBoundaries(t *testing.T) { {"party transfer", strings.Replace(validJSON, `"kind":"acquired","to":"party"`, `"kind":"transferred","from":"party","to":"Borin"`, 1), "holders are incompatible"}, {"self transfer", strings.Replace(validJSON, `"kind":"acquired","to":"party"`, `"kind":"transferred","from":"Aria","to":"aria"`, 1), "holders are incompatible"}, {"missing source refs", strings.Replace(validJSON, `,"source_refs":[{"source_id":"session","start_unit_id":1,"end_unit_id":1}]`, "", 1), "source_refs must contain"}, + {"empty source refs", strings.Replace(validJSON, `[{"source_id":"session","start_unit_id":1,"end_unit_id":1}]`, `[]`, 1), "source_refs must contain"}, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { diff --git a/internal/modules/dnd/extract/itemevents/assets.go b/internal/modules/dnd/extract/itemevents/assets.go new file mode 100644 index 0000000..4a29b5d --- /dev/null +++ b/internal/modules/dnd/extract/itemevents/assets.go @@ -0,0 +1,6 @@ +package itemevents + +import "embed" + +//go:embed assets/schemas/dnd_item_events_llm.v1.json assets/prompts/*.yaml assets/prompts/*.md +var embeddedAssets embed.FS diff --git a/internal/modules/dnd/extract/itemevents/assets/prompts/dnd.item_events.yaml b/internal/modules/dnd/extract/itemevents/assets/prompts/dnd.item_events.yaml new file mode 100644 index 0000000..66830d4 --- /dev/null +++ b/internal/modules/dnd/extract/itemevents/assets/prompts/dnd.item_events.yaml @@ -0,0 +1,42 @@ +id: dnd.item_events +version: "v1" +default_profile: gemini-2-flash +inputs: + - name: transcript + required: true + content_type: application/json + - name: players + required: false + content_type: text/plain + - name: party + required: false + content_type: text/plain + - name: glossary + required: false + content_type: text/plain +messages: + - role: system + content_file: ./sharedassets/common-dnd-system.md + - role: user + content_file: ./sharedassets/common-dnd-extraction-evidence.md + - role: user + content_file: ./sharedassets/common-dnd-identity.md + cache_control: + type: ephemeral + - role: user + content_file: ./sharedassets/common-dnd-references.md + cache_control: + type: ephemeral + - role: user + content_file: ./task.md + - role: user + content_file: ./instructions.md + cache_control: + type: ephemeral + - role: user + content_file: ./sharedassets/common-dnd-transcript.md +output: + format: json + validation_mode: json_schema + schema_path: dnd_item_events_llm.v1.json + repair_attempts: 0 diff --git a/internal/modules/dnd/extract/itemevents/assets/prompts/instructions.md b/internal/modules/dnd/extract/itemevents/assets/prompts/instructions.md new file mode 100644 index 0000000..aa7d29d --- /dev/null +++ b/internal/modules/dnd/extract/itemevents/assets/prompts/instructions.md @@ -0,0 +1,22 @@ +Return one event only when the transcript establishes a meaningful item or +currency occurrence. Use a concise observed item name and preserve the stated +currency denomination; include quantity only when the transcript explicitly +states it. + +Use `discovered` when the party learns of or encounters an item without +establishing possession. Use `acquired` when the party or a party member gains +possession. Use `lost` when party possession ends through a gift, sale, payment, +theft, abandonment, or destruction not caused by intended use. Use `consumed` +when intended use depletes an item or currency. Use `transferred` only when +possession moves between two distinct named party members. + +For `discovered`, omit both holders. For `acquired`, provide only `to`; for +`lost` and `consumed`, provide only `from`; and for `transferred`, provide both +`from` and `to`. Use `party` only for collective or unresolved party possession, +never for either side of a transfer. Do not emit a transfer for a gift, sale, or +payment outside the party. + +Ordinary non-depleting use is not an event. Do not infer acquisition from a +discovery, or discovery from an acquisition: emit both only when each is +independently established. Every event needs at least one narrow transcript +range. Return no lore, inventory totals, aliases, or unstated holder details. diff --git a/internal/modules/dnd/extract/itemevents/assets/prompts/task.md b/internal/modules/dnd/extract/itemevents/assets/prompts/task.md new file mode 100644 index 0000000..fe267c0 --- /dev/null +++ b/internal/modules/dnd/extract/itemevents/assets/prompts/task.md @@ -0,0 +1,7 @@ +Extract meaningful Dungeons & Dragons item and currency events established by +the transcript. Record only discoveries and changes in party possession, with +the transcript ranges that support each event. + +This is an event history, not an inventory or ledger. Do not calculate balances, +resolve item identity across records, or infer ownership that the transcript +does not establish. diff --git a/internal/modules/dnd/extract/itemevents/assets/schemas/dnd_item_events_llm.v1.json b/internal/modules/dnd/extract/itemevents/assets/schemas/dnd_item_events_llm.v1.json new file mode 100644 index 0000000..6defd07 --- /dev/null +++ b/internal/modules/dnd/extract/itemevents/assets/schemas/dnd_item_events_llm.v1.json @@ -0,0 +1,36 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "notarius.dnd.item_events.llm", + "type": "object", + "additionalProperties": false, + "required": ["events"], + "properties": { + "events": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": ["name", "kind", "source_refs"], + "properties": { + "name": {"type": "string"}, + "kind": {"type": "string"}, + "quantity": {"type": "integer"}, + "from": {"type": "string"}, + "to": {"type": "string"}, + "source_refs": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": ["start_segment", "end_segment"], + "properties": { + "start_segment": {"type": "integer"}, + "end_segment": {"type": "integer"} + } + } + } + } + } + } + } +} diff --git a/internal/modules/dnd/extract/itemevents/canonicalize.go b/internal/modules/dnd/extract/itemevents/canonicalize.go new file mode 100644 index 0000000..8f90109 --- /dev/null +++ b/internal/modules/dnd/extract/itemevents/canonicalize.go @@ -0,0 +1,99 @@ +package itemevents + +import ( + "sort" + + "gitea.maximumdirect.net/eric/notarius/internal/core/source" + "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd" + "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared" +) + +type orderedItemEventResponse struct { + value itemEventResponse + earliest int + hasEvidence bool +} + +func canonicalizeResponse(response *extractionResponse, order shared.SourceRefOrder, sourceID string) { + if response == nil { + return + } + ordered := make([]orderedItemEventResponse, len(response.Events)) + for index := range response.Events { + earliest, hasEvidence := canonicalizeItemEvent(&response.Events[index], order, sourceID) + ordered[index] = orderedItemEventResponse{ + value: response.Events[index], + earliest: earliest, + hasEvidence: hasEvidence, + } + } + sort.SliceStable(ordered, func(left, right int) bool { + if ordered[left].hasEvidence != ordered[right].hasEvidence { + return ordered[left].hasEvidence + } + if !ordered[left].hasEvidence { + return false + } + return ordered[left].earliest < ordered[right].earliest + }) + for index := range ordered { + response.Events[index] = ordered[index].value + } +} + +func canonicalizeItemEvent(event *itemEventResponse, order shared.SourceRefOrder, sourceID string) (int, bool) { + if event == nil { + return 0, false + } + refs := order.Canonicalize(itemEventSourceRefs(event.SourceRefs, sourceID)) + event.SourceRefs = itemEventResponseRefs(refs) + return order.EarliestValid(refs) +} + +func canonicalItemEventList(response extractionResponse, sourceID string) dnd.ItemEventList { + if response.Events == nil { + return dnd.ItemEventList{} + } + events := make([]dnd.ItemEvent, len(response.Events)) + for index, event := range response.Events { + events[index] = dnd.ItemEvent{ + Name: event.Name, + Kind: dnd.ItemEventKind(event.Kind), + Quantity: cloneQuantity(event.Quantity), + From: event.From, + To: event.To, + SourceRefs: itemEventSourceRefs(event.SourceRefs, sourceID), + } + } + return dnd.ItemEventList{Events: events} +} + +func itemEventSourceRefs(refs []itemEventSourceRefResponse, sourceID string) []source.SourceRef { + if refs == nil { + return nil + } + values := make([]source.SourceRef, len(refs)) + for index, ref := range refs { + values[index] = source.SourceRef{SourceID: sourceID, StartUnitID: ref.StartSegment, EndUnitID: ref.EndSegment} + } + return values +} + +func itemEventResponseRefs(refs []source.SourceRef) []itemEventSourceRefResponse { + if refs == nil { + return nil + } + values := make([]itemEventSourceRefResponse, len(refs)) + for index, ref := range refs { + values[index] = itemEventSourceRefResponse{StartSegment: ref.StartUnitID, EndSegment: ref.EndUnitID} + } + return values +} + +func cloneQuantity(value *int) *int { + if value == nil { + return nil + } + quantity := *value + return &quantity +} diff --git a/internal/modules/dnd/extract/itemevents/extractor.go b/internal/modules/dnd/extract/itemevents/extractor.go new file mode 100644 index 0000000..579ad25 --- /dev/null +++ b/internal/modules/dnd/extract/itemevents/extractor.go @@ -0,0 +1,163 @@ +// Package itemevents extracts source-grounded D&D item events. +package itemevents + +import ( + "context" + "fmt" + + "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" + "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" + "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd" + "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared" +) + +const Key = "dnd/item-events" + +const mappingPolicy = "dnd.item_events.extract_mapping.v1" + +var requiredCapabilities = []string{ + "chunks", + "source.transcript", +} + +var providedCapabilities = []string{ + "dnd.item_events", +} + +var referenceSlotDescriptions = shared.ReferenceSlotDescriptions{ + Glossary: "Optional campaign glossary reference material used only for item and holder disambiguation.", + Party: "Optional party roster reference material used only for item and holder disambiguation.", + Players: "Optional player list reference material used only for item and holder disambiguation.", + Roster: "Deprecated alias for party roster reference material used only for item and holder disambiguation.", +} + +func referenceSlots() []contracts.ReferenceSlot { + return shared.ReferenceSlots(referenceSlotDescriptions) +} + +var _ contracts.Extractor[dnd.ItemEventList] = (*Extractor)(nil) +var _ contracts.ManifestMetadataProvider = (*Extractor)(nil) +var _ pipeline.CheckpointFingerprintProvider = (*Extractor)(nil) + +type Options struct{} + +type Extractor struct { + llm contracts.StructuredLLMClient + promptSHA string + responseSchemaSHA string +} + +func New(llmClient contracts.StructuredLLMClient, _ Options, references ...contracts.ReferenceSet) (*Extractor, error) { + if llmClient == nil { + return nil, extractorErrorf("LLM client must not be nil") + } + if len(references) > 1 { + return nil, extractorErrorf("at most one reference set may be supplied") + } + promptSHA, err := scriptoriumPromptMetadata() + if err != nil { + return nil, extractorErrorf("load prompt metadata: %w", err) + } + responseSchema, err := loadResponseSchema() + if err != nil { + return nil, extractorErrorf("load response schema: %w", err) + } + return &Extractor{llm: llmClient, promptSHA: promptSHA, responseSchemaSHA: responseSchema.SHA256}, nil +} + +func (e *Extractor) Key() string { return Key } + +func (e *Extractor) ReferenceSlots() []contracts.ReferenceSlot { return referenceSlots() } + +func (e *Extractor) ManifestMetadata() map[string]any { + if e == nil { + return nil + } + return map[string]any{ + "prompt_id": PromptID, + "prompt_version": SchemaVersion, + "prompt_sha256": e.promptSHA, + "response_schema_key": string(ResponseSchemaKey), + "response_schema_id": ResponseSchemaID, + "response_schema_name": ResponseSchemaName, + "response_schema_version": SchemaVersion, + "response_schema_sha256": e.responseSchemaSHA, + "mapping_policy": mappingPolicy, + } +} + +func (e *Extractor) CheckpointFingerprints() []pipeline.CheckpointFingerprint { + if e == nil { + return nil + } + return []pipeline.CheckpointFingerprint{ + {Name: "prompt", Value: e.promptSHA}, + {Name: "response_schema", Value: e.responseSchemaSHA}, + {Name: "mapping_policy", Value: mappingPolicy}, + } +} + +func (e *Extractor) Extract(ctx context.Context, req contracts.TypedExtractionRequest) (contracts.TypedExtractionResult[dnd.ItemEventList], error) { + if e == nil { + return contracts.TypedExtractionResult[dnd.ItemEventList]{}, extractorErrorf("extractor must not be nil") + } + if e.llm == nil { + return contracts.TypedExtractionResult[dnd.ItemEventList]{}, extractorErrorf("LLM client must not be nil") + } + sourceInput, err := shared.PrepareChunkExtraction(ctx, req) + if err != nil { + return contracts.TypedExtractionResult[dnd.ItemEventList]{}, extractorErrorf("%w", err) + } + order := shared.NewSourceRefOrder(req.Source) + + var response extractionResponse + if _, err := e.llm.CompleteStructured(ctx, contracts.StructuredCompletionRequest{ + StageName: Key, + PromptID: PromptID, + PromptVersion: SchemaVersion, + ProfileID: req.LLMProfile, + SessionID: req.SessionID, + Inputs: shared.PromptInputs(sourceInput, req.References), + }, &response); err != nil { + return contracts.TypedExtractionResult[dnd.ItemEventList]{}, extractorErrorf("complete structured output: %w", err) + } + canonicalizeResponse(&response, order, req.Source.ID) + return contracts.TypedExtractionResult[dnd.ItemEventList]{Value: canonicalItemEventList(response, req.Source.ID)}, nil +} + +func ModuleSpec() pipeline.ModuleSpec { + return pipeline.ModuleSpec{ + Key: Key, + Stage: pipeline.StageExtract, + Requires: append([]string(nil), requiredCapabilities...), + Provides: append([]string(nil), providedCapabilities...), + ArtifactKind: dnd.ItemEventListKind, + ReferenceSlots: referenceSlots(), + } +} + +func Register(registry *pipeline.ExtractorRegistry) error { + return pipeline.RegisterExtractorBuilder(registry, ModuleSpec(), validateOptions, func(request pipeline.BuildRequest) (contracts.Extractor[dnd.ItemEventList], error) { + options, err := DecodeOptions(request.Options) + if err != nil { + return nil, err + } + return New(request.Dependencies.LLM, options, request.References) + }) +} + +func validateOptions(options map[string]any) error { + _, err := DecodeOptions(options) + return err +} + +func DecodeOptions(options map[string]any) (Options, error) { + if err := pipeline.RejectUnknownOptions(options); err != nil { + return Options{}, extractorErrorf("%w", err) + } + return Options{}, nil +} + +func extractorErrorf(format string, args ...any) error { + return fmt.Errorf("dnd item events extractor: "+format, args...) +} diff --git a/internal/modules/dnd/extract/itemevents/extractor_test.go b/internal/modules/dnd/extract/itemevents/extractor_test.go new file mode 100644 index 0000000..8c2cb54 --- /dev/null +++ b/internal/modules/dnd/extract/itemevents/extractor_test.go @@ -0,0 +1,147 @@ +package itemevents + +import ( + "context" + "errors" + "reflect" + "strings" + "testing" + + "gitea.maximumdirect.net/eric/notarius/internal/core/source" + "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" + "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd" +) + +func TestExtractMapsAllEventKindsAndCanonicalizesEvidence(t *testing.T) { + quantity := 20 + client := &fakeItemEventsLLMClient{response: extractionResponse{Events: []itemEventResponse{ + {Name: "Torch", Kind: "lost", From: "party", SourceRefs: responseRefs(5, 5)}, + {Name: "Moonblade", Kind: "transferred", From: "Aria", To: "Borin", SourceRefs: responseRefs(3, 3)}, + {Name: "Hidden Cache", Kind: "discovered", SourceRefs: []itemEventSourceRefResponse{{StartSegment: 1, EndSegment: 1}, {StartSegment: 1, EndSegment: 1}}}, + {Name: "Gold Pieces", Kind: "acquired", Quantity: &quantity, To: "party", SourceRefs: responseRefs(2, 2)}, + {Name: "Healing Potion", Kind: "consumed", From: "party", SourceRefs: responseRefs(4, 4)}, + }}} + + result, err := newExtractor(t, client).Extract(context.Background(), extractionRequest()) + if err != nil { + t.Fatalf("Extract() error = %v", err) + } + if got := []dnd.ItemEventKind{result.Value.Events[0].Kind, result.Value.Events[1].Kind, result.Value.Events[2].Kind, result.Value.Events[3].Kind, result.Value.Events[4].Kind}; !reflect.DeepEqual(got, []dnd.ItemEventKind{ + dnd.ItemEventKindDiscovered, dnd.ItemEventKindAcquired, dnd.ItemEventKindTransferred, dnd.ItemEventKindConsumed, dnd.ItemEventKindLost, + }) { + t.Fatalf("event kinds = %#v, want source order", got) + } + acquired := result.Value.Events[1] + if acquired.Name != "Gold Pieces" || acquired.To != "party" || acquired.Quantity == nil || *acquired.Quantity != 20 { + t.Fatalf("acquired event = %#v", acquired) + } + if result.Value.Events[0].Quantity != nil || result.Value.Events[0].From != "" || result.Value.Events[0].To != "" { + t.Fatalf("discovered event = %#v, want omitted optional fields", result.Value.Events[0]) + } + if refs := result.Value.Events[0].SourceRefs; len(refs) != 1 || refs[0] != (source.SourceRef{SourceID: "session-alpha", StartUnitID: 1, EndUnitID: 1}) { + t.Fatalf("source refs = %#v, want source ID attachment and deduplication", refs) + } + if acquired.Quantity == client.response.Events[3].Quantity { + t.Fatal("mapped quantity aliases private response") + } + if len(client.requests) != 1 { + t.Fatalf("LLM calls = %d, want 1", len(client.requests)) + } + request := client.requests[0] + if request.StageName != Key || request.PromptID != PromptID || request.PromptVersion != SchemaVersion || request.ProfileID != "profile-item-events" || request.SessionID != "session-123" { + t.Fatalf("LLM request identity = %#v", request) + } + if transcript := request.Inputs["transcript"]; string(transcript.Content) != string(extractionRequest().Chunk.Content) || transcript.Name != "transcript" { + t.Fatalf("transcript input = %#v", transcript) + } + for _, name := range []string{"players", "party", "glossary"} { + if input := request.Inputs[name]; string(input.Content) != " " { + t.Fatalf("absent %s input = %#v, want retained empty prompt material", name, input) + } + } +} + +func TestExtractPreservesInvalidCandidatesAndEmptyResults(t *testing.T) { + client := &fakeItemEventsLLMClient{content: []byte(`{"events":[{"name":"","kind":"transferred","quantity":0,"from":"party","to":"Party","source_refs":[{"start_segment":99,"end_segment":-1}]}]}`)} + result, err := newExtractor(t, client).Extract(context.Background(), extractionRequest()) + if err != nil { + t.Fatalf("Extract() error = %v", err) + } + event := result.Value.Events[0] + if event.Name != "" || event.Kind != "transferred" || event.Quantity == nil || *event.Quantity != 0 || event.From != "party" || event.To != "Party" { + t.Fatalf("invalid candidate = %#v, want values preserved", event) + } + if refs := event.SourceRefs; len(refs) != 1 || refs[0] != (source.SourceRef{SourceID: "session-alpha", StartUnitID: 99, EndUnitID: -1}) { + t.Fatalf("source refs = %#v, want invalid range preserved", refs) + } + + empty, err := newExtractor(t, &fakeItemEventsLLMClient{response: extractionResponse{Events: []itemEventResponse{}}}).Extract(context.Background(), extractionRequest()) + if err != nil || empty.Value.Events == nil || len(empty.Value.Events) != 0 { + t.Fatalf("empty provider result = %#v, %v; want valid empty list", empty.Value, err) + } +} + +func TestExtractUsesSourceDocumentOrderForCandidates(t *testing.T) { + client := &fakeItemEventsLLMClient{response: extractionResponse{Events: []itemEventResponse{ + {Name: "Later", Kind: "discovered", SourceRefs: responseRefs(10, 10)}, + {Name: "Earlier", Kind: "discovered", SourceRefs: responseRefs(100, 100)}, + }}} + req := extractionRequest() + req.Source.Units = []source.SourceUnit{{ID: 100}, {ID: 10}} + req.Chunk.Units = append([]source.SourceUnit(nil), req.Source.Units...) + req.Chunk.Ref = source.SourceRef{SourceID: req.Source.ID, StartUnitID: 100, EndUnitID: 10} + result, err := newExtractor(t, client).Extract(context.Background(), req) + if err != nil { + t.Fatal(err) + } + if got := []string{result.Value.Events[0].Name, result.Value.Events[1].Name}; !reflect.DeepEqual(got, []string{"Earlier", "Later"}) { + t.Fatalf("event order = %#v, want source document order", got) + } +} + +func TestExtractUsesOnlySupportedPromptInputs(t *testing.T) { + client := &fakeItemEventsLLMClient{response: extractionResponse{Events: []itemEventResponse{}}} + req := extractionRequest() + req.References = contracts.ReferenceSet{Slots: map[string]contracts.ResolvedReferenceSlot{ + "players": {Slot: contracts.ReferenceSlot{Name: "players"}, Items: []contracts.ReferenceItem{{SlotName: "players", Content: []byte("Dana: Aria")}}}, + "party": {Slot: contracts.ReferenceSlot{Name: "party"}, Items: []contracts.ReferenceItem{{SlotName: "party", Content: []byte("Aria: ranger")}}}, + "glossary": {Slot: contracts.ReferenceSlot{Name: "glossary"}, Items: []contracts.ReferenceItem{{SlotName: "glossary", Content: []byte("Moonblade: heirloom")}}}, + "npcs": {Slot: contracts.ReferenceSlot{Name: "npcs"}, Items: []contracts.ReferenceItem{{SlotName: "npcs", Content: []byte("must not be used")}}}, + }} + if _, err := newExtractor(t, client).Extract(context.Background(), req); err != nil { + t.Fatal(err) + } + inputs := client.requests[0].Inputs + if string(inputs["players"].Content) != "Dana: Aria" || string(inputs["party"].Content) != "Aria: ranger" || string(inputs["glossary"].Content) != "Moonblade: heirloom" { + t.Fatalf("reference inputs = %#v", inputs) + } + if _, ok := inputs["npcs"]; ok { + t.Fatalf("unexpected generated-lane input: %#v", inputs) + } +} + +func TestExtractRetainsLocalErrorContext(t *testing.T) { + request := extractionRequest() + extractor := newExtractor(t, &fakeItemEventsLLMClient{}) + var nilExtractor *Extractor + request.SourceInput = contracts.NewLLMInputMaterial("source", "application/json", []byte(`{"different":true}`), "sha256:other", "file:///other.json") + for _, test := range []struct { + name string + extractor *Extractor + req contracts.TypedExtractionRequest + want string + }{ + {"nil extractor", nilExtractor, extractionRequest(), "extractor"}, + {"nil client", &Extractor{}, extractionRequest(), "LLM client"}, + {"preflight", extractor, request, "must match chunk"}, + } { + t.Run(test.name, func(t *testing.T) { + if _, err := test.extractor.Extract(context.Background(), test.req); err == nil || !strings.Contains(err.Error(), "dnd item events") || !strings.Contains(err.Error(), test.want) { + t.Fatalf("Extract() error = %v", err) + } + }) + } + if _, err := newExtractor(t, &fakeItemEventsLLMClient{err: errors.New("provider unavailable")}).Extract(context.Background(), extractionRequest()); err == nil || !strings.Contains(err.Error(), "provider unavailable") { + t.Fatalf("provider error = %v", err) + } +} diff --git a/internal/modules/dnd/extract/itemevents/model.go b/internal/modules/dnd/extract/itemevents/model.go new file mode 100644 index 0000000..c91e6e8 --- /dev/null +++ b/internal/modules/dnd/extract/itemevents/model.go @@ -0,0 +1,19 @@ +package itemevents + +type extractionResponse struct { + Events []itemEventResponse `json:"events"` +} + +type itemEventResponse struct { + Name string `json:"name"` + Kind string `json:"kind"` + Quantity *int `json:"quantity,omitempty"` + From string `json:"from,omitempty"` + To string `json:"to,omitempty"` + SourceRefs []itemEventSourceRefResponse `json:"source_refs"` +} + +type itemEventSourceRefResponse struct { + StartSegment int `json:"start_segment"` + EndSegment int `json:"end_segment"` +} diff --git a/internal/modules/dnd/extract/itemevents/registry_test.go b/internal/modules/dnd/extract/itemevents/registry_test.go new file mode 100644 index 0000000..640ecbf --- /dev/null +++ b/internal/modules/dnd/extract/itemevents/registry_test.go @@ -0,0 +1,65 @@ +package itemevents + +import ( + "reflect" + "strings" + "testing" + + "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" + "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" + "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd" +) + +func TestConstructorSpecOptionsAndMetadata(t *testing.T) { + if _, err := New(nil, Options{}); err == nil || !strings.Contains(err.Error(), "LLM client") { + t.Fatalf("New(nil) error = %v", err) + } + if _, err := New(&fakeItemEventsLLMClient{}, Options{}, contracts.ReferenceSet{}, contracts.ReferenceSet{}); err == nil || !strings.Contains(err.Error(), "at most one reference set") { + t.Fatalf("New() error = %v", err) + } + want := pipeline.ModuleSpec{ + Key: Key, Stage: pipeline.StageExtract, Requires: []string{"chunks", "source.transcript"}, Provides: []string{"dnd.item_events"}, ArtifactKind: dnd.ItemEventListKind, + ReferenceSlots: []contracts.ReferenceSlot{ + {Name: "glossary", Description: referenceSlotDescriptions.Glossary, AcceptedMediaTypes: []string{"application/json", "application/x-yaml", "application/yaml", "text/markdown", "text/plain"}}, + {Name: "party", Description: referenceSlotDescriptions.Party, AcceptedMediaTypes: []string{"application/json", "application/x-yaml", "application/yaml", "text/markdown", "text/plain"}}, + {Name: "players", Description: referenceSlotDescriptions.Players, AcceptedMediaTypes: []string{"application/json", "application/x-yaml", "application/yaml", "text/markdown", "text/plain"}}, + {Name: "roster", Description: referenceSlotDescriptions.Roster, AcceptedMediaTypes: []string{"application/json", "application/x-yaml", "application/yaml", "text/markdown", "text/plain"}}, + }, + } + if got := ModuleSpec(); !reflect.DeepEqual(got, want) { + t.Fatalf("ModuleSpec() = %#v, want %#v", got, want) + } + if _, err := DecodeOptions(map[string]any{"unexpected": true}); err == nil || !strings.Contains(err.Error(), "unknown option") { + t.Fatalf("DecodeOptions() error = %v", err) + } + registry := pipeline.NewExtractorRegistry() + if err := Register(registry); err != nil { + t.Fatal(err) + } + if got, ok := registry.Spec(Key); !ok || !reflect.DeepEqual(got, want) { + t.Fatalf("registry spec = %#v, %t", got, ok) + } + + extractor := newExtractor(t, &fakeItemEventsLLMClient{}) + metadata := extractor.ManifestMetadata() + for key, want := range map[string]string{ + "prompt_id": PromptID, "prompt_version": SchemaVersion, "response_schema_key": string(ResponseSchemaKey), "response_schema_id": ResponseSchemaID, + "response_schema_name": ResponseSchemaName, "response_schema_version": SchemaVersion, "mapping_policy": mappingPolicy, + } { + if metadata[key] != want { + t.Fatalf("metadata[%q] = %#v, want %q", key, metadata[key], want) + } + } + for _, key := range []string{"prompt_sha256", "response_schema_sha256"} { + if value, ok := metadata[key].(string); !ok || !strings.HasPrefix(value, "sha256:") { + t.Fatalf("metadata[%q] = %#v", key, metadata[key]) + } + } + if got := extractor.CheckpointFingerprints(); !reflect.DeepEqual(got, []pipeline.CheckpointFingerprint{ + {Name: "prompt", Value: metadata["prompt_sha256"].(string)}, + {Name: "response_schema", Value: metadata["response_schema_sha256"].(string)}, + {Name: "mapping_policy", Value: mappingPolicy}, + }) { + t.Fatalf("CheckpointFingerprints() = %#v", got) + } +} diff --git a/internal/modules/dnd/extract/itemevents/schema.go b/internal/modules/dnd/extract/itemevents/schema.go new file mode 100644 index 0000000..10791b1 --- /dev/null +++ b/internal/modules/dnd/extract/itemevents/schema.go @@ -0,0 +1,21 @@ +package itemevents + +import "gitea.maximumdirect.net/eric/notarius/internal/framework/llm" + +const ( + PromptID = "dnd.item_events" + ResponseSchemaKey = llm.ResponseSchemaKey("dnd_item_events_llm") + ResponseSchemaID = "notarius.dnd.item_events.llm" + ResponseSchemaName = "notarius_dnd_item_events_llm_v1" + SchemaVersion = "v1" +) + +func loadResponseSchema() (llm.ResponseSchema, error) { + return llm.LoadResponseSchema(embeddedAssets, llm.ResponseSchemaDefinition{ + Key: ResponseSchemaKey, + ID: ResponseSchemaID, + Version: SchemaVersion, + Name: ResponseSchemaName, + AssetPath: "assets/schemas/dnd_item_events_llm.v1.json", + }) +} diff --git a/internal/modules/dnd/extract/itemevents/schema_test.go b/internal/modules/dnd/extract/itemevents/schema_test.go new file mode 100644 index 0000000..18497d4 --- /dev/null +++ b/internal/modules/dnd/extract/itemevents/schema_test.go @@ -0,0 +1,80 @@ +package itemevents + +import ( + "bytes" + "encoding/json" + "strings" + "testing" + + "github.com/santhosh-tekuri/jsonschema/v6" +) + +func TestResponseSchemaIsStrictlyStructuralAndPrivate(t *testing.T) { + schema, err := loadResponseSchema() + if err != nil { + t.Fatal(err) + } + if schema.Key != ResponseSchemaKey || schema.ID != ResponseSchemaID || schema.Name != ResponseSchemaName || schema.Version != SchemaVersion || !strings.HasPrefix(schema.SHA256, "sha256:") || !json.Valid(schema.JSONSchema) { + t.Fatalf("schema = %#v", schema) + } + valid := map[string]any{"events": []any{map[string]any{ + "name": "", "kind": "unsupported", "quantity": 0, "from": "party", "to": "Party", + "source_refs": []any{map[string]any{"start_segment": 0, "end_segment": -1}}, + }}} + content, err := json.Marshal(valid) + if err != nil { + t.Fatal(err) + } + if err := validateJSONSchema(content, schema.JSONSchema); err != nil { + t.Fatalf("private schema rejected validator-owned values: %v", err) + } + for _, test := range []struct { + name string + value map[string]any + }{ + {"missing events", map[string]any{}}, + {"missing event name", map[string]any{"events": []any{map[string]any{"kind": "acquired", "source_refs": []any{}}}}}, + {"unknown event field", map[string]any{"events": []any{map[string]any{"name": "Ring", "kind": "acquired", "source_refs": []any{}, "extra": true}}}}, + {"unknown reference field", map[string]any{"events": []any{map[string]any{"name": "Ring", "kind": "acquired", "source_refs": []any{map[string]any{"start_segment": 1, "end_segment": 1, "extra": true}}}}}}, + {"noninteger range", map[string]any{"events": []any{map[string]any{"name": "Ring", "kind": "acquired", "source_refs": []any{map[string]any{"start_segment": 1.5, "end_segment": 1}}}}}}, + } { + t.Run(test.name, func(t *testing.T) { + content, err := json.Marshal(test.value) + if err != nil { + t.Fatal(err) + } + if err := validateJSONSchema(content, schema.JSONSchema); err == nil { + t.Fatal("private schema accepted invalid structure") + } + }) + } + first := schema.JSONSchema + first[0] = '[' + second, err := loadResponseSchema() + if err != nil || !json.Valid(second.JSONSchema) || bytes.Equal(first, second.JSONSchema) { + t.Fatalf("defensive schema copy = %s, %v", second.JSONSchema, err) + } + if _, ok := second.DiagnosticsMap()["json_schema"]; ok { + t.Fatalf("schema diagnostics leaked content: %#v", second.DiagnosticsMap()) + } +} + +func validateJSONSchema(instanceContent, schemaContent []byte) error { + instance, err := jsonschema.UnmarshalJSON(bytes.NewReader(instanceContent)) + if err != nil { + return err + } + document, err := jsonschema.UnmarshalJSON(bytes.NewReader(schemaContent)) + if err != nil { + return err + } + compiler := jsonschema.NewCompiler() + if err := compiler.AddResource("schema.json", document); err != nil { + return err + } + compiled, err := compiler.Compile("schema.json") + if err != nil { + return err + } + return compiled.Validate(instance) +} diff --git a/internal/modules/dnd/extract/itemevents/scriptorium_assets.go b/internal/modules/dnd/extract/itemevents/scriptorium_assets.go new file mode 100644 index 0000000..74113fe --- /dev/null +++ b/internal/modules/dnd/extract/itemevents/scriptorium_assets.go @@ -0,0 +1,52 @@ +package itemevents + +import ( + "fmt" + "sync" + + "gitea.maximumdirect.net/eric/notarius/internal/framework/llm" + "gitea.maximumdirect.net/eric/notarius/internal/framework/promptfs" + "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared" +) + +const scriptoriumPromptRoot = "assets/prompts" + +var promptAssetManifest = shared.PromptAssetManifest{ + ModuleDir: "dnd.item_events", + ModuleFiles: []promptfs.ModulePromptFile{ + {Name: "dnd.item_events.yaml", Path: "assets/prompts/dnd.item_events.yaml"}, + {Name: "task.md", Path: "assets/prompts/task.md"}, + {Name: "instructions.md", Path: "assets/prompts/instructions.md"}, + }, + SharedFiles: []string{ + "common-dnd-system.md", + "common-dnd-extraction-evidence.md", + "common-dnd-identity.md", + "common-dnd-references.md", + "common-dnd-transcript.md", + }, +} + +func RegisterPromptAssets(registry *llm.AssetRegistry) error { + promptFS, err := promptAssetManifest.PromptFS(embeddedAssets) + if err != nil { + return fmt.Errorf("prepare item event prompt assets: %w", err) + } + if err := registry.RegisterPromptFS(promptFS, scriptoriumPromptRoot); err != nil { + return err + } + return registry.RegisterSchemaFS(embeddedAssets, "assets/schemas") +} + +func scriptoriumPromptMetadata() (string, error) { + scriptoriumPromptHashOnce.Do(func() { + scriptoriumPromptHash, scriptoriumPromptHashErr = promptAssetManifest.Hash(embeddedAssets) + }) + return scriptoriumPromptHash, scriptoriumPromptHashErr +} + +var ( + scriptoriumPromptHashOnce sync.Once + scriptoriumPromptHash string + scriptoriumPromptHashErr error +) diff --git a/internal/modules/dnd/extract/itemevents/scriptorium_assets_test.go b/internal/modules/dnd/extract/itemevents/scriptorium_assets_test.go new file mode 100644 index 0000000..c8f8867 --- /dev/null +++ b/internal/modules/dnd/extract/itemevents/scriptorium_assets_test.go @@ -0,0 +1,114 @@ +package itemevents + +import ( + "context" + "encoding/json" + "io/fs" + "reflect" + "strings" + "testing" + "time" + + "gitea.maximumdirect.net/eric/notarius/internal/framework/llm" + "gitea.maximumdirect.net/eric/scriptorium" +) + +func TestPromptAssetsUseSharedSequenceAndTranscriptLast(t *testing.T) { + registry := llm.NewAssetRegistry() + if err := RegisterPromptAssets(registry); err != nil { + t.Fatal(err) + } + options, err := registry.ScriptoriumOptions() + if err != nil { + t.Fatal(err) + } + options = append(options, scriptorium.WithProfiles(scriptorium.OpenAICompatibleProfile(scriptorium.OpenAICompatibleProfileConfig{ + ID: "item-events-test-profile", Endpoint: "http://127.0.0.1:1/v1", Model: "item-events-test-model", + }))) + engine, err := scriptorium.NewEngine(scriptorium.Config{Timeout: time.Second}, options...) + if err != nil { + t.Fatal(err) + } + prepared, err := engine.Prepare(context.Background(), scriptorium.RunRequest{ + PromptID: PromptID, PromptVersion: SchemaVersion, ProfileID: "item-events-test-profile", + Inputs: map[string]scriptorium.ArtifactRef{ + "transcript": scriptorium.InlineWithURI("file:///session.json", `{"segments":[1]}`), + "players": scriptorium.Inline("Dana: Aria"), + "party": scriptorium.Inline("Aria: ranger"), + "glossary": scriptorium.Inline("Moonblade: heirloom"), + }, + }) + if err != nil { + t.Fatal(err) + } + if prepared.PromptID != PromptID || prepared.OutputContract.SchemaPath != "dnd_item_events_llm.v1.json" { + t.Fatalf("prepared prompt = %#v", prepared) + } + want := []struct { + role string + cached bool + marker string + }{ + {"system", false, "Dungeons & Dragons gameplay transcripts"}, + {"user", false, "Transcript units are the only evidence"}, + {"user", true, "most specific supported in-world"}, + {"user", true, "Dana: Aria"}, + {"user", false, "item and currency events"}, + {"user", true, "Ordinary non-depleting use"}, + {"user", false, `{"segments":[1]}`}, + } + if len(prepared.Messages) != len(want) { + t.Fatalf("prompt messages = %d, want %d", len(prepared.Messages), len(want)) + } + for index, expected := range want { + message := prepared.Messages[index] + if message.Role != expected.role || !strings.Contains(message.Content, expected.marker) { + t.Fatalf("message %d = %#v", index, message) + } + if (message.CacheControl != nil) != expected.cached { + t.Fatalf("message %d cache control = %#v", index, message.CacheControl) + } + } + if strings.Contains(prepared.Messages[len(prepared.Messages)-1].Content, "Moonblade: heirloom") { + t.Fatal("transcript message contains optional reference content") + } +} + +func TestPromptAssetsDoNotLeakIntoMetadata(t *testing.T) { + hash, err := scriptoriumPromptMetadata() + if err != nil || !strings.HasPrefix(hash, "sha256:") { + t.Fatalf("scriptoriumPromptMetadata() = %q, %v", hash, err) + } + metadata := newExtractor(t, &fakeItemEventsLLMClient{}).ManifestMetadata() + payload, err := json.Marshal(metadata) + if err != nil { + t.Fatal(err) + } + for _, forbidden := range []string{"meaningful Dungeons", "common-dnd-system", "start_segment", "dnd_item_events_llm.v1.json"} { + if strings.Contains(string(payload), forbidden) { + t.Fatalf("metadata leaked raw asset content %q: %s", forbidden, payload) + } + } +} + +func TestPromptManifestReusesOnlySharedAssets(t *testing.T) { + want := []string{ + "common-dnd-system.md", + "common-dnd-extraction-evidence.md", + "common-dnd-identity.md", + "common-dnd-references.md", + "common-dnd-transcript.md", + } + if !reflect.DeepEqual(promptAssetManifest.SharedFiles, want) { + t.Fatalf("shared assets = %#v, want %#v", promptAssetManifest.SharedFiles, want) + } + for _, path := range []string{"assets/prompts/task.md", "assets/prompts/instructions.md"} { + content, err := fs.ReadFile(embeddedAssets, path) + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(content), "Transcript units are the only evidence") || strings.Contains(string(content), "Dungeons & Dragons gameplay transcripts") { + t.Fatalf("module asset %q copied shared prompt text", path) + } + } +} diff --git a/internal/modules/dnd/extract/itemevents/test_helpers_test.go b/internal/modules/dnd/extract/itemevents/test_helpers_test.go new file mode 100644 index 0000000..ebf0352 --- /dev/null +++ b/internal/modules/dnd/extract/itemevents/test_helpers_test.go @@ -0,0 +1,92 @@ +package itemevents + +import ( + "context" + "encoding/json" + "errors" + "testing" + + "gitea.maximumdirect.net/eric/notarius/internal/core/source" + "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" +) + +func extractionRequest() contracts.TypedExtractionRequest { + doc := sourceDocument() + chunk := &source.Chunk{ + ID: "session-alpha:chunk:0", + SourceID: doc.ID, + Index: 0, + Ref: source.SourceRef{SourceID: doc.ID, StartUnitID: 1, EndUnitID: 5}, + Content: []byte(`{"segments":[1,2,3,4,5]}`), + MediaType: "application/json", + Units: append([]source.SourceUnit(nil), doc.Units...), + } + return contracts.TypedExtractionRequest{ + Source: doc, + Chunk: chunk, + SourceInput: contracts.NewLLMInputMaterial("source", chunk.MediaType, chunk.Content, "sha256:chunk", "file:///session-alpha.json"), + SessionID: "session-123", + LLMProfile: "profile-item-events", + } +} + +func sourceDocument() *source.SourceDocument { + return &source.SourceDocument{ + ID: "session-alpha", Kind: "transcript", Format: "application/vnd.seriatim.minimal+json", Digest: "sha256:test", + Units: []source.SourceUnit{ + {ID: 1, Kind: "transcript_segment", Text: "The party discovers a hidden cache."}, + {ID: 2, Kind: "transcript_segment", Text: "They acquire 20 gold pieces."}, + {ID: 3, Kind: "transcript_segment", Text: "Aria gives Borin the moonblade."}, + {ID: 4, Kind: "transcript_segment", Text: "The party consumes a healing potion."}, + {ID: 5, Kind: "transcript_segment", Text: "The party loses a torch."}, + }, + } +} + +func responseRefs(start, end int) []itemEventSourceRefResponse { + return []itemEventSourceRefResponse{{StartSegment: start, EndSegment: end}} +} + +func newExtractor(t *testing.T, client contracts.StructuredLLMClient, references ...contracts.ReferenceSet) *Extractor { + t.Helper() + extractor, err := New(client, Options{}, references...) + if err != nil { + t.Fatalf("New() error = %v", err) + } + return extractor +} + +func cloneStructuredCompletionRequest(req contracts.StructuredCompletionRequest) contracts.StructuredCompletionRequest { + req.Inputs = req.Inputs.Clone() + return req +} + +type fakeItemEventsLLMClient struct { + response extractionResponse + content []byte + err error + requests []contracts.StructuredCompletionRequest +} + +func (client *fakeItemEventsLLMClient) CompleteStructured(_ context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) { + client.requests = append(client.requests, cloneStructuredCompletionRequest(req)) + if client.err != nil { + return contracts.StructuredCompletionResponse{}, client.err + } + target, ok := out.(*extractionResponse) + if !ok { + return contracts.StructuredCompletionResponse{}, errors.New("unexpected output target") + } + content := append([]byte(nil), client.content...) + if len(content) == 0 { + var err error + content, err = json.Marshal(client.response) + if err != nil { + return contracts.StructuredCompletionResponse{}, err + } + } + if err := json.Unmarshal(content, target); err != nil { + return contracts.StructuredCompletionResponse{}, err + } + return contracts.StructuredCompletionResponse{Content: content}, nil +}