Add D&D NPC extraction and validation
This commit is contained in:
6
internal/modules/dnd/extract/npcs/assets.go
Normal file
6
internal/modules/dnd/extract/npcs/assets.go
Normal file
@@ -0,0 +1,6 @@
|
||||
package npcs
|
||||
|
||||
import "embed"
|
||||
|
||||
//go:embed assets/schemas/dnd_npcs_llm.v1.json assets/prompts/*.yaml assets/prompts/*.md
|
||||
var embeddedAssets embed.FS
|
||||
@@ -0,0 +1,36 @@
|
||||
id: dnd.npcs
|
||||
version: "v1"
|
||||
default_profile: gemini-2-flash
|
||||
inputs:
|
||||
- name: transcript
|
||||
required: true
|
||||
content_type: application/json
|
||||
- name: players
|
||||
required: false
|
||||
content_type: text/plain
|
||||
- name: party
|
||||
required: false
|
||||
content_type: text/plain
|
||||
- name: glossary
|
||||
required: false
|
||||
content_type: text/plain
|
||||
messages:
|
||||
- role: system
|
||||
content_file: ./sharedassets/common-dnd-system.md
|
||||
- role: user
|
||||
content_file: ./sharedassets/common-dnd-transcript.md
|
||||
cache_control:
|
||||
type: ephemeral
|
||||
- role: user
|
||||
content_file: ./sharedassets/common-dnd-references.md
|
||||
cache_control:
|
||||
type: ephemeral
|
||||
- role: user
|
||||
content_file: ./task.md
|
||||
- role: user
|
||||
content_file: ./instructions.md
|
||||
output:
|
||||
format: json
|
||||
validation_mode: json_schema
|
||||
schema_path: dnd_npcs_llm.v1.json
|
||||
repair_attempts: 0
|
||||
@@ -0,0 +1,25 @@
|
||||
Return exactly one JSON object and no explanatory text.
|
||||
|
||||
For every NPC record, cite one or more transcript source-unit ranges that
|
||||
collectively support the canonical name, every alias, the description, and
|
||||
every relationship. Use integer start_unit_id and end_unit_id values from the
|
||||
transcript. Do not provide source_id; the extractor assigns it automatically.
|
||||
Use narrow ranges when evidence is not contiguous and do not bridge unrelated
|
||||
conversation with a broad range.
|
||||
|
||||
A canonical name must be the most specific in-world identity supported by the
|
||||
transcript. Never use a human player, transcript speaker, or GM name when an
|
||||
associated in-world character or creature is identified. Player, party, and
|
||||
glossary references may disambiguate identities, but they are not transcript
|
||||
evidence and cannot establish that an NPC appeared, acted, or was discussed.
|
||||
Do not return a person or entity mentioned only in those references.
|
||||
|
||||
Descriptions must be short session records, not biographies, statistics,
|
||||
alignment, motivations, or lore inferred from general D&D knowledge. Do not
|
||||
summarize every action or follow a participant through unrelated scenes.
|
||||
Relationships must be stated or directly demonstrated by cited transcript
|
||||
units, not inferred from game lore.
|
||||
|
||||
Return aliases and relationships as arrays, including empty arrays when there
|
||||
are none. Return only NPC records supported by the transcript and preserve
|
||||
observed display spelling.
|
||||
16
internal/modules/dnd/extract/npcs/assets/prompts/task.md
Normal file
16
internal/modules/dnd/extract/npcs/assets/prompts/task.md
Normal file
@@ -0,0 +1,16 @@
|
||||
Extract a concise Dungeons & Dragons non-player-character registry from the
|
||||
provided transcript.
|
||||
|
||||
Include an in-world non-PC participant when the transcript establishes that it
|
||||
appears, acts, speaks, or is materially discussed and gives it a proper name,
|
||||
a stable alias or title, or an individually useful distinguishing description.
|
||||
|
||||
Exclude human players, transcript speakers, the GM as an out-of-world person,
|
||||
player characters identified by the player or party references, incidental or
|
||||
hypothetical name drops, corrected transcription mistakes, characters mentioned
|
||||
only by reference material, indistinguishable crowds or groups, and temporary
|
||||
summoned creatures or spell effects without a persistent individual identity.
|
||||
|
||||
Return canonical in-world names rather than player names or transcript speaker
|
||||
names. Keep each description concise and limited to facts established by the
|
||||
transcript. Include only explicitly supported aliases and relationships.
|
||||
@@ -0,0 +1,77 @@
|
||||
{
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"$id": "notarius.dnd.npcs.llm",
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["npcs"],
|
||||
"properties": {
|
||||
"npcs": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": [
|
||||
"name",
|
||||
"aliases",
|
||||
"description",
|
||||
"relationships",
|
||||
"source_refs"
|
||||
],
|
||||
"properties": {
|
||||
"name": {
|
||||
"type": "string",
|
||||
"minLength": 1
|
||||
},
|
||||
"aliases": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "string",
|
||||
"minLength": 1
|
||||
}
|
||||
},
|
||||
"description": {
|
||||
"type": "string",
|
||||
"minLength": 1
|
||||
},
|
||||
"relationships": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["target", "relationship"],
|
||||
"properties": {
|
||||
"target": {
|
||||
"type": "string",
|
||||
"minLength": 1
|
||||
},
|
||||
"relationship": {
|
||||
"type": "string",
|
||||
"minLength": 1
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"source_refs": {
|
||||
"type": "array",
|
||||
"minItems": 1,
|
||||
"items": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["start_unit_id", "end_unit_id"],
|
||||
"properties": {
|
||||
"start_unit_id": {
|
||||
"type": "integer",
|
||||
"minimum": 1
|
||||
},
|
||||
"end_unit_id": {
|
||||
"type": "integer",
|
||||
"minimum": 1
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
153
internal/modules/dnd/extract/npcs/canonicalize.go
Normal file
153
internal/modules/dnd/extract/npcs/canonicalize.go
Normal file
@@ -0,0 +1,153 @@
|
||||
package npcs
|
||||
|
||||
import (
|
||||
"sort"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/npcs/identity"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared"
|
||||
)
|
||||
|
||||
func canonicalizeResponse(response *extractionResponse, doc *source.SourceDocument) {
|
||||
if response == nil {
|
||||
return
|
||||
}
|
||||
for index := range response.NPCs {
|
||||
canonicalizeNPC(&response.NPCs[index])
|
||||
}
|
||||
sort.SliceStable(response.NPCs, func(i, j int) bool {
|
||||
left, leftOK := earliestSourceUnit(doc, response.NPCs[i])
|
||||
right, rightOK := earliestSourceUnit(doc, response.NPCs[j])
|
||||
if leftOK != rightOK {
|
||||
return leftOK
|
||||
}
|
||||
if !leftOK {
|
||||
return false
|
||||
}
|
||||
return left < right
|
||||
})
|
||||
}
|
||||
|
||||
func canonicalizeNPC(npc *npcResponse) {
|
||||
if npc == nil {
|
||||
return
|
||||
}
|
||||
for index := range npc.SourceRefs {
|
||||
npc.SourceRefs[index].StartUnitID = canonicalUnitRef(npc.SourceRefs[index].StartUnitID)
|
||||
npc.SourceRefs[index].EndUnitID = canonicalUnitRef(npc.SourceRefs[index].EndUnitID)
|
||||
}
|
||||
sort.SliceStable(npc.SourceRefs, func(i, j int) bool {
|
||||
left := npc.SourceRefs[i]
|
||||
right := npc.SourceRefs[j]
|
||||
if unitSortValue(left.StartUnitID) != unitSortValue(right.StartUnitID) {
|
||||
return unitSortValue(left.StartUnitID) < unitSortValue(right.StartUnitID)
|
||||
}
|
||||
return unitSortValue(left.EndUnitID) < unitSortValue(right.EndUnitID)
|
||||
})
|
||||
npc.SourceRefs = dedupeSourceRefs(npc.SourceRefs)
|
||||
}
|
||||
|
||||
func canonicalUnitRef(ref shared.UnitRef) shared.UnitRef {
|
||||
value := ref.Int()
|
||||
if value <= 0 {
|
||||
return ref
|
||||
}
|
||||
return shared.UnitRefFromInt(value)
|
||||
}
|
||||
|
||||
func dedupeSourceRefs(refs []npcSourceRefResponse) []npcSourceRefResponse {
|
||||
if len(refs) < 2 {
|
||||
return refs
|
||||
}
|
||||
out := refs[:0]
|
||||
var previous npcSourceRefResponse
|
||||
for index, ref := range refs {
|
||||
if index > 0 && sameSourceRef(previous, ref) {
|
||||
continue
|
||||
}
|
||||
out = append(out, ref)
|
||||
previous = ref
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func sameSourceRef(left npcSourceRefResponse, right npcSourceRefResponse) bool {
|
||||
return left.StartUnitID.Int() == right.StartUnitID.Int() &&
|
||||
left.EndUnitID.Int() == right.EndUnitID.Int()
|
||||
}
|
||||
|
||||
func earliestSourceUnit(doc *source.SourceDocument, npc npcResponse) (int, bool) {
|
||||
for _, ref := range npc.SourceRefs {
|
||||
start := ref.StartUnitID.Int()
|
||||
end := ref.EndUnitID.Int()
|
||||
if start > 0 && end > 0 {
|
||||
startIndex, startOK := source.UnitIndex(doc, start)
|
||||
endIndex, endOK := source.UnitIndex(doc, end)
|
||||
if !startOK || !endOK || startIndex > endIndex {
|
||||
continue
|
||||
}
|
||||
return start, true
|
||||
}
|
||||
}
|
||||
return 0, false
|
||||
}
|
||||
|
||||
func unitSortValue(ref shared.UnitRef) int {
|
||||
value := ref.Int()
|
||||
if value <= 0 {
|
||||
return int(^uint(0) >> 1)
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
func canonicalNPCList(response extractionResponse, sourceID string) dnd.NPCList {
|
||||
if response.NPCs == nil {
|
||||
return dnd.NPCList{NPCs: nil}
|
||||
}
|
||||
npcs := make([]dnd.NPC, len(response.NPCs))
|
||||
for index, npc := range response.NPCs {
|
||||
npcs[index] = dnd.NPC{
|
||||
ID: identity.DeriveID(npc.Name),
|
||||
Name: npc.Name,
|
||||
Aliases: cloneStrings(npc.Aliases),
|
||||
Description: npc.Description,
|
||||
Relationships: cloneRelationships(npc.Relationships),
|
||||
SourceRefs: canonicalSourceRefs(npc.SourceRefs, sourceID),
|
||||
}
|
||||
}
|
||||
return dnd.NPCList{NPCs: npcs}
|
||||
}
|
||||
|
||||
func cloneStrings(values []string) []string {
|
||||
if values == nil {
|
||||
return nil
|
||||
}
|
||||
return append([]string{}, values...)
|
||||
}
|
||||
|
||||
func cloneRelationships(values []npcRelationshipResponse) []dnd.NPCRelationship {
|
||||
if values == nil {
|
||||
return nil
|
||||
}
|
||||
out := make([]dnd.NPCRelationship, len(values))
|
||||
for index, value := range values {
|
||||
out[index] = dnd.NPCRelationship{Target: value.Target, Relationship: value.Relationship}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func canonicalSourceRefs(values []npcSourceRefResponse, sourceID string) []source.SourceRef {
|
||||
if values == nil {
|
||||
return nil
|
||||
}
|
||||
out := make([]source.SourceRef, len(values))
|
||||
for index, value := range values {
|
||||
out[index] = source.SourceRef{
|
||||
SourceID: sourceID,
|
||||
StartUnitID: value.StartUnitID.Int(),
|
||||
EndUnitID: value.EndUnitID.Int(),
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
200
internal/modules/dnd/extract/npcs/extractor.go
Normal file
200
internal/modules/dnd/extract/npcs/extractor.go
Normal file
@@ -0,0 +1,200 @@
|
||||
package npcs
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/npcs/identity"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared"
|
||||
)
|
||||
|
||||
const Key = "dnd/npcs"
|
||||
|
||||
var requiredCapabilities = []string{
|
||||
"chunks",
|
||||
"source.transcript",
|
||||
}
|
||||
|
||||
var providedCapabilities = []string{
|
||||
"dnd.npcs",
|
||||
}
|
||||
|
||||
var referenceSlotDescriptions = shared.ReferenceSlotDescriptions{
|
||||
Glossary: "Optional campaign glossary reference material used only for NPC disambiguation.",
|
||||
Party: "Optional party roster reference material used only for NPC disambiguation.",
|
||||
Players: "Optional player list reference material used only for NPC disambiguation.",
|
||||
Roster: "Deprecated alias for party roster reference material used only for NPC disambiguation.",
|
||||
}
|
||||
|
||||
func referenceSlots() []contracts.ReferenceSlot {
|
||||
return shared.ReferenceSlots(referenceSlotDescriptions)
|
||||
}
|
||||
|
||||
var _ contracts.Extractor[dnd.NPCList] = (*Extractor)(nil)
|
||||
var _ contracts.ManifestMetadataProvider = (*Extractor)(nil)
|
||||
var _ pipeline.CheckpointFingerprintProvider = (*Extractor)(nil)
|
||||
|
||||
type Options struct{}
|
||||
|
||||
type Extractor struct {
|
||||
llm contracts.StructuredLLMClient
|
||||
promptSHA string
|
||||
responseSchemaSHA string
|
||||
}
|
||||
|
||||
func New(llmClient contracts.StructuredLLMClient, _ Options, references ...contracts.ReferenceSet) (*Extractor, error) {
|
||||
if llmClient == nil {
|
||||
return nil, extractorErrorf("LLM client must not be nil")
|
||||
}
|
||||
if len(references) > 1 {
|
||||
return nil, extractorErrorf("at most one reference set may be supplied")
|
||||
}
|
||||
promptSHA, err := scriptoriumPromptMetadata()
|
||||
if err != nil {
|
||||
return nil, extractorErrorf("load prompt metadata: %w", err)
|
||||
}
|
||||
responseSchema, err := loadResponseSchema()
|
||||
if err != nil {
|
||||
return nil, extractorErrorf("load response schema: %w", err)
|
||||
}
|
||||
return &Extractor{
|
||||
llm: llmClient,
|
||||
promptSHA: promptSHA,
|
||||
responseSchemaSHA: responseSchema.SHA256,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (e *Extractor) Key() string { return Key }
|
||||
|
||||
func (e *Extractor) ReferenceSlots() []contracts.ReferenceSlot { return referenceSlots() }
|
||||
|
||||
func (e *Extractor) ManifestMetadata() map[string]any {
|
||||
if e == nil {
|
||||
return nil
|
||||
}
|
||||
return map[string]any{
|
||||
"prompt_id": PromptID,
|
||||
"prompt_version": SchemaVersion,
|
||||
"prompt_sha256": e.promptSHA,
|
||||
"response_schema_key": string(ResponseSchemaKey),
|
||||
"response_schema_id": ResponseSchemaID,
|
||||
"response_schema_name": ResponseSchemaName,
|
||||
"response_schema_version": SchemaVersion,
|
||||
"response_schema_sha256": e.responseSchemaSHA,
|
||||
"identity_policy": identity.Policy,
|
||||
}
|
||||
}
|
||||
|
||||
func (e *Extractor) CheckpointFingerprints() []pipeline.CheckpointFingerprint {
|
||||
if e == nil {
|
||||
return nil
|
||||
}
|
||||
return []pipeline.CheckpointFingerprint{
|
||||
{Name: "prompt", Value: e.promptSHA},
|
||||
{Name: "response_schema", Value: e.responseSchemaSHA},
|
||||
{Name: "identity_policy", Value: identity.Policy},
|
||||
}
|
||||
}
|
||||
|
||||
func (e *Extractor) Extract(ctx context.Context, req contracts.TypedExtractionRequest) (contracts.TypedExtractionResult[dnd.NPCList], error) {
|
||||
if e == nil {
|
||||
return contracts.TypedExtractionResult[dnd.NPCList]{}, extractorErrorf("extractor must not be nil")
|
||||
}
|
||||
if e.llm == nil {
|
||||
return contracts.TypedExtractionResult[dnd.NPCList]{}, extractorErrorf("LLM client must not be nil")
|
||||
}
|
||||
if ctx == nil {
|
||||
return contracts.TypedExtractionResult[dnd.NPCList]{}, extractorErrorf("context must not be nil")
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return contracts.TypedExtractionResult[dnd.NPCList]{}, extractorErrorf("context error before extraction: %w", err)
|
||||
}
|
||||
if req.Source == nil {
|
||||
return contracts.TypedExtractionResult[dnd.NPCList]{}, extractorErrorf("source must not be nil")
|
||||
}
|
||||
if req.Chunk == nil {
|
||||
return contracts.TypedExtractionResult[dnd.NPCList]{}, extractorErrorf("chunk must not be nil")
|
||||
}
|
||||
if len(req.Chunk.Units) == 0 {
|
||||
return contracts.TypedExtractionResult[dnd.NPCList]{}, extractorErrorf("chunk %q units must not be empty", req.Chunk.ID)
|
||||
}
|
||||
sourceInput, err := chunkSourceInput(req)
|
||||
if err != nil {
|
||||
return contracts.TypedExtractionResult[dnd.NPCList]{}, err
|
||||
}
|
||||
|
||||
var response extractionResponse
|
||||
if _, err := e.llm.CompleteStructured(ctx, contracts.StructuredCompletionRequest{
|
||||
StageName: Key,
|
||||
PromptID: PromptID,
|
||||
PromptVersion: SchemaVersion,
|
||||
ProfileID: req.LLMProfile,
|
||||
SessionID: req.SessionID,
|
||||
Inputs: shared.PromptInputs(sourceInput, req.References),
|
||||
}, &response); err != nil {
|
||||
return contracts.TypedExtractionResult[dnd.NPCList]{}, extractorErrorf("complete structured output: %w", err)
|
||||
}
|
||||
canonicalizeResponse(&response, req.Source)
|
||||
return contracts.TypedExtractionResult[dnd.NPCList]{Value: canonicalNPCList(response, req.Source.ID)}, nil
|
||||
}
|
||||
|
||||
func chunkSourceInput(req contracts.TypedExtractionRequest) (contracts.LLMInputMaterial, error) {
|
||||
material := req.SourceInput.Clone()
|
||||
if len(material.Content) == 0 {
|
||||
material = contracts.NewLLMInputMaterial("source", req.Chunk.MediaType, req.Chunk.Content, "", "")
|
||||
}
|
||||
if !bytes.Equal(material.Content, req.Chunk.Content) {
|
||||
return contracts.LLMInputMaterial{}, extractorErrorf("source input must match chunk %q content", req.Chunk.ID)
|
||||
}
|
||||
if material.Name == "" {
|
||||
material.Name = "source"
|
||||
}
|
||||
if material.MediaType == "" {
|
||||
material.MediaType = req.Chunk.MediaType
|
||||
}
|
||||
if material.SizeBytes == 0 {
|
||||
material.SizeBytes = int64(len(material.Content))
|
||||
}
|
||||
return material, nil
|
||||
}
|
||||
|
||||
func ModuleSpec() pipeline.ModuleSpec {
|
||||
return pipeline.ModuleSpec{
|
||||
Key: Key,
|
||||
Stage: pipeline.StageExtract,
|
||||
Requires: append([]string(nil), requiredCapabilities...),
|
||||
Provides: append([]string(nil), providedCapabilities...),
|
||||
ArtifactKind: dnd.NPCListKind,
|
||||
ReferenceSlots: referenceSlots(),
|
||||
}
|
||||
}
|
||||
|
||||
func Register(registry *pipeline.ExtractorRegistry) error {
|
||||
return pipeline.RegisterExtractorBuilder(registry, ModuleSpec(), validateOptions, func(request pipeline.BuildRequest) (contracts.Extractor[dnd.NPCList], error) {
|
||||
options, err := DecodeOptions(request.Options)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return New(request.Dependencies.LLM, options, request.References)
|
||||
})
|
||||
}
|
||||
|
||||
func validateOptions(options map[string]any) error {
|
||||
_, err := DecodeOptions(options)
|
||||
return err
|
||||
}
|
||||
|
||||
func DecodeOptions(options map[string]any) (Options, error) {
|
||||
if err := pipeline.RejectUnknownOptions(options); err != nil {
|
||||
return Options{}, extractorErrorf("%w", err)
|
||||
}
|
||||
return Options{}, nil
|
||||
}
|
||||
|
||||
func extractorErrorf(format string, args ...any) error {
|
||||
return fmt.Errorf("dnd npcs extractor: "+format, args...)
|
||||
}
|
||||
110
internal/modules/dnd/extract/npcs/extractor_test.go
Normal file
110
internal/modules/dnd/extract/npcs/extractor_test.go
Normal file
@@ -0,0 +1,110 @@
|
||||
package npcs
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/npcs/identity"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared"
|
||||
)
|
||||
|
||||
func TestExtractReturnsCanonicalNPCListFromPrivateResponse(t *testing.T) {
|
||||
client := &fakeNPCsLLMClient{response: extractionResponse{NPCs: []npcResponse{
|
||||
{
|
||||
Name: "Captain Vale", Aliases: []string{"The Captain"}, Description: "A road captain.",
|
||||
Relationships: []npcRelationshipResponse{{Target: "Mira Thorn", Relationship: "reports to"}},
|
||||
SourceRefs: responseSourceRefs(3, 3),
|
||||
},
|
||||
{
|
||||
Name: "Mira Thorn", Aliases: []string{"The Greencloak"}, Description: "A guarded ranger.",
|
||||
Relationships: []npcRelationshipResponse{{Target: "Captain Vale", Relationship: "commands"}},
|
||||
SourceRefs: []npcSourceRefResponse{
|
||||
{StartUnitID: sharedUnitRef(2), EndUnitID: sharedUnitRef(2)},
|
||||
{StartUnitID: sharedUnitRef(1), EndUnitID: sharedUnitRef(2)},
|
||||
{StartUnitID: sharedUnitRef(1), EndUnitID: sharedUnitRef(2)},
|
||||
},
|
||||
},
|
||||
}}}
|
||||
|
||||
result, err := newExtractor(t, client).Extract(context.Background(), extractionRequest())
|
||||
if err != nil {
|
||||
t.Fatalf("Extract() error = %v, want nil", err)
|
||||
}
|
||||
want := dnd.NPCList{NPCs: []dnd.NPC{
|
||||
{ID: identity.DeriveID("Mira Thorn"), Name: "Mira Thorn", Aliases: []string{"The Greencloak"}, Description: "A guarded ranger.", Relationships: []dnd.NPCRelationship{{Target: "Captain Vale", Relationship: "commands"}}, SourceRefs: []source.SourceRef{{SourceID: "session-alpha", StartUnitID: 1, EndUnitID: 2}, {SourceID: "session-alpha", StartUnitID: 2, EndUnitID: 2}}},
|
||||
{ID: identity.DeriveID("Captain Vale"), Name: "Captain Vale", Aliases: []string{"The Captain"}, Description: "A road captain.", Relationships: []dnd.NPCRelationship{{Target: "Mira Thorn", Relationship: "reports to"}}, SourceRefs: []source.SourceRef{{SourceID: "session-alpha", StartUnitID: 3, EndUnitID: 3}}},
|
||||
}}
|
||||
if !reflect.DeepEqual(result.Value, want) {
|
||||
t.Fatalf("Value = %#v, want %#v", result.Value, want)
|
||||
}
|
||||
if len(client.requests) != 1 {
|
||||
t.Fatalf("LLM calls = %d, want 1", len(client.requests))
|
||||
}
|
||||
request := client.requests[0]
|
||||
if request.StageName != Key || request.PromptID != PromptID || request.PromptVersion != SchemaVersion || request.ProfileID != "profile-npcs" || request.SessionID != "session-123" {
|
||||
t.Fatalf("LLM request identity = %#v", request)
|
||||
}
|
||||
transcript := request.Inputs["transcript"]
|
||||
if transcript.Name != "transcript" || transcript.MediaType != "application/json" || transcript.Digest != "sha256:chunk" || transcript.OriginURI != "file:///session-alpha.json" || string(transcript.Content) != string(extractionRequest().Chunk.Content) {
|
||||
t.Fatalf("transcript input = %#v, want chunk-scoped material", transcript)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractPassesCampaignReferencesAsPromptInputs(t *testing.T) {
|
||||
client := &fakeNPCsLLMClient{response: extractionResponse{NPCs: []npcResponse{}}}
|
||||
req := extractionRequest()
|
||||
req.References = contracts.ReferenceSet{Slots: map[string]contracts.ResolvedReferenceSlot{
|
||||
"players": {Slot: contracts.ReferenceSlot{Name: "players"}, Items: []contracts.ReferenceItem{{SlotName: "players", Content: []byte("Dana: Mira")}}},
|
||||
"party": {Slot: contracts.ReferenceSlot{Name: "party"}, Items: []contracts.ReferenceItem{{SlotName: "party", Content: []byte("Mira: ranger")}}},
|
||||
"glossary": {Slot: contracts.ReferenceSlot{Name: "glossary"}, Items: []contracts.ReferenceItem{{SlotName: "glossary", Content: []byte("Greencloak: local title")}}},
|
||||
}}
|
||||
if _, err := newExtractor(t, client).Extract(context.Background(), req); err != nil {
|
||||
t.Fatalf("Extract() error = %v, want nil", err)
|
||||
}
|
||||
inputs := client.requests[0].Inputs
|
||||
if string(inputs["players"].Content) != "Dana: Mira" || string(inputs["party"].Content) != "Mira: ranger" || string(inputs["glossary"].Content) != "Greencloak: local title" {
|
||||
t.Fatalf("reference inputs = %#v", inputs)
|
||||
}
|
||||
if strings.Contains(string(inputs["transcript"].Content), "local title") {
|
||||
t.Fatal("transcript input contains reference content")
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractPreservesMalformedCandidatesForValidators(t *testing.T) {
|
||||
client := &fakeNPCsLLMClient{response: extractionResponse{NPCs: []npcResponse{{
|
||||
Name: "", Aliases: nil, Description: "", Relationships: nil,
|
||||
SourceRefs: []npcSourceRefResponse{{StartUnitID: sharedUnitRef(99), EndUnitID: shared.UnitRefFromString("missing")}, {StartUnitID: sharedUnitRef(99), EndUnitID: shared.UnitRefFromInt(0)}},
|
||||
}}}}
|
||||
result, err := newExtractor(t, client).Extract(context.Background(), extractionRequest())
|
||||
if err != nil {
|
||||
t.Fatalf("Extract() error = %v, want nil", err)
|
||||
}
|
||||
if len(result.Value.NPCs) != 1 || result.Value.NPCs[0].ID != "" || result.Value.NPCs[0].Name != "" || result.Value.NPCs[0].Aliases != nil || result.Value.NPCs[0].Relationships != nil {
|
||||
t.Fatalf("malformed candidate = %#v, want invalid values preserved", result.Value)
|
||||
}
|
||||
if refs := result.Value.NPCs[0].SourceRefs; len(refs) != 1 || refs[0].SourceID != "session-alpha" || refs[0].StartUnitID != 99 || refs[0].EndUnitID != 0 {
|
||||
t.Fatalf("malformed source refs = %#v, want invalid range preserved after exact deduplication", refs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractHandlesCancellationAndProviderErrors(t *testing.T) {
|
||||
request := extractionRequest()
|
||||
extractor := newExtractor(t, &fakeNPCsLLMClient{response: extractionResponse{NPCs: []npcResponse{}}})
|
||||
canceled, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
if _, err := extractor.Extract(canceled, request); err == nil || !strings.Contains(err.Error(), "context") {
|
||||
t.Fatalf("canceled Extract() error = %v, want context error", err)
|
||||
}
|
||||
_, err := newExtractor(t, &fakeNPCsLLMClient{err: errors.New("provider unavailable")}).Extract(context.Background(), request)
|
||||
if err == nil || !strings.Contains(err.Error(), "dnd npcs") || !strings.Contains(err.Error(), "provider unavailable") {
|
||||
t.Fatalf("provider Extract() error = %v, want contextual provider error", err)
|
||||
}
|
||||
}
|
||||
|
||||
func sharedUnitRef(value int) shared.UnitRef { return shared.UnitRefFromInt(value) }
|
||||
25
internal/modules/dnd/extract/npcs/model.go
Normal file
25
internal/modules/dnd/extract/npcs/model.go
Normal file
@@ -0,0 +1,25 @@
|
||||
package npcs
|
||||
|
||||
import "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared"
|
||||
|
||||
type extractionResponse struct {
|
||||
NPCs []npcResponse `json:"npcs"`
|
||||
}
|
||||
|
||||
type npcResponse struct {
|
||||
Name string `json:"name"`
|
||||
Aliases []string `json:"aliases"`
|
||||
Description string `json:"description"`
|
||||
Relationships []npcRelationshipResponse `json:"relationships"`
|
||||
SourceRefs []npcSourceRefResponse `json:"source_refs"`
|
||||
}
|
||||
|
||||
type npcRelationshipResponse struct {
|
||||
Target string `json:"target"`
|
||||
Relationship string `json:"relationship"`
|
||||
}
|
||||
|
||||
type npcSourceRefResponse struct {
|
||||
StartUnitID shared.UnitRef `json:"start_unit_id"`
|
||||
EndUnitID shared.UnitRef `json:"end_unit_id"`
|
||||
}
|
||||
100
internal/modules/dnd/extract/npcs/registry_test.go
Normal file
100
internal/modules/dnd/extract/npcs/registry_test.go
Normal file
@@ -0,0 +1,100 @@
|
||||
package npcs
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
||||
)
|
||||
|
||||
func TestNewRequiresLLMClientAndRejectsAmbiguousReferences(t *testing.T) {
|
||||
if _, err := New(nil, Options{}); err == nil || !strings.Contains(err.Error(), "LLM client") {
|
||||
t.Fatalf("New(nil) error = %v, want dependency error", err)
|
||||
}
|
||||
if _, err := New(&fakeNPCsLLMClient{}, Options{}, contracts.ReferenceSet{}, contracts.ReferenceSet{}); err == nil || !strings.Contains(err.Error(), "at most one reference set") {
|
||||
t.Fatalf("New() error = %v, want reference-set error", err)
|
||||
}
|
||||
if got := newExtractor(t, &fakeNPCsLLMClient{}).Key(); got != Key {
|
||||
t.Fatalf("Key() = %q, want %q", got, Key)
|
||||
}
|
||||
}
|
||||
|
||||
func TestModuleSpecAndReferenceSlots(t *testing.T) {
|
||||
got := ModuleSpec()
|
||||
want := pipeline.ModuleSpec{
|
||||
Key: Key,
|
||||
Stage: pipeline.StageExtract,
|
||||
Requires: []string{"chunks", "source.transcript"},
|
||||
Provides: []string{"dnd.npcs"},
|
||||
ArtifactKind: dnd.NPCListKind,
|
||||
ReferenceSlots: []contracts.ReferenceSlot{
|
||||
{Name: "glossary", Description: "Optional campaign glossary reference material used only for NPC disambiguation.", AcceptedMediaTypes: []string{"application/json", "application/x-yaml", "application/yaml", "text/markdown", "text/plain"}},
|
||||
{Name: "party", Description: "Optional party roster reference material used only for NPC disambiguation.", AcceptedMediaTypes: []string{"application/json", "application/x-yaml", "application/yaml", "text/markdown", "text/plain"}},
|
||||
{Name: "players", Description: "Optional player list reference material used only for NPC disambiguation.", AcceptedMediaTypes: []string{"application/json", "application/x-yaml", "application/yaml", "text/markdown", "text/plain"}},
|
||||
{Name: "roster", Description: "Deprecated alias for party roster reference material used only for NPC disambiguation.", AcceptedMediaTypes: []string{"application/json", "application/x-yaml", "application/yaml", "text/markdown", "text/plain"}},
|
||||
},
|
||||
}
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("ModuleSpec() = %#v, want %#v", got, want)
|
||||
}
|
||||
got.Requires[0] = "changed"
|
||||
got.Provides[0] = "changed"
|
||||
got.ReferenceSlots[0].AcceptedMediaTypes[0] = "changed"
|
||||
if !reflect.DeepEqual(ModuleSpec(), want) {
|
||||
t.Fatal("ModuleSpec() returned mutable shared slices")
|
||||
}
|
||||
extractor := newExtractor(t, &fakeNPCsLLMClient{})
|
||||
if !reflect.DeepEqual(extractor.ReferenceSlots(), want.ReferenceSlots) {
|
||||
t.Fatalf("ReferenceSlots() = %#v, want %#v", extractor.ReferenceSlots(), want.ReferenceSlots)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRegisterStoresTypedModuleSpec(t *testing.T) {
|
||||
registry := pipeline.NewExtractorRegistry()
|
||||
if err := Register(registry); err != nil {
|
||||
t.Fatalf("Register() error = %v, want nil", err)
|
||||
}
|
||||
got, ok := registry.Spec(Key)
|
||||
if !ok || !reflect.DeepEqual(got, ModuleSpec()) {
|
||||
t.Fatalf("registry.Spec(%q) = %#v, present = %t", Key, got, ok)
|
||||
}
|
||||
if err := Register(nil); err == nil || !strings.Contains(err.Error(), "extractor registry") {
|
||||
t.Fatalf("Register(nil) error = %v, want registry error", err)
|
||||
}
|
||||
if _, err := DecodeOptions(map[string]any{"unexpected": true}); err == nil || !strings.Contains(err.Error(), "unknown option") {
|
||||
t.Fatalf("DecodeOptions() error = %v, want strict options error", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractorMetadataAndCheckpointIdentity(t *testing.T) {
|
||||
extractor := newExtractor(t, &fakeNPCsLLMClient{})
|
||||
metadata := extractor.ManifestMetadata()
|
||||
for key, want := range map[string]string{
|
||||
"prompt_id": PromptID, "prompt_version": SchemaVersion,
|
||||
"response_schema_key": string(ResponseSchemaKey), "response_schema_id": ResponseSchemaID,
|
||||
"response_schema_name": ResponseSchemaName, "response_schema_version": SchemaVersion,
|
||||
"identity_policy": "dnd.npcs.identity.v1",
|
||||
} {
|
||||
if metadata[key] != want {
|
||||
t.Fatalf("metadata[%q] = %#v, want %q", key, metadata[key], want)
|
||||
}
|
||||
}
|
||||
for _, key := range []string{"prompt_sha256", "response_schema_sha256"} {
|
||||
if value, ok := metadata[key].(string); !ok || !strings.HasPrefix(value, "sha256:") {
|
||||
t.Fatalf("metadata[%q] = %#v, want hash", key, metadata[key])
|
||||
}
|
||||
}
|
||||
fingerprints := extractor.CheckpointFingerprints()
|
||||
want := map[string]string{"prompt": metadata["prompt_sha256"].(string), "response_schema": metadata["response_schema_sha256"].(string), "identity_policy": "dnd.npcs.identity.v1"}
|
||||
if len(fingerprints) != len(want) {
|
||||
t.Fatalf("CheckpointFingerprints() = %#v, want %d entries", fingerprints, len(want))
|
||||
}
|
||||
for _, fingerprint := range fingerprints {
|
||||
if fingerprint.Value != want[fingerprint.Name] {
|
||||
t.Fatalf("fingerprint %q = %q, want %#v", fingerprint.Name, fingerprint.Value, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
21
internal/modules/dnd/extract/npcs/schema.go
Normal file
21
internal/modules/dnd/extract/npcs/schema.go
Normal file
@@ -0,0 +1,21 @@
|
||||
package npcs
|
||||
|
||||
import "gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
||||
|
||||
const (
|
||||
PromptID = "dnd.npcs"
|
||||
ResponseSchemaKey = llm.ResponseSchemaKey("dnd_npcs_llm")
|
||||
ResponseSchemaID = "notarius.dnd.npcs.llm"
|
||||
ResponseSchemaName = "notarius_dnd_npcs_llm_v1"
|
||||
SchemaVersion = "v1"
|
||||
)
|
||||
|
||||
func loadResponseSchema() (llm.ResponseSchema, error) {
|
||||
return llm.LoadResponseSchema(embeddedAssets, llm.ResponseSchemaDefinition{
|
||||
Key: ResponseSchemaKey,
|
||||
ID: ResponseSchemaID,
|
||||
Version: SchemaVersion,
|
||||
Name: ResponseSchemaName,
|
||||
AssetPath: "assets/schemas/dnd_npcs_llm.v1.json",
|
||||
})
|
||||
}
|
||||
78
internal/modules/dnd/extract/npcs/schema_test.go
Normal file
78
internal/modules/dnd/extract/npcs/schema_test.go
Normal file
@@ -0,0 +1,78 @@
|
||||
package npcs
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/santhosh-tekuri/jsonschema/v6"
|
||||
)
|
||||
|
||||
func TestLoadResponseSchemaUsesPrivateNPCSchema(t *testing.T) {
|
||||
schema, err := loadResponseSchema()
|
||||
if err != nil {
|
||||
t.Fatalf("loadResponseSchema() error = %v, want nil", err)
|
||||
}
|
||||
if schema.Key != ResponseSchemaKey || schema.ID != ResponseSchemaID || schema.Version != SchemaVersion || schema.Name != ResponseSchemaName || !strings.HasPrefix(schema.SHA256, "sha256:") || !json.Valid(schema.JSONSchema) {
|
||||
t.Fatalf("schema = %#v, want private NPC schema identity", schema)
|
||||
}
|
||||
valid := map[string]any{"npcs": []any{map[string]any{
|
||||
"name": "Mira Thorn", "aliases": []any{}, "description": "A ranger.", "relationships": []any{},
|
||||
"source_refs": []any{map[string]any{"start_unit_id": 1, "end_unit_id": 2}},
|
||||
}}}
|
||||
validJSON, err := json.Marshal(valid)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := validateJSONSchema(validJSON, schema.JSONSchema); err != nil {
|
||||
t.Fatalf("valid private NPC response rejected: %v", err)
|
||||
}
|
||||
withID := map[string]any{"npcs": []any{map[string]any{
|
||||
"name": "Mira Thorn", "id": "assigned-later", "aliases": []any{}, "description": "A ranger.", "relationships": []any{},
|
||||
"source_refs": []any{map[string]any{"start_unit_id": 1, "end_unit_id": 2}},
|
||||
}}}
|
||||
withIDJSON, err := json.Marshal(withID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := validateJSONSchema(withIDJSON, schema.JSONSchema); err == nil {
|
||||
t.Fatal("private schema accepted framework-assigned id")
|
||||
}
|
||||
}
|
||||
|
||||
func TestResponseSchemaIsMutationSafeAndDiagnosticsRedactContent(t *testing.T) {
|
||||
first, err := loadResponseSchema()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
first.JSONSchema[0] = '['
|
||||
second, err := loadResponseSchema()
|
||||
if err != nil || !json.Valid(second.JSONSchema) || bytes.Equal(first.JSONSchema, second.JSONSchema) {
|
||||
t.Fatalf("second schema = %s, %v; want defensive copy", second.JSONSchema, err)
|
||||
}
|
||||
diagnostics := second.DiagnosticsMap()
|
||||
if _, ok := diagnostics["json_schema"]; ok {
|
||||
t.Fatalf("schema diagnostics included raw content: %#v", diagnostics)
|
||||
}
|
||||
}
|
||||
|
||||
func validateJSONSchema(instanceContent, schemaContent []byte) error {
|
||||
instance, err := jsonschema.UnmarshalJSON(bytes.NewReader(instanceContent))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
schemaDocument, err := jsonschema.UnmarshalJSON(bytes.NewReader(schemaContent))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
compiler := jsonschema.NewCompiler()
|
||||
if err := compiler.AddResource("schema.json", schemaDocument); err != nil {
|
||||
return err
|
||||
}
|
||||
compiled, err := compiler.Compile("schema.json")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return compiled.Validate(instance)
|
||||
}
|
||||
45
internal/modules/dnd/extract/npcs/scriptorium_assets.go
Normal file
45
internal/modules/dnd/extract/npcs/scriptorium_assets.go
Normal file
@@ -0,0 +1,45 @@
|
||||
package npcs
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sync"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/promptfs"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared"
|
||||
)
|
||||
|
||||
const scriptoriumPromptRoot = "assets/prompts"
|
||||
|
||||
func RegisterPromptAssets(registry *llm.AssetRegistry) error {
|
||||
promptFS, err := shared.ModulePromptFS("dnd.npcs", embeddedAssets, []promptfs.ModulePromptFile{
|
||||
{Name: "dnd.npcs.yaml", Path: "assets/prompts/dnd.npcs.yaml"},
|
||||
{Name: "task.md", Path: "assets/prompts/task.md"},
|
||||
{Name: "instructions.md", Path: "assets/prompts/instructions.md"},
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("prepare NPC prompt assets: %w", err)
|
||||
}
|
||||
if err := registry.RegisterPromptFS(promptFS, scriptoriumPromptRoot); err != nil {
|
||||
return err
|
||||
}
|
||||
return registry.RegisterSchemaFS(embeddedAssets, "assets/schemas")
|
||||
}
|
||||
|
||||
func scriptoriumPromptMetadata() (string, error) {
|
||||
scriptoriumPromptHashOnce.Do(func() {
|
||||
parts := append([]llm.AssetHashPart{
|
||||
{FS: embeddedAssets, Path: "assets/prompts/dnd.npcs.yaml"},
|
||||
{FS: embeddedAssets, Path: "assets/prompts/task.md"},
|
||||
{FS: embeddedAssets, Path: "assets/prompts/instructions.md"},
|
||||
}, append(shared.CommonHashParts(), shared.ReferenceHashParts()...)...)
|
||||
scriptoriumPromptHash, scriptoriumPromptHashErr = llm.HashAssets(parts)
|
||||
})
|
||||
return scriptoriumPromptHash, scriptoriumPromptHashErr
|
||||
}
|
||||
|
||||
var (
|
||||
scriptoriumPromptHashOnce sync.Once
|
||||
scriptoriumPromptHash string
|
||||
scriptoriumPromptHashErr error
|
||||
)
|
||||
68
internal/modules/dnd/extract/npcs/scriptorium_assets_test.go
Normal file
68
internal/modules/dnd/extract/npcs/scriptorium_assets_test.go
Normal file
@@ -0,0 +1,68 @@
|
||||
package npcs
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
||||
"gitea.maximumdirect.net/eric/scriptorium"
|
||||
)
|
||||
|
||||
func TestRegisterPromptAssetsAndPrepareNPCPrompt(t *testing.T) {
|
||||
registry := llm.NewAssetRegistry()
|
||||
if err := RegisterPromptAssets(registry); err != nil {
|
||||
t.Fatalf("RegisterPromptAssets() error = %v, want nil", err)
|
||||
}
|
||||
options, err := registry.ScriptoriumOptions()
|
||||
if err != nil {
|
||||
t.Fatalf("ScriptoriumOptions() error = %v, want nil", err)
|
||||
}
|
||||
options = append(options, scriptorium.WithProfiles(scriptorium.OpenAICompatibleProfile(scriptorium.OpenAICompatibleProfileConfig{
|
||||
ID: "npc-test-profile", Endpoint: "http://127.0.0.1:1/v1", Model: "npc-test-model",
|
||||
})))
|
||||
engine, err := scriptorium.NewEngine(scriptorium.Config{Timeout: time.Second}, options...)
|
||||
if err != nil {
|
||||
t.Fatalf("NewEngine() error = %v, want nil", err)
|
||||
}
|
||||
prepared, err := engine.Prepare(context.Background(), scriptorium.RunRequest{
|
||||
PromptID: PromptID, PromptVersion: SchemaVersion, ProfileID: "npc-test-profile",
|
||||
Inputs: map[string]scriptorium.ArtifactRef{
|
||||
"transcript": scriptorium.InlineWithURI("file:///session.json", `{"units":[1]}`),
|
||||
"players": scriptorium.Inline("Dana: Mira"),
|
||||
"party": scriptorium.Inline("Mira: ranger"),
|
||||
"glossary": scriptorium.Inline("Greencloak: title"),
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Prepare() error = %v, want nil", err)
|
||||
}
|
||||
if prepared.PromptID != PromptID || prepared.OutputContract.SchemaPath != "dnd_npcs_llm.v1.json" || len(prepared.Messages) != 5 {
|
||||
t.Fatalf("prepared prompt = %#v, want NPC prompt identity and wiring", prepared)
|
||||
}
|
||||
if !strings.Contains(prepared.Messages[1].Content, `{"units":[1]}`) || !strings.Contains(prepared.Messages[2].Content, "Dana: Mira") || !strings.Contains(prepared.Messages[2].Content, "Mira: ranger") {
|
||||
t.Fatalf("prepared prompt inputs do not include transcript/references")
|
||||
}
|
||||
if strings.Contains(prepared.Messages[3].Content, `{"units":[1]}`) || strings.Contains(prepared.Messages[4].Content, `{"units":[1]}`) {
|
||||
t.Fatal("task or instruction prompt leaked raw transcript")
|
||||
}
|
||||
}
|
||||
|
||||
func TestPromptMetadataAndDiagnosticsDoNotContainRawAssets(t *testing.T) {
|
||||
hash, err := scriptoriumPromptMetadata()
|
||||
if err != nil || !strings.HasPrefix(hash, "sha256:") {
|
||||
t.Fatalf("scriptoriumPromptMetadata() = %q, %v; want hash", hash, err)
|
||||
}
|
||||
metadata := newExtractor(t, &fakeNPCsLLMClient{}).ManifestMetadata()
|
||||
payload, err := json.Marshal(metadata)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, forbidden := range []string{"Include an in-world", "common-dnd-system", "source-unit", "dnd_npcs_llm.v1.json"} {
|
||||
if strings.Contains(string(payload), forbidden) {
|
||||
t.Fatalf("metadata leaked raw prompt/schema content %q: %s", forbidden, payload)
|
||||
}
|
||||
}
|
||||
}
|
||||
112
internal/modules/dnd/extract/npcs/test_helpers_test.go
Normal file
112
internal/modules/dnd/extract/npcs/test_helpers_test.go
Normal file
@@ -0,0 +1,112 @@
|
||||
package npcs
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared"
|
||||
)
|
||||
|
||||
func extractionRequest() contracts.TypedExtractionRequest {
|
||||
doc := sourceDocument()
|
||||
chunk := &source.Chunk{
|
||||
ID: "session-alpha:chunk:0",
|
||||
SourceID: doc.ID,
|
||||
Index: 0,
|
||||
Ref: source.SourceRef{SourceID: doc.ID, StartUnitID: 1, EndUnitID: 3},
|
||||
Content: []byte(`{"units":[1,2,3]}`),
|
||||
MediaType: "application/json",
|
||||
Units: append([]source.SourceUnit(nil), doc.Units...),
|
||||
Metadata: map[string]any{"ignored": "chunk metadata"},
|
||||
}
|
||||
return contracts.TypedExtractionRequest{
|
||||
Source: doc,
|
||||
Chunk: chunk,
|
||||
SourceInput: contracts.NewLLMInputMaterial("source", chunk.MediaType, chunk.Content, "sha256:chunk", "file:///session-alpha.json"),
|
||||
SessionID: "session-123",
|
||||
LLMProfile: "profile-npcs",
|
||||
}
|
||||
}
|
||||
|
||||
func sourceDocument() *source.SourceDocument {
|
||||
return &source.SourceDocument{
|
||||
ID: "session-alpha",
|
||||
Kind: "transcript",
|
||||
Format: "application/vnd.seriatim.minimal+json",
|
||||
Digest: "sha256:test",
|
||||
Units: []source.SourceUnit{
|
||||
{ID: 1, Kind: "transcript_segment", Text: "Mira Thorn greets the party."},
|
||||
{ID: 2, Kind: "transcript_segment", Text: "The Greencloak watches the northern road."},
|
||||
{ID: 3, Kind: "transcript_segment", Text: "Captain Vale reports to Mira Thorn."},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func responseSourceRefs(startUnitID, endUnitID int) []npcSourceRefResponse {
|
||||
return []npcSourceRefResponse{{StartUnitID: shared.UnitRefFromInt(startUnitID), EndUnitID: shared.UnitRefFromInt(endUnitID)}}
|
||||
}
|
||||
|
||||
func newExtractor(t *testing.T, client contracts.StructuredLLMClient, references ...contracts.ReferenceSet) *Extractor {
|
||||
t.Helper()
|
||||
extractor, err := New(client, Options{}, references...)
|
||||
if err != nil {
|
||||
t.Fatalf("New() error = %v, want nil", err)
|
||||
}
|
||||
return extractor
|
||||
}
|
||||
|
||||
func emptyChunkRequest(req contracts.TypedExtractionRequest) contracts.TypedExtractionRequest {
|
||||
req.Chunk = &source.Chunk{ID: req.Chunk.ID, SourceID: req.Chunk.SourceID, Index: req.Chunk.Index}
|
||||
return req
|
||||
}
|
||||
|
||||
func mismatchedSourceInputRequest(req contracts.TypedExtractionRequest) contracts.TypedExtractionRequest {
|
||||
req.SourceInput = contracts.NewLLMInputMaterial("source", "application/json", []byte(`{"different":true}`), "sha256:other", "file:///other.json")
|
||||
return req
|
||||
}
|
||||
|
||||
func cloneStructuredCompletionRequest(req contracts.StructuredCompletionRequest) contracts.StructuredCompletionRequest {
|
||||
req.Inputs = req.Inputs.Clone()
|
||||
if len(req.Vars) == 0 {
|
||||
req.Vars = nil
|
||||
return req
|
||||
}
|
||||
vars := make(map[string]any, len(req.Vars))
|
||||
for key, value := range req.Vars {
|
||||
vars[key] = value
|
||||
}
|
||||
req.Vars = vars
|
||||
return req
|
||||
}
|
||||
|
||||
type fakeNPCsLLMClient struct {
|
||||
response extractionResponse
|
||||
content []byte
|
||||
err error
|
||||
requests []contracts.StructuredCompletionRequest
|
||||
}
|
||||
|
||||
func (client *fakeNPCsLLMClient) CompleteStructured(_ context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
|
||||
client.requests = append(client.requests, cloneStructuredCompletionRequest(req))
|
||||
if client.err != nil {
|
||||
return contracts.StructuredCompletionResponse{}, client.err
|
||||
}
|
||||
target, ok := out.(*extractionResponse)
|
||||
if !ok {
|
||||
return contracts.StructuredCompletionResponse{}, errors.New("unexpected output target")
|
||||
}
|
||||
*target = client.response
|
||||
content := append([]byte(nil), client.content...)
|
||||
if len(content) == 0 {
|
||||
var err error
|
||||
content, err = json.Marshal(client.response)
|
||||
if err != nil {
|
||||
return contracts.StructuredCompletionResponse{}, err
|
||||
}
|
||||
}
|
||||
return contracts.StructuredCompletionResponse{Content: content}, nil
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
// Package diagnostics provides bounded, safe text for deterministic NPC
|
||||
// validator decisions and warnings.
|
||||
package diagnostics
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const (
|
||||
MaxIssues = 20
|
||||
MaxDisplayedRunes = 128
|
||||
MaxMessageBytes = 4096
|
||||
)
|
||||
|
||||
func Truncate(value string) string {
|
||||
runes := []rune(value)
|
||||
if len(runes) <= MaxDisplayedRunes {
|
||||
return value
|
||||
}
|
||||
return string(runes[:MaxDisplayedRunes-1]) + "…"
|
||||
}
|
||||
|
||||
func Quote(value string) string { return strconv.Quote(Truncate(value)) }
|
||||
|
||||
func Aggregate(prefix string, issues []string) string {
|
||||
displayed := make([]string, 0, min(len(issues), MaxIssues))
|
||||
for len(displayed) < len(issues) && len(displayed) < MaxIssues {
|
||||
issue := Truncate(issues[len(displayed)])
|
||||
candidate := aggregateMessage(prefix, append(displayed, issue), len(issues)-len(displayed)-1)
|
||||
if len([]byte(candidate)) > MaxMessageBytes {
|
||||
break
|
||||
}
|
||||
displayed = append(displayed, issue)
|
||||
}
|
||||
return aggregateMessage(prefix, displayed, len(issues)-len(displayed))
|
||||
}
|
||||
|
||||
func aggregateMessage(prefix string, issues []string, omitted int) string {
|
||||
message := prefix + ": " + strings.Join(issues, ", ")
|
||||
if omitted > 0 {
|
||||
message += fmt.Sprintf("; %d additional issue(s) omitted", omitted)
|
||||
}
|
||||
return message
|
||||
}
|
||||
|
||||
func min(left, right int) int {
|
||||
if left < right {
|
||||
return left
|
||||
}
|
||||
return right
|
||||
}
|
||||
121
internal/modules/dnd/validate/npcs/shape/validator.go
Normal file
121
internal/modules/dnd/validate/npcs/shape/validator.go
Normal file
@@ -0,0 +1,121 @@
|
||||
package shape
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/npcs/diagnostics"
|
||||
)
|
||||
|
||||
const (
|
||||
Key = "extract/dnd/npcs/shape"
|
||||
ReasonCode = "invalid_npc_shape"
|
||||
policy = "dnd.npcs.validator.shape.v1"
|
||||
)
|
||||
|
||||
type Options struct{}
|
||||
type Validator struct{}
|
||||
|
||||
var _ contracts.TypedValidator[dnd.NPCList] = (*Validator)(nil)
|
||||
var _ pipeline.CheckpointFingerprintProvider = (*Validator)(nil)
|
||||
|
||||
func New(Options) *Validator { return &Validator{} }
|
||||
func (v *Validator) Name() string { return Key }
|
||||
func (v *Validator) ExecutionClass() contracts.ExecutionClass {
|
||||
return contracts.ExecutionClassDeterministic
|
||||
}
|
||||
func (v *Validator) CheckpointFingerprints() []pipeline.CheckpointFingerprint {
|
||||
return []pipeline.CheckpointFingerprint{{Name: "policy", Value: policy}}
|
||||
}
|
||||
|
||||
func (v *Validator) Validate(_ context.Context, req contracts.TypedValidationRequest[dnd.NPCList]) (contracts.ValidationResult, error) {
|
||||
issues := issuesFor(req.Value)
|
||||
if len(issues) > 0 {
|
||||
return rejection(diagnostics.Aggregate("invalid NPC shape", issues)), nil
|
||||
}
|
||||
return contracts.ValidationResult{Approved: true}, nil
|
||||
}
|
||||
|
||||
func Validate(value dnd.NPCList) error {
|
||||
issues := issuesFor(value)
|
||||
if len(issues) == 0 {
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("%s", diagnostics.Aggregate("invalid NPC shape", issues))
|
||||
}
|
||||
|
||||
func issuesFor(value dnd.NPCList) []string {
|
||||
issues := make([]string, 0)
|
||||
if value.NPCs == nil {
|
||||
return []string{"npcs must be present"}
|
||||
}
|
||||
for index, npc := range value.NPCs {
|
||||
prefix := fmt.Sprintf("npcs[%d]", index)
|
||||
if strings.TrimSpace(npc.ID) == "" {
|
||||
issues = append(issues, prefix+".id must not be empty")
|
||||
}
|
||||
if strings.TrimSpace(npc.Name) == "" {
|
||||
issues = append(issues, prefix+".name must not be empty")
|
||||
}
|
||||
if npc.Aliases == nil {
|
||||
issues = append(issues, prefix+".aliases must be present")
|
||||
} else {
|
||||
for aliasIndex, alias := range npc.Aliases {
|
||||
if strings.TrimSpace(alias) == "" {
|
||||
issues = append(issues, fmt.Sprintf("%s.aliases[%d] must not be empty: %s", prefix, aliasIndex, diagnostics.Quote(alias)))
|
||||
}
|
||||
}
|
||||
}
|
||||
if strings.TrimSpace(npc.Description) == "" {
|
||||
issues = append(issues, prefix+".description must not be empty")
|
||||
}
|
||||
if npc.Relationships == nil {
|
||||
issues = append(issues, prefix+".relationships must be present")
|
||||
} else {
|
||||
for relationshipIndex, relationship := range npc.Relationships {
|
||||
relationshipPrefix := fmt.Sprintf("%s.relationships[%d]", prefix, relationshipIndex)
|
||||
if strings.TrimSpace(relationship.Target) == "" {
|
||||
issues = append(issues, relationshipPrefix+".target must not be empty: "+diagnostics.Quote(relationship.Target))
|
||||
}
|
||||
if strings.TrimSpace(relationship.Relationship) == "" {
|
||||
issues = append(issues, relationshipPrefix+".relationship must not be empty: "+diagnostics.Quote(relationship.Relationship))
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(npc.SourceRefs) == 0 {
|
||||
issues = append(issues, prefix+".source_refs must not be empty")
|
||||
}
|
||||
}
|
||||
return issues
|
||||
}
|
||||
|
||||
func Spec() pipeline.ValidatorSpec {
|
||||
return pipeline.ValidatorSpec{Key: Key, ExecutionClass: contracts.ExecutionClassDeterministic}
|
||||
}
|
||||
|
||||
func Register(registry *pipeline.ValidatorRegistry) error {
|
||||
return pipeline.RegisterTypedValidatorBuilder(registry, dnd.NPCListKind, Spec(), validateOptions, func(request pipeline.BuildRequest) (contracts.TypedValidator[dnd.NPCList], error) {
|
||||
options, err := DecodeOptions(request.Options)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return New(options), nil
|
||||
})
|
||||
}
|
||||
|
||||
func DecodeOptions(options map[string]any) (Options, error) {
|
||||
if err := pipeline.RejectUnknownOptions(options); err != nil {
|
||||
return Options{}, err
|
||||
}
|
||||
return Options{}, nil
|
||||
}
|
||||
|
||||
func validateOptions(options map[string]any) error { _, err := DecodeOptions(options); return err }
|
||||
|
||||
func rejection(message string) contracts.ValidationResult {
|
||||
return contracts.ValidationResult{Approved: false, ReasonCode: ReasonCode, Message: message}
|
||||
}
|
||||
92
internal/modules/dnd/validate/npcs/shape/validator_test.go
Normal file
92
internal/modules/dnd/validate/npcs/shape/validator_test.go
Normal file
@@ -0,0 +1,92 @@
|
||||
package shape
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
"unicode/utf8"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
||||
)
|
||||
|
||||
func TestValidatorApprovesWellFormedNPCPayload(t *testing.T) {
|
||||
result, err := New(Options{}).Validate(context.Background(), requestWithValue(validNPCList()))
|
||||
if err != nil || !result.Approved {
|
||||
t.Fatalf("Validate() = %#v, %v; want approval", result, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidatorRejectsRequiredShapeValues(t *testing.T) {
|
||||
value := validNPCList()
|
||||
value.NPCs[0].Aliases = nil
|
||||
result, err := New(Options{}).Validate(context.Background(), requestWithValue(value))
|
||||
if err != nil || result.Approved || result.ReasonCode != ReasonCode || !strings.Contains(result.Message, "aliases must be present") {
|
||||
t.Fatalf("Validate() = %#v, %v; want bounded shape rejection", result, err)
|
||||
}
|
||||
|
||||
missing := dnd.NPCList{}
|
||||
result, err = New(Options{}).Validate(context.Background(), requestWithValue(missing))
|
||||
if err != nil || result.Approved || result.ReasonCode != ReasonCode {
|
||||
t.Fatalf("missing Validate() = %#v, %v; want shape rejection", result, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidatorBoundsDiagnosticsAndQuotesUnicode(t *testing.T) {
|
||||
value := dnd.NPCList{NPCs: make([]dnd.NPC, 24)}
|
||||
long := strings.Repeat("火", 220) + "\n\t"
|
||||
for index := range value.NPCs {
|
||||
value.NPCs[index] = dnd.NPC{ID: "candidate", Name: long, Aliases: []string{"\n\t"}, Description: "", Relationships: []dnd.NPCRelationship{{Target: " ", Relationship: " "}}, SourceRefs: []source.SourceRef{}}
|
||||
}
|
||||
result, err := New(Options{}).Validate(context.Background(), requestWithValue(value))
|
||||
if err != nil || result.Approved || len([]byte(result.Message)) > diagnosticsMaxMessageBytes || !utf8.ValidString(result.Message) {
|
||||
t.Fatalf("Validate() = %#v, %v; want bounded valid UTF-8 rejection", result, err)
|
||||
}
|
||||
if strings.Count(result.Message, "npcs[") > diagnosticsMaxIssues || !strings.Contains(result.Message, "additional issue(s) omitted") || !strings.Contains(result.Message, `\n\t`) {
|
||||
t.Fatalf("message = %q, want bounded quoted diagnostics", result.Message)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidatorSpecCheckpointAndRegistration(t *testing.T) {
|
||||
if got := New(Options{}).CheckpointFingerprints(); len(got) != 1 || got[0].Name != "policy" || got[0].Value != "dnd.npcs.validator.shape.v1" {
|
||||
t.Fatalf("CheckpointFingerprints() = %#v, want local policy", got)
|
||||
}
|
||||
if spec := Spec(); spec.Key != Key || spec.ExecutionClass != contracts.ExecutionClassDeterministic {
|
||||
t.Fatalf("Spec() = %#v, want deterministic shape validator", spec)
|
||||
}
|
||||
registry := pipeline.NewValidatorRegistry()
|
||||
if err := Register(registry); err != nil {
|
||||
t.Fatalf("Register() error = %v", err)
|
||||
}
|
||||
if _, err := DecodeOptions(map[string]any{"unexpected": true}); err == nil {
|
||||
t.Fatal("DecodeOptions() accepted unknown option")
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidatorDoesNotMutateValue(t *testing.T) {
|
||||
value := validNPCList()
|
||||
before := value
|
||||
_, err := New(Options{}).Validate(context.Background(), requestWithValue(value))
|
||||
if err != nil || value.NPCs[0].Aliases[0] != before.NPCs[0].Aliases[0] {
|
||||
t.Fatalf("Validate() mutated value: %#v", value)
|
||||
}
|
||||
}
|
||||
|
||||
func requestWithValue(value dnd.NPCList) contracts.TypedValidationRequest[dnd.NPCList] {
|
||||
return contracts.TypedValidationRequest[dnd.NPCList]{Value: value}
|
||||
}
|
||||
|
||||
func validNPCList() dnd.NPCList {
|
||||
return dnd.NPCList{NPCs: []dnd.NPC{{
|
||||
ID: "candidate", Name: "Mira Thorn", Aliases: []string{"The Greencloak"}, Description: "A guarded ranger.",
|
||||
Relationships: []dnd.NPCRelationship{{Target: "Captain Vale", Relationship: "reports to"}},
|
||||
SourceRefs: []source.SourceRef{{SourceID: "session", StartUnitID: 1, EndUnitID: 1}},
|
||||
}}}
|
||||
}
|
||||
|
||||
const (
|
||||
diagnosticsMaxIssues = 20
|
||||
diagnosticsMaxMessageBytes = 4096
|
||||
)
|
||||
79
internal/modules/dnd/validate/npcs/source_refs/validator.go
Normal file
79
internal/modules/dnd/validate/npcs/source_refs/validator.go
Normal file
@@ -0,0 +1,79 @@
|
||||
package sourcerefs
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/npcs/diagnostics"
|
||||
npcshape "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/npcs/shape"
|
||||
)
|
||||
|
||||
const (
|
||||
Key = "extract/dnd/npcs/source_refs"
|
||||
ReasonCode = "invalid_npc_source_refs"
|
||||
policy = "dnd.npcs.validator.source_refs.v1"
|
||||
)
|
||||
|
||||
type Options struct{}
|
||||
type Validator struct{}
|
||||
|
||||
var _ contracts.TypedValidator[dnd.NPCList] = (*Validator)(nil)
|
||||
var _ pipeline.CheckpointFingerprintProvider = (*Validator)(nil)
|
||||
|
||||
func New(Options) *Validator { return &Validator{} }
|
||||
func (v *Validator) Name() string { return Key }
|
||||
func (v *Validator) ExecutionClass() contracts.ExecutionClass {
|
||||
return contracts.ExecutionClassDeterministic
|
||||
}
|
||||
func (v *Validator) CheckpointFingerprints() []pipeline.CheckpointFingerprint {
|
||||
return []pipeline.CheckpointFingerprint{{Name: "policy", Value: policy}}
|
||||
}
|
||||
|
||||
func (v *Validator) Validate(_ context.Context, req contracts.TypedValidationRequest[dnd.NPCList]) (contracts.ValidationResult, error) {
|
||||
if err := npcshape.Validate(req.Value); err != nil {
|
||||
return contracts.ValidationResult{Approved: true}, nil
|
||||
}
|
||||
issues := make([]string, 0)
|
||||
for npcIndex, npc := range req.Value.NPCs {
|
||||
for refIndex, ref := range npc.SourceRefs {
|
||||
if err := source.ValidateRef(req.Source, ref); err != nil {
|
||||
issues = append(issues, fmt.Sprintf("npcs[%d].source_refs[%d]: %s", npcIndex, refIndex, diagnostics.Truncate(err.Error())))
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(issues) == 0 {
|
||||
return contracts.ValidationResult{Approved: true}, nil
|
||||
}
|
||||
return rejection(diagnostics.Aggregate("invalid NPC source references", issues)), nil
|
||||
}
|
||||
|
||||
func Spec() pipeline.ValidatorSpec {
|
||||
return pipeline.ValidatorSpec{Key: Key, ExecutionClass: contracts.ExecutionClassDeterministic}
|
||||
}
|
||||
|
||||
func Register(registry *pipeline.ValidatorRegistry) error {
|
||||
return pipeline.RegisterTypedValidatorBuilder(registry, dnd.NPCListKind, Spec(), validateOptions, func(request pipeline.BuildRequest) (contracts.TypedValidator[dnd.NPCList], error) {
|
||||
options, err := DecodeOptions(request.Options)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return New(options), nil
|
||||
})
|
||||
}
|
||||
|
||||
func DecodeOptions(options map[string]any) (Options, error) {
|
||||
if err := pipeline.RejectUnknownOptions(options); err != nil {
|
||||
return Options{}, err
|
||||
}
|
||||
return Options{}, nil
|
||||
}
|
||||
|
||||
func validateOptions(options map[string]any) error { _, err := DecodeOptions(options); return err }
|
||||
|
||||
func rejection(message string) contracts.ValidationResult {
|
||||
return contracts.ValidationResult{Approved: false, ReasonCode: ReasonCode, Message: message}
|
||||
}
|
||||
@@ -0,0 +1,88 @@
|
||||
package sourcerefs
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
"unicode/utf8"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
||||
)
|
||||
|
||||
func TestValidatorApprovesValidSourceReferences(t *testing.T) {
|
||||
result, err := New(Options{}).Validate(context.Background(), requestWithValue(validDocument(), validNPCList()))
|
||||
if err != nil || !result.Approved || len(result.Warnings) != 0 {
|
||||
t.Fatalf("Validate() = %#v, %v; want approval", result, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidatorRejectsInvalidSourceReferences(t *testing.T) {
|
||||
value := validNPCList()
|
||||
value.NPCs[0].SourceRefs = []source.SourceRef{
|
||||
{SourceID: "foreign", StartUnitID: 1, EndUnitID: 1},
|
||||
{SourceID: "session", StartUnitID: 2, EndUnitID: 1},
|
||||
{SourceID: "session", StartUnitID: 99, EndUnitID: 99},
|
||||
}
|
||||
result, err := New(Options{}).Validate(context.Background(), requestWithValue(validDocument(), value))
|
||||
if err != nil || result.Approved || result.ReasonCode != ReasonCode || !strings.Contains(result.Message, "npcs[0].source_refs[0]") {
|
||||
t.Fatalf("Validate() = %#v, %v; want source-reference rejection", result, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidatorDefersMalformedShape(t *testing.T) {
|
||||
value := dnd.NPCList{NPCs: []dnd.NPC{{Name: "Mira Thorn"}}}
|
||||
result, err := New(Options{}).Validate(context.Background(), requestWithValue(validDocument(), value))
|
||||
if err != nil || !result.Approved || result.ReasonCode != "" || result.Message != "" {
|
||||
t.Fatalf("Validate() = %#v, %v; want shape deferral", result, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidatorBoundsDiagnosticsAndHandlesMissingDocument(t *testing.T) {
|
||||
value := validNPCList()
|
||||
value.NPCs[0].SourceRefs = make([]source.SourceRef, 24)
|
||||
for index := range value.NPCs[0].SourceRefs {
|
||||
value.NPCs[0].SourceRefs[index] = source.SourceRef{SourceID: strings.Repeat("火", 220) + "\n\t", StartUnitID: index + 1, EndUnitID: index + 1}
|
||||
}
|
||||
result, err := New(Options{}).Validate(context.Background(), requestWithValue(nil, value))
|
||||
if err != nil || result.Approved || len([]byte(result.Message)) > 4096 || !utf8.ValidString(result.Message) {
|
||||
t.Fatalf("Validate() = %#v, %v; want bounded missing-document rejection", result, err)
|
||||
}
|
||||
if !strings.Contains(result.Message, "additional issue(s) omitted") || !strings.Contains(result.Message, fmt.Sprintf("npcs[0].source_refs[%d]", 19)) {
|
||||
t.Fatalf("message = %q, want bounded aggregate diagnostics", result.Message)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidatorSpecCheckpointAndRegistration(t *testing.T) {
|
||||
if got := New(Options{}).CheckpointFingerprints(); len(got) != 1 || got[0].Name != "policy" || got[0].Value != "dnd.npcs.validator.source_refs.v1" {
|
||||
t.Fatalf("CheckpointFingerprints() = %#v, want local policy", got)
|
||||
}
|
||||
if Spec().ExecutionClass != contracts.ExecutionClassDeterministic {
|
||||
t.Fatalf("Spec() = %#v, want deterministic validator", Spec())
|
||||
}
|
||||
registry := pipeline.NewValidatorRegistry()
|
||||
if err := Register(registry); err != nil {
|
||||
t.Fatalf("Register() error = %v", err)
|
||||
}
|
||||
if _, err := DecodeOptions(map[string]any{"unexpected": true}); err == nil {
|
||||
t.Fatal("DecodeOptions() accepted unknown option")
|
||||
}
|
||||
}
|
||||
|
||||
func requestWithValue(doc *source.SourceDocument, value dnd.NPCList) contracts.TypedValidationRequest[dnd.NPCList] {
|
||||
return contracts.TypedValidationRequest[dnd.NPCList]{Source: doc, Value: value}
|
||||
}
|
||||
|
||||
func validDocument() *source.SourceDocument {
|
||||
return &source.SourceDocument{ID: "session", Kind: "transcript", Format: "application/json", Digest: "sha256:session", Units: []source.SourceUnit{
|
||||
{ID: 1, Kind: "message", Text: "Mira Thorn enters."},
|
||||
{ID: 2, Kind: "message", Text: "The ranger watches."},
|
||||
}}
|
||||
}
|
||||
|
||||
func validNPCList() dnd.NPCList {
|
||||
return dnd.NPCList{NPCs: []dnd.NPC{{ID: "candidate", Name: "Mira Thorn", Aliases: []string{"The Greencloak"}, Description: "A ranger.", Relationships: []dnd.NPCRelationship{}, SourceRefs: []source.SourceRef{{SourceID: "session", StartUnitID: 1, EndUnitID: 2}}}}}
|
||||
}
|
||||
@@ -0,0 +1,114 @@
|
||||
package sourcerelatedness
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/npcs/identity"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/npcs/diagnostics"
|
||||
npcshape "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/npcs/shape"
|
||||
)
|
||||
|
||||
const (
|
||||
Key = "extract/dnd/npcs/source_relatedness"
|
||||
WarningReasonCode = "npc_not_near_source"
|
||||
policy = "dnd.npcs.validator.source_relatedness.v1"
|
||||
)
|
||||
|
||||
type Options struct{}
|
||||
type Validator struct{}
|
||||
|
||||
var _ contracts.TypedValidator[dnd.NPCList] = (*Validator)(nil)
|
||||
var _ pipeline.CheckpointFingerprintProvider = (*Validator)(nil)
|
||||
|
||||
func New(Options) *Validator { return &Validator{} }
|
||||
func (v *Validator) Name() string { return Key }
|
||||
func (v *Validator) ExecutionClass() contracts.ExecutionClass {
|
||||
return contracts.ExecutionClassDeterministic
|
||||
}
|
||||
func (v *Validator) CheckpointFingerprints() []pipeline.CheckpointFingerprint {
|
||||
return []pipeline.CheckpointFingerprint{{Name: "policy", Value: policy}}
|
||||
}
|
||||
|
||||
func (v *Validator) Validate(_ context.Context, req contracts.TypedValidationRequest[dnd.NPCList]) (contracts.ValidationResult, error) {
|
||||
if err := npcshape.Validate(req.Value); err != nil {
|
||||
return contracts.ValidationResult{Approved: true}, nil
|
||||
}
|
||||
var warnings []contracts.Warning
|
||||
for npcIndex, npc := range req.Value.NPCs {
|
||||
if npcAppearsInCitedText(req.Source, npc) {
|
||||
continue
|
||||
}
|
||||
warnings = append(warnings, contracts.Warning{
|
||||
Scope: fmt.Sprintf("npcs[%d]", npcIndex),
|
||||
ReasonCode: WarningReasonCode,
|
||||
Message: fmt.Sprintf("NPC %s was not found in cited source text", diagnostics.Quote(npc.Name)),
|
||||
})
|
||||
}
|
||||
return contracts.ValidationResult{Approved: true, Warnings: warnings}, nil
|
||||
}
|
||||
|
||||
func npcAppearsInCitedText(doc *source.SourceDocument, npc dnd.NPC) bool {
|
||||
cited := citedTextKey(doc, npc.SourceRefs)
|
||||
if cited == "" {
|
||||
return false
|
||||
}
|
||||
if strings.Contains(cited, identity.ComparisonKey(npc.Name)) {
|
||||
return true
|
||||
}
|
||||
for _, alias := range npc.Aliases {
|
||||
if strings.Contains(cited, identity.ComparisonKey(alias)) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func citedTextKey(doc *source.SourceDocument, refs []source.SourceRef) string {
|
||||
if doc == nil {
|
||||
return ""
|
||||
}
|
||||
var builder strings.Builder
|
||||
for _, ref := range refs {
|
||||
if err := source.ValidateRef(doc, ref); err != nil {
|
||||
continue
|
||||
}
|
||||
start, _ := source.UnitIndex(doc, ref.StartUnitID)
|
||||
end, _ := source.UnitIndex(doc, ref.EndUnitID)
|
||||
for index := start; index <= end; index++ {
|
||||
if builder.Len() > 0 {
|
||||
builder.WriteByte(' ')
|
||||
}
|
||||
builder.WriteString(doc.Units[index].Text)
|
||||
}
|
||||
}
|
||||
return identity.ComparisonKey(builder.String())
|
||||
}
|
||||
|
||||
func Spec() pipeline.ValidatorSpec {
|
||||
return pipeline.ValidatorSpec{Key: Key, ExecutionClass: contracts.ExecutionClassDeterministic}
|
||||
}
|
||||
|
||||
func Register(registry *pipeline.ValidatorRegistry) error {
|
||||
return pipeline.RegisterTypedValidatorBuilder(registry, dnd.NPCListKind, Spec(), validateOptions, func(request pipeline.BuildRequest) (contracts.TypedValidator[dnd.NPCList], error) {
|
||||
options, err := DecodeOptions(request.Options)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return New(options), nil
|
||||
})
|
||||
}
|
||||
|
||||
func DecodeOptions(options map[string]any) (Options, error) {
|
||||
if err := pipeline.RejectUnknownOptions(options); err != nil {
|
||||
return Options{}, err
|
||||
}
|
||||
return Options{}, nil
|
||||
}
|
||||
|
||||
func validateOptions(options map[string]any) error { _, err := DecodeOptions(options); return err }
|
||||
@@ -0,0 +1,81 @@
|
||||
package sourcerelatedness
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
"unicode/utf8"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
||||
)
|
||||
|
||||
func TestValidatorMatchesCanonicalNamesAndAliasesWithUnicodeVariants(t *testing.T) {
|
||||
value := dnd.NPCList{NPCs: []dnd.NPC{
|
||||
{ID: "one", Name: "O'Rin Thorn", Aliases: []string{}, Description: "A ranger.", Relationships: []dnd.NPCRelationship{}, SourceRefs: []source.SourceRef{{SourceID: "session", StartUnitID: 1, EndUnitID: 1}}},
|
||||
{ID: "two", Name: "Missing Name", Aliases: []string{"The Greencloak"}, Description: "A guard.", Relationships: []dnd.NPCRelationship{}, SourceRefs: []source.SourceRef{{SourceID: "session", StartUnitID: 2, EndUnitID: 2}}},
|
||||
}}
|
||||
doc := &source.SourceDocument{ID: "session", Kind: "transcript", Format: "application/json", Digest: "sha256:session", Units: []source.SourceUnit{
|
||||
{ID: 1, Kind: "message", Text: " o’rin\u2003thorn appears."},
|
||||
{ID: 2, Kind: "message", Text: "The greencloak watches."},
|
||||
}}
|
||||
result, err := New(Options{}).Validate(context.Background(), contracts.TypedValidationRequest[dnd.NPCList]{Source: doc, Value: value})
|
||||
if err != nil || !result.Approved || len(result.Warnings) != 0 {
|
||||
t.Fatalf("Validate() = %#v, %v; want alias/canonical relatedness approval", result, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidatorWarnsAtMostOncePerNPCForUnrelatedCitations(t *testing.T) {
|
||||
value := dnd.NPCList{NPCs: []dnd.NPC{
|
||||
{ID: "one", Name: "Missing\nName", Aliases: []string{"Also Missing"}, Description: "A guard.", Relationships: []dnd.NPCRelationship{}, SourceRefs: []source.SourceRef{{SourceID: "session", StartUnitID: 1, EndUnitID: 1}, {SourceID: "session", StartUnitID: 1, EndUnitID: 1}}},
|
||||
}}
|
||||
result, err := New(Options{}).Validate(context.Background(), contracts.TypedValidationRequest[dnd.NPCList]{Source: relatednessDocument(), Value: value})
|
||||
if err != nil || !result.Approved || len(result.Warnings) != 1 {
|
||||
t.Fatalf("Validate() = %#v, %v; want one warning", result, err)
|
||||
}
|
||||
warning := result.Warnings[0]
|
||||
if warning.Scope != "npcs[0]" || warning.ReasonCode != WarningReasonCode || !strings.Contains(warning.Message, `Missing\nName`) || strings.Contains(warning.Message, "Missing\nName") || !utf8.ValidString(warning.Message) {
|
||||
t.Fatalf("warning = %#v, want safely quoted bounded warning", warning)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidatorDefersMalformedShapeAndInvalidRangesDoNotPanic(t *testing.T) {
|
||||
invalidShape := dnd.NPCList{NPCs: []dnd.NPC{{Name: "Mira Thorn"}}}
|
||||
result, err := New(Options{}).Validate(context.Background(), contracts.TypedValidationRequest[dnd.NPCList]{Source: relatednessDocument(), Value: invalidShape})
|
||||
if err != nil || !result.Approved || len(result.Warnings) != 0 {
|
||||
t.Fatalf("shape deferral = %#v, %v; want approval without warning", result, err)
|
||||
}
|
||||
invalidRange := dnd.NPCList{NPCs: []dnd.NPC{{ID: "one", Name: "Mira Thorn", Aliases: []string{}, Description: "A ranger.", Relationships: []dnd.NPCRelationship{}, SourceRefs: []source.SourceRef{{SourceID: "session", StartUnitID: 99, EndUnitID: 99}}}}}
|
||||
result, err = New(Options{}).Validate(context.Background(), contracts.TypedValidationRequest[dnd.NPCList]{Source: relatednessDocument(), Value: invalidRange})
|
||||
if err != nil || !result.Approved || len(result.Warnings) != 1 || result.Warnings[0].ReasonCode != WarningReasonCode {
|
||||
t.Fatalf("invalid-range relatedness = %#v, %v; want one warning", result, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidatorUsesOnlyTranscriptEvidenceAndRegistersPolicy(t *testing.T) {
|
||||
value := dnd.NPCList{NPCs: []dnd.NPC{{ID: "one", Name: "Opaque NPC", Aliases: []string{}, Description: "A guard.", Relationships: []dnd.NPCRelationship{}, SourceRefs: []source.SourceRef{{SourceID: "session", StartUnitID: 1, EndUnitID: 1}}}}}
|
||||
references := contracts.ReferenceSet{Slots: map[string]contracts.ResolvedReferenceSlot{"glossary": {Items: []contracts.ReferenceItem{{Content: []byte("Opaque NPC")}}}}}
|
||||
result, err := New(Options{}).Validate(context.Background(), contracts.TypedValidationRequest[dnd.NPCList]{Source: relatednessDocument(), References: references, Value: value})
|
||||
if err != nil || len(result.Warnings) != 1 {
|
||||
t.Fatalf("reference-only relatedness = %#v, %v; want warning", result, err)
|
||||
}
|
||||
if got := New(Options{}).CheckpointFingerprints(); len(got) != 1 || got[0].Name != "policy" || got[0].Value != "dnd.npcs.validator.source_relatedness.v1" {
|
||||
t.Fatalf("CheckpointFingerprints() = %#v, want local policy", got)
|
||||
}
|
||||
if Spec().ExecutionClass != contracts.ExecutionClassDeterministic {
|
||||
t.Fatalf("Spec() = %#v, want deterministic validator", Spec())
|
||||
}
|
||||
registry := pipeline.NewValidatorRegistry()
|
||||
if err := Register(registry); err != nil {
|
||||
t.Fatalf("Register() error = %v", err)
|
||||
}
|
||||
if _, err := DecodeOptions(map[string]any{"unexpected": true}); err == nil {
|
||||
t.Fatal("DecodeOptions() accepted unknown option")
|
||||
}
|
||||
}
|
||||
|
||||
func relatednessDocument() *source.SourceDocument {
|
||||
return &source.SourceDocument{ID: "session", Kind: "transcript", Format: "application/json", Digest: "sha256:session", Units: []source.SourceUnit{{ID: 1, Kind: "message", Text: "The party waits."}}}
|
||||
}
|
||||
Reference in New Issue
Block a user