Ground spell extraction with the effective catalog
This commit is contained in:
@@ -325,16 +325,20 @@ declares optional `players`, `party`, and `glossary` references for scene
|
|||||||
disambiguation, and accepts `roster` as a deprecated compatibility alias for
|
disambiguation, and accepts `roster` as a deprecated compatibility alias for
|
||||||
`party`.
|
`party`.
|
||||||
|
|
||||||
The `dnd/spells` extractor declares optional reference slots:
|
The `dnd/spells` extractor declares optional campaign reference slots:
|
||||||
|
|
||||||
- `players`
|
- `players`
|
||||||
- `party`
|
- `party`
|
||||||
- `glossary`
|
- `glossary`
|
||||||
- `roster` as a deprecated compatibility alias for `party`
|
- `roster` as a deprecated compatibility alias for `party`
|
||||||
|
|
||||||
Both modules accept UTF-8 plain text, Markdown, YAML, or JSON reference files.
|
These campaign slots accept UTF-8 plain text, Markdown, YAML, or JSON reference
|
||||||
The extractor uses references only as supporting disambiguation material; spell
|
files. The extractor also declares an optional `spell_catalog` slot that accepts
|
||||||
casts still must be present in the source transcript.
|
one UTF-8 `application/json` overlay bundle no larger than 1 MiB. The slot does
|
||||||
|
not allow multiple files. Its format is defined in the
|
||||||
|
[spell-catalog overlay contract](integrations/dnd-spell-catalog-overlays.md).
|
||||||
|
The extractor uses campaign references only as supporting disambiguation
|
||||||
|
material; spell casts still must be present in the source transcript.
|
||||||
|
|
||||||
## State Surfaces
|
## State Surfaces
|
||||||
|
|
||||||
|
|||||||
@@ -5,6 +5,12 @@ resolver. An overlay supplies campaign-specific spell names and aliases for
|
|||||||
recognition. It does not supply spell rules, levels, classes, effects, or
|
recognition. It does not supply spell rules, levels, classes, effects, or
|
||||||
source evidence.
|
source evidence.
|
||||||
|
|
||||||
|
The `dnd/spells` extractor accepts one optional UTF-8 `application/json` overlay
|
||||||
|
bundle through its `spell_catalog` reference slot. The framework materializes
|
||||||
|
that file relative to the configuration or command-line binding, enforces the
|
||||||
|
1 MiB slot limit, and records its origin and raw digest separately from the
|
||||||
|
effective catalog digest.
|
||||||
|
|
||||||
## Shape
|
## Shape
|
||||||
|
|
||||||
An overlay bundle has this shape:
|
An overlay bundle has this shape:
|
||||||
|
|||||||
@@ -103,6 +103,13 @@ return defensive copies, and expose a diagnostics map that omits schema bytes.
|
|||||||
The small framework registry contains only generic test schemas; production
|
The small framework registry contains only generic test schemas; production
|
||||||
schemas remain package-owned.
|
schemas remain package-owned.
|
||||||
|
|
||||||
|
The spell extractor's package-owned prompt declares a required
|
||||||
|
`application/json` `spell_catalog` input. The extractor generates that input
|
||||||
|
from its prepared effective catalog as `{"spell_names":[...]}` using sorted
|
||||||
|
canonical names only. Its input digest covers those generated bytes; manifests
|
||||||
|
record catalog identity and digest rather than names, aliases, overlay bytes,
|
||||||
|
or source metadata.
|
||||||
|
|
||||||
## Debug And Redaction Boundaries
|
## Debug And Redaction Boundaries
|
||||||
|
|
||||||
The pipeline may wrap the client with a debug recorder that captures prepared
|
The pipeline may wrap the client with a debug recorder that captures prepared
|
||||||
|
|||||||
@@ -153,7 +153,12 @@ earliest cited unit, and returns `dnd.SpellList`.
|
|||||||
|
|
||||||
The extractor owns its private model-response DTO, embedded prompt, LLM response
|
The extractor owns its private model-response DTO, embedded prompt, LLM response
|
||||||
schema, strict option decoder, injected shared LLM client, and prompt/schema
|
schema, strict option decoder, injected shared LLM client, and prompt/schema
|
||||||
manifest metadata. The separate `internal/modules/dnd/codec/spells` package
|
manifest metadata. During preparation it resolves the optional `spell_catalog`
|
||||||
|
reference into an immutable effective catalog and adds a generated
|
||||||
|
canonical-name-only JSON input to every structured completion request. Overlay
|
||||||
|
failures therefore stop construction before source parsing or an LLM call;
|
||||||
|
campaign references remain separate disambiguation inputs and never become
|
||||||
|
source evidence. The separate `internal/modules/dnd/codec/spells` package
|
||||||
owns the durable schema and stable JSON representation for artifact kind
|
owns the durable schema and stable JSON representation for artifact kind
|
||||||
`dnd/spell-list`. The runner keeps the result typed through validators and later
|
`dnd/spell-list`. The runner keeps the result typed through validators and later
|
||||||
stages, using the codec only for checkpoint, debug, and output boundaries.
|
stages, using the codec only for checkpoint, debug, and output boundaries.
|
||||||
|
|||||||
21
examples/dnd-spells-catalog.json
Normal file
21
examples/dnd-spells-catalog.json
Normal file
@@ -0,0 +1,21 @@
|
|||||||
|
{
|
||||||
|
"schema_version": "notarius.dnd.spell-catalog-overlay.v1",
|
||||||
|
"catalogs": [
|
||||||
|
{
|
||||||
|
"id": "notarius.example-campaign",
|
||||||
|
"ruleset": "dnd-5e-2014",
|
||||||
|
"source": {
|
||||||
|
"title": "Notarius example campaign spell names",
|
||||||
|
"version": "1",
|
||||||
|
"url": "",
|
||||||
|
"license": ""
|
||||||
|
},
|
||||||
|
"spells": [
|
||||||
|
{
|
||||||
|
"name": "Aegis of Emberfall",
|
||||||
|
"aliases": ["Emberfall Aegis"]
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
@@ -29,3 +29,5 @@ pipelines:
|
|||||||
extract:
|
extract:
|
||||||
module: dnd/spells
|
module: dnd/spells
|
||||||
retries: 2
|
retries: 2
|
||||||
|
references:
|
||||||
|
spell_catalog: ./dnd-spells-catalog.json
|
||||||
|
|||||||
@@ -10,6 +10,7 @@ import (
|
|||||||
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
|
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
|
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/core/debugbundle"
|
"gitea.maximumdirect.net/eric/notarius/internal/core/debugbundle"
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -18,9 +19,22 @@ func TestMaintainedExamplesLoadResolveAndList(t *testing.T) {
|
|||||||
for _, example := range maintainedExampleFiles(t) {
|
for _, example := range maintainedExampleFiles(t) {
|
||||||
t.Run(example.name, func(t *testing.T) {
|
t.Run(example.name, func(t *testing.T) {
|
||||||
cfg := loadMaintainedExample(t, example.path)
|
cfg := loadMaintainedExample(t, example.path)
|
||||||
if _, err := cfg.Resolve(resolveInputForMaintainedExample(components, "dnd-session")); err != nil {
|
effective, err := cfg.Resolve(resolveInputForMaintainedExample(components, "dnd-session"))
|
||||||
|
if err != nil {
|
||||||
t.Fatalf("resolve maintained example: %v", err)
|
t.Fatalf("resolve maintained example: %v", err)
|
||||||
}
|
}
|
||||||
|
materialized, _, err := pipeline.MaterializeReferences(effective.ResolvedPipeline, catalogFromRegistries(components.registries), pipeline.ReferenceMaterializationOptions{
|
||||||
|
ConfigPath: example.path,
|
||||||
|
WorkingDir: filepath.Dir(example.path),
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("materialize maintained example references: %v", err)
|
||||||
|
}
|
||||||
|
if example.name == "production" {
|
||||||
|
if len(materialized.ArtifactLanes) != 1 || len(materialized.ArtifactLanes[0].ExtractReferences.ReferenceSet.Slots["spell_catalog"].Items) != 1 {
|
||||||
|
t.Fatalf("production spell catalog reference was not materialized: %#v", materialized.ArtifactLanes)
|
||||||
|
}
|
||||||
|
}
|
||||||
var stdout, stderr strings.Builder
|
var stdout, stderr strings.Builder
|
||||||
code := RunWithOptions([]string{"pipelines", "list", "--config", example.path}, &stdout, &stderr, productionOptionsFromComponents(components))
|
code := RunWithOptions([]string{"pipelines", "list", "--config", example.path}, &stdout, &stderr, productionOptionsFromComponents(components))
|
||||||
if code != 0 || stdout.String() != "dnd-session\n" || stderr.Len() != 0 {
|
if code != 0 || stdout.String() != "dnd-session\n" || stderr.Len() != 0 {
|
||||||
|
|||||||
@@ -70,6 +70,7 @@ func TestProductionCatalogCoversMaintainedConfigurations(t *testing.T) {
|
|||||||
"dnd.scenes/sharedassets/common-dnd-transcript.md",
|
"dnd.scenes/sharedassets/common-dnd-transcript.md",
|
||||||
"dnd.scenes/task.md",
|
"dnd.scenes/task.md",
|
||||||
"dnd.spells/dnd.spells.yaml",
|
"dnd.spells/dnd.spells.yaml",
|
||||||
|
"dnd.spells/catalog.md",
|
||||||
"dnd.spells/instructions.md",
|
"dnd.spells/instructions.md",
|
||||||
"dnd.spells/sharedassets/common-dnd-references.md",
|
"dnd.spells/sharedassets/common-dnd-references.md",
|
||||||
"dnd.spells/sharedassets/common-dnd-system.md",
|
"dnd.spells/sharedassets/common-dnd-system.md",
|
||||||
|
|||||||
@@ -0,0 +1,6 @@
|
|||||||
|
The canonical spell-name catalog for this extraction is provided below as JSON.
|
||||||
|
Return spell names using the catalog's canonical spelling exactly. Aliases and
|
||||||
|
other campaign reference material are not part of this catalog input and must
|
||||||
|
not be copied into the output as spell names.
|
||||||
|
|
||||||
|
{{ input "spell_catalog" }}
|
||||||
@@ -5,6 +5,9 @@ inputs:
|
|||||||
- name: transcript
|
- name: transcript
|
||||||
required: true
|
required: true
|
||||||
content_type: application/json
|
content_type: application/json
|
||||||
|
- name: spell_catalog
|
||||||
|
required: true
|
||||||
|
content_type: application/json
|
||||||
- name: players
|
- name: players
|
||||||
required: false
|
required: false
|
||||||
content_type: text/plain
|
content_type: text/plain
|
||||||
@@ -25,6 +28,8 @@ messages:
|
|||||||
content_file: ./sharedassets/common-dnd-references.md
|
content_file: ./sharedassets/common-dnd-references.md
|
||||||
cache_control:
|
cache_control:
|
||||||
type: ephemeral
|
type: ephemeral
|
||||||
|
- role: user
|
||||||
|
content_file: ./catalog.md
|
||||||
- role: user
|
- role: user
|
||||||
content_file: ./task.md
|
content_file: ./task.md
|
||||||
- role: user
|
- role: user
|
||||||
|
|||||||
@@ -5,8 +5,12 @@ source_id automatically.
|
|||||||
Return only D&D spell-cast artifacts. For each spell cast, identify the in-world
|
Return only D&D spell-cast artifacts. For each spell cast, identify the in-world
|
||||||
caster, spell name, effect, narrative description, and source references.
|
caster, spell name, effect, narrative description, and source references.
|
||||||
|
|
||||||
Use player, party, and glossary reference material only to clarify source text.
|
Use the canonical spell-name catalog to select spell names. Do not return a
|
||||||
Do not return spells, casters, or effects that are mentioned only in reference
|
spell name absent from that catalog, even when it is suggested by general D&D
|
||||||
material.
|
knowledge or reference material.
|
||||||
|
|
||||||
|
Use player, party, and glossary reference material only to clarify source text;
|
||||||
|
references are not source evidence for a spell cast. Do not return spells,
|
||||||
|
casters, or effects that are mentioned only in reference material.
|
||||||
|
|
||||||
Return exactly one JSON object and no explanatory text.
|
Return exactly one JSON object and no explanatory text.
|
||||||
|
|||||||
@@ -3,3 +3,7 @@ Extract Dungeons & Dragons spell-cast artifacts from the provided transcript.
|
|||||||
Extract only spell casts that are supported by the transcript. Do not infer
|
Extract only spell casts that are supported by the transcript. Do not infer
|
||||||
spells from general D&D knowledge or from table chatter that does not identify a
|
spells from general D&D knowledge or from table chatter that does not identify a
|
||||||
spell being cast.
|
spell being cast.
|
||||||
|
|
||||||
|
Use the provided canonical spell-name catalog when naming each extracted spell.
|
||||||
|
Return the canonical catalog spelling exactly. The catalog is a recognition
|
||||||
|
aid; it does not establish that a spell was cast.
|
||||||
|
|||||||
29
internal/modules/dnd/extract/spells/catalog_prompt_input.go
Normal file
29
internal/modules/dnd/extract/spells/catalog_prompt_input.go
Normal file
@@ -0,0 +1,29 @@
|
|||||||
|
package spells
|
||||||
|
|
||||||
|
import (
|
||||||
|
"crypto/sha256"
|
||||||
|
"encoding/hex"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||||
|
spellcatalog "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/spells/catalog"
|
||||||
|
)
|
||||||
|
|
||||||
|
func newCatalogPromptInput(effective spellcatalog.EffectiveCatalog) (contracts.LLMInputMaterial, error) {
|
||||||
|
content, err := json.Marshal(struct {
|
||||||
|
SpellNames []string `json:"spell_names"`
|
||||||
|
}{SpellNames: effective.CanonicalNames()})
|
||||||
|
if err != nil {
|
||||||
|
return contracts.LLMInputMaterial{}, fmt.Errorf("encode canonical spell names: %w", err)
|
||||||
|
}
|
||||||
|
sum := sha256.Sum256(content)
|
||||||
|
digest := "sha256:" + hex.EncodeToString(sum[:])
|
||||||
|
return contracts.NewLLMInputMaterial(
|
||||||
|
spellcatalog.SpellCatalogReferenceSlot,
|
||||||
|
"application/json",
|
||||||
|
content,
|
||||||
|
digest,
|
||||||
|
"",
|
||||||
|
), nil
|
||||||
|
}
|
||||||
@@ -9,6 +9,7 @@ import (
|
|||||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared"
|
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared"
|
||||||
|
spellcatalog "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/spells/catalog"
|
||||||
)
|
)
|
||||||
|
|
||||||
const Key = "dnd/spells"
|
const Key = "dnd/spells"
|
||||||
@@ -31,19 +32,50 @@ var referenceSlotDescriptions = shared.ReferenceSlotDescriptions{
|
|||||||
Roster: "Deprecated alias for party roster reference material used only for disambiguation.",
|
Roster: "Deprecated alias for party roster reference material used only for disambiguation.",
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func referenceSlots() []contracts.ReferenceSlot {
|
||||||
|
slots := shared.ReferenceSlots(referenceSlotDescriptions)
|
||||||
|
return append(slots, contracts.ReferenceSlot{
|
||||||
|
Name: spellcatalog.SpellCatalogReferenceSlot,
|
||||||
|
Description: "Optional canonical spell-name catalog used for extraction grounding.",
|
||||||
|
AcceptedMediaTypes: []string{"application/json"},
|
||||||
|
MaxBytes: 1048576,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
var _ contracts.Extractor[dnd.SpellList] = (*Extractor)(nil)
|
var _ contracts.Extractor[dnd.SpellList] = (*Extractor)(nil)
|
||||||
|
|
||||||
type Options struct{}
|
type Options struct{}
|
||||||
|
|
||||||
type Extractor struct {
|
type Extractor struct {
|
||||||
llm contracts.StructuredLLMClient
|
llm contracts.StructuredLLMClient
|
||||||
|
effectiveCatalog spellcatalog.EffectiveCatalog
|
||||||
|
catalogPromptInput contracts.LLMInputMaterial
|
||||||
}
|
}
|
||||||
|
|
||||||
func New(llmClient contracts.StructuredLLMClient, _ Options) (*Extractor, error) {
|
func New(llmClient contracts.StructuredLLMClient, _ Options, references ...contracts.ReferenceSet) (*Extractor, error) {
|
||||||
if llmClient == nil {
|
if llmClient == nil {
|
||||||
return nil, extractorErrorf("LLM client must not be nil")
|
return nil, extractorErrorf("LLM client must not be nil")
|
||||||
}
|
}
|
||||||
return &Extractor{llm: llmClient}, nil
|
if len(references) > 1 {
|
||||||
|
return nil, extractorErrorf("at most one reference set may be supplied")
|
||||||
|
}
|
||||||
|
var referenceSet contracts.ReferenceSet
|
||||||
|
if len(references) == 1 {
|
||||||
|
referenceSet = references[0]
|
||||||
|
}
|
||||||
|
effectiveCatalog, err := spellcatalog.ResolveEffectiveCatalog(referenceSet)
|
||||||
|
if err != nil {
|
||||||
|
return nil, extractorErrorf("resolve effective spell catalog: %w", err)
|
||||||
|
}
|
||||||
|
catalogPromptInput, err := newCatalogPromptInput(effectiveCatalog)
|
||||||
|
if err != nil {
|
||||||
|
return nil, extractorErrorf("prepare spell catalog prompt input: %w", err)
|
||||||
|
}
|
||||||
|
return &Extractor{
|
||||||
|
llm: llmClient,
|
||||||
|
effectiveCatalog: effectiveCatalog,
|
||||||
|
catalogPromptInput: catalogPromptInput,
|
||||||
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (e *Extractor) Key() string {
|
func (e *Extractor) Key() string {
|
||||||
@@ -51,7 +83,7 @@ func (e *Extractor) Key() string {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (e *Extractor) ReferenceSlots() []contracts.ReferenceSlot {
|
func (e *Extractor) ReferenceSlots() []contracts.ReferenceSlot {
|
||||||
return shared.ReferenceSlots(referenceSlotDescriptions)
|
return referenceSlots()
|
||||||
}
|
}
|
||||||
|
|
||||||
func (e *Extractor) ManifestMetadata() map[string]any {
|
func (e *Extractor) ManifestMetadata() map[string]any {
|
||||||
@@ -63,6 +95,9 @@ func (e *Extractor) ManifestMetadata() map[string]any {
|
|||||||
"prompt_id": PromptID,
|
"prompt_id": PromptID,
|
||||||
"prompt_version": SchemaVersion,
|
"prompt_version": SchemaVersion,
|
||||||
"prompt_sha256": promptSHA,
|
"prompt_sha256": promptSHA,
|
||||||
|
"catalog_base_id": e.effectiveCatalog.BaseID(),
|
||||||
|
"catalog_digest": e.effectiveCatalog.Digest(),
|
||||||
|
"catalog_overlay_ids": e.effectiveCatalog.OverlayIDs(),
|
||||||
"response_schema_key": string(ResponseSchemaKey),
|
"response_schema_key": string(ResponseSchemaKey),
|
||||||
"response_schema_id": ResponseSchemaID,
|
"response_schema_id": ResponseSchemaID,
|
||||||
"response_schema_name": ResponseSchemaName,
|
"response_schema_name": ResponseSchemaName,
|
||||||
@@ -102,13 +137,15 @@ func (e *Extractor) Extract(ctx context.Context, req contracts.TypedExtractionRe
|
|||||||
}
|
}
|
||||||
|
|
||||||
var response extractionResponse
|
var response extractionResponse
|
||||||
|
inputs := shared.PromptInputs(sourceInput, req.References)
|
||||||
|
inputs[spellcatalog.SpellCatalogReferenceSlot] = e.catalogPromptInput.Clone()
|
||||||
if _, err := e.llm.CompleteStructured(ctx, contracts.StructuredCompletionRequest{
|
if _, err := e.llm.CompleteStructured(ctx, contracts.StructuredCompletionRequest{
|
||||||
StageName: Key,
|
StageName: Key,
|
||||||
PromptID: PromptID,
|
PromptID: PromptID,
|
||||||
PromptVersion: SchemaVersion,
|
PromptVersion: SchemaVersion,
|
||||||
ProfileID: req.LLMProfile,
|
ProfileID: req.LLMProfile,
|
||||||
SessionID: req.SessionID,
|
SessionID: req.SessionID,
|
||||||
Inputs: shared.PromptInputs(sourceInput, req.References),
|
Inputs: inputs,
|
||||||
}, &response); err != nil {
|
}, &response); err != nil {
|
||||||
return contracts.TypedExtractionResult[dnd.SpellList]{}, extractorErrorf("complete structured output: %w", err)
|
return contracts.TypedExtractionResult[dnd.SpellList]{}, extractorErrorf("complete structured output: %w", err)
|
||||||
}
|
}
|
||||||
@@ -143,7 +180,7 @@ func ModuleSpec() pipeline.ModuleSpec {
|
|||||||
Requires: append([]string(nil), requiredCapabilities...),
|
Requires: append([]string(nil), requiredCapabilities...),
|
||||||
Provides: append([]string(nil), providedCapabilities...),
|
Provides: append([]string(nil), providedCapabilities...),
|
||||||
ArtifactKind: dnd.SpellListKind,
|
ArtifactKind: dnd.SpellListKind,
|
||||||
ReferenceSlots: shared.ReferenceSlots(referenceSlotDescriptions),
|
ReferenceSlots: referenceSlots(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -153,7 +190,7 @@ func Register(registry *pipeline.ExtractorRegistry) error {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
return New(request.Dependencies.LLM, options)
|
return New(request.Dependencies.LLM, options, request.References)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -2,8 +2,10 @@ package spells
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"encoding/json"
|
||||||
"errors"
|
"errors"
|
||||||
"reflect"
|
"reflect"
|
||||||
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
@@ -11,6 +13,7 @@ import (
|
|||||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared"
|
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared"
|
||||||
|
spellcatalog "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/spells/catalog"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestExtractReturnsCanonicalSpellListFromPrivateResponse(t *testing.T) {
|
func TestExtractReturnsCanonicalSpellListFromPrivateResponse(t *testing.T) {
|
||||||
@@ -62,6 +65,78 @@ func TestExtractReturnsCanonicalSpellListFromPrivateResponse(t *testing.T) {
|
|||||||
if got := string(transcript.Content); got != string(req.Chunk.Content) {
|
if got := string(transcript.Content); got != string(req.Chunk.Content) {
|
||||||
t.Fatalf("transcript content = %q, want chunk content %q", got, req.Chunk.Content)
|
t.Fatalf("transcript content = %q, want chunk content %q", got, req.Chunk.Content)
|
||||||
}
|
}
|
||||||
|
catalogInput := llmReq.Inputs[spellcatalog.SpellCatalogReferenceSlot]
|
||||||
|
if catalogInput.Name != spellcatalog.SpellCatalogReferenceSlot || catalogInput.MediaType != "application/json" || catalogInput.OriginURI != "" || !strings.HasPrefix(catalogInput.Digest, "sha256:") {
|
||||||
|
t.Fatalf("catalog prompt input metadata = %#v", catalogInput)
|
||||||
|
}
|
||||||
|
var catalogPayload struct {
|
||||||
|
SpellNames []string `json:"spell_names"`
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(catalogInput.Content, &catalogPayload); err != nil {
|
||||||
|
t.Fatalf("decode catalog prompt input: %v", err)
|
||||||
|
}
|
||||||
|
base, err := spellcatalog.LoadSRD5E2014()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
wantNames := make([]string, 0, len(base.Spells()))
|
||||||
|
for _, spell := range base.Spells() {
|
||||||
|
wantNames = append(wantNames, spell.Name)
|
||||||
|
}
|
||||||
|
sort.Strings(wantNames)
|
||||||
|
if !reflect.DeepEqual(catalogPayload.SpellNames, wantNames) || !sort.StringsAreSorted(catalogPayload.SpellNames) {
|
||||||
|
t.Fatalf("catalog prompt names = %d entries, want sorted base catalog", len(catalogPayload.SpellNames))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExtractPromptUsesCanonicalOverlayNamesWithoutAliasesOrMetadata(t *testing.T) {
|
||||||
|
client := &fakeSpellsLLMClient{response: extractionResponse{SpellCasts: []spellCastResponse{}}}
|
||||||
|
if _, err := newExtractor(t, client, overlaySpellCatalogReference()).Extract(context.Background(), extractionRequest()); err != nil {
|
||||||
|
t.Fatalf("Extract() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
input := client.requests[0].Inputs[spellcatalog.SpellCatalogReferenceSlot]
|
||||||
|
content := string(input.Content)
|
||||||
|
for _, expected := range []string{"Aegis of Emberfall", `"spell_names"`} {
|
||||||
|
if !strings.Contains(content, expected) {
|
||||||
|
t.Fatalf("catalog prompt input = %q, want %q", content, expected)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, forbidden := range []string{"Emberfall Aegis", "Private campaign source", "file:///private-source.json", "private"} {
|
||||||
|
if strings.Contains(content, forbidden) {
|
||||||
|
t.Fatalf("catalog prompt input leaked %q: %s", forbidden, content)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
metadata := newExtractor(t, &fakeSpellsLLMClient{}, overlaySpellCatalogReference()).ManifestMetadata()
|
||||||
|
if metadata["catalog_base_id"] != spellcatalog.SRD5E2014ID {
|
||||||
|
t.Fatalf("catalog base metadata = %#v", metadata["catalog_base_id"])
|
||||||
|
}
|
||||||
|
if digest, ok := metadata["catalog_digest"].(string); !ok || !strings.HasPrefix(digest, "sha256:") {
|
||||||
|
t.Fatalf("catalog digest metadata = %#v", metadata["catalog_digest"])
|
||||||
|
}
|
||||||
|
if got, ok := metadata["catalog_overlay_ids"].([]string); !ok || !reflect.DeepEqual(got, []string{"campaign.example"}) {
|
||||||
|
t.Fatalf("catalog overlay metadata = %#v", metadata["catalog_overlay_ids"])
|
||||||
|
}
|
||||||
|
encoded, err := json.Marshal(metadata)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, forbidden := range []string{"Aegis of Emberfall", "Emberfall Aegis", "Private campaign source", "file:///private-source.json"} {
|
||||||
|
if strings.Contains(string(encoded), forbidden) {
|
||||||
|
t.Fatalf("manifest metadata leaked %q: %s", forbidden, encoded)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNewRejectsMalformedCatalogBeforeLLMCall(t *testing.T) {
|
||||||
|
client := &fakeSpellsLLMClient{}
|
||||||
|
_, err := New(client, Options{}, spellCatalogReference(`{"schema_version":"notarius.dnd.spell-catalog-overlay.v2","catalogs":[]}`))
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "resolve effective spell catalog") {
|
||||||
|
t.Fatalf("New() error = %v, want effective catalog error", err)
|
||||||
|
}
|
||||||
|
if len(client.requests) != 0 {
|
||||||
|
t.Fatalf("LLM calls = %d, want none during failed construction", len(client.requests))
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestExtractorManifestMetadataIncludesLLMSchemaProvenance(t *testing.T) {
|
func TestExtractorManifestMetadataIncludesLLMSchemaProvenance(t *testing.T) {
|
||||||
|
|||||||
@@ -54,6 +54,12 @@ func TestModuleSpec(t *testing.T) {
|
|||||||
Description: "Deprecated alias for party roster reference material used only for disambiguation.",
|
Description: "Deprecated alias for party roster reference material used only for disambiguation.",
|
||||||
AcceptedMediaTypes: []string{"application/json", "application/x-yaml", "application/yaml", "text/markdown", "text/plain"},
|
AcceptedMediaTypes: []string{"application/json", "application/x-yaml", "application/yaml", "text/markdown", "text/plain"},
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
Name: "spell_catalog",
|
||||||
|
Description: "Optional canonical spell-name catalog used for extraction grounding.",
|
||||||
|
AcceptedMediaTypes: []string{"application/json"},
|
||||||
|
MaxBytes: 1048576,
|
||||||
|
},
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
if !reflect.DeepEqual(got, want) {
|
if !reflect.DeepEqual(got, want) {
|
||||||
|
|||||||
@@ -14,6 +14,7 @@ const scriptoriumPromptRoot = "assets/prompts"
|
|||||||
func RegisterPromptAssets(registry *llm.AssetRegistry) error {
|
func RegisterPromptAssets(registry *llm.AssetRegistry) error {
|
||||||
promptFS, err := shared.ModulePromptFS("dnd.spells", embeddedAssets, []promptfs.ModulePromptFile{
|
promptFS, err := shared.ModulePromptFS("dnd.spells", embeddedAssets, []promptfs.ModulePromptFile{
|
||||||
{Name: "dnd.spells.yaml", Path: "assets/prompts/dnd.spells.yaml"},
|
{Name: "dnd.spells.yaml", Path: "assets/prompts/dnd.spells.yaml"},
|
||||||
|
{Name: "catalog.md", Path: "assets/prompts/catalog.md"},
|
||||||
{Name: "task.md", Path: "assets/prompts/task.md"},
|
{Name: "task.md", Path: "assets/prompts/task.md"},
|
||||||
{Name: "instructions.md", Path: "assets/prompts/instructions.md"},
|
{Name: "instructions.md", Path: "assets/prompts/instructions.md"},
|
||||||
})
|
})
|
||||||
@@ -30,6 +31,7 @@ func scriptoriumPromptMetadata() (string, error) {
|
|||||||
scriptoriumPromptHashOnce.Do(func() {
|
scriptoriumPromptHashOnce.Do(func() {
|
||||||
parts := append([]llm.AssetHashPart{
|
parts := append([]llm.AssetHashPart{
|
||||||
{FS: embeddedAssets, Path: "assets/prompts/dnd.spells.yaml"},
|
{FS: embeddedAssets, Path: "assets/prompts/dnd.spells.yaml"},
|
||||||
|
{FS: embeddedAssets, Path: "assets/prompts/catalog.md"},
|
||||||
{FS: embeddedAssets, Path: "assets/prompts/task.md"},
|
{FS: embeddedAssets, Path: "assets/prompts/task.md"},
|
||||||
{FS: embeddedAssets, Path: "assets/prompts/instructions.md"},
|
{FS: embeddedAssets, Path: "assets/prompts/instructions.md"},
|
||||||
}, append(shared.CommonHashParts(), shared.ReferenceHashParts()...)...)
|
}, append(shared.CommonHashParts(), shared.ReferenceHashParts()...)...)
|
||||||
|
|||||||
@@ -21,8 +21,8 @@ func TestScriptoriumPromptPreparesTranscriptReferencesAndTaskMessages(t *testing
|
|||||||
if prepared.OutputContract.SchemaPath != "dnd_spells_llm.v1.json" {
|
if prepared.OutputContract.SchemaPath != "dnd_spells_llm.v1.json" {
|
||||||
t.Fatalf("schema path = %q, want LLM-only schema", prepared.OutputContract.SchemaPath)
|
t.Fatalf("schema path = %q, want LLM-only schema", prepared.OutputContract.SchemaPath)
|
||||||
}
|
}
|
||||||
if got := len(prepared.Messages); got != 5 {
|
if got := len(prepared.Messages); got != 6 {
|
||||||
t.Fatalf("message count = %d, want 5", got)
|
t.Fatalf("message count = %d, want 6", got)
|
||||||
}
|
}
|
||||||
if !strings.Contains(prepared.Messages[1].Content, string(transcript)) {
|
if !strings.Contains(prepared.Messages[1].Content, string(transcript)) {
|
||||||
t.Fatalf("transcript message did not include source input")
|
t.Fatalf("transcript message did not include source input")
|
||||||
@@ -39,7 +39,10 @@ func TestScriptoriumPromptPreparesTranscriptReferencesAndTaskMessages(t *testing
|
|||||||
if !strings.Contains(prepared.Messages[2].Content, "Shield: abjuration") {
|
if !strings.Contains(prepared.Messages[2].Content, "Shield: abjuration") {
|
||||||
t.Fatalf("reference message missing glossary content")
|
t.Fatalf("reference message missing glossary content")
|
||||||
}
|
}
|
||||||
if strings.Contains(prepared.Messages[3].Content, string(transcript)) {
|
if !strings.Contains(prepared.Messages[3].Content, `{"spell_names":["Cure Wounds"]}`) {
|
||||||
|
t.Fatalf("catalog message missing canonical spell-name input: %s", prepared.Messages[3].Content)
|
||||||
|
}
|
||||||
|
if strings.Contains(prepared.Messages[4].Content, string(transcript)) {
|
||||||
t.Fatalf("task message leaked transcript bytes")
|
t.Fatalf("task message leaked transcript bytes")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -57,7 +60,7 @@ func TestScriptoriumPromptDiagnosticsOmitRawMaterials(t *testing.T) {
|
|||||||
transcript := []byte(`{"secret":"source text"}`)
|
transcript := []byte(`{"secret":"source text"}`)
|
||||||
reference := "private party note"
|
reference := "private party note"
|
||||||
prepared := prepareSpellsPrompt(t, transcript, "private player note", reference, " ")
|
prepared := prepareSpellsPrompt(t, transcript, "private player note", reference, " ")
|
||||||
metadata := newExtractor(t, &fakeSpellsLLMClient{}).ManifestMetadata()
|
metadata := newExtractor(t, &fakeSpellsLLMClient{}, overlaySpellCatalogReference()).ManifestMetadata()
|
||||||
|
|
||||||
payload, err := json.Marshal(map[string]any{
|
payload, err := json.Marshal(map[string]any{
|
||||||
"prepared": map[string]any{
|
"prepared": map[string]any{
|
||||||
@@ -80,6 +83,11 @@ func TestScriptoriumPromptDiagnosticsOmitRawMaterials(t *testing.T) {
|
|||||||
"source text",
|
"source text",
|
||||||
"private player note",
|
"private player note",
|
||||||
reference,
|
reference,
|
||||||
|
"Cure Wounds",
|
||||||
|
"Aegis of Emberfall",
|
||||||
|
"Emberfall Aegis",
|
||||||
|
"Private campaign source",
|
||||||
|
"file:///private-source.json",
|
||||||
`"properties"`,
|
`"properties"`,
|
||||||
"spell_casts",
|
"spell_casts",
|
||||||
} {
|
} {
|
||||||
@@ -119,10 +127,11 @@ func prepareSpellsPrompt(t *testing.T, transcript []byte, players string, party
|
|||||||
PromptVersion: SchemaVersion,
|
PromptVersion: SchemaVersion,
|
||||||
ProfileID: "spell-test-profile",
|
ProfileID: "spell-test-profile",
|
||||||
Inputs: map[string]scriptorium.ArtifactRef{
|
Inputs: map[string]scriptorium.ArtifactRef{
|
||||||
"transcript": scriptorium.InlineWithURI("file:///session.json", string(transcript)),
|
"transcript": scriptorium.InlineWithURI("file:///session.json", string(transcript)),
|
||||||
"players": scriptorium.Inline(players),
|
"spell_catalog": scriptorium.Inline(`{"spell_names":["Cure Wounds"]}`),
|
||||||
"party": scriptorium.Inline(party),
|
"players": scriptorium.Inline(players),
|
||||||
"glossary": scriptorium.Inline(glossary),
|
"party": scriptorium.Inline(party),
|
||||||
|
"glossary": scriptorium.Inline(glossary),
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
@@ -9,6 +9,7 @@ import (
|
|||||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared"
|
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared"
|
||||||
|
spellcatalog "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/spells/catalog"
|
||||||
)
|
)
|
||||||
|
|
||||||
func promptExtractionRequest() contracts.TypedExtractionRequest {
|
func promptExtractionRequest() contracts.TypedExtractionRequest {
|
||||||
@@ -109,15 +110,31 @@ func mismatchedSourceInputRequest(req contracts.TypedExtractionRequest) contract
|
|||||||
return req
|
return req
|
||||||
}
|
}
|
||||||
|
|
||||||
func newExtractor(t *testing.T, client contracts.StructuredLLMClient) *Extractor {
|
func newExtractor(t *testing.T, client contracts.StructuredLLMClient, references ...contracts.ReferenceSet) *Extractor {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
extractor, err := New(client, Options{})
|
extractor, err := New(client, Options{}, references...)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("New() error = %v, want nil", err)
|
t.Fatalf("New() error = %v, want nil", err)
|
||||||
}
|
}
|
||||||
return extractor
|
return extractor
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func spellCatalogReference(content string) contracts.ReferenceSet {
|
||||||
|
return contracts.ReferenceSet{Slots: map[string]contracts.ResolvedReferenceSlot{
|
||||||
|
spellcatalog.SpellCatalogReferenceSlot: {
|
||||||
|
Items: []contracts.ReferenceItem{{
|
||||||
|
SlotName: spellcatalog.SpellCatalogReferenceSlot,
|
||||||
|
MediaType: "application/json",
|
||||||
|
Content: []byte(content),
|
||||||
|
}},
|
||||||
|
},
|
||||||
|
}}
|
||||||
|
}
|
||||||
|
|
||||||
|
func overlaySpellCatalogReference() contracts.ReferenceSet {
|
||||||
|
return spellCatalogReference(`{"schema_version":"notarius.dnd.spell-catalog-overlay.v1","catalogs":[{"id":"campaign.example","ruleset":"dnd-5e-2014","source":{"title":"Private campaign source","version":"1","url":"file:///private-source.json","license":"private"},"spells":[{"name":"Aegis of Emberfall","aliases":["Emberfall Aegis"]}]}]}`)
|
||||||
|
}
|
||||||
|
|
||||||
type fakeSpellsLLMClient struct {
|
type fakeSpellsLLMClient struct {
|
||||||
response extractionResponse
|
response extractionResponse
|
||||||
content []byte
|
content []byte
|
||||||
|
|||||||
Reference in New Issue
Block a user