Ground spell extraction with the effective catalog
This commit is contained in:
@@ -2,8 +2,10 @@ package spells
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"reflect"
|
||||
"sort"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
@@ -11,6 +13,7 @@ import (
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared"
|
||||
spellcatalog "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/spells/catalog"
|
||||
)
|
||||
|
||||
func TestExtractReturnsCanonicalSpellListFromPrivateResponse(t *testing.T) {
|
||||
@@ -62,6 +65,78 @@ func TestExtractReturnsCanonicalSpellListFromPrivateResponse(t *testing.T) {
|
||||
if got := string(transcript.Content); got != string(req.Chunk.Content) {
|
||||
t.Fatalf("transcript content = %q, want chunk content %q", got, req.Chunk.Content)
|
||||
}
|
||||
catalogInput := llmReq.Inputs[spellcatalog.SpellCatalogReferenceSlot]
|
||||
if catalogInput.Name != spellcatalog.SpellCatalogReferenceSlot || catalogInput.MediaType != "application/json" || catalogInput.OriginURI != "" || !strings.HasPrefix(catalogInput.Digest, "sha256:") {
|
||||
t.Fatalf("catalog prompt input metadata = %#v", catalogInput)
|
||||
}
|
||||
var catalogPayload struct {
|
||||
SpellNames []string `json:"spell_names"`
|
||||
}
|
||||
if err := json.Unmarshal(catalogInput.Content, &catalogPayload); err != nil {
|
||||
t.Fatalf("decode catalog prompt input: %v", err)
|
||||
}
|
||||
base, err := spellcatalog.LoadSRD5E2014()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
wantNames := make([]string, 0, len(base.Spells()))
|
||||
for _, spell := range base.Spells() {
|
||||
wantNames = append(wantNames, spell.Name)
|
||||
}
|
||||
sort.Strings(wantNames)
|
||||
if !reflect.DeepEqual(catalogPayload.SpellNames, wantNames) || !sort.StringsAreSorted(catalogPayload.SpellNames) {
|
||||
t.Fatalf("catalog prompt names = %d entries, want sorted base catalog", len(catalogPayload.SpellNames))
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractPromptUsesCanonicalOverlayNamesWithoutAliasesOrMetadata(t *testing.T) {
|
||||
client := &fakeSpellsLLMClient{response: extractionResponse{SpellCasts: []spellCastResponse{}}}
|
||||
if _, err := newExtractor(t, client, overlaySpellCatalogReference()).Extract(context.Background(), extractionRequest()); err != nil {
|
||||
t.Fatalf("Extract() error = %v, want nil", err)
|
||||
}
|
||||
input := client.requests[0].Inputs[spellcatalog.SpellCatalogReferenceSlot]
|
||||
content := string(input.Content)
|
||||
for _, expected := range []string{"Aegis of Emberfall", `"spell_names"`} {
|
||||
if !strings.Contains(content, expected) {
|
||||
t.Fatalf("catalog prompt input = %q, want %q", content, expected)
|
||||
}
|
||||
}
|
||||
for _, forbidden := range []string{"Emberfall Aegis", "Private campaign source", "file:///private-source.json", "private"} {
|
||||
if strings.Contains(content, forbidden) {
|
||||
t.Fatalf("catalog prompt input leaked %q: %s", forbidden, content)
|
||||
}
|
||||
}
|
||||
|
||||
metadata := newExtractor(t, &fakeSpellsLLMClient{}, overlaySpellCatalogReference()).ManifestMetadata()
|
||||
if metadata["catalog_base_id"] != spellcatalog.SRD5E2014ID {
|
||||
t.Fatalf("catalog base metadata = %#v", metadata["catalog_base_id"])
|
||||
}
|
||||
if digest, ok := metadata["catalog_digest"].(string); !ok || !strings.HasPrefix(digest, "sha256:") {
|
||||
t.Fatalf("catalog digest metadata = %#v", metadata["catalog_digest"])
|
||||
}
|
||||
if got, ok := metadata["catalog_overlay_ids"].([]string); !ok || !reflect.DeepEqual(got, []string{"campaign.example"}) {
|
||||
t.Fatalf("catalog overlay metadata = %#v", metadata["catalog_overlay_ids"])
|
||||
}
|
||||
encoded, err := json.Marshal(metadata)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, forbidden := range []string{"Aegis of Emberfall", "Emberfall Aegis", "Private campaign source", "file:///private-source.json"} {
|
||||
if strings.Contains(string(encoded), forbidden) {
|
||||
t.Fatalf("manifest metadata leaked %q: %s", forbidden, encoded)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewRejectsMalformedCatalogBeforeLLMCall(t *testing.T) {
|
||||
client := &fakeSpellsLLMClient{}
|
||||
_, err := New(client, Options{}, spellCatalogReference(`{"schema_version":"notarius.dnd.spell-catalog-overlay.v2","catalogs":[]}`))
|
||||
if err == nil || !strings.Contains(err.Error(), "resolve effective spell catalog") {
|
||||
t.Fatalf("New() error = %v, want effective catalog error", err)
|
||||
}
|
||||
if len(client.requests) != 0 {
|
||||
t.Fatalf("LLM calls = %d, want none during failed construction", len(client.requests))
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractorManifestMetadataIncludesLLMSchemaProvenance(t *testing.T) {
|
||||
|
||||
Reference in New Issue
Block a user