290 lines
14 KiB
Go
290 lines
14 KiB
Go
package spells
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"reflect"
|
|
"sort"
|
|
"strings"
|
|
"testing"
|
|
|
|
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared"
|
|
spellcatalog "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/spells/catalog"
|
|
)
|
|
|
|
func TestExtractReturnsCanonicalSpellListFromPrivateResponse(t *testing.T) {
|
|
client := &fakeSpellsLLMClient{response: extractionResponse{SpellCasts: []spellCastResponse{
|
|
{
|
|
Caster: " Aria ",
|
|
Spell: " Cure Wounds ",
|
|
Effect: " Heals an injured ally. ",
|
|
NarrativeDescription: " Aria restores the fighter after the fight. ",
|
|
SourceRefs: responseSourceRefs(1, 2),
|
|
},
|
|
}}}
|
|
req := extractionRequest()
|
|
|
|
result, err := newExtractor(t, client).Extract(context.Background(), req)
|
|
if err != nil {
|
|
t.Fatalf("Extract() error = %v, want nil", err)
|
|
}
|
|
want := dnd.SpellList{SpellCasts: []dnd.SpellCast{
|
|
{
|
|
Caster: " Aria ",
|
|
Spell: " Cure Wounds ",
|
|
Effect: " Heals an injured ally. ",
|
|
NarrativeDescription: " Aria restores the fighter after the fight. ",
|
|
SourceRefs: []source.SourceRef{{SourceID: "session-alpha", StartUnitID: 1, EndUnitID: 2}},
|
|
},
|
|
}}
|
|
if !reflect.DeepEqual(result.Value, want) {
|
|
t.Fatalf("Value = %#v, want %#v", result.Value, want)
|
|
}
|
|
if len(result.Warnings) != 0 {
|
|
t.Fatalf("Warnings = %#v, want none", result.Warnings)
|
|
}
|
|
|
|
if len(client.requests) != 1 {
|
|
t.Fatalf("LLM calls = %d, want 1", len(client.requests))
|
|
}
|
|
llmReq := client.requests[0]
|
|
if llmReq.StageName != Key || llmReq.PromptID != PromptID || llmReq.PromptVersion != SchemaVersion {
|
|
t.Fatalf("LLM request identity = %#v, want spell prompt", llmReq)
|
|
}
|
|
if llmReq.SessionID != "session-123" || llmReq.ProfileID != "profile-spells" {
|
|
t.Fatalf("session/profile = %q/%q, want session-123/profile-spells", llmReq.SessionID, llmReq.ProfileID)
|
|
}
|
|
transcript := llmReq.Inputs["transcript"]
|
|
if transcript.Name != "transcript" || transcript.MediaType != "application/json" || transcript.Digest != "sha256:chunk" || transcript.OriginURI != "file:///session-alpha.json" {
|
|
t.Fatalf("transcript metadata = %#v", transcript)
|
|
}
|
|
if got := string(transcript.Content); got != string(req.Chunk.Content) {
|
|
t.Fatalf("transcript content = %q, want chunk content %q", got, req.Chunk.Content)
|
|
}
|
|
catalogInput := llmReq.Inputs[spellcatalog.SpellCatalogReferenceSlot]
|
|
if catalogInput.Name != spellcatalog.SpellCatalogReferenceSlot || catalogInput.MediaType != "application/json" || catalogInput.OriginURI != "" || !strings.HasPrefix(catalogInput.Digest, "sha256:") {
|
|
t.Fatalf("catalog prompt input metadata = %#v", catalogInput)
|
|
}
|
|
var catalogPayload struct {
|
|
SpellNames []string `json:"spell_names"`
|
|
}
|
|
if err := json.Unmarshal(catalogInput.Content, &catalogPayload); err != nil {
|
|
t.Fatalf("decode catalog prompt input: %v", err)
|
|
}
|
|
base, err := spellcatalog.LoadSRD5E2014()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
wantNames := make([]string, 0, len(base.Spells()))
|
|
for _, spell := range base.Spells() {
|
|
wantNames = append(wantNames, spell.Name)
|
|
}
|
|
sort.Strings(wantNames)
|
|
if !reflect.DeepEqual(catalogPayload.SpellNames, wantNames) || !sort.StringsAreSorted(catalogPayload.SpellNames) {
|
|
t.Fatalf("catalog prompt names = %d entries, want sorted base catalog", len(catalogPayload.SpellNames))
|
|
}
|
|
}
|
|
|
|
func TestExtractPromptUsesCanonicalOverlayNamesWithoutAliasesOrMetadata(t *testing.T) {
|
|
client := &fakeSpellsLLMClient{response: extractionResponse{SpellCasts: []spellCastResponse{}}}
|
|
if _, err := newExtractor(t, client, overlaySpellCatalogReference()).Extract(context.Background(), extractionRequest()); err != nil {
|
|
t.Fatalf("Extract() error = %v, want nil", err)
|
|
}
|
|
input := client.requests[0].Inputs[spellcatalog.SpellCatalogReferenceSlot]
|
|
content := string(input.Content)
|
|
for _, expected := range []string{"Aegis of Emberfall", `"spell_names"`} {
|
|
if !strings.Contains(content, expected) {
|
|
t.Fatalf("catalog prompt input = %q, want %q", content, expected)
|
|
}
|
|
}
|
|
for _, forbidden := range []string{"Emberfall Aegis", "Private campaign source", "file:///private-source.json", "private"} {
|
|
if strings.Contains(content, forbidden) {
|
|
t.Fatalf("catalog prompt input leaked %q: %s", forbidden, content)
|
|
}
|
|
}
|
|
|
|
metadata := newExtractor(t, &fakeSpellsLLMClient{}, overlaySpellCatalogReference()).ManifestMetadata()
|
|
if metadata["catalog_base_id"] != spellcatalog.SRD5E2014ID {
|
|
t.Fatalf("catalog base metadata = %#v", metadata["catalog_base_id"])
|
|
}
|
|
if digest, ok := metadata["catalog_digest"].(string); !ok || !strings.HasPrefix(digest, "sha256:") {
|
|
t.Fatalf("catalog digest metadata = %#v", metadata["catalog_digest"])
|
|
}
|
|
if got, ok := metadata["catalog_overlay_ids"].([]string); !ok || !reflect.DeepEqual(got, []string{"campaign.example"}) {
|
|
t.Fatalf("catalog overlay metadata = %#v", metadata["catalog_overlay_ids"])
|
|
}
|
|
fingerprints := newExtractor(t, &fakeSpellsLLMClient{}, overlaySpellCatalogReference()).CheckpointFingerprints()
|
|
if len(fingerprints) != 1 || fingerprints[0].Name != "effective_catalog" || fingerprints[0].Value != metadata["catalog_digest"] {
|
|
t.Fatalf("checkpoint fingerprints = %#v, want manifest catalog digest %#v", fingerprints, metadata["catalog_digest"])
|
|
}
|
|
encoded, err := json.Marshal(metadata)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for _, forbidden := range []string{"Aegis of Emberfall", "Emberfall Aegis", "Private campaign source", "file:///private-source.json"} {
|
|
if strings.Contains(string(encoded), forbidden) {
|
|
t.Fatalf("manifest metadata leaked %q: %s", forbidden, encoded)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestNewRejectsMalformedCatalogBeforeLLMCall(t *testing.T) {
|
|
client := &fakeSpellsLLMClient{}
|
|
_, err := New(client, Options{}, spellCatalogReference(`{"schema_version":"notarius.dnd.spell-catalog-overlay.v2","catalogs":[]}`))
|
|
if err == nil || !strings.Contains(err.Error(), "resolve effective spell catalog") {
|
|
t.Fatalf("New() error = %v, want effective catalog error", err)
|
|
}
|
|
if len(client.requests) != 0 {
|
|
t.Fatalf("LLM calls = %d, want none during failed construction", len(client.requests))
|
|
}
|
|
}
|
|
|
|
func TestExtractorManifestMetadataIncludesLLMSchemaProvenance(t *testing.T) {
|
|
metadata := newExtractor(t, &fakeSpellsLLMClient{}).ManifestMetadata()
|
|
tests := map[string]string{
|
|
"prompt_id": PromptID, "prompt_version": SchemaVersion,
|
|
"response_schema_key": string(ResponseSchemaKey), "response_schema_id": ResponseSchemaID,
|
|
"response_schema_name": ResponseSchemaName, "response_schema_version": SchemaVersion,
|
|
}
|
|
for key, want := range tests {
|
|
if metadata[key] != want {
|
|
t.Fatalf("metadata[%q] = %#v, want %q", key, metadata[key], want)
|
|
}
|
|
}
|
|
for _, key := range []string{"prompt_sha256", "response_schema_sha256"} {
|
|
value, ok := metadata[key].(string)
|
|
if !ok || !strings.HasPrefix(value, "sha256:") {
|
|
t.Fatalf("metadata[%q] = %#v, want sha256 value", key, metadata[key])
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestExtractPassesReferencesAsPromptInputs(t *testing.T) {
|
|
client := &fakeSpellsLLMClient{response: extractionResponse{SpellCasts: []spellCastResponse{}}}
|
|
req := extractionRequest()
|
|
req.References = contracts.ReferenceSet{Slots: map[string]contracts.ResolvedReferenceSlot{
|
|
"players": {Slot: contracts.ReferenceSlot{Name: "players"}, Items: []contracts.ReferenceItem{{SlotName: "players", Content: []byte("Alice: Aria Brightmantle")}}},
|
|
"party": {Slot: contracts.ReferenceSlot{Name: "party"}, Items: []contracts.ReferenceItem{{SlotName: "party", Content: []byte("Aria Brightmantle: party cleric")}}},
|
|
"glossary": {Slot: contracts.ReferenceSlot{Name: "glossary"}, Items: []contracts.ReferenceItem{{SlotName: "glossary", Content: []byte("Brightmantle: local temple name")}}},
|
|
}}
|
|
|
|
if _, err := newExtractor(t, client).Extract(context.Background(), req); err != nil {
|
|
t.Fatalf("Extract() error = %v, want nil", err)
|
|
}
|
|
inputs := client.requests[0].Inputs
|
|
if string(inputs["players"].Content) != "Alice: Aria Brightmantle" || string(inputs["party"].Content) != "Aria Brightmantle: party cleric" || string(inputs["glossary"].Content) != "Brightmantle: local temple name" {
|
|
t.Fatalf("reference inputs = %#v, want configured content", inputs)
|
|
}
|
|
if strings.Contains(string(inputs["transcript"].Content), "party cleric") {
|
|
t.Fatal("transcript input contains reference content")
|
|
}
|
|
}
|
|
|
|
func TestPromptInputsMapLegacyRosterReferenceToParty(t *testing.T) {
|
|
inputs := shared.PromptInputs(spellSourceInput(), contracts.ReferenceSet{Slots: map[string]contracts.ResolvedReferenceSlot{
|
|
"roster": {Slot: contracts.ReferenceSlot{Name: "roster"}, Items: []contracts.ReferenceItem{{SlotName: "roster", Content: []byte("Legacy roster text")}}},
|
|
}})
|
|
if got := string(inputs["party"].Content); got != "Legacy roster text" {
|
|
t.Fatalf("party input = %q, want legacy roster content", got)
|
|
}
|
|
if _, ok := inputs["roster"]; ok {
|
|
t.Fatal("roster prompt input was present; want only party input")
|
|
}
|
|
}
|
|
|
|
func TestExtractPreservesEmptyAndMalformedValuesForTypedValidators(t *testing.T) {
|
|
tests := []struct {
|
|
name string
|
|
response extractionResponse
|
|
wantNil bool
|
|
}{
|
|
{name: "empty", response: extractionResponse{SpellCasts: []spellCastResponse{}}},
|
|
{name: "missing", response: extractionResponse{}, wantNil: true},
|
|
}
|
|
for _, test := range tests {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
result, err := newExtractor(t, &fakeSpellsLLMClient{response: test.response}).Extract(context.Background(), extractionRequest())
|
|
if err != nil {
|
|
t.Fatalf("Extract() error = %v, want nil", err)
|
|
}
|
|
if (result.Value.SpellCasts == nil) != test.wantNil || len(result.Value.SpellCasts) != 0 {
|
|
t.Fatalf("SpellCasts = %#v, want empty with nil=%t", result.Value.SpellCasts, test.wantNil)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestExtractWrapsLLMClientError(t *testing.T) {
|
|
_, err := newExtractor(t, &fakeSpellsLLMClient{err: errors.New("provider unavailable")}).Extract(context.Background(), extractionRequest())
|
|
if err == nil || !strings.Contains(err.Error(), "dnd spells") || !strings.Contains(err.Error(), "provider unavailable") {
|
|
t.Fatalf("Extract() error = %v, want wrapped provider error", err)
|
|
}
|
|
}
|
|
|
|
func TestExtractRejectsInvalidRequests(t *testing.T) {
|
|
validReq := extractionRequest()
|
|
canceledCtx, cancel := context.WithCancel(context.Background())
|
|
cancel()
|
|
validExtractor := newExtractor(t, &fakeSpellsLLMClient{response: extractionResponse{SpellCasts: []spellCastResponse{}}})
|
|
tests := []struct {
|
|
name string
|
|
extractor *Extractor
|
|
ctx context.Context
|
|
req contracts.TypedExtractionRequest
|
|
want string
|
|
}{
|
|
{name: "nil extractor", ctx: context.Background(), req: validReq, want: "extractor"},
|
|
{name: "nil context", extractor: validExtractor, req: validReq, want: "context"},
|
|
{name: "canceled context", extractor: validExtractor, ctx: canceledCtx, req: validReq, want: "context"},
|
|
{name: "nil source", extractor: validExtractor, ctx: context.Background(), req: contracts.TypedExtractionRequest{Chunk: validReq.Chunk}, want: "source"},
|
|
{name: "nil chunk", extractor: validExtractor, ctx: context.Background(), req: contracts.TypedExtractionRequest{Source: validReq.Source}, want: "chunk"},
|
|
{name: "empty chunk units", extractor: validExtractor, ctx: context.Background(), req: emptyChunkRequest(validReq), want: "units"},
|
|
{name: "source input mismatches chunk", extractor: validExtractor, ctx: context.Background(), req: mismatchedSourceInputRequest(validReq), want: "must match chunk"},
|
|
}
|
|
for _, test := range tests {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
_, err := test.extractor.Extract(test.ctx, test.req)
|
|
if err == nil || !strings.Contains(err.Error(), "dnd spells") || !strings.Contains(err.Error(), test.want) {
|
|
t.Fatalf("Extract() error = %v, want %q context", err, test.want)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestExtractOrdersAndDeduplicatesEvidence(t *testing.T) {
|
|
client := &fakeSpellsLLMClient{response: extractionResponse{SpellCasts: []spellCastResponse{
|
|
{Caster: "Borin", Spell: "Fire Bolt", Effect: "Burns.", NarrativeDescription: "Second.", SourceRefs: responseSourceRefs(2, 2)},
|
|
{Caster: "Aria", Spell: "Cure Wounds", Effect: "Heals.", NarrativeDescription: "First.", SourceRefs: []spellSourceRefResponse{{StartUnitID: shared.UnitRefFromInt(1), EndUnitID: shared.UnitRefFromInt(2)}, {StartUnitID: shared.UnitRefFromInt(1), EndUnitID: shared.UnitRefFromInt(2)}}},
|
|
{Caster: "Narrator", Spell: "Unknown", Effect: "Unknown.", NarrativeDescription: "Uncited."},
|
|
}}}
|
|
result, err := newExtractor(t, client).Extract(context.Background(), extractionRequest())
|
|
if err != nil {
|
|
t.Fatalf("Extract() error = %v, want nil", err)
|
|
}
|
|
if got := []string{result.Value.SpellCasts[0].Spell, result.Value.SpellCasts[1].Spell, result.Value.SpellCasts[2].Spell}; !reflect.DeepEqual(got, []string{"Cure Wounds", "Fire Bolt", "Unknown"}) {
|
|
t.Fatalf("spell order = %#v, want evidence order", got)
|
|
}
|
|
if refs := result.Value.SpellCasts[0].SourceRefs; len(refs) != 1 || refs[0] != (source.SourceRef{SourceID: "session-alpha", StartUnitID: 1, EndUnitID: 2}) {
|
|
t.Fatalf("source refs = %#v, want one canonical ref", refs)
|
|
}
|
|
}
|
|
|
|
func TestExtractPreservesInvalidEvidenceForValidators(t *testing.T) {
|
|
client := &fakeSpellsLLMClient{response: extractionResponse{SpellCasts: []spellCastResponse{{
|
|
Caster: "Aria", Spell: "Cure Wounds", Effect: "Heals.", NarrativeDescription: "Aria heals.",
|
|
SourceRefs: []spellSourceRefResponse{{StartUnitID: shared.UnitRefFromInt(99), EndUnitID: shared.UnitRefFromString("missing")}},
|
|
}}}}
|
|
result, err := newExtractor(t, client).Extract(context.Background(), extractionRequest())
|
|
if err != nil {
|
|
t.Fatalf("Extract() error = %v, want nil", err)
|
|
}
|
|
ref := result.Value.SpellCasts[0].SourceRefs[0]
|
|
if ref != (source.SourceRef{SourceID: "session-alpha", StartUnitID: 99}) {
|
|
t.Fatalf("source ref = %#v, want canonical source with invalid range preserved", ref)
|
|
}
|
|
}
|