Introduce typed D&D spell artifacts
This commit is contained in:
@@ -2,47 +2,55 @@ package spells
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared"
|
||||
)
|
||||
|
||||
func TestExtractReturnsCanonicalOutputFromStructuredResponse(t *testing.T) {
|
||||
client := &fakeSpellsLLMClient{
|
||||
response: extractionResponse{
|
||||
SpellCasts: []spellCastResponse{
|
||||
{
|
||||
Caster: " Aria ",
|
||||
Spell: " Cure Wounds ",
|
||||
Effect: " Heals an injured ally. ",
|
||||
NarrativeDescription: " Aria restores the fighter after the fight. ",
|
||||
SourceRefs: responseSourceRefsInt("transcript", 1, 2),
|
||||
},
|
||||
},
|
||||
func TestExtractReturnsCanonicalSpellListFromPrivateResponse(t *testing.T) {
|
||||
client := &fakeSpellsLLMClient{response: extractionResponse{SpellCasts: []spellCastResponse{
|
||||
{
|
||||
Caster: " Aria ",
|
||||
Spell: " Cure Wounds ",
|
||||
Effect: " Heals an injured ally. ",
|
||||
NarrativeDescription: " Aria restores the fighter after the fight. ",
|
||||
SourceRefs: responseSourceRefs(1, 2),
|
||||
},
|
||||
content: []byte(`{"spell_casts":[{"caster":" Aria ","spell":" Cure Wounds ","effect":" Heals an injured ally. ","narrative_description":" Aria restores the fighter after the fight. ","source_refs":[{"source_id":"session-alpha","start_unit_id":1,"end_unit_id":2}]}],"raw_marker":true}`),
|
||||
}
|
||||
}}}
|
||||
req := extractionRequest()
|
||||
|
||||
extractReq := extractionRequestWithClient(client)
|
||||
result, err := New().Extract(context.Background(), extractReq)
|
||||
result, err := newExtractor(t, client).Extract(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("Extract() error = %v, want nil", err)
|
||||
}
|
||||
want := dnd.SpellList{SpellCasts: []dnd.SpellCast{
|
||||
{
|
||||
Caster: " Aria ",
|
||||
Spell: " Cure Wounds ",
|
||||
Effect: " Heals an injured ally. ",
|
||||
NarrativeDescription: " Aria restores the fighter after the fight. ",
|
||||
SourceRefs: []source.SourceRef{{SourceID: "session-alpha", StartUnitID: 1, EndUnitID: 2}},
|
||||
},
|
||||
}}
|
||||
if !reflect.DeepEqual(result.Value, want) {
|
||||
t.Fatalf("Value = %#v, want %#v", result.Value, want)
|
||||
}
|
||||
if len(result.Warnings) != 0 {
|
||||
t.Fatalf("Warnings = %#v, want none", result.Warnings)
|
||||
}
|
||||
|
||||
if len(client.requests) != 1 {
|
||||
t.Fatalf("LLM calls = %d, want 1", len(client.requests))
|
||||
}
|
||||
llmReq := client.requests[0]
|
||||
if llmReq.StageName != Key {
|
||||
t.Fatalf("StageName = %q, want %q", llmReq.StageName, Key)
|
||||
}
|
||||
if llmReq.PromptID != PromptID || llmReq.PromptVersion != SchemaVersion {
|
||||
t.Fatalf("prompt = %q/%q, want %q/%q", llmReq.PromptID, llmReq.PromptVersion, PromptID, SchemaVersion)
|
||||
if llmReq.StageName != Key || llmReq.PromptID != PromptID || llmReq.PromptVersion != SchemaVersion {
|
||||
t.Fatalf("LLM request identity = %#v, want spell prompt", llmReq)
|
||||
}
|
||||
if llmReq.SessionID != "session-123" || llmReq.ProfileID != "profile-spells" {
|
||||
t.Fatalf("session/profile = %q/%q, want session-123/profile-spells", llmReq.SessionID, llmReq.ProfileID)
|
||||
@@ -51,45 +59,17 @@ func TestExtractReturnsCanonicalOutputFromStructuredResponse(t *testing.T) {
|
||||
if transcript.Name != "transcript" || transcript.MediaType != "application/json" || transcript.Digest != "sha256:chunk" || transcript.OriginURI != "file:///session-alpha.json" {
|
||||
t.Fatalf("transcript metadata = %#v", transcript)
|
||||
}
|
||||
if got := string(transcript.Content); got != string(extractReq.Chunk.Content) {
|
||||
t.Fatalf("transcript content = %q, want chunk content %q", got, extractReq.Chunk.Content)
|
||||
}
|
||||
|
||||
if result.Output.Payload.MediaType != "application/json" {
|
||||
t.Fatalf("MediaType = %q, want application/json", result.Output.Payload.MediaType)
|
||||
}
|
||||
if result.Output.Schema.ID != ResponseSchemaID || result.Output.Schema.Name != ResponseSchemaName || result.Output.Schema.Version != SchemaVersion {
|
||||
t.Fatalf("schema = %#v, want response schema provenance", result.Output.Schema)
|
||||
}
|
||||
if !json.Valid(result.Output.Schema.JSONSchema) {
|
||||
t.Fatalf("schema JSON is invalid or missing: %s", result.Output.Schema.JSONSchema)
|
||||
}
|
||||
if strings.Contains(string(result.Output.Payload.Content), "raw_marker") {
|
||||
t.Fatalf("content = %q, want canonical payload without raw completion marker", result.Output.Payload.Content)
|
||||
}
|
||||
|
||||
var payload extractionResponse
|
||||
if err := json.Unmarshal(result.Output.Payload.Content, &payload); err != nil {
|
||||
t.Fatalf("Unmarshal(Content) error = %v, want nil", err)
|
||||
}
|
||||
if len(payload.SpellCasts) != 1 || payload.SpellCasts[0].Spell != " Cure Wounds " {
|
||||
t.Fatalf("payload = %#v, want structured response fields", payload)
|
||||
}
|
||||
if got := payload.SpellCasts[0].SourceRefs[0].SourceID; got != "session-alpha" {
|
||||
t.Fatalf("source_id = %q, want canonical source document ID", got)
|
||||
if got := string(transcript.Content); got != string(req.Chunk.Content) {
|
||||
t.Fatalf("transcript content = %q, want chunk content %q", got, req.Chunk.Content)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractorManifestMetadataIncludesPromptAndSchemaProvenance(t *testing.T) {
|
||||
metadata := New().ManifestMetadata()
|
||||
|
||||
func TestExtractorManifestMetadataIncludesLLMSchemaProvenance(t *testing.T) {
|
||||
metadata := newExtractor(t, &fakeSpellsLLMClient{}).ManifestMetadata()
|
||||
tests := map[string]string{
|
||||
"prompt_id": PromptID,
|
||||
"prompt_version": SchemaVersion,
|
||||
"response_schema_key": string(ResponseSchemaKey),
|
||||
"response_schema_id": ResponseSchemaID,
|
||||
"response_schema_name": ResponseSchemaName,
|
||||
"response_schema_version": SchemaVersion,
|
||||
"prompt_id": PromptID, "prompt_version": SchemaVersion,
|
||||
"response_schema_key": string(ResponseSchemaKey), "response_schema_id": ResponseSchemaID,
|
||||
"response_schema_name": ResponseSchemaName, "response_schema_version": SchemaVersion,
|
||||
}
|
||||
for key, want := range tests {
|
||||
if metadata[key] != want {
|
||||
@@ -106,373 +86,125 @@ func TestExtractorManifestMetadataIncludesPromptAndSchemaProvenance(t *testing.T
|
||||
|
||||
func TestExtractPassesReferencesAsPromptInputs(t *testing.T) {
|
||||
client := &fakeSpellsLLMClient{response: extractionResponse{SpellCasts: []spellCastResponse{}}}
|
||||
req := extractionRequestWithClient(client)
|
||||
req.References = contracts.ReferenceSet{
|
||||
Slots: map[string]contracts.ResolvedReferenceSlot{
|
||||
"players": {
|
||||
Slot: contracts.ReferenceSlot{Name: "players"},
|
||||
Items: []contracts.ReferenceItem{
|
||||
{SlotName: "players", Content: []byte("Alice: Aria Brightmantle")},
|
||||
},
|
||||
},
|
||||
"party": {
|
||||
Slot: contracts.ReferenceSlot{Name: "party"},
|
||||
Items: []contracts.ReferenceItem{
|
||||
{SlotName: "party", Content: []byte("Aria Brightmantle: party cleric")},
|
||||
},
|
||||
},
|
||||
"glossary": {
|
||||
Slot: contracts.ReferenceSlot{Name: "glossary"},
|
||||
Items: []contracts.ReferenceItem{
|
||||
{SlotName: "glossary", Content: []byte("Brightmantle: local temple name")},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
req := extractionRequest()
|
||||
req.References = contracts.ReferenceSet{Slots: map[string]contracts.ResolvedReferenceSlot{
|
||||
"players": {Slot: contracts.ReferenceSlot{Name: "players"}, Items: []contracts.ReferenceItem{{SlotName: "players", Content: []byte("Alice: Aria Brightmantle")}}},
|
||||
"party": {Slot: contracts.ReferenceSlot{Name: "party"}, Items: []contracts.ReferenceItem{{SlotName: "party", Content: []byte("Aria Brightmantle: party cleric")}}},
|
||||
"glossary": {Slot: contracts.ReferenceSlot{Name: "glossary"}, Items: []contracts.ReferenceItem{{SlotName: "glossary", Content: []byte("Brightmantle: local temple name")}}},
|
||||
}}
|
||||
|
||||
if _, err := New().Extract(context.Background(), req); err != nil {
|
||||
if _, err := newExtractor(t, client).Extract(context.Background(), req); err != nil {
|
||||
t.Fatalf("Extract() error = %v, want nil", err)
|
||||
}
|
||||
|
||||
if len(client.requests) != 1 {
|
||||
t.Fatalf("LLM calls = %d, want 1", len(client.requests))
|
||||
inputs := client.requests[0].Inputs
|
||||
if string(inputs["players"].Content) != "Alice: Aria Brightmantle" || string(inputs["party"].Content) != "Aria Brightmantle: party cleric" || string(inputs["glossary"].Content) != "Brightmantle: local temple name" {
|
||||
t.Fatalf("reference inputs = %#v, want configured content", inputs)
|
||||
}
|
||||
request := client.requests[0]
|
||||
if request.PromptID != PromptID || request.PromptVersion != SchemaVersion {
|
||||
t.Fatalf("prompt = %q/%q, want %q/%q", request.PromptID, request.PromptVersion, PromptID, SchemaVersion)
|
||||
}
|
||||
if got := string(request.Inputs["players"].Content); got != "Alice: Aria Brightmantle" {
|
||||
t.Fatalf("players input = %q, want reference content", got)
|
||||
}
|
||||
if got := string(request.Inputs["party"].Content); got != "Aria Brightmantle: party cleric" {
|
||||
t.Fatalf("party input = %q, want reference content", got)
|
||||
}
|
||||
if got := string(request.Inputs["glossary"].Content); got != "Brightmantle: local temple name" {
|
||||
t.Fatalf("glossary input = %q, want reference content", got)
|
||||
}
|
||||
if strings.Contains(string(request.Inputs["transcript"].Content), "Aria Brightmantle: party cleric") {
|
||||
t.Fatalf("transcript input contains reference content")
|
||||
if strings.Contains(string(inputs["transcript"].Content), "party cleric") {
|
||||
t.Fatal("transcript input contains reference content")
|
||||
}
|
||||
}
|
||||
|
||||
func TestPromptInputsMapLegacyRosterReferenceToParty(t *testing.T) {
|
||||
inputs := shared.PromptInputs(spellSourceInput(), contracts.ReferenceSet{
|
||||
Slots: map[string]contracts.ResolvedReferenceSlot{
|
||||
"roster": {
|
||||
Slot: contracts.ReferenceSlot{Name: "roster"},
|
||||
Items: []contracts.ReferenceItem{
|
||||
{SlotName: "roster", Content: []byte("Legacy roster text")},
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
inputs := shared.PromptInputs(spellSourceInput(), contracts.ReferenceSet{Slots: map[string]contracts.ResolvedReferenceSlot{
|
||||
"roster": {Slot: contracts.ReferenceSlot{Name: "roster"}, Items: []contracts.ReferenceItem{{SlotName: "roster", Content: []byte("Legacy roster text")}}},
|
||||
}})
|
||||
if got := string(inputs["party"].Content); got != "Legacy roster text" {
|
||||
t.Fatalf("party input = %q, want legacy roster content", got)
|
||||
}
|
||||
if _, ok := inputs["roster"]; ok {
|
||||
t.Fatalf("roster prompt input was present; want only party input")
|
||||
t.Fatal("roster prompt input was present; want only party input")
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractReturnsRawOutputForEmptyResponse(t *testing.T) {
|
||||
client := &fakeSpellsLLMClient{response: extractionResponse{SpellCasts: []spellCastResponse{}}}
|
||||
|
||||
result, err := New().Extract(context.Background(), extractionRequestWithClient(client))
|
||||
if err != nil {
|
||||
t.Fatalf("Extract() error = %v, want nil", err)
|
||||
}
|
||||
var payload extractionResponse
|
||||
if err := json.Unmarshal(result.Output.Payload.Content, &payload); err != nil {
|
||||
t.Fatalf("Unmarshal(Content) error = %v, want nil", err)
|
||||
}
|
||||
if len(payload.SpellCasts) != 0 {
|
||||
t.Fatalf("SpellCasts = %#v, want none", payload.SpellCasts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractReturnsCanonicalOutputForMalformedStructuredResponse(t *testing.T) {
|
||||
client := &fakeSpellsLLMClient{response: extractionResponse{}}
|
||||
|
||||
result, err := New().Extract(context.Background(), extractionRequestWithClient(client))
|
||||
if err != nil {
|
||||
t.Fatalf("Extract() error = %v, want nil", err)
|
||||
}
|
||||
if string(result.Output.Payload.Content) != `{"spell_casts":null}` {
|
||||
t.Fatalf("content = %s, want canonical structured output", result.Output.Payload.Content)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractWrapsLLMClientError(t *testing.T) {
|
||||
client := &fakeSpellsLLMClient{err: errors.New("provider unavailable")}
|
||||
|
||||
_, err := New().Extract(context.Background(), extractionRequestWithClient(client))
|
||||
if err == nil {
|
||||
t.Fatal("Extract() error = nil, want LLM error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "dnd spells") || !strings.Contains(err.Error(), "provider unavailable") {
|
||||
t.Fatalf("Extract() error = %q, want wrapped LLM context", err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractRejectsInvalidRequests(t *testing.T) {
|
||||
validClient := &fakeSpellsLLMClient{response: extractionResponse{SpellCasts: []spellCastResponse{}}}
|
||||
validReq := extractionRequestWithClient(validClient)
|
||||
canceledCtx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
|
||||
func TestExtractPreservesEmptyAndMalformedValuesForTypedValidators(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
extractor *Extractor
|
||||
ctx context.Context
|
||||
req contracts.ExtractionRequest
|
||||
want string
|
||||
name string
|
||||
response extractionResponse
|
||||
wantNil bool
|
||||
}{
|
||||
{name: "nil extractor", extractor: nil, ctx: context.Background(), req: validReq, want: "extractor"},
|
||||
{name: "nil context", extractor: New(), ctx: nil, req: validReq, want: "context"},
|
||||
{name: "canceled context", extractor: New(), ctx: canceledCtx, req: validReq, want: "context"},
|
||||
{name: "nil source", extractor: New(), ctx: context.Background(), req: contracts.ExtractionRequest{Chunk: validReq.Chunk, LLMClient: validReq.LLMClient}, want: "source"},
|
||||
{name: "nil chunk", extractor: New(), ctx: context.Background(), req: contracts.ExtractionRequest{Source: validReq.Source, LLMClient: validReq.LLMClient}, want: "chunk"},
|
||||
{name: "empty chunk units", extractor: New(), ctx: context.Background(), req: emptyChunkRequest(validReq), want: "units"},
|
||||
{name: "nil LLM client", extractor: New(), ctx: context.Background(), req: contracts.ExtractionRequest{Source: validReq.Source, Chunk: validReq.Chunk}, want: "LLM client"},
|
||||
{name: "source input mismatches chunk", extractor: New(), ctx: context.Background(), req: mismatchedSourceInputRequest(validReq), want: "must match chunk"},
|
||||
{name: "empty", response: extractionResponse{SpellCasts: []spellCastResponse{}}},
|
||||
{name: "missing", response: extractionResponse{}, wantNil: true},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
_, err := tt.extractor.Extract(tt.ctx, tt.req)
|
||||
if err == nil {
|
||||
t.Fatal("Extract() error = nil, want error")
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
result, err := newExtractor(t, &fakeSpellsLLMClient{response: test.response}).Extract(context.Background(), extractionRequest())
|
||||
if err != nil {
|
||||
t.Fatalf("Extract() error = %v, want nil", err)
|
||||
}
|
||||
if !strings.Contains(err.Error(), "dnd spells") || !strings.Contains(err.Error(), tt.want) {
|
||||
t.Fatalf("Extract() error = %q, want %q context", err.Error(), tt.want)
|
||||
if (result.Value.SpellCasts == nil) != test.wantNil || len(result.Value.SpellCasts) != 0 {
|
||||
t.Fatalf("SpellCasts = %#v, want empty with nil=%t", result.Value.SpellCasts, test.wantNil)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractOrdersSpellCastsByEarliestSourceUnit(t *testing.T) {
|
||||
client := &fakeSpellsLLMClient{
|
||||
response: extractionResponse{
|
||||
SpellCasts: []spellCastResponse{
|
||||
{
|
||||
Caster: "Bandit Shaman",
|
||||
Spell: "Fire Bolt",
|
||||
Effect: "Burns.",
|
||||
NarrativeDescription: "Second spell.",
|
||||
SourceRefs: responseSourceRefs("session-alpha", 2, 2),
|
||||
},
|
||||
{
|
||||
Caster: "Aria",
|
||||
Spell: "Cure Wounds",
|
||||
Effect: "Heals.",
|
||||
NarrativeDescription: "First spell.",
|
||||
SourceRefs: responseSourceRefs("session-alpha", 1, 1),
|
||||
},
|
||||
{
|
||||
Caster: "Narrator",
|
||||
Spell: "Unknown Spell",
|
||||
Effect: "No cited range.",
|
||||
NarrativeDescription: "This should sort after cited spell casts.",
|
||||
},
|
||||
},
|
||||
},
|
||||
func TestExtractWrapsLLMClientError(t *testing.T) {
|
||||
_, err := newExtractor(t, &fakeSpellsLLMClient{err: errors.New("provider unavailable")}).Extract(context.Background(), extractionRequest())
|
||||
if err == nil || !strings.Contains(err.Error(), "dnd spells") || !strings.Contains(err.Error(), "provider unavailable") {
|
||||
t.Fatalf("Extract() error = %v, want wrapped provider error", err)
|
||||
}
|
||||
}
|
||||
|
||||
result, err := New().Extract(context.Background(), extractionRequestWithClient(client))
|
||||
func TestExtractRejectsInvalidRequests(t *testing.T) {
|
||||
validReq := extractionRequest()
|
||||
canceledCtx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
validExtractor := newExtractor(t, &fakeSpellsLLMClient{response: extractionResponse{SpellCasts: []spellCastResponse{}}})
|
||||
tests := []struct {
|
||||
name string
|
||||
extractor *Extractor
|
||||
ctx context.Context
|
||||
req contracts.TypedExtractionRequest
|
||||
want string
|
||||
}{
|
||||
{name: "nil extractor", ctx: context.Background(), req: validReq, want: "extractor"},
|
||||
{name: "nil context", extractor: validExtractor, req: validReq, want: "context"},
|
||||
{name: "canceled context", extractor: validExtractor, ctx: canceledCtx, req: validReq, want: "context"},
|
||||
{name: "nil source", extractor: validExtractor, ctx: context.Background(), req: contracts.TypedExtractionRequest{Chunk: validReq.Chunk}, want: "source"},
|
||||
{name: "nil chunk", extractor: validExtractor, ctx: context.Background(), req: contracts.TypedExtractionRequest{Source: validReq.Source}, want: "chunk"},
|
||||
{name: "empty chunk units", extractor: validExtractor, ctx: context.Background(), req: emptyChunkRequest(validReq), want: "units"},
|
||||
{name: "source input mismatches chunk", extractor: validExtractor, ctx: context.Background(), req: mismatchedSourceInputRequest(validReq), want: "must match chunk"},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
_, err := test.extractor.Extract(test.ctx, test.req)
|
||||
if err == nil || !strings.Contains(err.Error(), "dnd spells") || !strings.Contains(err.Error(), test.want) {
|
||||
t.Fatalf("Extract() error = %v, want %q context", err, test.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractOrdersAndDeduplicatesEvidence(t *testing.T) {
|
||||
client := &fakeSpellsLLMClient{response: extractionResponse{SpellCasts: []spellCastResponse{
|
||||
{Caster: "Borin", Spell: "Fire Bolt", Effect: "Burns.", NarrativeDescription: "Second.", SourceRefs: responseSourceRefs(2, 2)},
|
||||
{Caster: "Aria", Spell: "Cure Wounds", Effect: "Heals.", NarrativeDescription: "First.", SourceRefs: []spellSourceRefResponse{{StartUnitID: shared.UnitRefFromInt(1), EndUnitID: shared.UnitRefFromInt(2)}, {StartUnitID: shared.UnitRefFromInt(1), EndUnitID: shared.UnitRefFromInt(2)}}},
|
||||
{Caster: "Narrator", Spell: "Unknown", Effect: "Unknown.", NarrativeDescription: "Uncited."},
|
||||
}}}
|
||||
result, err := newExtractor(t, client).Extract(context.Background(), extractionRequest())
|
||||
if err != nil {
|
||||
t.Fatalf("Extract() error = %v, want nil", err)
|
||||
}
|
||||
var payload extractionResponse
|
||||
if err := json.Unmarshal(result.Output.Payload.Content, &payload); err != nil {
|
||||
t.Fatalf("Unmarshal(Content) error = %v, want nil", err)
|
||||
if got := []string{result.Value.SpellCasts[0].Spell, result.Value.SpellCasts[1].Spell, result.Value.SpellCasts[2].Spell}; !reflect.DeepEqual(got, []string{"Cure Wounds", "Fire Bolt", "Unknown"}) {
|
||||
t.Fatalf("spell order = %#v, want evidence order", got)
|
||||
}
|
||||
if len(payload.SpellCasts) != 3 ||
|
||||
payload.SpellCasts[0].Spell != "Cure Wounds" ||
|
||||
payload.SpellCasts[1].Spell != "Fire Bolt" ||
|
||||
payload.SpellCasts[2].Spell != "Unknown Spell" {
|
||||
t.Fatalf("spell order = %#v, want earliest source-unit order with uncited spell last", payload.SpellCasts)
|
||||
if refs := result.Value.SpellCasts[0].SourceRefs; len(refs) != 1 || refs[0] != (source.SourceRef{SourceID: "session-alpha", StartUnitID: 1, EndUnitID: 2}) {
|
||||
t.Fatalf("source refs = %#v, want one canonical ref", refs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractCanonicalizesSourceRefs(t *testing.T) {
|
||||
client := &fakeSpellsLLMClient{
|
||||
response: extractionResponse{
|
||||
SpellCasts: []spellCastResponse{
|
||||
{
|
||||
Caster: "Aria",
|
||||
Spell: "Cure Wounds",
|
||||
Effect: "Heals.",
|
||||
NarrativeDescription: "Aria heals.",
|
||||
SourceRefs: []shared.SourceRefResponse{
|
||||
{SourceID: "gameplay_transcript", StartUnitID: shared.UnitRefFromInt(2), EndUnitID: shared.UnitRefFromInt(2)},
|
||||
{SourceID: "", StartUnitID: shared.UnitRefFromInt(1), EndUnitID: shared.UnitRefFromInt(2)},
|
||||
{SourceID: "transcript", StartUnitID: shared.UnitRefFromInt(1), EndUnitID: shared.UnitRefFromInt(2)},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
result, err := New().Extract(context.Background(), extractionRequestWithClient(client))
|
||||
func TestExtractPreservesInvalidEvidenceForValidators(t *testing.T) {
|
||||
client := &fakeSpellsLLMClient{response: extractionResponse{SpellCasts: []spellCastResponse{{
|
||||
Caster: "Aria", Spell: "Cure Wounds", Effect: "Heals.", NarrativeDescription: "Aria heals.",
|
||||
SourceRefs: []spellSourceRefResponse{{StartUnitID: shared.UnitRefFromInt(99), EndUnitID: shared.UnitRefFromString("missing")}},
|
||||
}}}}
|
||||
result, err := newExtractor(t, client).Extract(context.Background(), extractionRequest())
|
||||
if err != nil {
|
||||
t.Fatalf("Extract() error = %v, want nil", err)
|
||||
}
|
||||
var payload extractionResponse
|
||||
if err := json.Unmarshal(result.Output.Payload.Content, &payload); err != nil {
|
||||
t.Fatalf("Unmarshal(Content) error = %v, want nil", err)
|
||||
}
|
||||
refs := payload.SpellCasts[0].SourceRefs
|
||||
if len(refs) != 2 {
|
||||
t.Fatalf("source refs = %#v, want duplicate collapsed", refs)
|
||||
}
|
||||
for _, ref := range refs {
|
||||
if ref.SourceID != "session-alpha" {
|
||||
t.Fatalf("source ref = %#v, want canonical source_id", ref)
|
||||
}
|
||||
}
|
||||
if refs[0].StartUnitID.Int() != 1 || refs[0].EndUnitID.Int() != 2 ||
|
||||
refs[1].StartUnitID.Int() != 2 || refs[1].EndUnitID.Int() != 2 {
|
||||
t.Fatalf("source refs = %#v, want sorted unit ranges", refs)
|
||||
ref := result.Value.SpellCasts[0].SourceRefs[0]
|
||||
if ref != (source.SourceRef{SourceID: "session-alpha", StartUnitID: 99}) {
|
||||
t.Fatalf("source ref = %#v, want canonical source with invalid range preserved", ref)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractPreservesInvalidSourceRefsForValidators(t *testing.T) {
|
||||
client := &fakeSpellsLLMClient{
|
||||
response: extractionResponse{
|
||||
SpellCasts: []spellCastResponse{
|
||||
{
|
||||
Caster: "Aria",
|
||||
Spell: "Cure Wounds",
|
||||
Effect: "Heals.",
|
||||
NarrativeDescription: "Aria heals.",
|
||||
SourceRefs: []shared.SourceRefResponse{
|
||||
{SourceID: "transcript", StartUnitID: shared.UnitRefFromInt(99), EndUnitID: shared.UnitRefFromString("missing")},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
result, err := New().Extract(context.Background(), extractionRequestWithClient(client))
|
||||
if err != nil {
|
||||
t.Fatalf("Extract() error = %v, want nil", err)
|
||||
}
|
||||
var payload map[string][]map[string]any
|
||||
if err := json.Unmarshal(result.Output.Payload.Content, &payload); err != nil {
|
||||
t.Fatalf("Unmarshal(Content) error = %v, want nil", err)
|
||||
}
|
||||
ref := payload["spell_casts"][0]["source_refs"].([]any)[0].(map[string]any)
|
||||
if ref["source_id"] != "session-alpha" || ref["start_unit_id"] != float64(99) || ref["end_unit_id"] != "" {
|
||||
t.Fatalf("source ref = %#v, want source_id canonicalized without unit repair", ref)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractDefensivelyCopiesRawContent(t *testing.T) {
|
||||
client := &fakeSpellsLLMClient{
|
||||
response: extractionResponse{
|
||||
SpellCasts: []spellCastResponse{
|
||||
{
|
||||
Caster: "Aria",
|
||||
Spell: "Cure Wounds",
|
||||
Effect: "Heals.",
|
||||
NarrativeDescription: "Aria heals.",
|
||||
SourceRefs: responseSourceRefs("session-alpha", 1, 2),
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
result, err := New().Extract(context.Background(), extractionRequestWithClient(client))
|
||||
if err != nil {
|
||||
t.Fatalf("Extract() error = %v, want nil", err)
|
||||
}
|
||||
client.response.SpellCasts[0].SourceRefs[0].StartUnitID = shared.UnitRefFromInt(99)
|
||||
|
||||
var payload extractionResponse
|
||||
if err := json.Unmarshal(result.Output.Payload.Content, &payload); err != nil {
|
||||
t.Fatalf("Unmarshal(Content) error = %v, want nil", err)
|
||||
}
|
||||
if got := payload.SpellCasts[0].SourceRefs[0].StartUnitID.String(); got != "1" {
|
||||
t.Fatalf("source ref start = %q, want copied 1", got)
|
||||
}
|
||||
}
|
||||
|
||||
func extractionRequestWithClient(client contracts.StructuredLLMClient) contracts.ExtractionRequest {
|
||||
req := promptExtractionRequest()
|
||||
req.LLMClient = client
|
||||
req.SourceInput = spellChunkInput(req.Chunk)
|
||||
req.SessionID = "session-123"
|
||||
req.LLMProfile = "profile-spells"
|
||||
return req
|
||||
}
|
||||
|
||||
const spellTranscriptJSON = `{"id":"session-alpha","segments":[{"id":1,"text":"Aria raises her hand and casts Cure Wounds."}]}`
|
||||
|
||||
func spellSourceInput() contracts.LLMInputMaterial {
|
||||
return contracts.NewLLMInputMaterial("source", "application/json", []byte(spellTranscriptJSON), "sha256:transcript", "file:///session-alpha.json")
|
||||
}
|
||||
|
||||
func spellChunkInput(chunk *source.Chunk) contracts.LLMInputMaterial {
|
||||
return contracts.NewLLMInputMaterial("source", chunk.MediaType, chunk.Content, "sha256:chunk", "file:///session-alpha.json")
|
||||
}
|
||||
|
||||
func emptyChunkRequest(req contracts.ExtractionRequest) contracts.ExtractionRequest {
|
||||
req.Chunk = &source.Chunk{
|
||||
ID: req.Chunk.ID,
|
||||
SourceID: req.Chunk.SourceID,
|
||||
Index: req.Chunk.Index,
|
||||
}
|
||||
return req
|
||||
}
|
||||
|
||||
func mismatchedSourceInputRequest(req contracts.ExtractionRequest) contracts.ExtractionRequest {
|
||||
req.SourceInput = spellSourceInput()
|
||||
return req
|
||||
}
|
||||
|
||||
type fakeSpellsLLMClient struct {
|
||||
response extractionResponse
|
||||
content []byte
|
||||
err error
|
||||
requests []contracts.StructuredCompletionRequest
|
||||
}
|
||||
|
||||
func (client *fakeSpellsLLMClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
|
||||
client.requests = append(client.requests, cloneStructuredCompletionRequest(req))
|
||||
if client.err != nil {
|
||||
return contracts.StructuredCompletionResponse{}, client.err
|
||||
}
|
||||
|
||||
target, ok := out.(*extractionResponse)
|
||||
if !ok {
|
||||
return contracts.StructuredCompletionResponse{}, errors.New("unexpected output target")
|
||||
}
|
||||
*target = client.response
|
||||
content := append([]byte(nil), client.content...)
|
||||
if len(content) == 0 {
|
||||
var err error
|
||||
content, err = json.Marshal(client.response)
|
||||
if err != nil {
|
||||
return contracts.StructuredCompletionResponse{}, err
|
||||
}
|
||||
}
|
||||
return contracts.StructuredCompletionResponse{Content: content}, nil
|
||||
}
|
||||
|
||||
func cloneStructuredCompletionRequest(req contracts.StructuredCompletionRequest) contracts.StructuredCompletionRequest {
|
||||
req.Inputs = req.Inputs.Clone()
|
||||
req.Vars = cloneVars(req.Vars)
|
||||
return req
|
||||
}
|
||||
|
||||
func cloneVars(in map[string]any) map[string]any {
|
||||
if len(in) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make(map[string]any, len(in))
|
||||
for key, value := range in {
|
||||
out[key] = value
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user