227 lines
7.6 KiB
Go
227 lines
7.6 KiB
Go
package spells
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"os"
|
|
"strings"
|
|
"testing"
|
|
|
|
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/modules/input/seriatim"
|
|
)
|
|
|
|
func TestRunnerProcessesSeriatimInputWithDNDSpellsExtractor(t *testing.T) {
|
|
raw := readDNDSpellsFixture(t)
|
|
expectedDoc := parseDNDSpellsFixture(t, raw)
|
|
resolved := resolveDNDSpellsPipeline(t)
|
|
llmClient := &fakeSpellsLLMClient{
|
|
response: extractionResponse{
|
|
SpellCasts: []spellCastResponse{
|
|
{
|
|
Caster: "Aria",
|
|
Spell: "Cure Wounds",
|
|
Effect: "Heals an injured ally.",
|
|
NarrativeDescription: "Aria restores the fighter after the fight.",
|
|
SourceRefs: []source.SourceRef{
|
|
{SourceID: expectedDoc.ID, StartUnitID: "seg-001", EndUnitID: "seg-001"},
|
|
},
|
|
},
|
|
{
|
|
Caster: "Borin",
|
|
Spell: "Fire Bolt",
|
|
Effect: "Scorches the wight.",
|
|
NarrativeDescription: "Borin hurls fire at the wight.",
|
|
SourceRefs: []source.SourceRef{
|
|
{SourceID: expectedDoc.ID, StartUnitID: "seg-003", EndUnitID: "seg-003"},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
|
|
output, err := pipeline.New(dndSpellsRunnerRegistries(t)).Run(context.Background(), pipeline.RunInput{
|
|
Pipeline: resolved.ResolvedPipeline,
|
|
RawInput: raw,
|
|
LLMClient: llmClient,
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("Run() error = %v, want nil", err)
|
|
}
|
|
|
|
if len(output.Approved) != 2 {
|
|
t.Fatalf("len(Approved) = %d, want 2", len(output.Approved))
|
|
}
|
|
var first, second SpellCast
|
|
if err := json.Unmarshal(output.Approved[0].Payload, &first); err != nil {
|
|
t.Fatalf("Unmarshal(first payload) error = %v, want nil", err)
|
|
}
|
|
if err := json.Unmarshal(output.Approved[1].Payload, &second); err != nil {
|
|
t.Fatalf("Unmarshal(second payload) error = %v, want nil", err)
|
|
}
|
|
if first.Spell != "Cure Wounds" || second.Spell != "Fire Bolt" {
|
|
t.Fatalf("approved spell order = %q, %q; want response order", first.Spell, second.Spell)
|
|
}
|
|
if first.Caster != "Aria" || second.Caster != "Borin" {
|
|
t.Fatalf("approved casters = %q, %q; want spell data", first.Caster, second.Caster)
|
|
}
|
|
for _, artifact := range output.Approved {
|
|
if artifact.ExtractorKey != Key || artifact.ArtifactType != ArtifactType || artifact.SchemaVersion != SchemaVersion {
|
|
t.Fatalf("approved artifact envelope = %#v, want dnd spells envelope", artifact)
|
|
}
|
|
if len(artifact.SourceRefs) != 1 {
|
|
t.Fatalf("len(SourceRefs) = %d, want 1", len(artifact.SourceRefs))
|
|
}
|
|
if err := source.ValidateRef(expectedDoc, artifact.SourceRefs[0]); err != nil {
|
|
t.Fatalf("ValidateRef() error = %v, want nil", err)
|
|
}
|
|
}
|
|
|
|
if output.Manifest.InputModule != seriatim.Key {
|
|
t.Fatalf("manifest input module = %q, want %q", output.Manifest.InputModule, seriatim.Key)
|
|
}
|
|
if output.Manifest.ValidationStatus != "approved" {
|
|
t.Fatalf("ValidationStatus = %q, want approved", output.Manifest.ValidationStatus)
|
|
}
|
|
if len(output.Manifest.ArtifactLanes) != 1 {
|
|
t.Fatalf("len(ArtifactLanes) = %d, want 1", len(output.Manifest.ArtifactLanes))
|
|
}
|
|
lane := output.Manifest.ArtifactLanes[0]
|
|
if lane.ID != "spells" || lane.Extractor != Key {
|
|
t.Fatalf("manifest lane = %#v, want spells lane with dnd/spells extractor", lane)
|
|
}
|
|
extractorMetadata, ok := lane.Metadata["extractor"].(map[string]any)
|
|
if !ok {
|
|
t.Fatalf("manifest lane metadata = %#v, want extractor metadata", lane.Metadata)
|
|
}
|
|
if extractorMetadata["prompt_id"] != PromptID ||
|
|
extractorMetadata["response_schema_key"] != string(ResponseSchemaKey) ||
|
|
extractorMetadata["response_schema_name"] != ResponseSchemaName {
|
|
t.Fatalf("extractor metadata = %#v, want prompt/schema identifiers", extractorMetadata)
|
|
}
|
|
if len(output.OutputFiles) != 1 {
|
|
t.Fatalf("len(OutputFiles) = %d, want 1", len(output.OutputFiles))
|
|
}
|
|
if output.OutputFiles[0].ContentType != "application/json" {
|
|
t.Fatalf("ContentType = %q, want application/json", output.OutputFiles[0].ContentType)
|
|
}
|
|
}
|
|
|
|
func TestRunnerRejectsDNDSpellCastWithInvalidSourceRef(t *testing.T) {
|
|
raw := readDNDSpellsFixture(t)
|
|
resolved := resolveDNDSpellsPipeline(t)
|
|
llmClient := &fakeSpellsLLMClient{
|
|
response: extractionResponse{
|
|
SpellCasts: []spellCastResponse{
|
|
{
|
|
Caster: "Aria",
|
|
Spell: "Cure Wounds",
|
|
Effect: "Heals an injured ally.",
|
|
NarrativeDescription: "Aria restores the fighter after the fight.",
|
|
SourceRefs: []source.SourceRef{
|
|
{SourceID: "spell-session", StartUnitID: "seg-999", EndUnitID: "seg-999"},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
|
|
output, err := pipeline.New(dndSpellsRunnerRegistries(t)).Run(context.Background(), pipeline.RunInput{
|
|
Pipeline: resolved.ResolvedPipeline,
|
|
RawInput: raw,
|
|
LLMClient: llmClient,
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("Run() error = %v, want nil", err)
|
|
}
|
|
if len(output.Approved) != 0 {
|
|
t.Fatalf("len(Approved) = %d, want 0", len(output.Approved))
|
|
}
|
|
if len(output.Rejected) != 1 {
|
|
t.Fatalf("len(Rejected) = %d, want 1", len(output.Rejected))
|
|
}
|
|
rejected := output.Rejected[0]
|
|
if rejected.ValidatorName != sourceRefValidatorName {
|
|
t.Fatalf("ValidatorName = %q, want %q", rejected.ValidatorName, sourceRefValidatorName)
|
|
}
|
|
if rejected.ReasonCode != reasonInvalidSourceRef {
|
|
t.Fatalf("ReasonCode = %q, want %q", rejected.ReasonCode, reasonInvalidSourceRef)
|
|
}
|
|
if output.Manifest.ValidationStatus != "rejected" {
|
|
t.Fatalf("ValidationStatus = %q, want rejected", output.Manifest.ValidationStatus)
|
|
}
|
|
}
|
|
|
|
func TestRunnerFailsWhenDNDSpellsExtractorReturnsMalformedOutput(t *testing.T) {
|
|
raw := readDNDSpellsFixture(t)
|
|
resolved := resolveDNDSpellsPipeline(t)
|
|
llmClient := &fakeSpellsLLMClient{response: extractionResponse{}}
|
|
|
|
output, err := pipeline.New(dndSpellsRunnerRegistries(t)).Run(context.Background(), pipeline.RunInput{
|
|
Pipeline: resolved.ResolvedPipeline,
|
|
RawInput: raw,
|
|
LLMClient: llmClient,
|
|
})
|
|
if err == nil {
|
|
t.Fatal("Run() error = nil, want malformed extraction error")
|
|
}
|
|
if !strings.Contains(err.Error(), "extract lane") ||
|
|
!strings.Contains(err.Error(), "dnd spells") ||
|
|
!strings.Contains(err.Error(), "spell_casts") {
|
|
t.Fatalf("Run() error = %q, want D&D spells extraction context", err.Error())
|
|
}
|
|
if output.Manifest.ValidationStatus != "failed" {
|
|
t.Fatalf("ValidationStatus = %q, want failed", output.Manifest.ValidationStatus)
|
|
}
|
|
}
|
|
|
|
func resolveDNDSpellsPipeline(t *testing.T) config.EffectiveConfig {
|
|
t.Helper()
|
|
|
|
resolved, err := loadDNDSpellsPipelineConfig(t).Resolve(config.ResolveInput{
|
|
PipelineID: "dnd-spells-fixture",
|
|
Catalog: dndSpellsTestCatalog(t, dndSpellsCatalogSpecs{}),
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("Resolve() error = %v, want nil", err)
|
|
}
|
|
return resolved
|
|
}
|
|
|
|
func dndSpellsRunnerRegistries(t *testing.T) pipeline.Registries {
|
|
t.Helper()
|
|
|
|
catalog := dndSpellsTestCatalog(t, dndSpellsCatalogSpecs{})
|
|
return pipeline.Registries{
|
|
Inputs: catalog.Inputs,
|
|
Chunkers: catalog.Chunkers,
|
|
Extractors: catalog.Extractors,
|
|
Mergers: catalog.Mergers,
|
|
Normalizers: catalog.Normalizers,
|
|
Outputs: catalog.Outputs,
|
|
}
|
|
}
|
|
|
|
func readDNDSpellsFixture(t *testing.T) []byte {
|
|
t.Helper()
|
|
|
|
raw, err := os.ReadFile("testdata/seriatim_spell_session.json")
|
|
if err != nil {
|
|
t.Fatalf("ReadFile(seriatim_spell_session.json) error = %v, want nil", err)
|
|
}
|
|
return raw
|
|
}
|
|
|
|
func parseDNDSpellsFixture(t *testing.T, raw []byte) *source.SourceDocument {
|
|
t.Helper()
|
|
|
|
doc, err := seriatim.New().Parse(context.Background(), contracts.ParseRequest{Raw: raw})
|
|
if err != nil {
|
|
t.Fatalf("Parse() error = %v, want nil", err)
|
|
}
|
|
return doc
|
|
}
|