package spells import ( "context" "encoding/json" "os" "strings" "testing" "gitea.maximumdirect.net/eric/notarius/internal/core/config" "gitea.maximumdirect.net/eric/notarius/internal/core/source" "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" "gitea.maximumdirect.net/eric/notarius/internal/modules/input/seriatim" ) func TestRunnerProcessesSeriatimInputWithDNDSpellsExtractor(t *testing.T) { raw := readDNDSpellsFixture(t) expectedDoc := parseDNDSpellsFixture(t, raw) resolved := resolveDNDSpellsPipeline(t) llmClient := &fakeSpellsLLMClient{ response: extractionResponse{ SpellCasts: []spellCastResponse{ { Caster: "Aria", Spell: "Cure Wounds", Effect: "Heals an injured ally.", NarrativeDescription: "Aria restores the fighter after the fight.", SourceRefs: []source.SourceRef{ {SourceID: expectedDoc.ID, StartUnitID: "seg-001", EndUnitID: "seg-001"}, }, }, { Caster: "Borin", Spell: "Fire Bolt", Effect: "Scorches the wight.", NarrativeDescription: "Borin hurls fire at the wight.", SourceRefs: []source.SourceRef{ {SourceID: expectedDoc.ID, StartUnitID: "seg-003", EndUnitID: "seg-003"}, }, }, }, }, } output, err := pipeline.New(dndSpellsRunnerRegistries(t)).Run(context.Background(), pipeline.RunInput{ Pipeline: resolved.ResolvedPipeline, RawInput: raw, LLMClient: llmClient, }) if err != nil { t.Fatalf("Run() error = %v, want nil", err) } if len(output.Approved) != 2 { t.Fatalf("len(Approved) = %d, want 2", len(output.Approved)) } var first, second SpellCast if err := json.Unmarshal(output.Approved[0].Payload, &first); err != nil { t.Fatalf("Unmarshal(first payload) error = %v, want nil", err) } if err := json.Unmarshal(output.Approved[1].Payload, &second); err != nil { t.Fatalf("Unmarshal(second payload) error = %v, want nil", err) } if first.Spell != "Cure Wounds" || second.Spell != "Fire Bolt" { t.Fatalf("approved spell order = %q, %q; want response order", first.Spell, second.Spell) } if first.Caster != "Aria" || second.Caster != "Borin" { t.Fatalf("approved casters = %q, %q; want spell data", first.Caster, second.Caster) } for _, artifact := range output.Approved { if artifact.ExtractorKey != Key || artifact.ArtifactType != ArtifactType || artifact.SchemaVersion != SchemaVersion { t.Fatalf("approved artifact envelope = %#v, want dnd spells envelope", artifact) } if len(artifact.SourceRefs) != 1 { t.Fatalf("len(SourceRefs) = %d, want 1", len(artifact.SourceRefs)) } if err := source.ValidateRef(expectedDoc, artifact.SourceRefs[0]); err != nil { t.Fatalf("ValidateRef() error = %v, want nil", err) } } if output.Manifest.InputModule != seriatim.Key { t.Fatalf("manifest input module = %q, want %q", output.Manifest.InputModule, seriatim.Key) } if output.Manifest.ValidationStatus != "approved" { t.Fatalf("ValidationStatus = %q, want approved", output.Manifest.ValidationStatus) } if len(output.Manifest.ArtifactLanes) != 1 { t.Fatalf("len(ArtifactLanes) = %d, want 1", len(output.Manifest.ArtifactLanes)) } lane := output.Manifest.ArtifactLanes[0] if lane.ID != "spells" || lane.Extractor != Key { t.Fatalf("manifest lane = %#v, want spells lane with dnd/spells extractor", lane) } extractorMetadata, ok := lane.Metadata["extractor"].(map[string]any) if !ok { t.Fatalf("manifest lane metadata = %#v, want extractor metadata", lane.Metadata) } if extractorMetadata["prompt_id"] != PromptID || extractorMetadata["response_schema_key"] != string(ResponseSchemaKey) || extractorMetadata["response_schema_name"] != ResponseSchemaName { t.Fatalf("extractor metadata = %#v, want prompt/schema identifiers", extractorMetadata) } if len(output.OutputFiles) != 1 { t.Fatalf("len(OutputFiles) = %d, want 1", len(output.OutputFiles)) } if output.OutputFiles[0].ContentType != "application/json" { t.Fatalf("ContentType = %q, want application/json", output.OutputFiles[0].ContentType) } } func TestRunnerPassesRosterAndGlossaryReferencesToDNDSpellsPrompt(t *testing.T) { raw := readDNDSpellsFixture(t) expectedDoc := parseDNDSpellsFixture(t, raw) resolved := resolveDNDSpellsPipeline(t) resolved.ResolvedPipeline.ArtifactLanes[0].ExtractReferences.ReferenceSet = dndSpellsReferenceSet( "Aria: party cleric\nBorin: fighter", "Fire Bolt: evocation cantrip", ) llmClient := &fakeSpellsLLMClient{ response: extractionResponse{ SpellCasts: []spellCastResponse{ { Caster: "Borin", Spell: "Fire Bolt", Effect: "Scorches the wight.", NarrativeDescription: "Borin hurls fire at the wight.", SourceRefs: []source.SourceRef{ {SourceID: expectedDoc.ID, StartUnitID: "seg-003", EndUnitID: "seg-003"}, }, }, }, }, } output, err := pipeline.New(dndSpellsRunnerRegistries(t)).Run(context.Background(), pipeline.RunInput{ Pipeline: resolved.ResolvedPipeline, RawInput: raw, LLMClient: llmClient, }) if err != nil { t.Fatalf("Run() error = %v, want nil", err) } if len(output.Approved) != 1 { t.Fatalf("len(Approved) = %d, want 1", len(output.Approved)) } if len(output.Manifest.References) != 2 { t.Fatalf("manifest references = %#v, want roster and glossary provenance", output.Manifest.References) } if len(llmClient.requests) != 1 { t.Fatalf("LLM calls = %d, want 1", len(llmClient.requests)) } request := llmClient.requests[0] if request.PromptID != PromptID || request.PromptVersion != SchemaVersion { t.Fatalf("prompt = %q/%q, want %q/%q", request.PromptID, request.PromptVersion, PromptID, SchemaVersion) } if got := string(request.Inputs["roster"].Content); got != "Aria: party cleric\nBorin: fighter" { t.Fatalf("roster input = %q, want reference text", got) } if got := string(request.Inputs["glossary"].Content); got != "Fire Bolt: evocation cantrip" { t.Fatalf("glossary input = %q, want reference text", got) } } func TestRunnerDoesNotExtractSpellMentionedOnlyInRoster(t *testing.T) { raw := readDNDSpellsFixture(t) resolved := resolveDNDSpellsPipeline(t) resolved.ResolvedPipeline.ArtifactLanes[0].ExtractReferences.ReferenceSet = dndSpellsReferenceSet( "Mira: wizard who can cast Lightning Bolt", "", ) llmClient := &fakeSpellsLLMClient{ response: extractionResponse{SpellCasts: []spellCastResponse{}}, } output, err := pipeline.New(dndSpellsRunnerRegistries(t)).Run(context.Background(), pipeline.RunInput{ Pipeline: resolved.ResolvedPipeline, RawInput: raw, LLMClient: llmClient, }) if err != nil { t.Fatalf("Run() error = %v, want nil", err) } if len(output.Approved) != 0 { t.Fatalf("approved artifacts = %#v, want no roster-only spell casts", output.Approved) } if len(llmClient.requests) != 1 { t.Fatalf("LLM calls = %d, want 1", len(llmClient.requests)) } request := llmClient.requests[0] if request.PromptID != PromptID || request.PromptVersion != SchemaVersion { t.Fatalf("prompt = %q/%q, want %q/%q", request.PromptID, request.PromptVersion, PromptID, SchemaVersion) } if got := string(request.Inputs["roster"].Content); !strings.Contains(got, "Lightning Bolt") { t.Fatalf("roster input = %q, want roster-only spell in reference input", got) } if output.Manifest.ValidationStatus != "approved" { t.Fatalf("ValidationStatus = %q, want approved empty extraction", output.Manifest.ValidationStatus) } } func TestRunnerRejectsDNDSpellCastWithInvalidSourceRef(t *testing.T) { raw := readDNDSpellsFixture(t) resolved := resolveDNDSpellsPipeline(t) llmClient := &fakeSpellsLLMClient{ response: extractionResponse{ SpellCasts: []spellCastResponse{ { Caster: "Aria", Spell: "Cure Wounds", Effect: "Heals an injured ally.", NarrativeDescription: "Aria restores the fighter after the fight.", SourceRefs: []source.SourceRef{ {SourceID: "spell-session", StartUnitID: "seg-999", EndUnitID: "seg-999"}, }, }, }, }, } output, err := pipeline.New(dndSpellsRunnerRegistries(t)).Run(context.Background(), pipeline.RunInput{ Pipeline: resolved.ResolvedPipeline, RawInput: raw, LLMClient: llmClient, }) if err != nil { t.Fatalf("Run() error = %v, want nil", err) } if len(output.Approved) != 0 { t.Fatalf("len(Approved) = %d, want 0", len(output.Approved)) } if len(output.Rejected) != 1 { t.Fatalf("len(Rejected) = %d, want 1", len(output.Rejected)) } rejected := output.Rejected[0] if rejected.ValidatorName != sourceRefValidatorName { t.Fatalf("ValidatorName = %q, want %q", rejected.ValidatorName, sourceRefValidatorName) } if rejected.ReasonCode != reasonInvalidSourceRef { t.Fatalf("ReasonCode = %q, want %q", rejected.ReasonCode, reasonInvalidSourceRef) } if output.Manifest.ValidationStatus != "rejected" { t.Fatalf("ValidationStatus = %q, want rejected", output.Manifest.ValidationStatus) } } func dndSpellsReferenceSet(roster string, glossary string) contracts.ReferenceSet { slots := make(map[string]contracts.ResolvedReferenceSlot) if strings.TrimSpace(roster) != "" { slots["roster"] = contracts.ResolvedReferenceSlot{ Slot: contracts.ReferenceSlot{Name: "roster"}, Items: []contracts.ReferenceItem{ { SlotName: "roster", MediaType: "text/plain; charset=utf-8", Content: []byte(roster), Digest: "sha256:roster", Origin: contracts.ReferenceOrigin{Type: "file", URI: "file:///tmp/roster.txt"}, SizeBytes: int64(len(roster)), BindingSource: contracts.ReferenceBindingSourceConfig, }, }, } } if strings.TrimSpace(glossary) != "" { slots["glossary"] = contracts.ResolvedReferenceSlot{ Slot: contracts.ReferenceSlot{Name: "glossary"}, Items: []contracts.ReferenceItem{ { SlotName: "glossary", MediaType: "text/plain; charset=utf-8", Content: []byte(glossary), Digest: "sha256:glossary", Origin: contracts.ReferenceOrigin{Type: "file", URI: "file:///tmp/glossary.txt"}, SizeBytes: int64(len(glossary)), BindingSource: contracts.ReferenceBindingSourceConfig, }, }, } } return contracts.ReferenceSet{Slots: slots} } func TestRunnerFailsWhenDNDSpellsExtractorReturnsMalformedOutput(t *testing.T) { raw := readDNDSpellsFixture(t) resolved := resolveDNDSpellsPipeline(t) llmClient := &fakeSpellsLLMClient{response: extractionResponse{}} output, err := pipeline.New(dndSpellsRunnerRegistries(t)).Run(context.Background(), pipeline.RunInput{ Pipeline: resolved.ResolvedPipeline, RawInput: raw, LLMClient: llmClient, }) if err == nil { t.Fatal("Run() error = nil, want malformed extraction error") } if !strings.Contains(err.Error(), "extract lane") || !strings.Contains(err.Error(), "dnd spells") || !strings.Contains(err.Error(), "spell_casts") { t.Fatalf("Run() error = %q, want D&D spells extraction context", err.Error()) } if output.Manifest.ValidationStatus != "failed" { t.Fatalf("ValidationStatus = %q, want failed", output.Manifest.ValidationStatus) } } func resolveDNDSpellsPipeline(t *testing.T) config.EffectiveConfig { t.Helper() resolved, err := loadDNDSpellsPipelineConfig(t).Resolve(config.ResolveInput{ PipelineID: "dnd-spells-fixture", Catalog: dndSpellsTestCatalog(t, dndSpellsCatalogSpecs{}), }) if err != nil { t.Fatalf("Resolve() error = %v, want nil", err) } return resolved } func dndSpellsRunnerRegistries(t *testing.T) pipeline.Registries { t.Helper() catalog := dndSpellsTestCatalog(t, dndSpellsCatalogSpecs{}) return pipeline.Registries{ Inputs: catalog.Inputs, Chunkers: catalog.Chunkers, Extractors: catalog.Extractors, Mergers: catalog.Mergers, Normalizers: catalog.Normalizers, Outputs: catalog.Outputs, } } func readDNDSpellsFixture(t *testing.T) []byte { t.Helper() raw, err := os.ReadFile("testdata/seriatim_spell_session.json") if err != nil { t.Fatalf("ReadFile(seriatim_spell_session.json) error = %v, want nil", err) } return raw } func parseDNDSpellsFixture(t *testing.T, raw []byte) *source.SourceDocument { t.Helper() doc, err := seriatim.New().Parse(context.Background(), contracts.ParseRequest{Raw: raw}) if err != nil { t.Fatalf("Parse() error = %v, want nil", err) } return doc }