Add D&D spells pipeline integration tests

This commit is contained in:
2026-07-03 23:47:06 +00:00
parent c51c2d7fe5
commit 373ba13562
4 changed files with 530 additions and 0 deletions

View File

@@ -0,0 +1,276 @@
package spells
import (
"context"
"os"
"strings"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
"gitea.maximumdirect.net/eric/notarius/internal/modules/input/seriatim"
)
func TestPipelineConfigLoadsAndResolvesWithDNDSpellsExtractor(t *testing.T) {
data, err := os.ReadFile("testdata/pipeline.yml")
if err != nil {
t.Fatalf("ReadFile(pipeline.yml) error = %v, want nil", err)
}
fileCfg, err := config.ParseFileConfigYAML(data)
if err != nil {
t.Fatalf("ParseFileConfigYAML() error = %v, want nil", err)
}
cfg := config.Default()
if err := cfg.ApplyFileConfig(fileCfg); err != nil {
t.Fatalf("ApplyFileConfig() error = %v, want nil", err)
}
resolved, err := cfg.Resolve(config.ResolveInput{
PipelineID: "dnd-spells-fixture",
Catalog: dndSpellsTestCatalog(t, dndSpellsCatalogSpecs{}),
})
if err != nil {
t.Fatalf("Resolve() error = %v, want nil", err)
}
if len(resolved.ResolvedPipeline.ArtifactLanes) != 1 {
t.Fatalf("len(ArtifactLanes) = %d, want 1", len(resolved.ResolvedPipeline.ArtifactLanes))
}
lane := resolved.ResolvedPipeline.ArtifactLanes[0]
if lane.ID != "spells" {
t.Fatalf("lane ID = %q, want spells", lane.ID)
}
if lane.Extract.Module != Key {
t.Fatalf("extract module = %q, want %q", lane.Extract.Module, Key)
}
if resolved.ResolvedPipeline.Digest == "" {
t.Fatal("resolved digest is empty")
}
again, err := cfg.Resolve(config.ResolveInput{
PipelineID: "dnd-spells-fixture",
Catalog: dndSpellsTestCatalog(t, dndSpellsCatalogSpecs{}),
})
if err != nil {
t.Fatalf("second Resolve() error = %v, want nil", err)
}
if resolved.ResolvedPipeline.Digest != again.ResolvedPipeline.Digest {
t.Fatalf("resolved digest = %q, second digest = %q; want stable digest", resolved.ResolvedPipeline.Digest, again.ResolvedPipeline.Digest)
}
}
func TestPipelineConfigRejectsMissingTranscriptCapabilityForDNDSpells(t *testing.T) {
inputSpec := seriatim.ModuleSpec()
inputSpec.Provides = withoutCapability(inputSpec.Provides, "source.transcript")
chunkSpec := dndSpellsChunkerSpec()
chunkSpec.Requires = nil
_, err := loadDNDSpellsPipelineConfig(t).Resolve(config.ResolveInput{
PipelineID: "dnd-spells-fixture",
Catalog: dndSpellsTestCatalog(t, dndSpellsCatalogSpecs{
input: inputSpec,
chunk: chunkSpec,
}),
})
if err == nil {
t.Fatal("Resolve() error = nil, want missing capability error")
}
if !strings.Contains(err.Error(), "missing capability") ||
!strings.Contains(err.Error(), "source.transcript") ||
!strings.Contains(err.Error(), Key) {
t.Fatalf("Resolve() error = %q, want dnd/spells missing source.transcript capability", err.Error())
}
}
func TestPipelineConfigRejectsMissingSpellCastsCapabilityForAppendOrder(t *testing.T) {
extractorSpec := ModuleSpec()
extractorSpec.Provides = withoutCapability(extractorSpec.Provides, "dnd.spell_casts")
_, err := loadDNDSpellsPipelineConfig(t).Resolve(config.ResolveInput{
PipelineID: "dnd-spells-fixture",
Catalog: dndSpellsTestCatalog(t, dndSpellsCatalogSpecs{
extractor: extractorSpec,
}),
})
if err == nil {
t.Fatal("Resolve() error = nil, want missing capability error")
}
if !strings.Contains(err.Error(), "missing capability") ||
!strings.Contains(err.Error(), "dnd.spell_casts") ||
!strings.Contains(err.Error(), pipeline.DefaultMergeModule) {
t.Fatalf("Resolve() error = %q, want appendorder missing dnd.spell_casts capability", err.Error())
}
}
func TestPipelineConfigRejectsUnknownLaneSelection(t *testing.T) {
_, err := loadDNDSpellsPipelineConfig(t).Resolve(config.ResolveInput{
PipelineID: "dnd-spells-fixture",
Only: []string{"missing"},
Catalog: dndSpellsTestCatalog(t, dndSpellsCatalogSpecs{}),
})
if err == nil {
t.Fatal("Resolve() error = nil, want unknown lane error")
}
if !strings.Contains(err.Error(), "selected artifact lane") || !strings.Contains(err.Error(), "missing") {
t.Fatalf("Resolve() error = %q, want unknown lane context", err.Error())
}
}
func loadDNDSpellsPipelineConfig(t *testing.T) config.Config {
t.Helper()
data, err := os.ReadFile("testdata/pipeline.yml")
if err != nil {
t.Fatalf("ReadFile(pipeline.yml) error = %v, want nil", err)
}
fileCfg, err := config.ParseFileConfigYAML(data)
if err != nil {
t.Fatalf("ParseFileConfigYAML() error = %v, want nil", err)
}
cfg := config.Default()
if err := cfg.ApplyFileConfig(fileCfg); err != nil {
t.Fatalf("ApplyFileConfig() error = %v, want nil", err)
}
return cfg
}
type dndSpellsCatalogSpecs struct {
input pipeline.ModuleSpec
chunk pipeline.ModuleSpec
extractor pipeline.ModuleSpec
}
func dndSpellsTestCatalog(t *testing.T, specs dndSpellsCatalogSpecs) pipeline.ModuleCatalog {
t.Helper()
inputs := pipeline.NewInputAdapterRegistry()
chunkers := pipeline.NewChunkerRegistry()
extractors := pipeline.NewExtractorRegistry()
mergers := pipeline.NewMergerRegistry()
normalizers := pipeline.NewNormalizerRegistry()
outputs := pipeline.NewOutputEncoderRegistry()
if specs.input.Key == "" {
if err := seriatim.Register(inputs); err != nil {
t.Fatalf("register seriatim input: %v", err)
}
} else if err := inputs.RegisterWithSpec(specs.input, func() (contracts.InputAdapter, error) {
return seriatim.New(), nil
}); err != nil {
t.Fatalf("register seriatim input override: %v", err)
}
chunkSpec := specs.chunk
if chunkSpec.Key == "" {
chunkSpec = dndSpellsChunkerSpec()
}
if err := chunkers.RegisterWithSpec(chunkSpec, func() (contracts.Chunker, error) {
return dndSpellsChunker{}, nil
}); err != nil {
t.Fatalf("register chunker: %v", err)
}
if specs.extractor.Key == "" {
if err := Register(extractors); err != nil {
t.Fatalf("register dnd spells extractor: %v", err)
}
} else if err := extractors.RegisterWithSpec(specs.extractor, func() (contracts.Extractor, error) {
return New(), nil
}); err != nil {
t.Fatalf("register dnd spells extractor override: %v", err)
}
if err := mergers.RegisterWithSpec(pipeline.ModuleSpec{
Key: pipeline.DefaultMergeModule,
Stage: pipeline.StageMerge,
Requires: []string{"dnd.spell_casts"},
}, func() (contracts.Merger, error) {
return pipeline.AppendOrderMerger{}, nil
}); err != nil {
t.Fatalf("register merger: %v", err)
}
if err := normalizers.RegisterWithSpec(pipeline.ModuleSpec{
Key: pipeline.DefaultNormalizeModule,
Stage: pipeline.StageNormalize,
}, func() (contracts.Normalizer, error) {
return pipeline.NoopNormalizer{}, nil
}); err != nil {
t.Fatalf("register normalizer: %v", err)
}
if err := outputs.RegisterWithSpec(pipeline.ModuleSpec{
Key: pipeline.DefaultOutputModule,
Stage: pipeline.StageOutput,
}, func() (contracts.OutputEncoder, error) {
return dndSpellsOutput{}, nil
}); err != nil {
t.Fatalf("register output: %v", err)
}
return pipeline.ModuleCatalog{
Inputs: inputs,
Chunkers: chunkers,
Extractors: extractors,
Mergers: mergers,
Normalizers: normalizers,
Outputs: outputs,
}
}
func dndSpellsChunkerSpec() pipeline.ModuleSpec {
return pipeline.ModuleSpec{
Key: "fake/chunk",
Stage: pipeline.StageChunk,
Requires: []string{"source.transcript"},
Provides: []string{"chunks"},
}
}
type dndSpellsChunker struct{}
func (dndSpellsChunker) Key() string {
return "fake/chunk"
}
func (dndSpellsChunker) Chunk(ctx context.Context, req contracts.ChunkRequest) (contracts.ChunkResult, error) {
return contracts.ChunkResult{
Chunks: []contracts.SourceChunk{
{
ID: req.Source.ID + ":chunk:0",
SourceID: req.Source.ID,
Index: 0,
Units: append([]source.SourceUnit(nil), req.Source.Units...),
},
},
}, nil
}
type dndSpellsOutput struct{}
func (dndSpellsOutput) Key() string {
return pipeline.DefaultOutputModule
}
func (dndSpellsOutput) Encode(ctx context.Context, req contracts.OutputRequest) (contracts.OutputResult, error) {
return contracts.OutputResult{
Bytes: []byte(`{"encoded":true}`),
ContentType: "application/json",
}, nil
}
func withoutCapability(capabilities []string, capability string) []string {
filtered := make([]string, 0, len(capabilities))
for _, candidate := range capabilities {
if candidate != capability {
filtered = append(filtered, candidate)
}
}
return filtered
}
var (
_ contracts.Chunker = dndSpellsChunker{}
_ contracts.OutputEncoder = dndSpellsOutput{}
)

View File

@@ -0,0 +1,214 @@
package spells
import (
"context"
"encoding/json"
"os"
"strings"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
"gitea.maximumdirect.net/eric/notarius/internal/modules/input/seriatim"
)
func TestRunnerProcessesSeriatimInputWithDNDSpellsExtractor(t *testing.T) {
raw := readDNDSpellsFixture(t)
expectedDoc := parseDNDSpellsFixture(t, raw)
resolved := resolveDNDSpellsPipeline(t)
llmClient := &fakeSpellsLLMClient{
response: extractionResponse{
SpellCasts: []spellCastResponse{
{
Caster: "Aria",
Spell: "Cure Wounds",
Effect: "Heals an injured ally.",
NarrativeDescription: "Aria restores the fighter after the fight.",
SourceRefs: []source.SourceRef{
{SourceID: expectedDoc.ID, StartUnitID: "seg-001", EndUnitID: "seg-001"},
},
},
{
Caster: "Borin",
Spell: "Fire Bolt",
Effect: "Scorches the wight.",
NarrativeDescription: "Borin hurls fire at the wight.",
SourceRefs: []source.SourceRef{
{SourceID: expectedDoc.ID, StartUnitID: "seg-003", EndUnitID: "seg-003"},
},
},
},
},
}
output, err := pipeline.New(dndSpellsRunnerRegistries(t)).Run(context.Background(), pipeline.RunInput{
Pipeline: resolved.ResolvedPipeline,
RawInput: raw,
LLMClient: llmClient,
})
if err != nil {
t.Fatalf("Run() error = %v, want nil", err)
}
if len(output.Approved) != 2 {
t.Fatalf("len(Approved) = %d, want 2", len(output.Approved))
}
var first, second SpellCast
if err := json.Unmarshal(output.Approved[0].Payload, &first); err != nil {
t.Fatalf("Unmarshal(first payload) error = %v, want nil", err)
}
if err := json.Unmarshal(output.Approved[1].Payload, &second); err != nil {
t.Fatalf("Unmarshal(second payload) error = %v, want nil", err)
}
if first.Spell != "Cure Wounds" || second.Spell != "Fire Bolt" {
t.Fatalf("approved spell order = %q, %q; want response order", first.Spell, second.Spell)
}
if first.Caster != "Aria" || second.Caster != "Borin" {
t.Fatalf("approved casters = %q, %q; want spell data", first.Caster, second.Caster)
}
for _, artifact := range output.Approved {
if artifact.ExtractorKey != Key || artifact.ArtifactType != ArtifactType || artifact.SchemaVersion != SchemaVersion {
t.Fatalf("approved artifact envelope = %#v, want dnd spells envelope", artifact)
}
if len(artifact.SourceRefs) != 1 {
t.Fatalf("len(SourceRefs) = %d, want 1", len(artifact.SourceRefs))
}
if err := source.ValidateRef(expectedDoc, artifact.SourceRefs[0]); err != nil {
t.Fatalf("ValidateRef() error = %v, want nil", err)
}
}
if output.Manifest.InputModule != seriatim.Key {
t.Fatalf("manifest input module = %q, want %q", output.Manifest.InputModule, seriatim.Key)
}
if output.Manifest.ValidationStatus != "approved" {
t.Fatalf("ValidationStatus = %q, want approved", output.Manifest.ValidationStatus)
}
if len(output.Manifest.ArtifactLanes) != 1 {
t.Fatalf("len(ArtifactLanes) = %d, want 1", len(output.Manifest.ArtifactLanes))
}
lane := output.Manifest.ArtifactLanes[0]
if lane.ID != "spells" || lane.Extractor != Key {
t.Fatalf("manifest lane = %#v, want spells lane with dnd/spells extractor", lane)
}
if output.ContentType != "application/json" {
t.Fatalf("ContentType = %q, want application/json", output.ContentType)
}
}
func TestRunnerRejectsDNDSpellCastWithInvalidSourceRef(t *testing.T) {
raw := readDNDSpellsFixture(t)
resolved := resolveDNDSpellsPipeline(t)
llmClient := &fakeSpellsLLMClient{
response: extractionResponse{
SpellCasts: []spellCastResponse{
{
Caster: "Aria",
Spell: "Cure Wounds",
Effect: "Heals an injured ally.",
NarrativeDescription: "Aria restores the fighter after the fight.",
SourceRefs: []source.SourceRef{
{SourceID: "spell-session", StartUnitID: "seg-999", EndUnitID: "seg-999"},
},
},
},
},
}
output, err := pipeline.New(dndSpellsRunnerRegistries(t)).Run(context.Background(), pipeline.RunInput{
Pipeline: resolved.ResolvedPipeline,
RawInput: raw,
LLMClient: llmClient,
})
if err != nil {
t.Fatalf("Run() error = %v, want nil", err)
}
if len(output.Approved) != 0 {
t.Fatalf("len(Approved) = %d, want 0", len(output.Approved))
}
if len(output.Rejected) != 1 {
t.Fatalf("len(Rejected) = %d, want 1", len(output.Rejected))
}
rejected := output.Rejected[0]
if rejected.ValidatorName != sourceRefValidatorName {
t.Fatalf("ValidatorName = %q, want %q", rejected.ValidatorName, sourceRefValidatorName)
}
if rejected.ReasonCode != reasonInvalidSourceRef {
t.Fatalf("ReasonCode = %q, want %q", rejected.ReasonCode, reasonInvalidSourceRef)
}
if output.Manifest.ValidationStatus != "rejected" {
t.Fatalf("ValidationStatus = %q, want rejected", output.Manifest.ValidationStatus)
}
}
func TestRunnerFailsWhenDNDSpellsExtractorReturnsMalformedOutput(t *testing.T) {
raw := readDNDSpellsFixture(t)
resolved := resolveDNDSpellsPipeline(t)
llmClient := &fakeSpellsLLMClient{response: extractionResponse{}}
output, err := pipeline.New(dndSpellsRunnerRegistries(t)).Run(context.Background(), pipeline.RunInput{
Pipeline: resolved.ResolvedPipeline,
RawInput: raw,
LLMClient: llmClient,
})
if err == nil {
t.Fatal("Run() error = nil, want malformed extraction error")
}
if !strings.Contains(err.Error(), "extract lane") ||
!strings.Contains(err.Error(), "dnd spells") ||
!strings.Contains(err.Error(), "spell_casts") {
t.Fatalf("Run() error = %q, want D&D spells extraction context", err.Error())
}
if output.Manifest.ValidationStatus != "failed" {
t.Fatalf("ValidationStatus = %q, want failed", output.Manifest.ValidationStatus)
}
}
func resolveDNDSpellsPipeline(t *testing.T) config.EffectiveConfig {
t.Helper()
resolved, err := loadDNDSpellsPipelineConfig(t).Resolve(config.ResolveInput{
PipelineID: "dnd-spells-fixture",
Catalog: dndSpellsTestCatalog(t, dndSpellsCatalogSpecs{}),
})
if err != nil {
t.Fatalf("Resolve() error = %v, want nil", err)
}
return resolved
}
func dndSpellsRunnerRegistries(t *testing.T) pipeline.Registries {
t.Helper()
catalog := dndSpellsTestCatalog(t, dndSpellsCatalogSpecs{})
return pipeline.Registries{
Inputs: catalog.Inputs,
Chunkers: catalog.Chunkers,
Extractors: catalog.Extractors,
Mergers: catalog.Mergers,
Normalizers: catalog.Normalizers,
Outputs: catalog.Outputs,
}
}
func readDNDSpellsFixture(t *testing.T) []byte {
t.Helper()
raw, err := os.ReadFile("testdata/seriatim_spell_session.json")
if err != nil {
t.Fatalf("ReadFile(seriatim_spell_session.json) error = %v, want nil", err)
}
return raw
}
func parseDNDSpellsFixture(t *testing.T, raw []byte) *source.SourceDocument {
t.Helper()
doc, err := seriatim.New().Parse(context.Background(), contracts.ParseRequest{Raw: raw})
if err != nil {
t.Fatalf("Parse() error = %v, want nil", err)
}
return doc
}

View File

@@ -0,0 +1,11 @@
version: 1
pipelines:
dnd-spells-fixture:
input: seriatim
chunk: fake/chunk
artifacts:
spells:
extract: dnd/spells
merge: appendorder
normalize: noop
output: json

View File

@@ -0,0 +1,29 @@
{
"metadata": {
"id": "spell-session",
"title": "Synthetic D&D spell session"
},
"segments": [
{
"id": "seg-001",
"start": 0,
"end": 4,
"speaker": "Alice",
"text": "Aria raises her holy symbol and casts Cure Wounds."
},
{
"id": "seg-002",
"start": 4,
"end": 8,
"speaker": "DM",
"text": "The bandit mage casts Shield as the blow lands."
},
{
"id": "seg-003",
"start": 8,
"end": 12,
"speaker": "Bob",
"text": "Borin points at the wight and casts Fire Bolt."
}
]
}