From 373ba13562f84292fb635625dcb1f5567df3e441 Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Fri, 3 Jul 2026 23:47:06 +0000 Subject: [PATCH] Add D&D spells pipeline integration tests --- .../modules/extract/dnd/spells/config_test.go | 276 ++++++++++++++++++ .../modules/extract/dnd/spells/runner_test.go | 214 ++++++++++++++ .../extract/dnd/spells/testdata/pipeline.yml | 11 + .../testdata/seriatim_spell_session.json | 29 ++ 4 files changed, 530 insertions(+) create mode 100644 internal/modules/extract/dnd/spells/config_test.go create mode 100644 internal/modules/extract/dnd/spells/runner_test.go create mode 100644 internal/modules/extract/dnd/spells/testdata/pipeline.yml create mode 100644 internal/modules/extract/dnd/spells/testdata/seriatim_spell_session.json diff --git a/internal/modules/extract/dnd/spells/config_test.go b/internal/modules/extract/dnd/spells/config_test.go new file mode 100644 index 0000000..b692c83 --- /dev/null +++ b/internal/modules/extract/dnd/spells/config_test.go @@ -0,0 +1,276 @@ +package spells + +import ( + "context" + "os" + "strings" + "testing" + + "gitea.maximumdirect.net/eric/notarius/internal/core/config" + "gitea.maximumdirect.net/eric/notarius/internal/core/source" + "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" + "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" + "gitea.maximumdirect.net/eric/notarius/internal/modules/input/seriatim" +) + +func TestPipelineConfigLoadsAndResolvesWithDNDSpellsExtractor(t *testing.T) { + data, err := os.ReadFile("testdata/pipeline.yml") + if err != nil { + t.Fatalf("ReadFile(pipeline.yml) error = %v, want nil", err) + } + fileCfg, err := config.ParseFileConfigYAML(data) + if err != nil { + t.Fatalf("ParseFileConfigYAML() error = %v, want nil", err) + } + + cfg := config.Default() + if err := cfg.ApplyFileConfig(fileCfg); err != nil { + t.Fatalf("ApplyFileConfig() error = %v, want nil", err) + } + + resolved, err := cfg.Resolve(config.ResolveInput{ + PipelineID: "dnd-spells-fixture", + Catalog: dndSpellsTestCatalog(t, dndSpellsCatalogSpecs{}), + }) + if err != nil { + t.Fatalf("Resolve() error = %v, want nil", err) + } + + if len(resolved.ResolvedPipeline.ArtifactLanes) != 1 { + t.Fatalf("len(ArtifactLanes) = %d, want 1", len(resolved.ResolvedPipeline.ArtifactLanes)) + } + lane := resolved.ResolvedPipeline.ArtifactLanes[0] + if lane.ID != "spells" { + t.Fatalf("lane ID = %q, want spells", lane.ID) + } + if lane.Extract.Module != Key { + t.Fatalf("extract module = %q, want %q", lane.Extract.Module, Key) + } + if resolved.ResolvedPipeline.Digest == "" { + t.Fatal("resolved digest is empty") + } + + again, err := cfg.Resolve(config.ResolveInput{ + PipelineID: "dnd-spells-fixture", + Catalog: dndSpellsTestCatalog(t, dndSpellsCatalogSpecs{}), + }) + if err != nil { + t.Fatalf("second Resolve() error = %v, want nil", err) + } + if resolved.ResolvedPipeline.Digest != again.ResolvedPipeline.Digest { + t.Fatalf("resolved digest = %q, second digest = %q; want stable digest", resolved.ResolvedPipeline.Digest, again.ResolvedPipeline.Digest) + } +} + +func TestPipelineConfigRejectsMissingTranscriptCapabilityForDNDSpells(t *testing.T) { + inputSpec := seriatim.ModuleSpec() + inputSpec.Provides = withoutCapability(inputSpec.Provides, "source.transcript") + chunkSpec := dndSpellsChunkerSpec() + chunkSpec.Requires = nil + + _, err := loadDNDSpellsPipelineConfig(t).Resolve(config.ResolveInput{ + PipelineID: "dnd-spells-fixture", + Catalog: dndSpellsTestCatalog(t, dndSpellsCatalogSpecs{ + input: inputSpec, + chunk: chunkSpec, + }), + }) + if err == nil { + t.Fatal("Resolve() error = nil, want missing capability error") + } + if !strings.Contains(err.Error(), "missing capability") || + !strings.Contains(err.Error(), "source.transcript") || + !strings.Contains(err.Error(), Key) { + t.Fatalf("Resolve() error = %q, want dnd/spells missing source.transcript capability", err.Error()) + } +} + +func TestPipelineConfigRejectsMissingSpellCastsCapabilityForAppendOrder(t *testing.T) { + extractorSpec := ModuleSpec() + extractorSpec.Provides = withoutCapability(extractorSpec.Provides, "dnd.spell_casts") + + _, err := loadDNDSpellsPipelineConfig(t).Resolve(config.ResolveInput{ + PipelineID: "dnd-spells-fixture", + Catalog: dndSpellsTestCatalog(t, dndSpellsCatalogSpecs{ + extractor: extractorSpec, + }), + }) + if err == nil { + t.Fatal("Resolve() error = nil, want missing capability error") + } + if !strings.Contains(err.Error(), "missing capability") || + !strings.Contains(err.Error(), "dnd.spell_casts") || + !strings.Contains(err.Error(), pipeline.DefaultMergeModule) { + t.Fatalf("Resolve() error = %q, want appendorder missing dnd.spell_casts capability", err.Error()) + } +} + +func TestPipelineConfigRejectsUnknownLaneSelection(t *testing.T) { + _, err := loadDNDSpellsPipelineConfig(t).Resolve(config.ResolveInput{ + PipelineID: "dnd-spells-fixture", + Only: []string{"missing"}, + Catalog: dndSpellsTestCatalog(t, dndSpellsCatalogSpecs{}), + }) + if err == nil { + t.Fatal("Resolve() error = nil, want unknown lane error") + } + if !strings.Contains(err.Error(), "selected artifact lane") || !strings.Contains(err.Error(), "missing") { + t.Fatalf("Resolve() error = %q, want unknown lane context", err.Error()) + } +} + +func loadDNDSpellsPipelineConfig(t *testing.T) config.Config { + t.Helper() + + data, err := os.ReadFile("testdata/pipeline.yml") + if err != nil { + t.Fatalf("ReadFile(pipeline.yml) error = %v, want nil", err) + } + fileCfg, err := config.ParseFileConfigYAML(data) + if err != nil { + t.Fatalf("ParseFileConfigYAML() error = %v, want nil", err) + } + cfg := config.Default() + if err := cfg.ApplyFileConfig(fileCfg); err != nil { + t.Fatalf("ApplyFileConfig() error = %v, want nil", err) + } + return cfg +} + +type dndSpellsCatalogSpecs struct { + input pipeline.ModuleSpec + chunk pipeline.ModuleSpec + extractor pipeline.ModuleSpec +} + +func dndSpellsTestCatalog(t *testing.T, specs dndSpellsCatalogSpecs) pipeline.ModuleCatalog { + t.Helper() + + inputs := pipeline.NewInputAdapterRegistry() + chunkers := pipeline.NewChunkerRegistry() + extractors := pipeline.NewExtractorRegistry() + mergers := pipeline.NewMergerRegistry() + normalizers := pipeline.NewNormalizerRegistry() + outputs := pipeline.NewOutputEncoderRegistry() + + if specs.input.Key == "" { + if err := seriatim.Register(inputs); err != nil { + t.Fatalf("register seriatim input: %v", err) + } + } else if err := inputs.RegisterWithSpec(specs.input, func() (contracts.InputAdapter, error) { + return seriatim.New(), nil + }); err != nil { + t.Fatalf("register seriatim input override: %v", err) + } + + chunkSpec := specs.chunk + if chunkSpec.Key == "" { + chunkSpec = dndSpellsChunkerSpec() + } + if err := chunkers.RegisterWithSpec(chunkSpec, func() (contracts.Chunker, error) { + return dndSpellsChunker{}, nil + }); err != nil { + t.Fatalf("register chunker: %v", err) + } + + if specs.extractor.Key == "" { + if err := Register(extractors); err != nil { + t.Fatalf("register dnd spells extractor: %v", err) + } + } else if err := extractors.RegisterWithSpec(specs.extractor, func() (contracts.Extractor, error) { + return New(), nil + }); err != nil { + t.Fatalf("register dnd spells extractor override: %v", err) + } + + if err := mergers.RegisterWithSpec(pipeline.ModuleSpec{ + Key: pipeline.DefaultMergeModule, + Stage: pipeline.StageMerge, + Requires: []string{"dnd.spell_casts"}, + }, func() (contracts.Merger, error) { + return pipeline.AppendOrderMerger{}, nil + }); err != nil { + t.Fatalf("register merger: %v", err) + } + if err := normalizers.RegisterWithSpec(pipeline.ModuleSpec{ + Key: pipeline.DefaultNormalizeModule, + Stage: pipeline.StageNormalize, + }, func() (contracts.Normalizer, error) { + return pipeline.NoopNormalizer{}, nil + }); err != nil { + t.Fatalf("register normalizer: %v", err) + } + if err := outputs.RegisterWithSpec(pipeline.ModuleSpec{ + Key: pipeline.DefaultOutputModule, + Stage: pipeline.StageOutput, + }, func() (contracts.OutputEncoder, error) { + return dndSpellsOutput{}, nil + }); err != nil { + t.Fatalf("register output: %v", err) + } + + return pipeline.ModuleCatalog{ + Inputs: inputs, + Chunkers: chunkers, + Extractors: extractors, + Mergers: mergers, + Normalizers: normalizers, + Outputs: outputs, + } +} + +func dndSpellsChunkerSpec() pipeline.ModuleSpec { + return pipeline.ModuleSpec{ + Key: "fake/chunk", + Stage: pipeline.StageChunk, + Requires: []string{"source.transcript"}, + Provides: []string{"chunks"}, + } +} + +type dndSpellsChunker struct{} + +func (dndSpellsChunker) Key() string { + return "fake/chunk" +} + +func (dndSpellsChunker) Chunk(ctx context.Context, req contracts.ChunkRequest) (contracts.ChunkResult, error) { + return contracts.ChunkResult{ + Chunks: []contracts.SourceChunk{ + { + ID: req.Source.ID + ":chunk:0", + SourceID: req.Source.ID, + Index: 0, + Units: append([]source.SourceUnit(nil), req.Source.Units...), + }, + }, + }, nil +} + +type dndSpellsOutput struct{} + +func (dndSpellsOutput) Key() string { + return pipeline.DefaultOutputModule +} + +func (dndSpellsOutput) Encode(ctx context.Context, req contracts.OutputRequest) (contracts.OutputResult, error) { + return contracts.OutputResult{ + Bytes: []byte(`{"encoded":true}`), + ContentType: "application/json", + }, nil +} + +func withoutCapability(capabilities []string, capability string) []string { + filtered := make([]string, 0, len(capabilities)) + for _, candidate := range capabilities { + if candidate != capability { + filtered = append(filtered, candidate) + } + } + return filtered +} + +var ( + _ contracts.Chunker = dndSpellsChunker{} + _ contracts.OutputEncoder = dndSpellsOutput{} +) diff --git a/internal/modules/extract/dnd/spells/runner_test.go b/internal/modules/extract/dnd/spells/runner_test.go new file mode 100644 index 0000000..74ce0b1 --- /dev/null +++ b/internal/modules/extract/dnd/spells/runner_test.go @@ -0,0 +1,214 @@ +package spells + +import ( + "context" + "encoding/json" + "os" + "strings" + "testing" + + "gitea.maximumdirect.net/eric/notarius/internal/core/config" + "gitea.maximumdirect.net/eric/notarius/internal/core/source" + "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" + "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" + "gitea.maximumdirect.net/eric/notarius/internal/modules/input/seriatim" +) + +func TestRunnerProcessesSeriatimInputWithDNDSpellsExtractor(t *testing.T) { + raw := readDNDSpellsFixture(t) + expectedDoc := parseDNDSpellsFixture(t, raw) + resolved := resolveDNDSpellsPipeline(t) + llmClient := &fakeSpellsLLMClient{ + response: extractionResponse{ + SpellCasts: []spellCastResponse{ + { + Caster: "Aria", + Spell: "Cure Wounds", + Effect: "Heals an injured ally.", + NarrativeDescription: "Aria restores the fighter after the fight.", + SourceRefs: []source.SourceRef{ + {SourceID: expectedDoc.ID, StartUnitID: "seg-001", EndUnitID: "seg-001"}, + }, + }, + { + Caster: "Borin", + Spell: "Fire Bolt", + Effect: "Scorches the wight.", + NarrativeDescription: "Borin hurls fire at the wight.", + SourceRefs: []source.SourceRef{ + {SourceID: expectedDoc.ID, StartUnitID: "seg-003", EndUnitID: "seg-003"}, + }, + }, + }, + }, + } + + output, err := pipeline.New(dndSpellsRunnerRegistries(t)).Run(context.Background(), pipeline.RunInput{ + Pipeline: resolved.ResolvedPipeline, + RawInput: raw, + LLMClient: llmClient, + }) + if err != nil { + t.Fatalf("Run() error = %v, want nil", err) + } + + if len(output.Approved) != 2 { + t.Fatalf("len(Approved) = %d, want 2", len(output.Approved)) + } + var first, second SpellCast + if err := json.Unmarshal(output.Approved[0].Payload, &first); err != nil { + t.Fatalf("Unmarshal(first payload) error = %v, want nil", err) + } + if err := json.Unmarshal(output.Approved[1].Payload, &second); err != nil { + t.Fatalf("Unmarshal(second payload) error = %v, want nil", err) + } + if first.Spell != "Cure Wounds" || second.Spell != "Fire Bolt" { + t.Fatalf("approved spell order = %q, %q; want response order", first.Spell, second.Spell) + } + if first.Caster != "Aria" || second.Caster != "Borin" { + t.Fatalf("approved casters = %q, %q; want spell data", first.Caster, second.Caster) + } + for _, artifact := range output.Approved { + if artifact.ExtractorKey != Key || artifact.ArtifactType != ArtifactType || artifact.SchemaVersion != SchemaVersion { + t.Fatalf("approved artifact envelope = %#v, want dnd spells envelope", artifact) + } + if len(artifact.SourceRefs) != 1 { + t.Fatalf("len(SourceRefs) = %d, want 1", len(artifact.SourceRefs)) + } + if err := source.ValidateRef(expectedDoc, artifact.SourceRefs[0]); err != nil { + t.Fatalf("ValidateRef() error = %v, want nil", err) + } + } + + if output.Manifest.InputModule != seriatim.Key { + t.Fatalf("manifest input module = %q, want %q", output.Manifest.InputModule, seriatim.Key) + } + if output.Manifest.ValidationStatus != "approved" { + t.Fatalf("ValidationStatus = %q, want approved", output.Manifest.ValidationStatus) + } + if len(output.Manifest.ArtifactLanes) != 1 { + t.Fatalf("len(ArtifactLanes) = %d, want 1", len(output.Manifest.ArtifactLanes)) + } + lane := output.Manifest.ArtifactLanes[0] + if lane.ID != "spells" || lane.Extractor != Key { + t.Fatalf("manifest lane = %#v, want spells lane with dnd/spells extractor", lane) + } + if output.ContentType != "application/json" { + t.Fatalf("ContentType = %q, want application/json", output.ContentType) + } +} + +func TestRunnerRejectsDNDSpellCastWithInvalidSourceRef(t *testing.T) { + raw := readDNDSpellsFixture(t) + resolved := resolveDNDSpellsPipeline(t) + llmClient := &fakeSpellsLLMClient{ + response: extractionResponse{ + SpellCasts: []spellCastResponse{ + { + Caster: "Aria", + Spell: "Cure Wounds", + Effect: "Heals an injured ally.", + NarrativeDescription: "Aria restores the fighter after the fight.", + SourceRefs: []source.SourceRef{ + {SourceID: "spell-session", StartUnitID: "seg-999", EndUnitID: "seg-999"}, + }, + }, + }, + }, + } + + output, err := pipeline.New(dndSpellsRunnerRegistries(t)).Run(context.Background(), pipeline.RunInput{ + Pipeline: resolved.ResolvedPipeline, + RawInput: raw, + LLMClient: llmClient, + }) + if err != nil { + t.Fatalf("Run() error = %v, want nil", err) + } + if len(output.Approved) != 0 { + t.Fatalf("len(Approved) = %d, want 0", len(output.Approved)) + } + if len(output.Rejected) != 1 { + t.Fatalf("len(Rejected) = %d, want 1", len(output.Rejected)) + } + rejected := output.Rejected[0] + if rejected.ValidatorName != sourceRefValidatorName { + t.Fatalf("ValidatorName = %q, want %q", rejected.ValidatorName, sourceRefValidatorName) + } + if rejected.ReasonCode != reasonInvalidSourceRef { + t.Fatalf("ReasonCode = %q, want %q", rejected.ReasonCode, reasonInvalidSourceRef) + } + if output.Manifest.ValidationStatus != "rejected" { + t.Fatalf("ValidationStatus = %q, want rejected", output.Manifest.ValidationStatus) + } +} + +func TestRunnerFailsWhenDNDSpellsExtractorReturnsMalformedOutput(t *testing.T) { + raw := readDNDSpellsFixture(t) + resolved := resolveDNDSpellsPipeline(t) + llmClient := &fakeSpellsLLMClient{response: extractionResponse{}} + + output, err := pipeline.New(dndSpellsRunnerRegistries(t)).Run(context.Background(), pipeline.RunInput{ + Pipeline: resolved.ResolvedPipeline, + RawInput: raw, + LLMClient: llmClient, + }) + if err == nil { + t.Fatal("Run() error = nil, want malformed extraction error") + } + if !strings.Contains(err.Error(), "extract lane") || + !strings.Contains(err.Error(), "dnd spells") || + !strings.Contains(err.Error(), "spell_casts") { + t.Fatalf("Run() error = %q, want D&D spells extraction context", err.Error()) + } + if output.Manifest.ValidationStatus != "failed" { + t.Fatalf("ValidationStatus = %q, want failed", output.Manifest.ValidationStatus) + } +} + +func resolveDNDSpellsPipeline(t *testing.T) config.EffectiveConfig { + t.Helper() + + resolved, err := loadDNDSpellsPipelineConfig(t).Resolve(config.ResolveInput{ + PipelineID: "dnd-spells-fixture", + Catalog: dndSpellsTestCatalog(t, dndSpellsCatalogSpecs{}), + }) + if err != nil { + t.Fatalf("Resolve() error = %v, want nil", err) + } + return resolved +} + +func dndSpellsRunnerRegistries(t *testing.T) pipeline.Registries { + t.Helper() + + catalog := dndSpellsTestCatalog(t, dndSpellsCatalogSpecs{}) + return pipeline.Registries{ + Inputs: catalog.Inputs, + Chunkers: catalog.Chunkers, + Extractors: catalog.Extractors, + Mergers: catalog.Mergers, + Normalizers: catalog.Normalizers, + Outputs: catalog.Outputs, + } +} + +func readDNDSpellsFixture(t *testing.T) []byte { + t.Helper() + + raw, err := os.ReadFile("testdata/seriatim_spell_session.json") + if err != nil { + t.Fatalf("ReadFile(seriatim_spell_session.json) error = %v, want nil", err) + } + return raw +} + +func parseDNDSpellsFixture(t *testing.T, raw []byte) *source.SourceDocument { + t.Helper() + + doc, err := seriatim.New().Parse(context.Background(), contracts.ParseRequest{Raw: raw}) + if err != nil { + t.Fatalf("Parse() error = %v, want nil", err) + } + return doc +} diff --git a/internal/modules/extract/dnd/spells/testdata/pipeline.yml b/internal/modules/extract/dnd/spells/testdata/pipeline.yml new file mode 100644 index 0000000..1d40403 --- /dev/null +++ b/internal/modules/extract/dnd/spells/testdata/pipeline.yml @@ -0,0 +1,11 @@ +version: 1 +pipelines: + dnd-spells-fixture: + input: seriatim + chunk: fake/chunk + artifacts: + spells: + extract: dnd/spells + merge: appendorder + normalize: noop + output: json diff --git a/internal/modules/extract/dnd/spells/testdata/seriatim_spell_session.json b/internal/modules/extract/dnd/spells/testdata/seriatim_spell_session.json new file mode 100644 index 0000000..091049c --- /dev/null +++ b/internal/modules/extract/dnd/spells/testdata/seriatim_spell_session.json @@ -0,0 +1,29 @@ +{ + "metadata": { + "id": "spell-session", + "title": "Synthetic D&D spell session" + }, + "segments": [ + { + "id": "seg-001", + "start": 0, + "end": 4, + "speaker": "Alice", + "text": "Aria raises her holy symbol and casts Cure Wounds." + }, + { + "id": "seg-002", + "start": 4, + "end": 8, + "speaker": "DM", + "text": "The bandit mage casts Shield as the blow lands." + }, + { + "id": "seg-003", + "start": 8, + "end": 12, + "speaker": "Bob", + "text": "Borin points at the wight and casts Fire Bolt." + } + ] +}