Organize D&D extensions by domain
This commit is contained in:
289
internal/modules/integration/dnd_spells_config_test.go
Normal file
289
internal/modules/integration/dnd_spells_config_test.go
Normal file
@@ -0,0 +1,289 @@
|
||||
package integration_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/spells"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/generic/merge/appendorder"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/generic/normalize/noop"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/seriatim/input/transcript"
|
||||
)
|
||||
|
||||
func TestPipelineConfigLoadsAndResolvesWithDNDSpellsExtractor(t *testing.T) {
|
||||
data, err := os.ReadFile("testdata/pipeline.yml")
|
||||
if err != nil {
|
||||
t.Fatalf("ReadFile(pipeline.yml) error = %v, want nil", err)
|
||||
}
|
||||
fileCfg, err := config.ParseFileConfigYAML(data)
|
||||
if err != nil {
|
||||
t.Fatalf("ParseFileConfigYAML() error = %v, want nil", err)
|
||||
}
|
||||
|
||||
cfg := config.Default()
|
||||
if err := cfg.ApplyFileConfig(fileCfg); err != nil {
|
||||
t.Fatalf("ApplyFileConfig() error = %v, want nil", err)
|
||||
}
|
||||
|
||||
resolved, err := cfg.Resolve(config.ResolveInput{
|
||||
PipelineID: "dnd-spells-fixture",
|
||||
Catalog: dndSpellsTestCatalog(t, dndSpellsCatalogSpecs{}),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Resolve() error = %v, want nil", err)
|
||||
}
|
||||
|
||||
if len(resolved.ResolvedPipeline.ArtifactLanes) != 1 {
|
||||
t.Fatalf("len(ArtifactLanes) = %d, want 1", len(resolved.ResolvedPipeline.ArtifactLanes))
|
||||
}
|
||||
lane := resolved.ResolvedPipeline.ArtifactLanes[0]
|
||||
if lane.ID != "spells" {
|
||||
t.Fatalf("lane ID = %q, want spells", lane.ID)
|
||||
}
|
||||
if lane.Extract.Module != spells.Key {
|
||||
t.Fatalf("extract module = %q, want %q", lane.Extract.Module, spells.Key)
|
||||
}
|
||||
if resolved.ResolvedPipeline.Digest == "" {
|
||||
t.Fatal("resolved digest is empty")
|
||||
}
|
||||
|
||||
again, err := cfg.Resolve(config.ResolveInput{
|
||||
PipelineID: "dnd-spells-fixture",
|
||||
Catalog: dndSpellsTestCatalog(t, dndSpellsCatalogSpecs{}),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("second Resolve() error = %v, want nil", err)
|
||||
}
|
||||
if resolved.ResolvedPipeline.Digest != again.ResolvedPipeline.Digest {
|
||||
t.Fatalf("resolved digest = %q, second digest = %q; want stable digest", resolved.ResolvedPipeline.Digest, again.ResolvedPipeline.Digest)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPipelineConfigRejectsMissingTranscriptCapabilityForDNDSpells(t *testing.T) {
|
||||
inputSpec := transcript.ModuleSpec()
|
||||
inputSpec.Provides = withoutCapability(inputSpec.Provides, "source.transcript")
|
||||
chunkSpec := dndSpellsChunkerSpec()
|
||||
chunkSpec.Requires = nil
|
||||
|
||||
_, err := loadDNDSpellsPipelineConfig(t).Resolve(config.ResolveInput{
|
||||
PipelineID: "dnd-spells-fixture",
|
||||
Catalog: dndSpellsTestCatalog(t, dndSpellsCatalogSpecs{
|
||||
input: inputSpec,
|
||||
chunk: chunkSpec,
|
||||
}),
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("Resolve() error = nil, want missing capability error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "missing capability") ||
|
||||
!strings.Contains(err.Error(), "source.transcript") ||
|
||||
!strings.Contains(err.Error(), spells.Key) {
|
||||
t.Fatalf("Resolve() error = %q, want dnd/spells missing source.transcript capability", err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
func TestPipelineConfigRejectsMissingSpellCastsCapabilityForAppendOrder(t *testing.T) {
|
||||
extractorSpec := spells.ModuleSpec()
|
||||
extractorSpec.Provides = withoutCapability(extractorSpec.Provides, "dnd.spell_casts")
|
||||
|
||||
_, err := loadDNDSpellsPipelineConfig(t).Resolve(config.ResolveInput{
|
||||
PipelineID: "dnd-spells-fixture",
|
||||
Catalog: dndSpellsTestCatalog(t, dndSpellsCatalogSpecs{
|
||||
extractor: extractorSpec,
|
||||
}),
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("Resolve() error = nil, want missing capability error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "missing capability") ||
|
||||
!strings.Contains(err.Error(), "dnd.spell_casts") ||
|
||||
!strings.Contains(err.Error(), pipeline.DefaultMergeModule) {
|
||||
t.Fatalf("Resolve() error = %q, want appendorder missing dnd.spell_casts capability", err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
func TestPipelineConfigRejectsUnknownLaneSelection(t *testing.T) {
|
||||
_, err := loadDNDSpellsPipelineConfig(t).Resolve(config.ResolveInput{
|
||||
PipelineID: "dnd-spells-fixture",
|
||||
Only: []string{"missing"},
|
||||
Catalog: dndSpellsTestCatalog(t, dndSpellsCatalogSpecs{}),
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("Resolve() error = nil, want unknown lane error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "selected artifact lane") || !strings.Contains(err.Error(), "missing") {
|
||||
t.Fatalf("Resolve() error = %q, want unknown lane context", err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
func loadDNDSpellsPipelineConfig(t *testing.T) config.Config {
|
||||
t.Helper()
|
||||
|
||||
data, err := os.ReadFile("testdata/pipeline.yml")
|
||||
if err != nil {
|
||||
t.Fatalf("ReadFile(pipeline.yml) error = %v, want nil", err)
|
||||
}
|
||||
fileCfg, err := config.ParseFileConfigYAML(data)
|
||||
if err != nil {
|
||||
t.Fatalf("ParseFileConfigYAML() error = %v, want nil", err)
|
||||
}
|
||||
cfg := config.Default()
|
||||
if err := cfg.ApplyFileConfig(fileCfg); err != nil {
|
||||
t.Fatalf("ApplyFileConfig() error = %v, want nil", err)
|
||||
}
|
||||
return cfg
|
||||
}
|
||||
|
||||
type dndSpellsCatalogSpecs struct {
|
||||
input pipeline.ModuleSpec
|
||||
chunk pipeline.ModuleSpec
|
||||
extractor pipeline.ModuleSpec
|
||||
}
|
||||
|
||||
func dndSpellsTestCatalog(t *testing.T, specs dndSpellsCatalogSpecs) pipeline.ModuleCatalog {
|
||||
t.Helper()
|
||||
|
||||
inputs := pipeline.NewInputAdapterRegistry()
|
||||
chunkers := pipeline.NewChunkerRegistry()
|
||||
extractors := pipeline.NewExtractorRegistry()
|
||||
mergers := pipeline.NewMergerRegistry()
|
||||
normalizers := pipeline.NewNormalizerRegistry()
|
||||
outputs := pipeline.NewOutputEncoderRegistry()
|
||||
|
||||
if specs.input.Key == "" {
|
||||
if err := transcript.Register(inputs); err != nil {
|
||||
t.Fatalf("register seriatim input: %v", err)
|
||||
}
|
||||
} else if err := inputs.RegisterWithSpec(specs.input, func() (contracts.InputAdapter, error) {
|
||||
return transcript.New(), nil
|
||||
}); err != nil {
|
||||
t.Fatalf("register seriatim input override: %v", err)
|
||||
}
|
||||
|
||||
chunkSpec := specs.chunk
|
||||
if chunkSpec.Key == "" {
|
||||
chunkSpec = dndSpellsChunkerSpec()
|
||||
}
|
||||
if err := chunkers.RegisterWithSpec(chunkSpec, func() (contracts.Chunker, error) {
|
||||
return dndSpellsChunker{}, nil
|
||||
}); err != nil {
|
||||
t.Fatalf("register chunker: %v", err)
|
||||
}
|
||||
|
||||
if specs.extractor.Key == "" {
|
||||
if err := spells.Register(extractors); err != nil {
|
||||
t.Fatalf("register dnd spells extractor: %v", err)
|
||||
}
|
||||
} else if err := extractors.RegisterWithSpec(specs.extractor, func() (contracts.Extractor, error) {
|
||||
return spells.New(), nil
|
||||
}); err != nil {
|
||||
t.Fatalf("register dnd spells extractor override: %v", err)
|
||||
}
|
||||
|
||||
if err := mergers.RegisterWithSpec(pipeline.ModuleSpec{
|
||||
Key: pipeline.DefaultMergeModule,
|
||||
Stage: pipeline.StageMerge,
|
||||
Requires: []string{"dnd.spell_casts"},
|
||||
}, func() (contracts.Merger, error) {
|
||||
return appendorder.New(), nil
|
||||
}); err != nil {
|
||||
t.Fatalf("register merger: %v", err)
|
||||
}
|
||||
if err := normalizers.RegisterWithSpec(pipeline.ModuleSpec{
|
||||
Key: pipeline.DefaultNormalizeModule,
|
||||
Stage: pipeline.StageNormalize,
|
||||
}, func() (contracts.Normalizer, error) {
|
||||
return noop.New(), nil
|
||||
}); err != nil {
|
||||
t.Fatalf("register normalizer: %v", err)
|
||||
}
|
||||
if err := outputs.RegisterWithSpec(pipeline.ModuleSpec{
|
||||
Key: pipeline.DefaultOutputModule,
|
||||
Stage: pipeline.StageOutput,
|
||||
}, func() (contracts.OutputEncoder, error) {
|
||||
return dndSpellsOutput{}, nil
|
||||
}); err != nil {
|
||||
t.Fatalf("register output: %v", err)
|
||||
}
|
||||
|
||||
return pipeline.ModuleCatalog{
|
||||
Inputs: inputs,
|
||||
Chunkers: chunkers,
|
||||
Extractors: extractors,
|
||||
Mergers: mergers,
|
||||
Normalizers: normalizers,
|
||||
ValidatorChains: pipeline.NewValidatorChainRegistry(),
|
||||
Outputs: outputs,
|
||||
}
|
||||
}
|
||||
|
||||
func dndSpellsChunkerSpec() pipeline.ModuleSpec {
|
||||
return pipeline.ModuleSpec{
|
||||
Key: "fake/chunk",
|
||||
Stage: pipeline.StageChunk,
|
||||
Requires: []string{"source.transcript"},
|
||||
Provides: []string{"chunks"},
|
||||
}
|
||||
}
|
||||
|
||||
type dndSpellsChunker struct{}
|
||||
|
||||
func (dndSpellsChunker) Key() string {
|
||||
return "fake/chunk"
|
||||
}
|
||||
|
||||
func (dndSpellsChunker) ReferenceSlots() []contracts.ReferenceSlot {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (dndSpellsChunker) Chunk(ctx context.Context, req contracts.ChunkRequest) (contracts.ChunkResult, error) {
|
||||
return contracts.ChunkResult{
|
||||
Chunks: []contracts.SourceChunk{
|
||||
{
|
||||
ID: req.Source.ID + ":chunk:0",
|
||||
SourceID: req.Source.ID,
|
||||
Index: 0,
|
||||
StartUnitID: req.Source.Units[0].ID,
|
||||
EndUnitID: req.Source.Units[len(req.Source.Units)-1].ID,
|
||||
Content: []byte(`{"units":[1,2,3]}`),
|
||||
MediaType: "application/json",
|
||||
Units: append([]source.SourceUnit(nil), req.Source.Units...),
|
||||
},
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
type dndSpellsOutput struct{}
|
||||
|
||||
func (dndSpellsOutput) Key() string {
|
||||
return pipeline.DefaultOutputModule
|
||||
}
|
||||
|
||||
func (dndSpellsOutput) Encode(ctx context.Context, req contracts.OutputRequest) (contracts.OutputResult, error) {
|
||||
return contracts.OutputResult{
|
||||
Files: []contracts.OutputFile{
|
||||
{Name: "output.json", ContentType: "application/json", Bytes: []byte(`{"encoded":true}`)},
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
func withoutCapability(capabilities []string, capability string) []string {
|
||||
filtered := make([]string, 0, len(capabilities))
|
||||
for _, candidate := range capabilities {
|
||||
if candidate != capability {
|
||||
filtered = append(filtered, candidate)
|
||||
}
|
||||
}
|
||||
return filtered
|
||||
}
|
||||
|
||||
var (
|
||||
_ contracts.Chunker = dndSpellsChunker{}
|
||||
_ contracts.OutputEncoder = dndSpellsOutput{}
|
||||
)
|
||||
63
internal/modules/integration/dnd_spells_helpers_test.go
Normal file
63
internal/modules/integration/dnd_spells_helpers_test.go
Normal file
@@ -0,0 +1,63 @@
|
||||
package integration_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared"
|
||||
)
|
||||
|
||||
type extractionResponse struct {
|
||||
SpellCasts []spellCastResponse `json:"spell_casts"`
|
||||
}
|
||||
|
||||
type spellCastResponse struct {
|
||||
Caster string `json:"caster"`
|
||||
Spell string `json:"spell"`
|
||||
Effect string `json:"effect"`
|
||||
NarrativeDescription string `json:"narrative_description"`
|
||||
SourceRefs []shared.SourceRefResponse `json:"source_refs"`
|
||||
}
|
||||
|
||||
type fakeSpellsLLMClient struct {
|
||||
response extractionResponse
|
||||
requests []contracts.StructuredCompletionRequest
|
||||
}
|
||||
|
||||
func (client *fakeSpellsLLMClient) CompleteStructured(_ context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
|
||||
client.requests = append(client.requests, cloneStructuredCompletionRequest(req))
|
||||
content, err := json.Marshal(client.response)
|
||||
if err != nil {
|
||||
return contracts.StructuredCompletionResponse{}, err
|
||||
}
|
||||
if err := json.Unmarshal(content, out); err != nil {
|
||||
return contracts.StructuredCompletionResponse{}, fmt.Errorf("populate structured target: %w", err)
|
||||
}
|
||||
return contracts.StructuredCompletionResponse{Content: content}, nil
|
||||
}
|
||||
|
||||
func responseSourceRefs(sourceID string, startUnitID int, endUnitID int) []shared.SourceRefResponse {
|
||||
return []shared.SourceRefResponse{
|
||||
{
|
||||
SourceID: sourceID,
|
||||
StartUnitID: shared.UnitRefFromInt(startUnitID),
|
||||
EndUnitID: shared.UnitRefFromInt(endUnitID),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func cloneStructuredCompletionRequest(req contracts.StructuredCompletionRequest) contracts.StructuredCompletionRequest {
|
||||
req.Inputs = req.Inputs.Clone()
|
||||
if len(req.Vars) == 0 {
|
||||
req.Vars = nil
|
||||
return req
|
||||
}
|
||||
vars := make(map[string]any, len(req.Vars))
|
||||
for key, value := range req.Vars {
|
||||
vars[key] = value
|
||||
}
|
||||
req.Vars = vars
|
||||
return req
|
||||
}
|
||||
362
internal/modules/integration/dnd_spells_runner_test.go
Normal file
362
internal/modules/integration/dnd_spells_runner_test.go
Normal file
@@ -0,0 +1,362 @@
|
||||
package integration_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/spells"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/seriatim/input/transcript"
|
||||
)
|
||||
|
||||
func TestRunnerProcessesSeriatimInputWithDNDSpellsExtractor(t *testing.T) {
|
||||
raw := readDNDSpellsFixture(t)
|
||||
expectedDoc := parseDNDSpellsFixture(t, raw)
|
||||
resolved := resolveDNDSpellsPipeline(t)
|
||||
llmClient := &fakeSpellsLLMClient{
|
||||
response: extractionResponse{
|
||||
SpellCasts: []spellCastResponse{
|
||||
{
|
||||
Caster: "Aria",
|
||||
Spell: "Cure Wounds",
|
||||
Effect: "Heals an injured ally.",
|
||||
NarrativeDescription: "Aria restores the fighter after the fight.",
|
||||
SourceRefs: responseSourceRefs(expectedDoc.ID, 1, 1),
|
||||
},
|
||||
{
|
||||
Caster: "Borin",
|
||||
Spell: "Fire Bolt",
|
||||
Effect: "Scorches the wight.",
|
||||
NarrativeDescription: "Borin hurls fire at the wight.",
|
||||
SourceRefs: responseSourceRefs(expectedDoc.ID, 3, 3),
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
output, err := pipeline.New(dndSpellsRunnerRegistries(t)).Run(context.Background(), pipeline.RunInput{
|
||||
Pipeline: resolved.ResolvedPipeline,
|
||||
RawInput: raw,
|
||||
LLMClient: llmClient,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v, want nil", err)
|
||||
}
|
||||
|
||||
if len(output.NormalizeOutputs) != 1 {
|
||||
t.Fatalf("len(NormalizeOutputs) = %d, want 1", len(output.NormalizeOutputs))
|
||||
}
|
||||
rawOutput := output.NormalizeOutputs[0]
|
||||
if rawOutput.LaneID != "spells" || rawOutput.Schema.ID != spells.ResponseSchemaID || rawOutput.Schema.Version != spells.SchemaVersion {
|
||||
t.Fatalf("raw output envelope = %#v, want dnd spells schema on spells lane", rawOutput)
|
||||
}
|
||||
response := decodeRunnerSpellResponse(t, rawOutput.Payload.Content)
|
||||
if len(response.SpellCasts) != 2 {
|
||||
t.Fatalf("len(spell_casts) = %d, want 2", len(response.SpellCasts))
|
||||
}
|
||||
first, second := response.SpellCasts[0], response.SpellCasts[1]
|
||||
if first.Spell != "Cure Wounds" || second.Spell != "Fire Bolt" {
|
||||
t.Fatalf("spell order = %q, %q; want source-unit order", first.Spell, second.Spell)
|
||||
}
|
||||
if first.Caster != "Aria" || second.Caster != "Borin" {
|
||||
t.Fatalf("casters = %q, %q; want spell data", first.Caster, second.Caster)
|
||||
}
|
||||
for _, spell := range response.SpellCasts {
|
||||
if len(spell.SourceRefs) != 1 {
|
||||
t.Fatalf("len(SourceRefs) = %d, want 1", len(spell.SourceRefs))
|
||||
}
|
||||
if spell.SourceRefs[0].SourceID != expectedDoc.ID {
|
||||
t.Fatalf("SourceID = %q, want fixture document ID", spell.SourceRefs[0].SourceID)
|
||||
}
|
||||
}
|
||||
|
||||
if output.Manifest.InputModule != transcript.Key {
|
||||
t.Fatalf("manifest input module = %q, want %q", output.Manifest.InputModule, transcript.Key)
|
||||
}
|
||||
if output.Manifest.ValidationStatus != "approved" {
|
||||
t.Fatalf("ValidationStatus = %q, want approved", output.Manifest.ValidationStatus)
|
||||
}
|
||||
if len(output.Manifest.ArtifactLanes) != 1 {
|
||||
t.Fatalf("len(ArtifactLanes) = %d, want 1", len(output.Manifest.ArtifactLanes))
|
||||
}
|
||||
lane := output.Manifest.ArtifactLanes[0]
|
||||
if lane.ID != "spells" || lane.Extractor != spells.Key {
|
||||
t.Fatalf("manifest lane = %#v, want spells lane with dnd/spells extractor", lane)
|
||||
}
|
||||
extractorMetadata, ok := lane.Metadata["extractor"].(map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf("manifest lane metadata = %#v, want extractor metadata", lane.Metadata)
|
||||
}
|
||||
if extractorMetadata["prompt_id"] != spells.PromptID ||
|
||||
extractorMetadata["response_schema_key"] != string(spells.ResponseSchemaKey) ||
|
||||
extractorMetadata["response_schema_name"] != spells.ResponseSchemaName {
|
||||
t.Fatalf("extractor metadata = %#v, want prompt/schema identifiers", extractorMetadata)
|
||||
}
|
||||
if len(output.OutputFiles) != 1 {
|
||||
t.Fatalf("len(OutputFiles) = %d, want 1", len(output.OutputFiles))
|
||||
}
|
||||
if output.OutputFiles[0].ContentType != "application/json" {
|
||||
t.Fatalf("ContentType = %q, want application/json", output.OutputFiles[0].ContentType)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerPassesPartyAndGlossaryReferencesToDNDSpellsPrompt(t *testing.T) {
|
||||
raw := readDNDSpellsFixture(t)
|
||||
expectedDoc := parseDNDSpellsFixture(t, raw)
|
||||
resolved := resolveDNDSpellsPipeline(t)
|
||||
resolved.ResolvedPipeline.ArtifactLanes[0].ExtractReferences.ReferenceSet = dndSpellsReferenceSet(
|
||||
"Aria: party cleric\nBorin: fighter",
|
||||
"Fire Bolt: evocation cantrip",
|
||||
)
|
||||
llmClient := &fakeSpellsLLMClient{
|
||||
response: extractionResponse{
|
||||
SpellCasts: []spellCastResponse{
|
||||
{
|
||||
Caster: "Borin",
|
||||
Spell: "Fire Bolt",
|
||||
Effect: "Scorches the wight.",
|
||||
NarrativeDescription: "Borin hurls fire at the wight.",
|
||||
SourceRefs: responseSourceRefs(expectedDoc.ID, 3, 3),
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
output, err := pipeline.New(dndSpellsRunnerRegistries(t)).Run(context.Background(), pipeline.RunInput{
|
||||
Pipeline: resolved.ResolvedPipeline,
|
||||
RawInput: raw,
|
||||
LLMClient: llmClient,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v, want nil", err)
|
||||
}
|
||||
|
||||
if len(output.NormalizeOutputs) != 1 {
|
||||
t.Fatalf("len(NormalizeOutputs) = %d, want 1", len(output.NormalizeOutputs))
|
||||
}
|
||||
if len(output.Manifest.References) != 2 {
|
||||
t.Fatalf("manifest references = %#v, want party and glossary provenance", output.Manifest.References)
|
||||
}
|
||||
if len(llmClient.requests) != 1 {
|
||||
t.Fatalf("LLM calls = %d, want 1", len(llmClient.requests))
|
||||
}
|
||||
request := llmClient.requests[0]
|
||||
if request.PromptID != spells.PromptID || request.PromptVersion != spells.SchemaVersion {
|
||||
t.Fatalf("prompt = %q/%q, want %q/%q", request.PromptID, request.PromptVersion, spells.PromptID, spells.SchemaVersion)
|
||||
}
|
||||
if got := string(request.Inputs["party"].Content); got != "Aria: party cleric\nBorin: fighter" {
|
||||
t.Fatalf("party input = %q, want reference text", got)
|
||||
}
|
||||
if got := string(request.Inputs["glossary"].Content); got != "Fire Bolt: evocation cantrip" {
|
||||
t.Fatalf("glossary input = %q, want reference text", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerDoesNotExtractSpellMentionedOnlyInPartyReference(t *testing.T) {
|
||||
raw := readDNDSpellsFixture(t)
|
||||
resolved := resolveDNDSpellsPipeline(t)
|
||||
resolved.ResolvedPipeline.ArtifactLanes[0].ExtractReferences.ReferenceSet = dndSpellsReferenceSet(
|
||||
"Mira: wizard who can cast Lightning Bolt",
|
||||
"",
|
||||
)
|
||||
llmClient := &fakeSpellsLLMClient{
|
||||
response: extractionResponse{SpellCasts: []spellCastResponse{}},
|
||||
}
|
||||
|
||||
output, err := pipeline.New(dndSpellsRunnerRegistries(t)).Run(context.Background(), pipeline.RunInput{
|
||||
Pipeline: resolved.ResolvedPipeline,
|
||||
RawInput: raw,
|
||||
LLMClient: llmClient,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v, want nil", err)
|
||||
}
|
||||
|
||||
if len(output.NormalizeOutputs) != 1 {
|
||||
t.Fatalf("len(NormalizeOutputs) = %d, want empty spell response output", len(output.NormalizeOutputs))
|
||||
}
|
||||
response := decodeRunnerSpellResponse(t, output.NormalizeOutputs[0].Payload.Content)
|
||||
if len(response.SpellCasts) != 0 {
|
||||
t.Fatalf("spell_casts = %#v, want no party-reference-only spell casts", response.SpellCasts)
|
||||
}
|
||||
if len(llmClient.requests) != 1 {
|
||||
t.Fatalf("LLM calls = %d, want 1", len(llmClient.requests))
|
||||
}
|
||||
request := llmClient.requests[0]
|
||||
if request.PromptID != spells.PromptID || request.PromptVersion != spells.SchemaVersion {
|
||||
t.Fatalf("prompt = %q/%q, want %q/%q", request.PromptID, request.PromptVersion, spells.PromptID, spells.SchemaVersion)
|
||||
}
|
||||
if got := string(request.Inputs["party"].Content); !strings.Contains(got, "Lightning Bolt") {
|
||||
t.Fatalf("party input = %q, want party-reference-only spell in reference input", got)
|
||||
}
|
||||
if output.Manifest.ValidationStatus != "approved" {
|
||||
t.Fatalf("ValidationStatus = %q, want approved empty extraction", output.Manifest.ValidationStatus)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerCarriesDNDSpellCastWithInvalidSourceRefAsRawOutput(t *testing.T) {
|
||||
raw := readDNDSpellsFixture(t)
|
||||
resolved := resolveDNDSpellsPipeline(t)
|
||||
llmClient := &fakeSpellsLLMClient{
|
||||
response: extractionResponse{
|
||||
SpellCasts: []spellCastResponse{
|
||||
{
|
||||
Caster: "Aria",
|
||||
Spell: "Cure Wounds",
|
||||
Effect: "Heals an injured ally.",
|
||||
NarrativeDescription: "Aria restores the fighter after the fight.",
|
||||
SourceRefs: responseSourceRefs("spell-session", 999, 999),
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
output, err := pipeline.New(dndSpellsRunnerRegistries(t)).Run(context.Background(), pipeline.RunInput{
|
||||
Pipeline: resolved.ResolvedPipeline,
|
||||
RawInput: raw,
|
||||
LLMClient: llmClient,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v, want nil", err)
|
||||
}
|
||||
if len(output.NormalizeOutputs) != 1 {
|
||||
t.Fatalf("len(NormalizeOutputs) = %d, want 1", len(output.NormalizeOutputs))
|
||||
}
|
||||
response := decodeRunnerSpellResponse(t, output.NormalizeOutputs[0].Payload.Content)
|
||||
if len(response.SpellCasts) != 1 {
|
||||
t.Fatalf("len(spell_casts) = %d, want 1", len(response.SpellCasts))
|
||||
}
|
||||
if response.SpellCasts[0].SourceRefs[0].SourceID != "spell-session" {
|
||||
t.Fatalf("SourceID = %q, want raw invalid source ref preserved", response.SpellCasts[0].SourceRefs[0].SourceID)
|
||||
}
|
||||
if len(output.Rejected) != 0 {
|
||||
t.Fatalf("len(Rejected) = %d, want 0", len(output.Rejected))
|
||||
}
|
||||
if output.Manifest.ValidationStatus != "approved" {
|
||||
t.Fatalf("ValidationStatus = %q, want approved", output.Manifest.ValidationStatus)
|
||||
}
|
||||
}
|
||||
|
||||
func dndSpellsReferenceSet(party string, glossary string) contracts.ReferenceSet {
|
||||
slots := make(map[string]contracts.ResolvedReferenceSlot)
|
||||
if strings.TrimSpace(party) != "" {
|
||||
slots["party"] = contracts.ResolvedReferenceSlot{
|
||||
Slot: contracts.ReferenceSlot{Name: "party"},
|
||||
Items: []contracts.ReferenceItem{
|
||||
{
|
||||
SlotName: "party",
|
||||
MediaType: "text/plain; charset=utf-8",
|
||||
Content: []byte(party),
|
||||
Digest: "sha256:party",
|
||||
Origin: contracts.ReferenceOrigin{Type: "file", URI: "file:///tmp/party.txt"},
|
||||
SizeBytes: int64(len(party)),
|
||||
BindingSource: contracts.ReferenceBindingSourceConfig,
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
if strings.TrimSpace(glossary) != "" {
|
||||
slots["glossary"] = contracts.ResolvedReferenceSlot{
|
||||
Slot: contracts.ReferenceSlot{Name: "glossary"},
|
||||
Items: []contracts.ReferenceItem{
|
||||
{
|
||||
SlotName: "glossary",
|
||||
MediaType: "text/plain; charset=utf-8",
|
||||
Content: []byte(glossary),
|
||||
Digest: "sha256:glossary",
|
||||
Origin: contracts.ReferenceOrigin{Type: "file", URI: "file:///tmp/glossary.txt"},
|
||||
SizeBytes: int64(len(glossary)),
|
||||
BindingSource: contracts.ReferenceBindingSourceConfig,
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
return contracts.ReferenceSet{Slots: slots}
|
||||
}
|
||||
|
||||
func TestRunnerCarriesMalformedDNDSpellsExtractorOutput(t *testing.T) {
|
||||
raw := readDNDSpellsFixture(t)
|
||||
resolved := resolveDNDSpellsPipeline(t)
|
||||
llmClient := &fakeSpellsLLMClient{response: extractionResponse{}}
|
||||
|
||||
output, err := pipeline.New(dndSpellsRunnerRegistries(t)).Run(context.Background(), pipeline.RunInput{
|
||||
Pipeline: resolved.ResolvedPipeline,
|
||||
RawInput: raw,
|
||||
LLMClient: llmClient,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v, want nil", err)
|
||||
}
|
||||
if len(output.NormalizeOutputs) != 1 {
|
||||
t.Fatalf("len(NormalizeOutputs) = %d, want raw output", len(output.NormalizeOutputs))
|
||||
}
|
||||
if string(output.NormalizeOutputs[0].Payload.Content) != `{"spell_casts":null}` {
|
||||
t.Fatalf("content = %s, want canonical structured output", output.NormalizeOutputs[0].Payload.Content)
|
||||
}
|
||||
if output.Manifest.ValidationStatus != "approved" {
|
||||
t.Fatalf("ValidationStatus = %q, want approved", output.Manifest.ValidationStatus)
|
||||
}
|
||||
}
|
||||
|
||||
func resolveDNDSpellsPipeline(t *testing.T) config.EffectiveConfig {
|
||||
t.Helper()
|
||||
|
||||
resolved, err := loadDNDSpellsPipelineConfig(t).Resolve(config.ResolveInput{
|
||||
PipelineID: "dnd-spells-fixture",
|
||||
Catalog: dndSpellsTestCatalog(t, dndSpellsCatalogSpecs{}),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Resolve() error = %v, want nil", err)
|
||||
}
|
||||
return resolved
|
||||
}
|
||||
|
||||
func dndSpellsRunnerRegistries(t *testing.T) pipeline.Registries {
|
||||
t.Helper()
|
||||
|
||||
catalog := dndSpellsTestCatalog(t, dndSpellsCatalogSpecs{})
|
||||
return pipeline.Registries{
|
||||
Inputs: catalog.Inputs,
|
||||
Chunkers: catalog.Chunkers,
|
||||
Extractors: catalog.Extractors,
|
||||
Mergers: catalog.Mergers,
|
||||
Normalizers: catalog.Normalizers,
|
||||
Outputs: catalog.Outputs,
|
||||
}
|
||||
}
|
||||
|
||||
func readDNDSpellsFixture(t *testing.T) []byte {
|
||||
t.Helper()
|
||||
|
||||
raw, err := os.ReadFile("testdata/seriatim_spell_session.json")
|
||||
if err != nil {
|
||||
t.Fatalf("ReadFile(seriatim_spell_session.json) error = %v, want nil", err)
|
||||
}
|
||||
return raw
|
||||
}
|
||||
|
||||
func parseDNDSpellsFixture(t *testing.T, raw []byte) *source.SourceDocument {
|
||||
t.Helper()
|
||||
|
||||
doc, err := transcript.New().Parse(context.Background(), contracts.ParseRequest{Raw: raw})
|
||||
if err != nil {
|
||||
t.Fatalf("Parse() error = %v, want nil", err)
|
||||
}
|
||||
return doc
|
||||
}
|
||||
|
||||
func decodeRunnerSpellResponse(t *testing.T, raw []byte) extractionResponse {
|
||||
t.Helper()
|
||||
|
||||
var response extractionResponse
|
||||
if err := json.Unmarshal(raw, &response); err != nil {
|
||||
t.Fatalf("Unmarshal(raw output) error = %v, want nil", err)
|
||||
}
|
||||
return response
|
||||
}
|
||||
11
internal/modules/integration/testdata/pipeline.yml
vendored
Normal file
11
internal/modules/integration/testdata/pipeline.yml
vendored
Normal file
@@ -0,0 +1,11 @@
|
||||
version: 2
|
||||
pipelines:
|
||||
dnd-spells-fixture:
|
||||
input: seriatim
|
||||
chunk: fake/chunk
|
||||
artifacts:
|
||||
spells:
|
||||
extract: dnd/spells
|
||||
merge: appendorder
|
||||
normalize: noop
|
||||
output: json
|
||||
29
internal/modules/integration/testdata/seriatim_spell_session.json
vendored
Normal file
29
internal/modules/integration/testdata/seriatim_spell_session.json
vendored
Normal file
@@ -0,0 +1,29 @@
|
||||
{
|
||||
"metadata": {
|
||||
"id": "spell-session",
|
||||
"title": "Synthetic D&D spell session"
|
||||
},
|
||||
"segments": [
|
||||
{
|
||||
"id": 1,
|
||||
"start": 0,
|
||||
"end": 4,
|
||||
"speaker": "Alice",
|
||||
"text": "Aria raises her holy symbol and casts Cure Wounds."
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"start": 4,
|
||||
"end": 8,
|
||||
"speaker": "DM",
|
||||
"text": "The bandit mage casts Shield as the blow lands."
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"start": 8,
|
||||
"end": 12,
|
||||
"speaker": "Bob",
|
||||
"text": "Borin points at the wight and casts Fire Bolt."
|
||||
}
|
||||
]
|
||||
}
|
||||
Reference in New Issue
Block a user