289 lines
8.9 KiB
Go
289 lines
8.9 KiB
Go
package spells
|
|
|
|
import (
|
|
"context"
|
|
"os"
|
|
"strings"
|
|
"testing"
|
|
|
|
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/modules/input/seriatim"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/modules/merge/appendorder"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/modules/normalize/noop"
|
|
)
|
|
|
|
func TestPipelineConfigLoadsAndResolvesWithDNDSpellsExtractor(t *testing.T) {
|
|
data, err := os.ReadFile("testdata/pipeline.yml")
|
|
if err != nil {
|
|
t.Fatalf("ReadFile(pipeline.yml) error = %v, want nil", err)
|
|
}
|
|
fileCfg, err := config.ParseFileConfigYAML(data)
|
|
if err != nil {
|
|
t.Fatalf("ParseFileConfigYAML() error = %v, want nil", err)
|
|
}
|
|
|
|
cfg := config.Default()
|
|
if err := cfg.ApplyFileConfig(fileCfg); err != nil {
|
|
t.Fatalf("ApplyFileConfig() error = %v, want nil", err)
|
|
}
|
|
|
|
resolved, err := cfg.Resolve(config.ResolveInput{
|
|
PipelineID: "dnd-spells-fixture",
|
|
Catalog: dndSpellsTestCatalog(t, dndSpellsCatalogSpecs{}),
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("Resolve() error = %v, want nil", err)
|
|
}
|
|
|
|
if len(resolved.ResolvedPipeline.ArtifactLanes) != 1 {
|
|
t.Fatalf("len(ArtifactLanes) = %d, want 1", len(resolved.ResolvedPipeline.ArtifactLanes))
|
|
}
|
|
lane := resolved.ResolvedPipeline.ArtifactLanes[0]
|
|
if lane.ID != "spells" {
|
|
t.Fatalf("lane ID = %q, want spells", lane.ID)
|
|
}
|
|
if lane.Extract.Module != Key {
|
|
t.Fatalf("extract module = %q, want %q", lane.Extract.Module, Key)
|
|
}
|
|
if resolved.ResolvedPipeline.Digest == "" {
|
|
t.Fatal("resolved digest is empty")
|
|
}
|
|
|
|
again, err := cfg.Resolve(config.ResolveInput{
|
|
PipelineID: "dnd-spells-fixture",
|
|
Catalog: dndSpellsTestCatalog(t, dndSpellsCatalogSpecs{}),
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("second Resolve() error = %v, want nil", err)
|
|
}
|
|
if resolved.ResolvedPipeline.Digest != again.ResolvedPipeline.Digest {
|
|
t.Fatalf("resolved digest = %q, second digest = %q; want stable digest", resolved.ResolvedPipeline.Digest, again.ResolvedPipeline.Digest)
|
|
}
|
|
}
|
|
|
|
func TestPipelineConfigRejectsMissingTranscriptCapabilityForDNDSpells(t *testing.T) {
|
|
inputSpec := seriatim.ModuleSpec()
|
|
inputSpec.Provides = withoutCapability(inputSpec.Provides, "source.transcript")
|
|
chunkSpec := dndSpellsChunkerSpec()
|
|
chunkSpec.Requires = nil
|
|
|
|
_, err := loadDNDSpellsPipelineConfig(t).Resolve(config.ResolveInput{
|
|
PipelineID: "dnd-spells-fixture",
|
|
Catalog: dndSpellsTestCatalog(t, dndSpellsCatalogSpecs{
|
|
input: inputSpec,
|
|
chunk: chunkSpec,
|
|
}),
|
|
})
|
|
if err == nil {
|
|
t.Fatal("Resolve() error = nil, want missing capability error")
|
|
}
|
|
if !strings.Contains(err.Error(), "missing capability") ||
|
|
!strings.Contains(err.Error(), "source.transcript") ||
|
|
!strings.Contains(err.Error(), Key) {
|
|
t.Fatalf("Resolve() error = %q, want dnd/spells missing source.transcript capability", err.Error())
|
|
}
|
|
}
|
|
|
|
func TestPipelineConfigRejectsMissingSpellCastsCapabilityForAppendOrder(t *testing.T) {
|
|
extractorSpec := ModuleSpec()
|
|
extractorSpec.Provides = withoutCapability(extractorSpec.Provides, "dnd.spell_casts")
|
|
|
|
_, err := loadDNDSpellsPipelineConfig(t).Resolve(config.ResolveInput{
|
|
PipelineID: "dnd-spells-fixture",
|
|
Catalog: dndSpellsTestCatalog(t, dndSpellsCatalogSpecs{
|
|
extractor: extractorSpec,
|
|
}),
|
|
})
|
|
if err == nil {
|
|
t.Fatal("Resolve() error = nil, want missing capability error")
|
|
}
|
|
if !strings.Contains(err.Error(), "missing capability") ||
|
|
!strings.Contains(err.Error(), "dnd.spell_casts") ||
|
|
!strings.Contains(err.Error(), pipeline.DefaultMergeModule) {
|
|
t.Fatalf("Resolve() error = %q, want appendorder missing dnd.spell_casts capability", err.Error())
|
|
}
|
|
}
|
|
|
|
func TestPipelineConfigRejectsUnknownLaneSelection(t *testing.T) {
|
|
_, err := loadDNDSpellsPipelineConfig(t).Resolve(config.ResolveInput{
|
|
PipelineID: "dnd-spells-fixture",
|
|
Only: []string{"missing"},
|
|
Catalog: dndSpellsTestCatalog(t, dndSpellsCatalogSpecs{}),
|
|
})
|
|
if err == nil {
|
|
t.Fatal("Resolve() error = nil, want unknown lane error")
|
|
}
|
|
if !strings.Contains(err.Error(), "selected artifact lane") || !strings.Contains(err.Error(), "missing") {
|
|
t.Fatalf("Resolve() error = %q, want unknown lane context", err.Error())
|
|
}
|
|
}
|
|
|
|
func loadDNDSpellsPipelineConfig(t *testing.T) config.Config {
|
|
t.Helper()
|
|
|
|
data, err := os.ReadFile("testdata/pipeline.yml")
|
|
if err != nil {
|
|
t.Fatalf("ReadFile(pipeline.yml) error = %v, want nil", err)
|
|
}
|
|
fileCfg, err := config.ParseFileConfigYAML(data)
|
|
if err != nil {
|
|
t.Fatalf("ParseFileConfigYAML() error = %v, want nil", err)
|
|
}
|
|
cfg := config.Default()
|
|
if err := cfg.ApplyFileConfig(fileCfg); err != nil {
|
|
t.Fatalf("ApplyFileConfig() error = %v, want nil", err)
|
|
}
|
|
return cfg
|
|
}
|
|
|
|
type dndSpellsCatalogSpecs struct {
|
|
input pipeline.ModuleSpec
|
|
chunk pipeline.ModuleSpec
|
|
extractor pipeline.ModuleSpec
|
|
}
|
|
|
|
func dndSpellsTestCatalog(t *testing.T, specs dndSpellsCatalogSpecs) pipeline.ModuleCatalog {
|
|
t.Helper()
|
|
|
|
inputs := pipeline.NewInputAdapterRegistry()
|
|
chunkers := pipeline.NewChunkerRegistry()
|
|
extractors := pipeline.NewExtractorRegistry()
|
|
mergers := pipeline.NewMergerRegistry()
|
|
normalizers := pipeline.NewNormalizerRegistry()
|
|
outputs := pipeline.NewOutputEncoderRegistry()
|
|
|
|
if specs.input.Key == "" {
|
|
if err := seriatim.Register(inputs); err != nil {
|
|
t.Fatalf("register seriatim input: %v", err)
|
|
}
|
|
} else if err := inputs.RegisterWithSpec(specs.input, func() (contracts.InputAdapter, error) {
|
|
return seriatim.New(), nil
|
|
}); err != nil {
|
|
t.Fatalf("register seriatim input override: %v", err)
|
|
}
|
|
|
|
chunkSpec := specs.chunk
|
|
if chunkSpec.Key == "" {
|
|
chunkSpec = dndSpellsChunkerSpec()
|
|
}
|
|
if err := chunkers.RegisterWithSpec(chunkSpec, func() (contracts.Chunker, error) {
|
|
return dndSpellsChunker{}, nil
|
|
}); err != nil {
|
|
t.Fatalf("register chunker: %v", err)
|
|
}
|
|
|
|
if specs.extractor.Key == "" {
|
|
if err := Register(extractors); err != nil {
|
|
t.Fatalf("register dnd spells extractor: %v", err)
|
|
}
|
|
} else if err := extractors.RegisterWithSpec(specs.extractor, func() (contracts.Extractor, error) {
|
|
return New(), nil
|
|
}); err != nil {
|
|
t.Fatalf("register dnd spells extractor override: %v", err)
|
|
}
|
|
|
|
if err := mergers.RegisterWithSpec(pipeline.ModuleSpec{
|
|
Key: pipeline.DefaultMergeModule,
|
|
Stage: pipeline.StageMerge,
|
|
Requires: []string{"dnd.spell_casts"},
|
|
}, func() (contracts.Merger, error) {
|
|
return appendorder.New(), nil
|
|
}); err != nil {
|
|
t.Fatalf("register merger: %v", err)
|
|
}
|
|
if err := normalizers.RegisterWithSpec(pipeline.ModuleSpec{
|
|
Key: pipeline.DefaultNormalizeModule,
|
|
Stage: pipeline.StageNormalize,
|
|
}, func() (contracts.Normalizer, error) {
|
|
return noop.New(), nil
|
|
}); err != nil {
|
|
t.Fatalf("register normalizer: %v", err)
|
|
}
|
|
if err := outputs.RegisterWithSpec(pipeline.ModuleSpec{
|
|
Key: pipeline.DefaultOutputModule,
|
|
Stage: pipeline.StageOutput,
|
|
}, func() (contracts.OutputEncoder, error) {
|
|
return dndSpellsOutput{}, nil
|
|
}); err != nil {
|
|
t.Fatalf("register output: %v", err)
|
|
}
|
|
|
|
return pipeline.ModuleCatalog{
|
|
Inputs: inputs,
|
|
Chunkers: chunkers,
|
|
Extractors: extractors,
|
|
Mergers: mergers,
|
|
Normalizers: normalizers,
|
|
ValidatorChains: pipeline.NewValidatorChainRegistry(),
|
|
Outputs: outputs,
|
|
}
|
|
}
|
|
|
|
func dndSpellsChunkerSpec() pipeline.ModuleSpec {
|
|
return pipeline.ModuleSpec{
|
|
Key: "fake/chunk",
|
|
Stage: pipeline.StageChunk,
|
|
Requires: []string{"source.transcript"},
|
|
Provides: []string{"chunks"},
|
|
}
|
|
}
|
|
|
|
type dndSpellsChunker struct{}
|
|
|
|
func (dndSpellsChunker) Key() string {
|
|
return "fake/chunk"
|
|
}
|
|
|
|
func (dndSpellsChunker) ReferenceSlots() []contracts.ReferenceSlot {
|
|
return nil
|
|
}
|
|
|
|
func (dndSpellsChunker) Chunk(ctx context.Context, req contracts.ChunkRequest) (contracts.ChunkResult, error) {
|
|
return contracts.ChunkResult{
|
|
Chunks: []contracts.SourceChunk{
|
|
{
|
|
ID: req.Source.ID + ":chunk:0",
|
|
SourceID: req.Source.ID,
|
|
Index: 0,
|
|
StartUnitID: req.Source.Units[0].ID,
|
|
EndUnitID: req.Source.Units[len(req.Source.Units)-1].ID,
|
|
Content: []byte(`{"units":[1,2,3]}`),
|
|
MediaType: "application/json",
|
|
Units: append([]source.SourceUnit(nil), req.Source.Units...),
|
|
},
|
|
},
|
|
}, nil
|
|
}
|
|
|
|
type dndSpellsOutput struct{}
|
|
|
|
func (dndSpellsOutput) Key() string {
|
|
return pipeline.DefaultOutputModule
|
|
}
|
|
|
|
func (dndSpellsOutput) Encode(ctx context.Context, req contracts.OutputRequest) (contracts.OutputResult, error) {
|
|
return contracts.OutputResult{
|
|
Files: []contracts.OutputFile{
|
|
{Name: "output.json", ContentType: "application/json", Bytes: []byte(`{"encoded":true}`)},
|
|
},
|
|
}, nil
|
|
}
|
|
|
|
func withoutCapability(capabilities []string, capability string) []string {
|
|
filtered := make([]string, 0, len(capabilities))
|
|
for _, candidate := range capabilities {
|
|
if candidate != capability {
|
|
filtered = append(filtered, candidate)
|
|
}
|
|
}
|
|
return filtered
|
|
}
|
|
|
|
var (
|
|
_ contracts.Chunker = dndSpellsChunker{}
|
|
_ contracts.OutputEncoder = dndSpellsOutput{}
|
|
)
|