308 lines
10 KiB
Go
308 lines
10 KiB
Go
package integration_test
|
|
|
|
import (
|
|
"context"
|
|
"os"
|
|
"strings"
|
|
"testing"
|
|
|
|
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
|
spellcodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/spells"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/spells"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/modules/generic/merge/appendorder"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/modules/generic/normalize/noop"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/modules/seriatim/input/transcript"
|
|
)
|
|
|
|
func TestPipelineConfigLoadsAndResolvesWithDNDSpellsExtractor(t *testing.T) {
|
|
data, err := os.ReadFile("testdata/pipeline.yml")
|
|
if err != nil {
|
|
t.Fatalf("ReadFile(pipeline.yml) error = %v, want nil", err)
|
|
}
|
|
fileCfg, err := config.ParseFileConfigYAML(data)
|
|
if err != nil {
|
|
t.Fatalf("ParseFileConfigYAML() error = %v, want nil", err)
|
|
}
|
|
|
|
cfg := config.Default()
|
|
if err := cfg.ApplyFileConfig(fileCfg); err != nil {
|
|
t.Fatalf("ApplyFileConfig() error = %v, want nil", err)
|
|
}
|
|
|
|
resolved, err := cfg.Resolve(config.ResolveInput{
|
|
PipelineID: "dnd-spells-fixture",
|
|
Catalog: dndSpellsTestCatalog(t, dndSpellsCatalogSpecs{}),
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("Resolve() error = %v, want nil", err)
|
|
}
|
|
|
|
if len(resolved.ResolvedPipeline.ArtifactLanes) != 1 {
|
|
t.Fatalf("len(ArtifactLanes) = %d, want 1", len(resolved.ResolvedPipeline.ArtifactLanes))
|
|
}
|
|
lane := resolved.ResolvedPipeline.ArtifactLanes[0]
|
|
if lane.ID != "spells" {
|
|
t.Fatalf("lane ID = %q, want spells", lane.ID)
|
|
}
|
|
if lane.Extract.Module != spells.Key {
|
|
t.Fatalf("extract module = %q, want %q", lane.Extract.Module, spells.Key)
|
|
}
|
|
if resolved.ResolvedPipeline.Digest == "" {
|
|
t.Fatal("resolved digest is empty")
|
|
}
|
|
|
|
again, err := cfg.Resolve(config.ResolveInput{
|
|
PipelineID: "dnd-spells-fixture",
|
|
Catalog: dndSpellsTestCatalog(t, dndSpellsCatalogSpecs{}),
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("second Resolve() error = %v, want nil", err)
|
|
}
|
|
if resolved.ResolvedPipeline.Digest != again.ResolvedPipeline.Digest {
|
|
t.Fatalf("resolved digest = %q, second digest = %q; want stable digest", resolved.ResolvedPipeline.Digest, again.ResolvedPipeline.Digest)
|
|
}
|
|
}
|
|
|
|
func TestPipelineConfigRejectsMissingTranscriptCapabilityForDNDSpells(t *testing.T) {
|
|
inputSpec := transcript.ModuleSpec()
|
|
inputSpec.Provides = withoutCapability(inputSpec.Provides, "source.transcript")
|
|
chunkSpec := dndSpellsChunkerSpec()
|
|
chunkSpec.Requires = nil
|
|
|
|
_, err := loadDNDSpellsPipelineConfig(t).Resolve(config.ResolveInput{
|
|
PipelineID: "dnd-spells-fixture",
|
|
Catalog: dndSpellsTestCatalog(t, dndSpellsCatalogSpecs{
|
|
input: inputSpec,
|
|
chunk: chunkSpec,
|
|
}),
|
|
})
|
|
if err == nil {
|
|
t.Fatal("Resolve() error = nil, want missing capability error")
|
|
}
|
|
if !strings.Contains(err.Error(), "missing capability") ||
|
|
!strings.Contains(err.Error(), "source.transcript") ||
|
|
!strings.Contains(err.Error(), spells.Key) {
|
|
t.Fatalf("Resolve() error = %q, want dnd/spells missing source.transcript capability", err.Error())
|
|
}
|
|
}
|
|
|
|
func TestPipelineConfigRejectsMissingSpellCastsCapabilityForAppendOrder(t *testing.T) {
|
|
extractorSpec := spells.ModuleSpec()
|
|
extractorSpec.Provides = withoutCapability(extractorSpec.Provides, "dnd.spell_casts")
|
|
|
|
_, err := loadDNDSpellsPipelineConfig(t).Resolve(config.ResolveInput{
|
|
PipelineID: "dnd-spells-fixture",
|
|
Catalog: dndSpellsTestCatalog(t, dndSpellsCatalogSpecs{
|
|
extractor: extractorSpec,
|
|
}),
|
|
})
|
|
if err == nil {
|
|
t.Fatal("Resolve() error = nil, want missing capability error")
|
|
}
|
|
if !strings.Contains(err.Error(), "missing capability") ||
|
|
!strings.Contains(err.Error(), "dnd.spell_casts") ||
|
|
!strings.Contains(err.Error(), pipeline.DefaultMergeModule) {
|
|
t.Fatalf("Resolve() error = %q, want appendorder missing dnd.spell_casts capability", err.Error())
|
|
}
|
|
}
|
|
|
|
func TestPipelineConfigRejectsUnknownLaneSelection(t *testing.T) {
|
|
_, err := loadDNDSpellsPipelineConfig(t).Resolve(config.ResolveInput{
|
|
PipelineID: "dnd-spells-fixture",
|
|
Only: []string{"missing"},
|
|
Catalog: dndSpellsTestCatalog(t, dndSpellsCatalogSpecs{}),
|
|
})
|
|
if err == nil {
|
|
t.Fatal("Resolve() error = nil, want unknown lane error")
|
|
}
|
|
if !strings.Contains(err.Error(), "selected artifact lane") || !strings.Contains(err.Error(), "missing") {
|
|
t.Fatalf("Resolve() error = %q, want unknown lane context", err.Error())
|
|
}
|
|
}
|
|
|
|
func loadDNDSpellsPipelineConfig(t *testing.T) config.Config {
|
|
t.Helper()
|
|
|
|
data, err := os.ReadFile("testdata/pipeline.yml")
|
|
if err != nil {
|
|
t.Fatalf("ReadFile(pipeline.yml) error = %v, want nil", err)
|
|
}
|
|
fileCfg, err := config.ParseFileConfigYAML(data)
|
|
if err != nil {
|
|
t.Fatalf("ParseFileConfigYAML() error = %v, want nil", err)
|
|
}
|
|
cfg := config.Default()
|
|
if err := cfg.ApplyFileConfig(fileCfg); err != nil {
|
|
t.Fatalf("ApplyFileConfig() error = %v, want nil", err)
|
|
}
|
|
return cfg
|
|
}
|
|
|
|
type dndSpellsCatalogSpecs struct {
|
|
input pipeline.ModuleSpec
|
|
chunk pipeline.ModuleSpec
|
|
extractor pipeline.ModuleSpec
|
|
}
|
|
|
|
func dndSpellsTestCatalog(t *testing.T, specs dndSpellsCatalogSpecs) pipeline.ModuleCatalog {
|
|
t.Helper()
|
|
|
|
inputs := pipeline.NewInputAdapterRegistry()
|
|
chunkers := pipeline.NewChunkerRegistry()
|
|
extractors := pipeline.NewExtractorRegistry()
|
|
codecs := pipeline.NewArtifactCodecRegistry()
|
|
mergers := pipeline.NewMergerRegistry()
|
|
normalizers := pipeline.NewNormalizerRegistry()
|
|
outputs := pipeline.NewOutputEncoderRegistry()
|
|
|
|
if specs.input.Key == "" {
|
|
if err := transcript.Register(inputs); err != nil {
|
|
t.Fatalf("register seriatim input: %v", err)
|
|
}
|
|
} else if err := inputs.RegisterWithSpec(specs.input, func() (contracts.InputAdapter, error) {
|
|
return transcript.New(), nil
|
|
}); err != nil {
|
|
t.Fatalf("register seriatim input override: %v", err)
|
|
}
|
|
|
|
chunkSpec := specs.chunk
|
|
if chunkSpec.Key == "" {
|
|
chunkSpec = dndSpellsChunkerSpec()
|
|
}
|
|
if err := chunkers.RegisterWithSpec(chunkSpec, func() (contracts.Chunker, error) {
|
|
return dndSpellsChunker{}, nil
|
|
}); err != nil {
|
|
t.Fatalf("register chunker: %v", err)
|
|
}
|
|
|
|
codec := spellcodec.New()
|
|
if err := pipeline.RegisterArtifactCodec(codecs, codec); err != nil {
|
|
t.Fatalf("register dnd spells codec: %v", err)
|
|
}
|
|
if specs.extractor.Key == "" {
|
|
if err := spells.Register(extractors); err != nil {
|
|
t.Fatalf("register dnd spells extractor: %v", err)
|
|
}
|
|
} else {
|
|
specs.extractor.ArtifactKind = dnd.SpellListKind
|
|
if err := pipeline.RegisterExtractor[dnd.SpellList](extractors, specs.extractor, func() (contracts.Extractor[dnd.SpellList], error) {
|
|
return configExtractor{key: specs.extractor.Key}, nil
|
|
}); err != nil {
|
|
t.Fatalf("register dnd spells extractor override: %v", err)
|
|
}
|
|
}
|
|
|
|
if err := pipeline.RegisterMerger[dnd.SpellList](mergers, pipeline.ModuleSpec{
|
|
Key: pipeline.DefaultMergeModule,
|
|
Stage: pipeline.StageMerge,
|
|
ArtifactKind: dnd.SpellListKind,
|
|
Requires: []string{"dnd.spell_casts"},
|
|
}, func() (contracts.Merger[dnd.SpellList], error) {
|
|
return appendorder.NewTyped(appendSpellLists)
|
|
}); err != nil {
|
|
t.Fatalf("register merger: %v", err)
|
|
}
|
|
if err := pipeline.RegisterNormalizer[dnd.SpellList](normalizers, pipeline.ModuleSpec{
|
|
Key: pipeline.DefaultNormalizeModule,
|
|
Stage: pipeline.StageNormalize,
|
|
ArtifactKind: dnd.SpellListKind,
|
|
}, func() (contracts.Normalizer[dnd.SpellList], error) {
|
|
return noop.NewTyped[dnd.SpellList](), nil
|
|
}); err != nil {
|
|
t.Fatalf("register normalizer: %v", err)
|
|
}
|
|
if err := outputs.RegisterWithSpec(pipeline.ModuleSpec{
|
|
Key: pipeline.DefaultOutputModule,
|
|
Stage: pipeline.StageOutput,
|
|
}, func() (contracts.OutputEncoder, error) {
|
|
return dndSpellsOutput{}, nil
|
|
}); err != nil {
|
|
t.Fatalf("register output: %v", err)
|
|
}
|
|
|
|
return pipeline.ModuleCatalog{
|
|
Inputs: inputs,
|
|
Chunkers: chunkers,
|
|
ArtifactCodecs: codecs,
|
|
Extractors: extractors,
|
|
Mergers: mergers,
|
|
Normalizers: normalizers,
|
|
ValidatorChains: pipeline.NewValidatorChainRegistry(),
|
|
Outputs: outputs,
|
|
}
|
|
}
|
|
|
|
type configExtractor struct{ key string }
|
|
|
|
func (extractor configExtractor) Key() string { return extractor.key }
|
|
func (configExtractor) ReferenceSlots() []contracts.ReferenceSlot { return nil }
|
|
func (configExtractor) Extract(context.Context, contracts.TypedExtractionRequest) (contracts.TypedExtractionResult[dnd.SpellList], error) {
|
|
return contracts.TypedExtractionResult[dnd.SpellList]{}, nil
|
|
}
|
|
|
|
func appendSpellLists(values []dnd.SpellList) (dnd.SpellList, error) {
|
|
combined := dnd.SpellList{SpellCasts: []dnd.SpellCast{}}
|
|
for _, value := range values {
|
|
combined.SpellCasts = append(combined.SpellCasts, value.SpellCasts...)
|
|
}
|
|
return combined, nil
|
|
}
|
|
|
|
func dndSpellsChunkerSpec() pipeline.ModuleSpec {
|
|
return pipeline.ModuleSpec{
|
|
Key: "fake/chunk",
|
|
Stage: pipeline.StageChunk,
|
|
Requires: []string{"source.transcript"},
|
|
Provides: []string{"chunks"},
|
|
}
|
|
}
|
|
|
|
type dndSpellsChunker struct{}
|
|
|
|
func (dndSpellsChunker) Key() string {
|
|
return "fake/chunk"
|
|
}
|
|
|
|
func (dndSpellsChunker) ReferenceSlots() []contracts.ReferenceSlot {
|
|
return nil
|
|
}
|
|
|
|
func (dndSpellsChunker) Plan(ctx context.Context, req contracts.ChunkRequest) (contracts.ChunkPlanResult, error) {
|
|
return contracts.ChunkPlanResult{
|
|
Plan: source.ChunkPlan{SourceDigest: req.Source.Digest, Ranges: []source.ChunkRange{{StartUnitID: req.Source.Units[0].ID, EndUnitID: req.Source.Units[len(req.Source.Units)-1].ID}}},
|
|
}, nil
|
|
}
|
|
|
|
type dndSpellsOutput struct{}
|
|
|
|
func (dndSpellsOutput) Key() string {
|
|
return pipeline.DefaultOutputModule
|
|
}
|
|
|
|
func (dndSpellsOutput) Encode(ctx context.Context, req contracts.OutputRequest) (contracts.OutputResult, error) {
|
|
return contracts.OutputResult{
|
|
Files: []contracts.OutputFile{
|
|
{Name: "output.json", ContentType: "application/json", Bytes: []byte(`{"encoded":true}`)},
|
|
},
|
|
}, nil
|
|
}
|
|
|
|
func withoutCapability(capabilities []string, capability string) []string {
|
|
filtered := make([]string, 0, len(capabilities))
|
|
for _, candidate := range capabilities {
|
|
if candidate != capability {
|
|
filtered = append(filtered, candidate)
|
|
}
|
|
}
|
|
return filtered
|
|
}
|
|
|
|
var (
|
|
_ contracts.Chunker = dndSpellsChunker{}
|
|
_ contracts.OutputEncoder = dndSpellsOutput{}
|
|
)
|