Remove legacy raw pipeline contracts
This commit is contained in:
@@ -250,7 +250,7 @@ pipelines:
|
||||
}
|
||||
lane := profile.Artifacts["events"]
|
||||
if !reflect.DeepEqual(lane.References, map[string]string{"lore": "./lore.md", "roster": "./legacy-roster.yml"}) {
|
||||
t.Fatalf("legacy lane references = %#v, want trimmed map", lane.References)
|
||||
t.Fatalf("lane references = %#v, want trimmed map", lane.References)
|
||||
}
|
||||
wantExtract := map[string]string{
|
||||
"glossary": "./glossary.md",
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
@@ -497,6 +498,11 @@ func fakeCatalog(t *testing.T, overrides ...pipeline.ModuleSpec) pipeline.Module
|
||||
for _, override := range overrides {
|
||||
specs[override.Key] = override
|
||||
}
|
||||
for _, key := range []string{"fake/extract", "appendorder", "noop"} {
|
||||
spec := specs[key]
|
||||
spec.ArtifactKind = fakeArtifactKind
|
||||
specs[key] = spec
|
||||
}
|
||||
|
||||
inputs := pipeline.NewInputAdapterRegistry()
|
||||
chunkers := pipeline.NewChunkerRegistry()
|
||||
@@ -515,10 +521,15 @@ func fakeCatalog(t *testing.T, overrides ...pipeline.ModuleSpec) pipeline.Module
|
||||
mustRegisterValidator(t, validators, specs["fake/llm-validator"])
|
||||
mustRegisterOutput(t, outputs, specs["json"])
|
||||
|
||||
codecs := pipeline.NewArtifactCodecRegistry()
|
||||
if err := pipeline.RegisterArtifactCodec(codecs, fakeArtifactCodec{}); err != nil {
|
||||
t.Fatalf("register artifact codec: %v", err)
|
||||
}
|
||||
|
||||
return pipeline.ModuleCatalog{
|
||||
Inputs: inputs,
|
||||
Chunkers: chunkers,
|
||||
ArtifactCodecs: pipeline.NewArtifactCodecRegistry(),
|
||||
ArtifactCodecs: codecs,
|
||||
Extractors: extractors,
|
||||
Mergers: mergers,
|
||||
Normalizers: normalizers,
|
||||
@@ -544,21 +555,32 @@ func mustRegisterChunker(t *testing.T, registry *pipeline.ChunkerRegistry, spec
|
||||
|
||||
func mustRegisterExtractor(t *testing.T, registry *pipeline.ExtractorRegistry, spec pipeline.ModuleSpec) {
|
||||
t.Helper()
|
||||
if err := registry.RegisterLegacyRawWithSpec(spec, func() (contracts.LegacyRawExtractor, error) { return nil, nil }); err != nil {
|
||||
validateOptions := func(options map[string]any) error {
|
||||
if err := pipeline.RejectUnknownOptions(options, "temperature"); err != nil {
|
||||
return err
|
||||
}
|
||||
if value, ok := options["temperature"]; ok {
|
||||
if _, ok := value.(float64); !ok {
|
||||
return fmt.Errorf("temperature must be a number")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if err := pipeline.RegisterExtractorBuilder[fakeArtifact](registry, spec, validateOptions, func(pipeline.BuildRequest) (contracts.Extractor[fakeArtifact], error) { return nil, nil }); err != nil {
|
||||
t.Fatalf("register extractor: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func mustRegisterMerger(t *testing.T, registry *pipeline.MergerRegistry, spec pipeline.ModuleSpec) {
|
||||
t.Helper()
|
||||
if err := registry.RegisterLegacyRawWithSpec(spec, func() (contracts.LegacyRawMerger, error) { return nil, nil }); err != nil {
|
||||
if err := pipeline.RegisterMerger[fakeArtifact](registry, spec, func() (contracts.Merger[fakeArtifact], error) { return nil, nil }); err != nil {
|
||||
t.Fatalf("register merger: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func mustRegisterNormalizer(t *testing.T, registry *pipeline.NormalizerRegistry, spec pipeline.ModuleSpec) {
|
||||
t.Helper()
|
||||
if err := registry.RegisterLegacyRawWithSpec(spec, func() (contracts.LegacyRawNormalizer, error) { return nil, nil }); err != nil {
|
||||
if err := pipeline.RegisterNormalizer[fakeArtifact](registry, spec, func() (contracts.Normalizer[fakeArtifact], error) { return nil, nil }); err != nil {
|
||||
t.Fatalf("register normalizer: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -570,11 +592,32 @@ func mustRegisterValidator(t *testing.T, registry *pipeline.ValidatorRegistry, s
|
||||
executionClass = contracts.ExecutionClassLLMBacked
|
||||
}
|
||||
validatorSpec := pipeline.ValidatorSpec{Key: spec.Key, ExecutionClass: executionClass}
|
||||
if err := registry.RegisterLegacyRawWithSpec(validatorSpec, func() (contracts.LegacyRawValidator, error) { return nil, nil }); err != nil {
|
||||
if err := pipeline.RegisterTypedValidator[fakeArtifact](registry, fakeArtifactKind, validatorSpec, func() (contracts.TypedValidator[fakeArtifact], error) { return nil, nil }); err != nil {
|
||||
t.Fatalf("register validator: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
const fakeArtifactKind contracts.ArtifactKind = "test/artifact"
|
||||
|
||||
type fakeArtifact string
|
||||
|
||||
type fakeArtifactCodec struct{}
|
||||
|
||||
func (fakeArtifactCodec) Kind() contracts.ArtifactKind { return fakeArtifactKind }
|
||||
func (fakeArtifactCodec) Schema() contracts.ArtifactSchema {
|
||||
return contracts.ArtifactSchema{ID: "urn:notarius:test:artifact", Name: "Test artifact", Version: "1", JSONSchema: []byte(`{"type":"string"}`)}
|
||||
}
|
||||
func (fakeArtifactCodec) MediaType() string { return "application/json" }
|
||||
func (fakeArtifactCodec) Encode(value fakeArtifact) ([]byte, error) {
|
||||
return []byte(fmt.Sprintf("%q", value)), nil
|
||||
}
|
||||
func (fakeArtifactCodec) Decode(content []byte) (fakeArtifact, error) {
|
||||
if len(content) < 2 {
|
||||
return "", fmt.Errorf("invalid test artifact")
|
||||
}
|
||||
return fakeArtifact(content[1 : len(content)-1]), nil
|
||||
}
|
||||
|
||||
func mustRegisterOutput(t *testing.T, registry *pipeline.OutputEncoderRegistry, spec pipeline.ModuleSpec) {
|
||||
t.Helper()
|
||||
if err := registry.RegisterWithSpec(spec, func() (contracts.OutputEncoder, error) { return nil, nil }); err != nil {
|
||||
|
||||
Reference in New Issue
Block a user