Remove legacy raw pipeline contracts
This commit is contained in:
@@ -52,7 +52,7 @@ func (c *Codec) Encode(value dnd.SpellList) ([]byte, error) {
|
||||
}
|
||||
|
||||
// EncodeCandidate provides the same stable representation before typed
|
||||
// validators have approved a value on the temporary raw downstream path.
|
||||
// validators have approved a value.
|
||||
func (c *Codec) EncodeCandidate(value dnd.SpellList) ([]byte, error) {
|
||||
content, err := json.Marshal(value)
|
||||
if err != nil {
|
||||
@@ -73,7 +73,7 @@ func (c *Codec) Decode(content []byte) (dnd.SpellList, error) {
|
||||
}
|
||||
|
||||
// DecodeCandidate reads the durable representation before semantic validators
|
||||
// have approved it on the temporary raw runner path.
|
||||
// have approved it.
|
||||
func (c *Codec) DecodeCandidate(content []byte) (dnd.SpellList, error) {
|
||||
decoder := json.NewDecoder(bytes.NewReader(content))
|
||||
decoder.DisallowUnknownFields()
|
||||
|
||||
@@ -39,18 +39,6 @@ type Extractor struct {
|
||||
llm contracts.StructuredLLMClient
|
||||
}
|
||||
|
||||
type rawAdapter struct {
|
||||
extractor *Extractor
|
||||
codec RawAdapterCodec
|
||||
}
|
||||
|
||||
type RawAdapterCodec interface {
|
||||
contracts.ArtifactCodec[dnd.SpellList]
|
||||
EncodeCandidate(dnd.SpellList) ([]byte, error)
|
||||
}
|
||||
|
||||
var _ contracts.LegacyRawExtractor = (*rawAdapter)(nil)
|
||||
|
||||
func New(llmClient contracts.StructuredLLMClient, _ Options) (*Extractor, error) {
|
||||
if llmClient == nil {
|
||||
return nil, extractorErrorf("LLM client must not be nil")
|
||||
@@ -169,66 +157,6 @@ func Register(registry *pipeline.ExtractorRegistry) error {
|
||||
})
|
||||
}
|
||||
|
||||
// RegisterWithRawAdapter keeps existing raw downstream implementations usable
|
||||
// while the extractor itself produces the canonical typed artifact.
|
||||
func RegisterWithRawAdapter(registry *pipeline.ExtractorRegistry, codec RawAdapterCodec) error {
|
||||
if codec == nil {
|
||||
return extractorErrorf("artifact codec must not be nil")
|
||||
}
|
||||
build := func(request pipeline.BuildRequest) (*Extractor, error) {
|
||||
options, err := DecodeOptions(request.Options)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return New(request.Dependencies.LLM, options)
|
||||
}
|
||||
return pipeline.RegisterExtractorBuilderWithRawAdapter(registry, ModuleSpec(), validateOptions,
|
||||
func(request pipeline.BuildRequest) (contracts.Extractor[dnd.SpellList], error) {
|
||||
return build(request)
|
||||
},
|
||||
func(request pipeline.BuildRequest) (contracts.LegacyRawExtractor, error) {
|
||||
extractor, err := build(request)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &rawAdapter{extractor: extractor, codec: codec}, nil
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
func (adapter *rawAdapter) Key() string { return Key }
|
||||
|
||||
func (adapter *rawAdapter) ReferenceSlots() []contracts.ReferenceSlot {
|
||||
return adapter.extractor.ReferenceSlots()
|
||||
}
|
||||
|
||||
func (adapter *rawAdapter) ManifestMetadata() map[string]any {
|
||||
return adapter.extractor.ManifestMetadata()
|
||||
}
|
||||
|
||||
func (adapter *rawAdapter) Extract(ctx context.Context, req contracts.ExtractionRequest) (contracts.ExtractionResult, error) {
|
||||
result, err := adapter.extractor.Extract(ctx, contracts.TypedExtractionRequest{
|
||||
Source: req.Source, Chunk: req.Chunk, AmbientContext: req.AmbientContext,
|
||||
SourceInput: req.SourceInput, SessionID: req.SessionID, References: req.References,
|
||||
LLMProfile: req.LLMProfile, Metadata: req.Metadata,
|
||||
})
|
||||
if err != nil {
|
||||
return contracts.ExtractionResult{}, err
|
||||
}
|
||||
content, err := adapter.codec.EncodeCandidate(result.Value)
|
||||
if err != nil {
|
||||
return contracts.ExtractionResult{}, extractorErrorf("encode canonical output: %w", err)
|
||||
}
|
||||
schema := adapter.codec.Schema()
|
||||
return contracts.ExtractionResult{
|
||||
Output: contracts.ExtractOutput{
|
||||
Schema: contracts.ResponseSchema{ID: schema.ID, Name: schema.Name, Version: schema.Version, JSONSchema: append([]byte(nil), schema.JSONSchema...)},
|
||||
Payload: contracts.RawPayload{Content: content, MediaType: adapter.codec.MediaType(), Metadata: map[string]any{"spell_cast_count": len(result.Value.SpellCasts)}},
|
||||
},
|
||||
Warnings: result.Warnings,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func validateOptions(options map[string]any) error {
|
||||
_, err := DecodeOptions(options)
|
||||
return err
|
||||
|
||||
@@ -76,9 +76,6 @@ func TestRegisterMakesExtractorBuildable(t *testing.T) {
|
||||
t.Fatalf("Register() error = %v, want nil", err)
|
||||
}
|
||||
|
||||
if _, err := registry.BuildLegacyRaw(Key); err == nil || !strings.Contains(err.Error(), "legacy raw") {
|
||||
t.Fatalf("BuildLegacyRaw() error = %v, want typed registration error", err)
|
||||
}
|
||||
if _, err := DecodeOptions(map[string]any{"unexpected": true}); err == nil || !strings.Contains(err.Error(), "unknown option") {
|
||||
t.Fatalf("DecodeOptions() error = %v, want unknown option error", err)
|
||||
}
|
||||
|
||||
@@ -15,10 +15,6 @@ const ReasonCode = "invalid_spell_shape"
|
||||
|
||||
type Options struct{}
|
||||
type Validator struct{}
|
||||
type legacyValidator struct{ codec decoder }
|
||||
type decoder interface {
|
||||
DecodeCandidate([]byte) (dnd.SpellList, error)
|
||||
}
|
||||
|
||||
var _ contracts.TypedValidator[dnd.SpellList] = (*Validator)(nil)
|
||||
|
||||
@@ -58,18 +54,6 @@ func Validate(value dnd.SpellList) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (v *legacyValidator) Name() string { return Key }
|
||||
func (v *legacyValidator) ExecutionClass() contracts.ExecutionClass {
|
||||
return contracts.ExecutionClassDeterministic
|
||||
}
|
||||
func (v *legacyValidator) Validate(ctx context.Context, req contracts.ValidationRequest) (contracts.ValidationResult, error) {
|
||||
value, err := v.codec.DecodeCandidate(req.Payload.Content)
|
||||
if err != nil {
|
||||
return rejection(err.Error()), nil
|
||||
}
|
||||
return New(Options{}).Validate(ctx, contracts.TypedValidationRequest[dnd.SpellList]{Value: value})
|
||||
}
|
||||
|
||||
func Spec() pipeline.ValidatorSpec {
|
||||
return pipeline.ValidatorSpec{Key: Key, ExecutionClass: contracts.ExecutionClassDeterministic}
|
||||
}
|
||||
@@ -82,17 +66,6 @@ func Register(registry *pipeline.ValidatorRegistry) error {
|
||||
return New(options), nil
|
||||
})
|
||||
}
|
||||
func RegisterLegacy(registry *pipeline.ValidatorRegistry, codec decoder) error {
|
||||
if codec == nil {
|
||||
return fmt.Errorf("spell shape validator codec must not be nil")
|
||||
}
|
||||
return registry.RegisterLegacyRawBuilderWithSpec(Spec(), validateOptions, func(request pipeline.BuildRequest) (contracts.LegacyRawValidator, error) {
|
||||
if _, err := DecodeOptions(request.Options); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &legacyValidator{codec: codec}, nil
|
||||
})
|
||||
}
|
||||
func DecodeOptions(options map[string]any) (Options, error) {
|
||||
if err := pipeline.RejectUnknownOptions(options); err != nil {
|
||||
return Options{}, err
|
||||
|
||||
@@ -16,10 +16,6 @@ const ReasonCode = "invalid_source_refs"
|
||||
|
||||
type Options struct{}
|
||||
type Validator struct{}
|
||||
type legacyValidator struct{ codec decoder }
|
||||
type decoder interface {
|
||||
DecodeCandidate([]byte) (dnd.SpellList, error)
|
||||
}
|
||||
|
||||
var _ contracts.TypedValidator[dnd.SpellList] = (*Validator)(nil)
|
||||
|
||||
@@ -41,20 +37,6 @@ func (v *Validator) Validate(_ context.Context, req contracts.TypedValidationReq
|
||||
}
|
||||
return contracts.ValidationResult{Approved: true}, nil
|
||||
}
|
||||
func (v *legacyValidator) Name() string { return Key }
|
||||
func (v *legacyValidator) ExecutionClass() contracts.ExecutionClass {
|
||||
return contracts.ExecutionClassDeterministic
|
||||
}
|
||||
func (v *legacyValidator) Validate(ctx context.Context, req contracts.ValidationRequest) (contracts.ValidationResult, error) {
|
||||
value, err := v.codec.DecodeCandidate(req.Payload.Content)
|
||||
if err != nil {
|
||||
return rejection(err.Error()), nil
|
||||
}
|
||||
if err := spellshape.Validate(value); err != nil {
|
||||
return rejection(err.Error()), nil
|
||||
}
|
||||
return New(Options{}).Validate(ctx, contracts.TypedValidationRequest[dnd.SpellList]{Source: req.Source, Value: value})
|
||||
}
|
||||
func Spec() pipeline.ValidatorSpec {
|
||||
return pipeline.ValidatorSpec{Key: Key, ExecutionClass: contracts.ExecutionClassDeterministic}
|
||||
}
|
||||
@@ -67,17 +49,6 @@ func Register(registry *pipeline.ValidatorRegistry) error {
|
||||
return New(options), nil
|
||||
})
|
||||
}
|
||||
func RegisterLegacy(registry *pipeline.ValidatorRegistry, codec decoder) error {
|
||||
if codec == nil {
|
||||
return fmt.Errorf("spell source references validator codec must not be nil")
|
||||
}
|
||||
return registry.RegisterLegacyRawBuilderWithSpec(Spec(), validateOptions, func(request pipeline.BuildRequest) (contracts.LegacyRawValidator, error) {
|
||||
if _, err := DecodeOptions(request.Options); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &legacyValidator{codec: codec}, nil
|
||||
})
|
||||
}
|
||||
func DecodeOptions(options map[string]any) (Options, error) {
|
||||
if err := pipeline.RejectUnknownOptions(options); err != nil {
|
||||
return Options{}, err
|
||||
|
||||
@@ -17,10 +17,6 @@ const WarningReasonCode = "spell_not_near_source"
|
||||
|
||||
type Options struct{}
|
||||
type Validator struct{}
|
||||
type legacyValidator struct{ codec decoder }
|
||||
type decoder interface {
|
||||
DecodeCandidate([]byte) (dnd.SpellList, error)
|
||||
}
|
||||
|
||||
var _ contracts.TypedValidator[dnd.SpellList] = (*Validator)(nil)
|
||||
|
||||
@@ -41,20 +37,6 @@ func (v *Validator) Validate(_ context.Context, req contracts.TypedValidationReq
|
||||
}
|
||||
return contracts.ValidationResult{Approved: true, Warnings: warnings}, nil
|
||||
}
|
||||
func (v *legacyValidator) Name() string { return Key }
|
||||
func (v *legacyValidator) ExecutionClass() contracts.ExecutionClass {
|
||||
return contracts.ExecutionClassDeterministic
|
||||
}
|
||||
func (v *legacyValidator) Validate(ctx context.Context, req contracts.ValidationRequest) (contracts.ValidationResult, error) {
|
||||
value, err := v.codec.DecodeCandidate(req.Payload.Content)
|
||||
if err != nil {
|
||||
return contracts.ValidationResult{Approved: true}, nil
|
||||
}
|
||||
if err := spellshape.Validate(value); err != nil {
|
||||
return contracts.ValidationResult{Approved: true}, nil
|
||||
}
|
||||
return New(Options{}).Validate(ctx, contracts.TypedValidationRequest[dnd.SpellList]{Source: req.Source, Value: value})
|
||||
}
|
||||
func spellAppearsInCitedText(doc *source.SourceDocument, spell dnd.SpellCast) bool {
|
||||
name := strings.ToLower(strings.TrimSpace(spell.Spell))
|
||||
if name == "" {
|
||||
@@ -100,17 +82,6 @@ func Register(registry *pipeline.ValidatorRegistry) error {
|
||||
return New(options), nil
|
||||
})
|
||||
}
|
||||
func RegisterLegacy(registry *pipeline.ValidatorRegistry, codec decoder) error {
|
||||
if codec == nil {
|
||||
return fmt.Errorf("spell source relatedness validator codec must not be nil")
|
||||
}
|
||||
return registry.RegisterLegacyRawBuilderWithSpec(Spec(), validateOptions, func(request pipeline.BuildRequest) (contracts.LegacyRawValidator, error) {
|
||||
if _, err := DecodeOptions(request.Options); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &legacyValidator{codec: codec}, nil
|
||||
})
|
||||
}
|
||||
func DecodeOptions(options map[string]any) (Options, error) {
|
||||
if err := pipeline.RejectUnknownOptions(options); err != nil {
|
||||
return Options{}, err
|
||||
|
||||
@@ -1,234 +1,15 @@
|
||||
package appendorder
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"mime"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
)
|
||||
|
||||
const Key = "appendorder"
|
||||
|
||||
var _ contracts.LegacyRawMerger = (*Merger)(nil)
|
||||
|
||||
type Merger struct{}
|
||||
|
||||
func New() *Merger {
|
||||
return &Merger{}
|
||||
}
|
||||
|
||||
func (m *Merger) Key() string {
|
||||
return Key
|
||||
}
|
||||
|
||||
func (m *Merger) Merge(ctx context.Context, req contracts.MergeRequest) (contracts.MergeResult, error) {
|
||||
if m == nil {
|
||||
return contracts.MergeResult{}, mergerErrorf("merger must not be nil")
|
||||
}
|
||||
if ctx == nil {
|
||||
return contracts.MergeResult{}, mergerErrorf("context must not be nil")
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return contracts.MergeResult{}, mergerErrorf("context error before merge: %w", err)
|
||||
}
|
||||
|
||||
outputs, err := orderedOutputs(req.ExtractOutputs)
|
||||
if err != nil {
|
||||
return contracts.MergeResult{}, err
|
||||
}
|
||||
if len(outputs) == 1 {
|
||||
payload := cloneRawPayload(outputs[0].Payload)
|
||||
return contracts.MergeResult{
|
||||
Output: contracts.MergeOutput{
|
||||
LaneID: req.LaneID,
|
||||
MergerKey: Key,
|
||||
SourceID: outputs[0].SourceID,
|
||||
Schema: outputs[0].Schema,
|
||||
Payload: payload,
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
content, err := mergedContent(outputs)
|
||||
if err != nil {
|
||||
return contracts.MergeResult{}, err
|
||||
}
|
||||
return contracts.MergeResult{
|
||||
Output: contracts.MergeOutput{
|
||||
LaneID: req.LaneID,
|
||||
MergerKey: Key,
|
||||
SourceID: sourceID(outputs),
|
||||
Schema: commonSchema(outputs),
|
||||
Payload: contracts.RawPayload{
|
||||
Content: content,
|
||||
MediaType: "application/json",
|
||||
},
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
func ModuleSpec() pipeline.ModuleSpec {
|
||||
return pipeline.ModuleSpec{
|
||||
Key: Key,
|
||||
Stage: pipeline.StageMerge,
|
||||
Provides: []string{"merged"},
|
||||
}
|
||||
}
|
||||
|
||||
func Register(registry *pipeline.MergerRegistry) error {
|
||||
return registry.RegisterLegacyRawWithSpec(ModuleSpec(), func() (contracts.LegacyRawMerger, error) {
|
||||
return New(), nil
|
||||
})
|
||||
}
|
||||
|
||||
func orderedOutputs(outputs []contracts.ExtractOutput) ([]contracts.ExtractOutput, error) {
|
||||
ordered := make([]contracts.ExtractOutput, 0, len(outputs))
|
||||
for _, output := range outputs {
|
||||
if !isJSONMediaType(output.Payload.MediaType) {
|
||||
return nil, mergerErrorf("extract output for chunk %q has unsupported media type %q", output.ChunkID, output.Payload.MediaType)
|
||||
}
|
||||
if !json.Valid(output.Payload.Content) {
|
||||
return nil, mergerErrorf("extract output for chunk %q contains invalid JSON", output.ChunkID)
|
||||
}
|
||||
ordered = append(ordered, cloneExtractOutput(output))
|
||||
}
|
||||
sort.SliceStable(ordered, func(i, j int) bool {
|
||||
return ordered[i].ChunkIndex < ordered[j].ChunkIndex
|
||||
})
|
||||
return ordered, nil
|
||||
}
|
||||
|
||||
func mergedContent(outputs []contracts.ExtractOutput) ([]byte, error) {
|
||||
values := make([]any, 0, len(outputs))
|
||||
objects := make([]map[string]any, 0, len(outputs))
|
||||
for _, output := range outputs {
|
||||
var value any
|
||||
if err := json.Unmarshal(output.Payload.Content, &value); err != nil {
|
||||
return nil, mergerErrorf("decode extract output for chunk %q: %w", output.ChunkID, err)
|
||||
}
|
||||
values = append(values, value)
|
||||
object, ok := value.(map[string]any)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
objects = append(objects, object)
|
||||
}
|
||||
|
||||
if len(objects) == len(outputs) {
|
||||
if field, ok := commonArrayField(objects); ok {
|
||||
merged := make([]any, 0)
|
||||
for _, object := range objects {
|
||||
items := object[field].([]any)
|
||||
merged = append(merged, items...)
|
||||
}
|
||||
return marshalMerged(map[string]any{field: merged})
|
||||
}
|
||||
}
|
||||
return marshalMerged(values)
|
||||
}
|
||||
|
||||
func commonArrayField(objects []map[string]any) (string, bool) {
|
||||
if len(objects) == 0 {
|
||||
return "", false
|
||||
}
|
||||
candidates := map[string]struct{}{}
|
||||
for key, value := range objects[0] {
|
||||
if _, ok := value.([]any); ok {
|
||||
candidates[key] = struct{}{}
|
||||
}
|
||||
}
|
||||
for _, object := range objects[1:] {
|
||||
for key := range candidates {
|
||||
if _, ok := object[key].([]any); !ok {
|
||||
delete(candidates, key)
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(candidates) != 1 {
|
||||
return "", false
|
||||
}
|
||||
for key := range candidates {
|
||||
return key, true
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
|
||||
func marshalMerged(value any) ([]byte, error) {
|
||||
content, err := json.Marshal(value)
|
||||
if err != nil {
|
||||
return nil, mergerErrorf("encode merged output: %w", err)
|
||||
}
|
||||
return content, nil
|
||||
}
|
||||
|
||||
func isJSONMediaType(mediaType string) bool {
|
||||
base, _, err := mime.ParseMediaType(strings.TrimSpace(mediaType))
|
||||
if err != nil {
|
||||
base = strings.TrimSpace(mediaType)
|
||||
}
|
||||
return strings.EqualFold(base, "application/json")
|
||||
}
|
||||
|
||||
func sourceID(outputs []contracts.ExtractOutput) string {
|
||||
for _, output := range outputs {
|
||||
if output.SourceID != "" {
|
||||
return output.SourceID
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func commonSchema(outputs []contracts.ExtractOutput) contracts.ResponseSchema {
|
||||
if len(outputs) == 0 {
|
||||
return contracts.ResponseSchema{}
|
||||
}
|
||||
schema := outputs[0].Schema
|
||||
for _, output := range outputs[1:] {
|
||||
if !sameResponseSchema(output.Schema, schema) {
|
||||
return contracts.ResponseSchema{}
|
||||
}
|
||||
}
|
||||
return schema
|
||||
}
|
||||
|
||||
func sameResponseSchema(left contracts.ResponseSchema, right contracts.ResponseSchema) bool {
|
||||
return left.ID == right.ID && left.Name == right.Name && left.Version == right.Version && string(left.JSONSchema) == string(right.JSONSchema)
|
||||
}
|
||||
|
||||
func cloneExtractOutput(output contracts.ExtractOutput) contracts.ExtractOutput {
|
||||
output.Schema = cloneResponseSchema(output.Schema)
|
||||
output.Payload = cloneRawPayload(output.Payload)
|
||||
return output
|
||||
}
|
||||
|
||||
func cloneResponseSchema(schema contracts.ResponseSchema) contracts.ResponseSchema {
|
||||
schema.JSONSchema = append([]byte(nil), schema.JSONSchema...)
|
||||
return schema
|
||||
}
|
||||
|
||||
func cloneRawPayload(payload contracts.RawPayload) contracts.RawPayload {
|
||||
return contracts.RawPayload{
|
||||
Content: append([]byte(nil), payload.Content...),
|
||||
MediaType: payload.MediaType,
|
||||
Metadata: cloneMetadata(payload.Metadata),
|
||||
Warnings: append([]contracts.Warning(nil), payload.Warnings...),
|
||||
}
|
||||
}
|
||||
|
||||
func cloneMetadata(metadata map[string]any) map[string]any {
|
||||
if len(metadata) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make(map[string]any, len(metadata))
|
||||
for key, value := range metadata {
|
||||
out[key] = value
|
||||
}
|
||||
return out
|
||||
return pipeline.ModuleSpec{Key: Key, Stage: pipeline.StageMerge, Provides: []string{"merged"}}
|
||||
}
|
||||
|
||||
func mergerErrorf(format string, args ...any) error {
|
||||
|
||||
@@ -1,215 +0,0 @@
|
||||
package appendorder
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
)
|
||||
|
||||
func TestModuleSpecAndRegister(t *testing.T) {
|
||||
want := pipeline.ModuleSpec{
|
||||
Key: Key,
|
||||
Stage: pipeline.StageMerge,
|
||||
Provides: []string{"merged"},
|
||||
}
|
||||
if got := ModuleSpec(); !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("ModuleSpec() = %#v, want %#v", got, want)
|
||||
}
|
||||
|
||||
registry := pipeline.NewMergerRegistry()
|
||||
if err := Register(registry); err != nil {
|
||||
t.Fatalf("Register() error = %v, want nil", err)
|
||||
}
|
||||
spec, ok := registry.Spec(Key)
|
||||
if !ok {
|
||||
t.Fatalf("Spec(%q) ok = false, want true", Key)
|
||||
}
|
||||
if !reflect.DeepEqual(spec, want) {
|
||||
t.Fatalf("registered spec = %#v, want %#v", spec, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMergePassesThroughSingleExtractOutput(t *testing.T) {
|
||||
input := extractOutput("chunk-0", 0, `{"name":"original"}`)
|
||||
|
||||
result, err := New().Merge(context.Background(), contracts.MergeRequest{
|
||||
LaneID: "events",
|
||||
ExtractOutputs: []contracts.ExtractOutput{input},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Merge() error = %v, want nil", err)
|
||||
}
|
||||
|
||||
if result.Output.LaneID != "events" || result.Output.MergerKey != Key {
|
||||
t.Fatalf("output provenance = %#v, want lane and merger", result.Output)
|
||||
}
|
||||
if string(result.Output.Payload.Content) != `{"name":"original"}` {
|
||||
t.Fatalf("content = %s, want original content", result.Output.Payload.Content)
|
||||
}
|
||||
if result.Output.Payload.Metadata["name"] != "chunk-0" {
|
||||
t.Fatalf("metadata = %#v, want original metadata", result.Output.Payload.Metadata)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMergeDefensivelyCopiesRawPayload(t *testing.T) {
|
||||
input := extractOutput("chunk-0", 0, `{"name":"original"}`)
|
||||
|
||||
result, err := New().Merge(context.Background(), contracts.MergeRequest{
|
||||
LaneID: "events",
|
||||
ExtractOutputs: []contracts.ExtractOutput{input},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Merge() error = %v, want nil", err)
|
||||
}
|
||||
|
||||
input.Payload.Content[0] = '['
|
||||
input.Payload.Metadata["name"] = "changed"
|
||||
|
||||
if string(result.Output.Payload.Content) != `{"name":"original"}` {
|
||||
t.Fatalf("content changed after input mutation: %s", result.Output.Payload.Content)
|
||||
}
|
||||
if result.Output.Payload.Metadata["name"] != "chunk-0" {
|
||||
t.Fatalf("metadata changed after input mutation: %#v", result.Output.Payload.Metadata)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMergeConcatenatesCommonTopLevelArrayFieldInChunkOrder(t *testing.T) {
|
||||
result, err := New().Merge(context.Background(), contracts.MergeRequest{
|
||||
LaneID: "events",
|
||||
ExtractOutputs: []contracts.ExtractOutput{
|
||||
extractOutput("chunk-1", 1, `{"events":[{"name":"second"}]}`),
|
||||
extractOutput("chunk-0", 0, `{"events":[{"name":"first"}]}`),
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Merge() error = %v, want nil", err)
|
||||
}
|
||||
if result.Output.Payload.MediaType != "application/json" {
|
||||
t.Fatalf("MediaType = %q, want application/json", result.Output.Payload.MediaType)
|
||||
}
|
||||
|
||||
var decoded struct {
|
||||
Events []struct {
|
||||
Name string `json:"name"`
|
||||
} `json:"events"`
|
||||
}
|
||||
if err := json.Unmarshal(result.Output.Payload.Content, &decoded); err != nil {
|
||||
t.Fatalf("Unmarshal() error = %v, want nil", err)
|
||||
}
|
||||
if len(decoded.Events) != 2 || decoded.Events[0].Name != "first" || decoded.Events[1].Name != "second" {
|
||||
t.Fatalf("events = %#v, want concatenated chunk order", decoded.Events)
|
||||
}
|
||||
if result.Output.Schema.ID != "schema-id" {
|
||||
t.Fatalf("schema = %#v, want common extract schema", result.Output.Schema)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMergeFallsBackToOrderedJSONValueArrayWhenShapesDiffer(t *testing.T) {
|
||||
result, err := New().Merge(context.Background(), contracts.MergeRequest{
|
||||
LaneID: "events",
|
||||
ExtractOutputs: []contracts.ExtractOutput{
|
||||
extractOutput("chunk-1", 1, `{"notes":["second"]}`),
|
||||
extractOutput("chunk-0", 0, `{"events":[{"name":"first"}]}`),
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Merge() error = %v, want nil", err)
|
||||
}
|
||||
|
||||
var decoded []map[string]any
|
||||
if err := json.Unmarshal(result.Output.Payload.Content, &decoded); err != nil {
|
||||
t.Fatalf("Unmarshal() error = %v, want nil", err)
|
||||
}
|
||||
if len(decoded) != 2 {
|
||||
t.Fatalf("len(decoded) = %d, want 2", len(decoded))
|
||||
}
|
||||
if _, ok := decoded[0]["events"]; !ok {
|
||||
t.Fatalf("decoded[0] = %#v, want first chunk value", decoded[0])
|
||||
}
|
||||
if _, ok := decoded[1]["notes"]; !ok {
|
||||
t.Fatalf("decoded[1] = %#v, want second chunk value", decoded[1])
|
||||
}
|
||||
}
|
||||
|
||||
func TestMergeRejectsInvalidJSONAndNonJSONMediaTypes(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
output contracts.ExtractOutput
|
||||
want string
|
||||
}{
|
||||
{
|
||||
name: "invalid JSON",
|
||||
output: extractOutput("chunk-0", 0, `{"events":[`),
|
||||
want: "invalid JSON",
|
||||
},
|
||||
{
|
||||
name: "non JSON media type",
|
||||
output: func() contracts.ExtractOutput {
|
||||
output := extractOutput("chunk-0", 0, `{"events":[]}`)
|
||||
output.Payload.MediaType = "text/plain"
|
||||
return output
|
||||
}(),
|
||||
want: "unsupported media type",
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
_, err := New().Merge(context.Background(), contracts.MergeRequest{
|
||||
LaneID: "events",
|
||||
ExtractOutputs: []contracts.ExtractOutput{test.output},
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("Merge() error = nil, want error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), test.want) {
|
||||
t.Fatalf("Merge() error = %q, want %q", err.Error(), test.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestTypedMergeUsesRequestOrderForReusableValueType(t *testing.T) {
|
||||
type notes struct{ Values []string }
|
||||
merger, err := NewTyped(func(values []notes) (notes, error) {
|
||||
var combined notes
|
||||
for _, value := range values {
|
||||
combined.Values = append(combined.Values, value.Values...)
|
||||
}
|
||||
return combined, nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("NewTyped() error = %v", err)
|
||||
}
|
||||
result, err := merger.Merge(context.Background(), contracts.TypedMergeRequest[notes]{ExtractOutputs: []contracts.ExtractArtifact[notes]{
|
||||
{ChunkIndex: 4, Value: notes{Values: []string{"first"}}},
|
||||
{ChunkIndex: 1, Value: notes{Values: []string{"second"}}},
|
||||
}})
|
||||
if err != nil {
|
||||
t.Fatalf("Merge() error = %v", err)
|
||||
}
|
||||
if got := result.Value.Values; !reflect.DeepEqual(got, []string{"first", "second"}) {
|
||||
t.Fatalf("Values = %#v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func extractOutput(chunkID string, chunkIndex int, content string) contracts.ExtractOutput {
|
||||
return contracts.ExtractOutput{
|
||||
LaneID: "events",
|
||||
ExtractorKey: "extract",
|
||||
SourceID: "source-1",
|
||||
ChunkID: chunkID,
|
||||
ChunkIndex: chunkIndex,
|
||||
Schema: contracts.ResponseSchema{ID: "schema-id", Name: "schema-name", Version: "v1"},
|
||||
Payload: contracts.RawPayload{
|
||||
Content: []byte(content),
|
||||
MediaType: "application/json",
|
||||
Metadata: map[string]any{"name": chunkID},
|
||||
},
|
||||
}
|
||||
}
|
||||
25
internal/modules/generic/merge/appendorder/typed_test.go
Normal file
25
internal/modules/generic/merge/appendorder/typed_test.go
Normal file
@@ -0,0 +1,25 @@
|
||||
package appendorder
|
||||
|
||||
import (
|
||||
"context"
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
)
|
||||
|
||||
func TestTypedMergerCombinesValuesInFrameworkOrder(t *testing.T) {
|
||||
merger, err := NewTyped(func(values []string) (string, error) {
|
||||
if !reflect.DeepEqual(values, []string{"first", "second"}) {
|
||||
t.Fatalf("values=%#v", values)
|
||||
}
|
||||
return values[0] + values[1], nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
result, err := merger.Merge(context.Background(), contracts.TypedMergeRequest[string]{ExtractOutputs: []contracts.ExtractArtifact[string]{{ChunkIndex: 0, Value: "first"}, {ChunkIndex: 1, Value: "second"}}})
|
||||
if err != nil || result.Value != "firstsecond" {
|
||||
t.Fatalf("result=%#v err=%v", result, err)
|
||||
}
|
||||
}
|
||||
@@ -1,85 +1,15 @@
|
||||
package noop
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
)
|
||||
|
||||
const Key = "noop"
|
||||
|
||||
var _ contracts.LegacyRawNormalizer = (*Normalizer)(nil)
|
||||
|
||||
type Normalizer struct{}
|
||||
|
||||
func New() *Normalizer {
|
||||
return &Normalizer{}
|
||||
}
|
||||
|
||||
func (n *Normalizer) Key() string {
|
||||
return Key
|
||||
}
|
||||
|
||||
func (n *Normalizer) ReferenceSlots() []contracts.ReferenceSlot {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (n *Normalizer) Normalize(ctx context.Context, req contracts.NormalizeRequest) (contracts.NormalizeResult, error) {
|
||||
if n == nil {
|
||||
return contracts.NormalizeResult{}, normalizerErrorf("normalizer must not be nil")
|
||||
}
|
||||
if ctx == nil {
|
||||
return contracts.NormalizeResult{}, normalizerErrorf("context must not be nil")
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return contracts.NormalizeResult{}, normalizerErrorf("context error before normalize: %w", err)
|
||||
}
|
||||
return contracts.NormalizeResult{
|
||||
Output: contracts.NormalizeOutput{
|
||||
LaneID: req.LaneID,
|
||||
NormalizerKey: Key,
|
||||
SourceID: req.MergeOutput.SourceID,
|
||||
Schema: req.MergeOutput.Schema,
|
||||
Payload: cloneRawPayload(req.MergeOutput.Payload),
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
func ModuleSpec() pipeline.ModuleSpec {
|
||||
return pipeline.ModuleSpec{
|
||||
Key: Key,
|
||||
Stage: pipeline.StageNormalize,
|
||||
Requires: []string{"merged"},
|
||||
Provides: []string{"normalized"},
|
||||
}
|
||||
}
|
||||
|
||||
func Register(registry *pipeline.NormalizerRegistry) error {
|
||||
return registry.RegisterLegacyRawWithSpec(ModuleSpec(), func() (contracts.LegacyRawNormalizer, error) {
|
||||
return New(), nil
|
||||
})
|
||||
}
|
||||
|
||||
func cloneRawPayload(payload contracts.RawPayload) contracts.RawPayload {
|
||||
return contracts.RawPayload{
|
||||
Content: append([]byte(nil), payload.Content...),
|
||||
MediaType: payload.MediaType,
|
||||
Metadata: cloneMetadata(payload.Metadata),
|
||||
Warnings: append([]contracts.Warning(nil), payload.Warnings...),
|
||||
}
|
||||
}
|
||||
|
||||
func cloneMetadata(metadata map[string]any) map[string]any {
|
||||
if len(metadata) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make(map[string]any, len(metadata))
|
||||
for key, value := range metadata {
|
||||
out[key] = value
|
||||
}
|
||||
return out
|
||||
return pipeline.ModuleSpec{Key: Key, Stage: pipeline.StageNormalize, Requires: []string{"merged"}, Provides: []string{"normalized"}}
|
||||
}
|
||||
|
||||
func normalizerErrorf(format string, args ...any) error {
|
||||
|
||||
@@ -1,112 +0,0 @@
|
||||
package noop
|
||||
|
||||
import (
|
||||
"context"
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
)
|
||||
|
||||
func TestModuleSpecAndRegister(t *testing.T) {
|
||||
want := pipeline.ModuleSpec{
|
||||
Key: Key,
|
||||
Stage: pipeline.StageNormalize,
|
||||
Requires: []string{"merged"},
|
||||
Provides: []string{"normalized"},
|
||||
}
|
||||
if got := ModuleSpec(); !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("ModuleSpec() = %#v, want %#v", got, want)
|
||||
}
|
||||
|
||||
registry := pipeline.NewNormalizerRegistry()
|
||||
if err := Register(registry); err != nil {
|
||||
t.Fatalf("Register() error = %v, want nil", err)
|
||||
}
|
||||
spec, ok := registry.Spec(Key)
|
||||
if !ok {
|
||||
t.Fatalf("Spec(%q) ok = false, want true", Key)
|
||||
}
|
||||
if !reflect.DeepEqual(spec, want) {
|
||||
t.Fatalf("registered spec = %#v, want %#v", spec, want)
|
||||
}
|
||||
normalizer, err := registry.BuildLegacyRaw(Key)
|
||||
if err != nil {
|
||||
t.Fatalf("Build(%q) error = %v, want nil", Key, err)
|
||||
}
|
||||
if slots := normalizer.ReferenceSlots(); len(slots) != 0 {
|
||||
t.Fatalf("ReferenceSlots() = %#v, want none", slots)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizePassesThroughMergeOutput(t *testing.T) {
|
||||
input := mergeOutput(`{"name":"original"}`)
|
||||
|
||||
result, err := New().Normalize(context.Background(), contracts.NormalizeRequest{
|
||||
LaneID: "events",
|
||||
MergeOutput: input,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Normalize() error = %v, want nil", err)
|
||||
}
|
||||
|
||||
if result.Output.LaneID != "events" || result.Output.NormalizerKey != Key {
|
||||
t.Fatalf("output provenance = %#v, want lane and normalizer", result.Output)
|
||||
}
|
||||
if string(result.Output.Payload.Content) != `{"name":"original"}` {
|
||||
t.Fatalf("content = %s, want original content", result.Output.Payload.Content)
|
||||
}
|
||||
if result.Output.Payload.Metadata["name"] != "original" {
|
||||
t.Fatalf("metadata = %#v, want original metadata", result.Output.Payload.Metadata)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeDefensivelyCopiesRawPayload(t *testing.T) {
|
||||
input := mergeOutput(`{"name":"original"}`)
|
||||
|
||||
result, err := New().Normalize(context.Background(), contracts.NormalizeRequest{
|
||||
LaneID: "events",
|
||||
MergeOutput: input,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Normalize() error = %v, want nil", err)
|
||||
}
|
||||
|
||||
input.Payload.Content[0] = '['
|
||||
input.Payload.Metadata["name"] = "changed"
|
||||
|
||||
if string(result.Output.Payload.Content) != `{"name":"original"}` {
|
||||
t.Fatalf("content changed after input mutation: %s", result.Output.Payload.Content)
|
||||
}
|
||||
if result.Output.Payload.Metadata["name"] != "original" {
|
||||
t.Fatalf("metadata changed after input mutation: %#v", result.Output.Payload.Metadata)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTypedNormalizePassesThroughReusableValueType(t *testing.T) {
|
||||
type score struct{ Value int }
|
||||
result, err := NewTyped[score]().Normalize(context.Background(), contracts.TypedNormalizeRequest[score]{
|
||||
MergeOutput: contracts.MergeArtifact[score]{Value: score{Value: 7}},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Normalize() error = %v", err)
|
||||
}
|
||||
if result.Value.Value != 7 {
|
||||
t.Fatalf("Value = %d, want 7", result.Value.Value)
|
||||
}
|
||||
}
|
||||
|
||||
func mergeOutput(content string) contracts.MergeOutput {
|
||||
return contracts.MergeOutput{
|
||||
LaneID: "events",
|
||||
MergerKey: "merge",
|
||||
SourceID: "source-1",
|
||||
Schema: contracts.ResponseSchema{ID: "schema-id", Name: "schema-name", Version: "v1"},
|
||||
Payload: contracts.RawPayload{
|
||||
Content: []byte(content),
|
||||
MediaType: "application/json",
|
||||
Metadata: map[string]any{"name": "original"},
|
||||
},
|
||||
}
|
||||
}
|
||||
16
internal/modules/generic/normalize/noop/typed_test.go
Normal file
16
internal/modules/generic/normalize/noop/typed_test.go
Normal file
@@ -0,0 +1,16 @@
|
||||
package noop
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
)
|
||||
|
||||
func TestTypedNormalizerPreservesValue(t *testing.T) {
|
||||
normalizer := NewTyped[string]()
|
||||
result, err := normalizer.Normalize(context.Background(), contracts.TypedNormalizeRequest[string]{MergeOutput: contracts.MergeArtifact[string]{Value: "value"}})
|
||||
if err != nil || result.Value != "value" {
|
||||
t.Fatalf("result=%#v err=%v", result, err)
|
||||
}
|
||||
}
|
||||
@@ -253,15 +253,6 @@ func cloneNormalizeOutputs(outputs []contracts.SerializedOutput) []contracts.Ser
|
||||
return out
|
||||
}
|
||||
|
||||
func cloneRawPayload(payload contracts.RawPayload) contracts.RawPayload {
|
||||
return contracts.RawPayload{
|
||||
Content: append([]byte(nil), payload.Content...),
|
||||
MediaType: payload.MediaType,
|
||||
Metadata: cloneMetadata(payload.Metadata),
|
||||
Warnings: append([]contracts.Warning(nil), payload.Warnings...),
|
||||
}
|
||||
}
|
||||
|
||||
func cloneRejected(rejected []contracts.RejectedOutput) []contracts.RejectedOutput {
|
||||
if len(rejected) == 0 {
|
||||
return []contracts.RejectedOutput{}
|
||||
|
||||
@@ -7,8 +7,6 @@ import (
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/generic/chunk/units"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/generic/merge/appendorder"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/generic/normalize/noop"
|
||||
jsonoutput "gitea.maximumdirect.net/eric/notarius/internal/modules/generic/output/json"
|
||||
alwaysaccept "gitea.maximumdirect.net/eric/notarius/internal/modules/generic/validate/always_accept"
|
||||
alwaysreject "gitea.maximumdirect.net/eric/notarius/internal/modules/generic/validate/always_reject"
|
||||
@@ -27,8 +25,6 @@ func Register(registries pipeline.Registries, assets *llm.AssetRegistry) error {
|
||||
register func() error
|
||||
}{
|
||||
{name: "generic chunker", register: func() error { return units.Register(registries.Chunkers) }},
|
||||
{name: "appendorder merger", register: func() error { return appendorder.Register(registries.Mergers) }},
|
||||
{name: "noop normalizer", register: func() error { return noop.Register(registries.Normalizers) }},
|
||||
{name: "always accept validator", register: func() error { return alwaysaccept.Register(registries.Validators) }},
|
||||
{name: "always reject validator", register: func() error { return alwaysreject.Register(registries.Validators) }},
|
||||
{name: "valid json validator", register: func() error { return validjson.Register(registries.Validators) }},
|
||||
|
||||
@@ -14,8 +14,8 @@ func TestRegisterAddsGenericFamily(t *testing.T) {
|
||||
t.Fatalf("Register() error = %v, want nil", err)
|
||||
}
|
||||
assertKeys(t, "chunkers", registries.Chunkers.RegisteredKeys(), []string{"generic"})
|
||||
assertKeys(t, "mergers", registries.Mergers.RegisteredKeys(), []string{"appendorder"})
|
||||
assertKeys(t, "normalizers", registries.Normalizers.RegisteredKeys(), []string{"noop"})
|
||||
assertKeys(t, "mergers", registries.Mergers.RegisteredKeys(), nil)
|
||||
assertKeys(t, "normalizers", registries.Normalizers.RegisteredKeys(), nil)
|
||||
assertKeys(t, "validators", registries.Validators.RegisteredKeys(), []string{
|
||||
"generic/always_accept",
|
||||
"generic/always_reject",
|
||||
|
||||
@@ -12,7 +12,6 @@ const Key = "generic/always_accept"
|
||||
type Options struct{}
|
||||
type ChunkValidator struct{}
|
||||
type TypedValidator[T any] struct{}
|
||||
type legacyValidator struct{}
|
||||
|
||||
var _ contracts.ChunkValidator = (*ChunkValidator)(nil)
|
||||
|
||||
@@ -35,33 +34,17 @@ func (v *TypedValidator[T]) Validate(context.Context, contracts.TypedValidationR
|
||||
return contracts.ValidationResult{Approved: true}, nil
|
||||
}
|
||||
|
||||
func (v *legacyValidator) Name() string { return Key }
|
||||
func (v *legacyValidator) ExecutionClass() contracts.ExecutionClass {
|
||||
return contracts.ExecutionClassDeterministic
|
||||
}
|
||||
func (v *legacyValidator) Validate(context.Context, contracts.ValidationRequest) (contracts.ValidationResult, error) {
|
||||
return contracts.ValidationResult{Approved: true}, nil
|
||||
}
|
||||
|
||||
func Spec() pipeline.ValidatorSpec {
|
||||
return pipeline.ValidatorSpec{Key: Key, ExecutionClass: contracts.ExecutionClassDeterministic}
|
||||
}
|
||||
|
||||
func Register(registry *pipeline.ValidatorRegistry) error {
|
||||
if err := pipeline.RegisterChunkValidatorBuilder(registry, Spec(), validateOptions, func(request pipeline.BuildRequest) (contracts.ChunkValidator, error) {
|
||||
return pipeline.RegisterChunkValidatorBuilder(registry, Spec(), validateOptions, func(request pipeline.BuildRequest) (contracts.ChunkValidator, error) {
|
||||
options, err := DecodeOptions(request.Options)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return NewChunk(options), nil
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
return registry.RegisterLegacyRawBuilderWithSpec(Spec(), validateOptions, func(request pipeline.BuildRequest) (contracts.LegacyRawValidator, error) {
|
||||
if _, err := DecodeOptions(request.Options); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &legacyValidator{}, nil
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -30,11 +30,7 @@ func TestSpecAndRegister(t *testing.T) {
|
||||
if err := Register(registry); err != nil {
|
||||
t.Fatalf("Register() error = %v, want nil", err)
|
||||
}
|
||||
validator, err := registry.BuildLegacyRaw(Key)
|
||||
if err != nil {
|
||||
t.Fatalf("Build(%q) error = %v, want nil", Key, err)
|
||||
}
|
||||
if validator.Name() != Key {
|
||||
t.Fatalf("Name() = %q, want %q", validator.Name(), Key)
|
||||
if registered, ok := registry.Spec(Key); !ok || registered.Key != Key {
|
||||
t.Fatalf("Spec(%q) = %#v, %v", Key, registered, ok)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -13,7 +13,6 @@ const ReasonCode = "always_reject"
|
||||
type Options struct{}
|
||||
type ChunkValidator struct{}
|
||||
type TypedValidator[T any] struct{}
|
||||
type legacyValidator struct{}
|
||||
|
||||
func NewChunk(Options) *ChunkValidator { return &ChunkValidator{} }
|
||||
func NewTyped[T any](Options) *TypedValidator[T] { return &TypedValidator[T]{} }
|
||||
@@ -35,32 +34,16 @@ func (v *TypedValidator[T]) ExecutionClass() contracts.ExecutionClass {
|
||||
func (v *TypedValidator[T]) Validate(context.Context, contracts.TypedValidationRequest[T]) (contracts.ValidationResult, error) {
|
||||
return rejection(), nil
|
||||
}
|
||||
func (v *legacyValidator) Name() string { return Key }
|
||||
func (v *legacyValidator) ExecutionClass() contracts.ExecutionClass {
|
||||
return contracts.ExecutionClassDeterministic
|
||||
}
|
||||
func (v *legacyValidator) Validate(context.Context, contracts.ValidationRequest) (contracts.ValidationResult, error) {
|
||||
return rejection(), nil
|
||||
}
|
||||
|
||||
func Spec() pipeline.ValidatorSpec {
|
||||
return pipeline.ValidatorSpec{Key: Key, ExecutionClass: contracts.ExecutionClassDeterministic}
|
||||
}
|
||||
func Register(registry *pipeline.ValidatorRegistry) error {
|
||||
if err := pipeline.RegisterChunkValidatorBuilder(registry, Spec(), validateOptions, func(request pipeline.BuildRequest) (contracts.ChunkValidator, error) {
|
||||
return pipeline.RegisterChunkValidatorBuilder(registry, Spec(), validateOptions, func(request pipeline.BuildRequest) (contracts.ChunkValidator, error) {
|
||||
options, err := DecodeOptions(request.Options)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return NewChunk(options), nil
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
return registry.RegisterLegacyRawBuilderWithSpec(Spec(), validateOptions, func(request pipeline.BuildRequest) (contracts.LegacyRawValidator, error) {
|
||||
if _, err := DecodeOptions(request.Options); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &legacyValidator{}, nil
|
||||
})
|
||||
}
|
||||
func RegisterTyped[T any](registry *pipeline.ValidatorRegistry, kind contracts.ArtifactKind) error {
|
||||
|
||||
@@ -33,11 +33,7 @@ func TestSpecAndRegister(t *testing.T) {
|
||||
if err := Register(registry); err != nil {
|
||||
t.Fatalf("Register() error = %v, want nil", err)
|
||||
}
|
||||
validator, err := registry.BuildLegacyRaw(Key)
|
||||
if err != nil {
|
||||
t.Fatalf("Build(%q) error = %v, want nil", Key, err)
|
||||
}
|
||||
if validator.Name() != Key {
|
||||
t.Fatalf("Name() = %q, want %q", validator.Name(), Key)
|
||||
if registered, ok := registry.Spec(Key); !ok || registered.Key != Key {
|
||||
t.Fatalf("Spec(%q) = %#v, %v", Key, registered, ok)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -15,10 +15,7 @@ type Options struct{}
|
||||
|
||||
type Validator struct{}
|
||||
|
||||
type legacyValidator struct{}
|
||||
|
||||
var _ contracts.SerializedValidator = (*Validator)(nil)
|
||||
var _ contracts.LegacyRawValidator = (*legacyValidator)(nil)
|
||||
|
||||
func New(Options) *Validator { return &Validator{} }
|
||||
|
||||
@@ -32,16 +29,6 @@ func (v *Validator) Validate(_ context.Context, req contracts.SerializedValidati
|
||||
return validate(req.Content), nil
|
||||
}
|
||||
|
||||
func (v *legacyValidator) Name() string { return Key }
|
||||
|
||||
func (v *legacyValidator) ExecutionClass() contracts.ExecutionClass {
|
||||
return contracts.ExecutionClassDeterministic
|
||||
}
|
||||
|
||||
func (v *legacyValidator) Validate(_ context.Context, req contracts.ValidationRequest) (contracts.ValidationResult, error) {
|
||||
return validate(req.Payload.Content), nil
|
||||
}
|
||||
|
||||
func validate(content []byte) contracts.ValidationResult {
|
||||
if !json.Valid(content) {
|
||||
return contracts.ValidationResult{Approved: false, ReasonCode: ReasonCodeInvalidJSON, Message: "payload is not valid JSON"}
|
||||
@@ -54,7 +41,7 @@ func Spec() pipeline.ValidatorSpec {
|
||||
}
|
||||
|
||||
func Register(registry *pipeline.ValidatorRegistry) error {
|
||||
if err := pipeline.RegisterSerializedValidatorBuilder(registry, pipeline.SerializedValidatorSpec{
|
||||
return pipeline.RegisterSerializedValidatorBuilder(registry, pipeline.SerializedValidatorSpec{
|
||||
ValidatorSpec: Spec(), SupportsChunks: true, SupportsArtifacts: true,
|
||||
}, validateOptions, func(request pipeline.BuildRequest) (contracts.SerializedValidator, error) {
|
||||
options, err := DecodeOptions(request.Options)
|
||||
@@ -62,14 +49,6 @@ func Register(registry *pipeline.ValidatorRegistry) error {
|
||||
return nil, err
|
||||
}
|
||||
return New(options), nil
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
return registry.RegisterLegacyRawBuilderWithSpec(Spec(), validateOptions, func(request pipeline.BuildRequest) (contracts.LegacyRawValidator, error) {
|
||||
if _, err := DecodeOptions(request.Options); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &legacyValidator{}, nil
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -47,12 +47,8 @@ func TestSpecAndRegister(t *testing.T) {
|
||||
if err := Register(registry); err != nil {
|
||||
t.Fatalf("Register() error = %v, want nil", err)
|
||||
}
|
||||
validator, err := registry.BuildLegacyRaw(Key)
|
||||
if err != nil {
|
||||
t.Fatalf("Build(%q) error = %v, want nil", Key, err)
|
||||
}
|
||||
if validator.Name() != Key {
|
||||
t.Fatalf("Name() = %q, want %q", validator.Name(), Key)
|
||||
if registered, ok := registry.Spec(Key); !ok || registered.Key != Key {
|
||||
t.Fatalf("Spec(%q) = %#v, %v", Key, registered, ok)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -17,10 +17,8 @@ const ReasonCodeSchemaInvalid = "json_schema_invalid"
|
||||
|
||||
type Options struct{}
|
||||
type Validator struct{}
|
||||
type legacyValidator struct{}
|
||||
|
||||
var _ contracts.SerializedValidator = (*Validator)(nil)
|
||||
var _ contracts.LegacyRawValidator = (*legacyValidator)(nil)
|
||||
|
||||
func New(Options) *Validator { return &Validator{} }
|
||||
func (v *Validator) Name() string { return Key }
|
||||
@@ -32,14 +30,6 @@ func (v *Validator) Validate(_ context.Context, req contracts.SerializedValidati
|
||||
return validate(req.Content, req.Schema.JSONSchema)
|
||||
}
|
||||
|
||||
func (v *legacyValidator) Name() string { return Key }
|
||||
func (v *legacyValidator) ExecutionClass() contracts.ExecutionClass {
|
||||
return contracts.ExecutionClassDeterministic
|
||||
}
|
||||
func (v *legacyValidator) Validate(_ context.Context, req contracts.ValidationRequest) (contracts.ValidationResult, error) {
|
||||
return validate(req.Payload.Content, req.Schema.JSONSchema)
|
||||
}
|
||||
|
||||
func validate(content, schemaContent []byte) (contracts.ValidationResult, error) {
|
||||
if len(schemaContent) == 0 {
|
||||
return contracts.ValidationResult{}, fmt.Errorf("response schema content is not available")
|
||||
@@ -71,7 +61,7 @@ func Spec() pipeline.ValidatorSpec {
|
||||
}
|
||||
|
||||
func Register(registry *pipeline.ValidatorRegistry) error {
|
||||
if err := pipeline.RegisterSerializedValidatorBuilder(registry, pipeline.SerializedValidatorSpec{
|
||||
return pipeline.RegisterSerializedValidatorBuilder(registry, pipeline.SerializedValidatorSpec{
|
||||
ValidatorSpec: Spec(), SupportsChunks: true, SupportsArtifacts: true,
|
||||
}, validateOptions, func(request pipeline.BuildRequest) (contracts.SerializedValidator, error) {
|
||||
options, err := DecodeOptions(request.Options)
|
||||
@@ -79,14 +69,6 @@ func Register(registry *pipeline.ValidatorRegistry) error {
|
||||
return nil, err
|
||||
}
|
||||
return New(options), nil
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
return registry.RegisterLegacyRawBuilderWithSpec(Spec(), validateOptions, func(request pipeline.BuildRequest) (contracts.LegacyRawValidator, error) {
|
||||
if _, err := DecodeOptions(request.Options); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &legacyValidator{}, nil
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -74,12 +74,8 @@ func TestSpecAndRegister(t *testing.T) {
|
||||
if err := Register(registry); err != nil {
|
||||
t.Fatalf("Register() error = %v, want nil", err)
|
||||
}
|
||||
validator, err := registry.BuildLegacyRaw(Key)
|
||||
if err != nil {
|
||||
t.Fatalf("Build(%q) error = %v, want nil", Key, err)
|
||||
}
|
||||
if validator.Name() != Key {
|
||||
t.Fatalf("Name() = %q, want %q", validator.Name(), Key)
|
||||
if registered, ok := registry.Spec(Key); !ok || registered.Key != Key {
|
||||
t.Fatalf("Spec(%q) = %#v, %v", Key, registered, ok)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
||||
spellcodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/spells"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/spells"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/generic/merge/appendorder"
|
||||
@@ -178,37 +179,39 @@ func dndSpellsTestCatalog(t *testing.T, specs dndSpellsCatalogSpecs) pipeline.Mo
|
||||
t.Fatalf("register chunker: %v", err)
|
||||
}
|
||||
|
||||
codec := spellcodec.New()
|
||||
if err := pipeline.RegisterArtifactCodec(codecs, codec); err != nil {
|
||||
t.Fatalf("register dnd spells codec: %v", err)
|
||||
}
|
||||
if specs.extractor.Key == "" {
|
||||
codec := spellcodec.New()
|
||||
if err := pipeline.RegisterArtifactCodec(codecs, codec); err != nil {
|
||||
t.Fatalf("register dnd spells codec: %v", err)
|
||||
}
|
||||
if err := spells.RegisterWithRawAdapter(extractors, codec); err != nil {
|
||||
if err := spells.Register(extractors); err != nil {
|
||||
t.Fatalf("register dnd spells extractor: %v", err)
|
||||
}
|
||||
} else {
|
||||
specs.extractor.ArtifactKind = ""
|
||||
if err := extractors.RegisterLegacyRawWithSpec(specs.extractor, func() (contracts.LegacyRawExtractor, error) {
|
||||
return configLegacyExtractor{key: specs.extractor.Key}, nil
|
||||
specs.extractor.ArtifactKind = dnd.SpellListKind
|
||||
if err := pipeline.RegisterExtractor[dnd.SpellList](extractors, specs.extractor, func() (contracts.Extractor[dnd.SpellList], error) {
|
||||
return configExtractor{key: specs.extractor.Key}, nil
|
||||
}); err != nil {
|
||||
t.Fatalf("register dnd spells extractor override: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
if err := mergers.RegisterLegacyRawWithSpec(pipeline.ModuleSpec{
|
||||
Key: pipeline.DefaultMergeModule,
|
||||
Stage: pipeline.StageMerge,
|
||||
Requires: []string{"dnd.spell_casts"},
|
||||
}, func() (contracts.LegacyRawMerger, error) {
|
||||
return appendorder.New(), nil
|
||||
if err := pipeline.RegisterMerger[dnd.SpellList](mergers, pipeline.ModuleSpec{
|
||||
Key: pipeline.DefaultMergeModule,
|
||||
Stage: pipeline.StageMerge,
|
||||
ArtifactKind: dnd.SpellListKind,
|
||||
Requires: []string{"dnd.spell_casts"},
|
||||
}, func() (contracts.Merger[dnd.SpellList], error) {
|
||||
return appendorder.NewTyped(appendSpellLists)
|
||||
}); err != nil {
|
||||
t.Fatalf("register merger: %v", err)
|
||||
}
|
||||
if err := normalizers.RegisterLegacyRawWithSpec(pipeline.ModuleSpec{
|
||||
Key: pipeline.DefaultNormalizeModule,
|
||||
Stage: pipeline.StageNormalize,
|
||||
}, func() (contracts.LegacyRawNormalizer, error) {
|
||||
return noop.New(), nil
|
||||
if err := pipeline.RegisterNormalizer[dnd.SpellList](normalizers, pipeline.ModuleSpec{
|
||||
Key: pipeline.DefaultNormalizeModule,
|
||||
Stage: pipeline.StageNormalize,
|
||||
ArtifactKind: dnd.SpellListKind,
|
||||
}, func() (contracts.Normalizer[dnd.SpellList], error) {
|
||||
return noop.NewTyped[dnd.SpellList](), nil
|
||||
}); err != nil {
|
||||
t.Fatalf("register normalizer: %v", err)
|
||||
}
|
||||
@@ -233,12 +236,20 @@ func dndSpellsTestCatalog(t *testing.T, specs dndSpellsCatalogSpecs) pipeline.Mo
|
||||
}
|
||||
}
|
||||
|
||||
type configLegacyExtractor struct{ key string }
|
||||
type configExtractor struct{ key string }
|
||||
|
||||
func (extractor configLegacyExtractor) Key() string { return extractor.key }
|
||||
func (configLegacyExtractor) ReferenceSlots() []contracts.ReferenceSlot { return nil }
|
||||
func (configLegacyExtractor) Extract(context.Context, contracts.ExtractionRequest) (contracts.ExtractionResult, error) {
|
||||
return contracts.ExtractionResult{}, nil
|
||||
func (extractor configExtractor) Key() string { return extractor.key }
|
||||
func (configExtractor) ReferenceSlots() []contracts.ReferenceSlot { return nil }
|
||||
func (configExtractor) Extract(context.Context, contracts.TypedExtractionRequest) (contracts.TypedExtractionResult[dnd.SpellList], error) {
|
||||
return contracts.TypedExtractionResult[dnd.SpellList]{}, nil
|
||||
}
|
||||
|
||||
func appendSpellLists(values []dnd.SpellList) (dnd.SpellList, error) {
|
||||
combined := dnd.SpellList{SpellCasts: []dnd.SpellCast{}}
|
||||
for _, value := range values {
|
||||
combined.SpellCasts = append(combined.SpellCasts, value.SpellCasts...)
|
||||
}
|
||||
return combined, nil
|
||||
}
|
||||
|
||||
func dndSpellsChunkerSpec() pipeline.ModuleSpec {
|
||||
|
||||
@@ -60,11 +60,11 @@ func TestRunnerProcessesSeriatimInputWithDNDSpellsExtractor(t *testing.T) {
|
||||
if len(output.NormalizeOutputs) != 1 {
|
||||
t.Fatalf("len(NormalizeOutputs) = %d, want 1", len(output.NormalizeOutputs))
|
||||
}
|
||||
rawOutput := output.NormalizeOutputs[0]
|
||||
if rawOutput.LaneID != "spells" || rawOutput.Artifact.Schema.ID != spells.ResponseSchemaID || rawOutput.Artifact.Schema.Version != spells.SchemaVersion {
|
||||
t.Fatalf("raw output envelope = %#v, want dnd spells schema on spells lane", rawOutput)
|
||||
serializedOutput := output.NormalizeOutputs[0]
|
||||
if serializedOutput.LaneID != "spells" || serializedOutput.Artifact.Schema.ID != spells.ResponseSchemaID || serializedOutput.Artifact.Schema.Version != spells.SchemaVersion {
|
||||
t.Fatalf("serialized output envelope = %#v, want dnd spells schema on spells lane", serializedOutput)
|
||||
}
|
||||
response := decodeRunnerSpellResponse(t, rawOutput.Artifact.Content)
|
||||
response := decodeRunnerSpellResponse(t, serializedOutput.Artifact.Content)
|
||||
if len(response.SpellCasts) != 2 {
|
||||
t.Fatalf("len(spell_casts) = %d, want 2", len(response.SpellCasts))
|
||||
}
|
||||
@@ -204,7 +204,7 @@ func TestRunnerDoesNotExtractSpellMentionedOnlyInPartyReference(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerCarriesDNDSpellCastWithInvalidSourceRefAsRawOutput(t *testing.T) {
|
||||
func TestRunnerCarriesDNDSpellCastWithInvalidSourceRefToSerializedOutput(t *testing.T) {
|
||||
raw := readDNDSpellsFixture(t)
|
||||
resolved := resolveDNDSpellsPipeline(t)
|
||||
llmClient := &fakeSpellsLLMClient{
|
||||
@@ -235,7 +235,7 @@ func TestRunnerCarriesDNDSpellCastWithInvalidSourceRefAsRawOutput(t *testing.T)
|
||||
t.Fatalf("len(spell_casts) = %d, want 1", len(response.SpellCasts))
|
||||
}
|
||||
if response.SpellCasts[0].SourceRefs[0].SourceID != "spell-session" {
|
||||
t.Fatalf("SourceID = %q, want raw invalid source ref preserved", response.SpellCasts[0].SourceRefs[0].SourceID)
|
||||
t.Fatalf("SourceID = %q, want invalid source ref preserved", response.SpellCasts[0].SourceRefs[0].SourceID)
|
||||
}
|
||||
if len(output.Rejected) != 0 {
|
||||
t.Fatalf("len(Rejected) = %d, want 0", len(output.Rejected))
|
||||
@@ -282,7 +282,7 @@ func dndSpellsReferenceSet(party string, glossary string) contracts.ReferenceSet
|
||||
return contracts.ReferenceSet{Slots: slots}
|
||||
}
|
||||
|
||||
func TestRunnerCarriesMalformedDNDSpellsExtractorOutput(t *testing.T) {
|
||||
func TestRunnerRejectsMalformedDNDSpellsArtifactAtSerializationBoundary(t *testing.T) {
|
||||
raw := readDNDSpellsFixture(t)
|
||||
resolved := resolveDNDSpellsPipeline(t)
|
||||
llmClient := &fakeSpellsLLMClient{response: extractionResponse{}}
|
||||
@@ -290,17 +290,11 @@ func TestRunnerCarriesMalformedDNDSpellsExtractorOutput(t *testing.T) {
|
||||
output, err := runPreparedPipeline(t, dndSpellsRunnerRegistries(t), resolved.ResolvedPipeline, llmClient, pipeline.RunInput{
|
||||
RawInput: raw,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v, want nil", err)
|
||||
if err == nil || !strings.Contains(err.Error(), "spell_casts must be present") {
|
||||
t.Fatalf("Run() error = %v, want invalid spell-list serialization error", err)
|
||||
}
|
||||
if len(output.NormalizeOutputs) != 1 {
|
||||
t.Fatalf("len(NormalizeOutputs) = %d, want raw output", len(output.NormalizeOutputs))
|
||||
}
|
||||
if string(output.NormalizeOutputs[0].Artifact.Content) != `{"spell_casts":null}` {
|
||||
t.Fatalf("content = %s, want canonical structured output", output.NormalizeOutputs[0].Artifact.Content)
|
||||
}
|
||||
if output.Manifest.ValidationStatus != "approved" {
|
||||
t.Fatalf("ValidationStatus = %q, want approved", output.Manifest.ValidationStatus)
|
||||
if len(output.NormalizeOutputs) != 0 {
|
||||
t.Fatalf("len(NormalizeOutputs) = %d, want no serialized malformed artifact", len(output.NormalizeOutputs))
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -2,12 +2,14 @@ package transcript
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/generic/merge/appendorder"
|
||||
@@ -131,29 +133,36 @@ func seriatimTestCatalog(t *testing.T, inputSpec pipeline.ModuleSpec) pipeline.M
|
||||
Provides: []string{"chunks"},
|
||||
})
|
||||
mustRegisterExtractor(t, extractors, pipeline.ModuleSpec{
|
||||
Key: "fake/extract",
|
||||
Stage: pipeline.StageExtract,
|
||||
Requires: []string{"chunks", "transcript.speaker", "transcript.timestamps"},
|
||||
Provides: []string{"fake.artifacts"},
|
||||
Key: "fake/extract",
|
||||
Stage: pipeline.StageExtract,
|
||||
ArtifactKind: seriatimArtifactKind,
|
||||
Requires: []string{"chunks", "transcript.speaker", "transcript.timestamps"},
|
||||
Provides: []string{"fake.artifacts"},
|
||||
})
|
||||
mustRegisterMerger(t, mergers, pipeline.ModuleSpec{
|
||||
Key: pipeline.DefaultMergeModule,
|
||||
Stage: pipeline.StageMerge,
|
||||
Requires: []string{"fake.artifacts"},
|
||||
Key: pipeline.DefaultMergeModule,
|
||||
Stage: pipeline.StageMerge,
|
||||
ArtifactKind: seriatimArtifactKind,
|
||||
Requires: []string{"fake.artifacts"},
|
||||
})
|
||||
mustRegisterNormalizer(t, normalizers, pipeline.ModuleSpec{
|
||||
Key: pipeline.DefaultNormalizeModule,
|
||||
Stage: pipeline.StageNormalize,
|
||||
Key: pipeline.DefaultNormalizeModule,
|
||||
Stage: pipeline.StageNormalize,
|
||||
ArtifactKind: seriatimArtifactKind,
|
||||
})
|
||||
mustRegisterOutput(t, outputs, pipeline.ModuleSpec{
|
||||
Key: pipeline.DefaultOutputModule,
|
||||
Stage: pipeline.StageOutput,
|
||||
})
|
||||
|
||||
codecs := pipeline.NewArtifactCodecRegistry()
|
||||
if err := pipeline.RegisterArtifactCodec(codecs, seriatimArtifactCodec{}); err != nil {
|
||||
t.Fatalf("register artifact codec: %v", err)
|
||||
}
|
||||
return pipeline.ModuleCatalog{
|
||||
Inputs: inputs,
|
||||
Chunkers: chunkers,
|
||||
ArtifactCodecs: pipeline.NewArtifactCodecRegistry(),
|
||||
ArtifactCodecs: codecs,
|
||||
Extractors: extractors,
|
||||
Mergers: mergers,
|
||||
Normalizers: normalizers,
|
||||
@@ -173,7 +182,7 @@ func mustRegisterChunker(t *testing.T, registry *pipeline.ChunkerRegistry, spec
|
||||
|
||||
func mustRegisterExtractor(t *testing.T, registry *pipeline.ExtractorRegistry, spec pipeline.ModuleSpec) {
|
||||
t.Helper()
|
||||
if err := registry.RegisterLegacyRawWithSpec(spec, func() (contracts.LegacyRawExtractor, error) {
|
||||
if err := pipeline.RegisterExtractor[seriatimArtifact](registry, spec, func() (contracts.Extractor[seriatimArtifact], error) {
|
||||
return fakeExtractor{}, nil
|
||||
}); err != nil {
|
||||
t.Fatalf("register extractor: %v", err)
|
||||
@@ -182,8 +191,13 @@ func mustRegisterExtractor(t *testing.T, registry *pipeline.ExtractorRegistry, s
|
||||
|
||||
func mustRegisterMerger(t *testing.T, registry *pipeline.MergerRegistry, spec pipeline.ModuleSpec) {
|
||||
t.Helper()
|
||||
if err := registry.RegisterLegacyRawWithSpec(spec, func() (contracts.LegacyRawMerger, error) {
|
||||
return appendorder.New(), nil
|
||||
if err := pipeline.RegisterMerger[seriatimArtifact](registry, spec, func() (contracts.Merger[seriatimArtifact], error) {
|
||||
return appendorder.NewTyped(func(values []seriatimArtifact) (seriatimArtifact, error) {
|
||||
if len(values) == 0 {
|
||||
return seriatimArtifact{}, nil
|
||||
}
|
||||
return values[0], nil
|
||||
})
|
||||
}); err != nil {
|
||||
t.Fatalf("register merger: %v", err)
|
||||
}
|
||||
@@ -191,8 +205,8 @@ func mustRegisterMerger(t *testing.T, registry *pipeline.MergerRegistry, spec pi
|
||||
|
||||
func mustRegisterNormalizer(t *testing.T, registry *pipeline.NormalizerRegistry, spec pipeline.ModuleSpec) {
|
||||
t.Helper()
|
||||
if err := registry.RegisterLegacyRawWithSpec(spec, func() (contracts.LegacyRawNormalizer, error) {
|
||||
return noop.New(), nil
|
||||
if err := pipeline.RegisterNormalizer[seriatimArtifact](registry, spec, func() (contracts.Normalizer[seriatimArtifact], error) {
|
||||
return noop.NewTyped[seriatimArtifact](), nil
|
||||
}); err != nil {
|
||||
t.Fatalf("register normalizer: %v", err)
|
||||
}
|
||||
@@ -223,8 +237,8 @@ func (fakeExtractor) Key() string { return "fake/extract" }
|
||||
|
||||
func (fakeExtractor) ReferenceSlots() []contracts.ReferenceSlot { return nil }
|
||||
|
||||
func (fakeExtractor) Extract(ctx context.Context, req contracts.ExtractionRequest) (contracts.ExtractionResult, error) {
|
||||
return contracts.ExtractionResult{}, nil
|
||||
func (fakeExtractor) Extract(ctx context.Context, req contracts.TypedExtractionRequest) (contracts.TypedExtractionResult[seriatimArtifact], error) {
|
||||
return contracts.TypedExtractionResult[seriatimArtifact]{}, nil
|
||||
}
|
||||
|
||||
type fakeOutput struct{}
|
||||
@@ -246,7 +260,30 @@ func withoutCapability(capabilities []string, capability string) []string {
|
||||
}
|
||||
|
||||
var (
|
||||
_ contracts.Chunker = fakeChunker{}
|
||||
_ contracts.LegacyRawExtractor = fakeExtractor{}
|
||||
_ contracts.OutputEncoder = fakeOutput{}
|
||||
_ contracts.Chunker = fakeChunker{}
|
||||
_ contracts.Extractor[seriatimArtifact] = fakeExtractor{}
|
||||
_ contracts.OutputEncoder = fakeOutput{}
|
||||
)
|
||||
|
||||
const seriatimArtifactKind contracts.ArtifactKind = "test/seriatim-event"
|
||||
|
||||
type seriatimArtifact struct {
|
||||
Value string `json:"value"`
|
||||
SourceRefs []source.SourceRef `json:"source_refs"`
|
||||
}
|
||||
|
||||
type seriatimArtifactCodec struct{}
|
||||
|
||||
func (seriatimArtifactCodec) Kind() contracts.ArtifactKind { return seriatimArtifactKind }
|
||||
func (seriatimArtifactCodec) Schema() contracts.ArtifactSchema {
|
||||
return contracts.ArtifactSchema{ID: "fake.event", Name: "fake_event", Version: "v1", JSONSchema: []byte(`{"type":"object"}`)}
|
||||
}
|
||||
func (seriatimArtifactCodec) MediaType() string { return "application/json" }
|
||||
func (seriatimArtifactCodec) Encode(value seriatimArtifact) ([]byte, error) {
|
||||
return json.Marshal(value)
|
||||
}
|
||||
func (seriatimArtifactCodec) Decode(content []byte) (seriatimArtifact, error) {
|
||||
var value seriatimArtifact
|
||||
err := json.Unmarshal(content, &value)
|
||||
return value, err
|
||||
}
|
||||
|
||||
@@ -55,16 +55,16 @@ func TestRunnerProcessesSeriatimInputWithFakeModules(t *testing.T) {
|
||||
t.Fatalf("len(NormalizeOutputs) = %d, want 1", len(output.NormalizeOutputs))
|
||||
}
|
||||
|
||||
rawOutput := output.NormalizeOutputs[0]
|
||||
if rawOutput.LaneID != "events" || rawOutput.NormalizerKey != pipeline.DefaultNormalizeModule || rawOutput.Artifact.Schema.ID != "fake.event" || rawOutput.Artifact.Schema.Version != "v1" {
|
||||
t.Fatalf("raw output envelope = %#v, want fake extractor envelope", rawOutput)
|
||||
serializedOutput := output.NormalizeOutputs[0]
|
||||
if serializedOutput.LaneID != "events" || serializedOutput.NormalizerKey != pipeline.DefaultNormalizeModule || serializedOutput.Artifact.Schema.ID != "fake.event" || serializedOutput.Artifact.Schema.Version != "v1" {
|
||||
t.Fatalf("serialized output envelope = %#v, want fake extractor envelope", serializedOutput)
|
||||
}
|
||||
var payload struct {
|
||||
Value string `json:"value"`
|
||||
SourceRefs []source.SourceRef `json:"source_refs"`
|
||||
}
|
||||
if err := json.Unmarshal(rawOutput.Artifact.Content, &payload); err != nil {
|
||||
t.Fatalf("Unmarshal(raw output) error = %v, want nil", err)
|
||||
if err := json.Unmarshal(serializedOutput.Artifact.Content, &payload); err != nil {
|
||||
t.Fatalf("Unmarshal(serialized output) error = %v, want nil", err)
|
||||
}
|
||||
if len(payload.SourceRefs) != 1 {
|
||||
t.Fatalf("len(SourceRefs) = %d, want 1", len(payload.SourceRefs))
|
||||
@@ -115,7 +115,7 @@ func configResolveInput(t *testing.T) config.ResolveInput {
|
||||
}
|
||||
}
|
||||
|
||||
func seriatimRunnerRegistries(t *testing.T, extractor contracts.LegacyRawExtractor) pipeline.Registries {
|
||||
func seriatimRunnerRegistries(t *testing.T, extractor contracts.Extractor[seriatimArtifact]) pipeline.Registries {
|
||||
t.Helper()
|
||||
|
||||
inputs := pipeline.NewInputAdapterRegistry()
|
||||
@@ -133,19 +133,20 @@ func seriatimRunnerRegistries(t *testing.T, extractor contracts.LegacyRawExtract
|
||||
}); err != nil {
|
||||
t.Fatalf("register chunker: %v", err)
|
||||
}
|
||||
if err := extractors.RegisterLegacyRaw("fake/extract", func() (contracts.LegacyRawExtractor, error) {
|
||||
if err := pipeline.RegisterExtractor[seriatimArtifact](extractors, pipeline.ModuleSpec{Key: "fake/extract", Stage: pipeline.StageExtract, ArtifactKind: seriatimArtifactKind, Requires: []string{"chunks", "transcript.speaker", "transcript.timestamps"}, Provides: []string{"fake.artifacts"}}, func() (contracts.Extractor[seriatimArtifact], error) {
|
||||
return extractor, nil
|
||||
}); err != nil {
|
||||
t.Fatalf("register extractor: %v", err)
|
||||
}
|
||||
if err := mergers.RegisterLegacyRaw(pipeline.DefaultMergeModule, func() (contracts.LegacyRawMerger, error) {
|
||||
return appendorder.New(), nil
|
||||
if err := appendorder.RegisterTyped(mergers, seriatimArtifactKind, func(values []seriatimArtifact) (seriatimArtifact, error) {
|
||||
if len(values) == 0 {
|
||||
return seriatimArtifact{}, nil
|
||||
}
|
||||
return values[0], nil
|
||||
}); err != nil {
|
||||
t.Fatalf("register merger: %v", err)
|
||||
}
|
||||
if err := normalizers.RegisterLegacyRaw(pipeline.DefaultNormalizeModule, func() (contracts.LegacyRawNormalizer, error) {
|
||||
return noop.New(), nil
|
||||
}); err != nil {
|
||||
if err := noop.RegisterTyped[seriatimArtifact](normalizers, seriatimArtifactKind); err != nil {
|
||||
t.Fatalf("register normalizer: %v", err)
|
||||
}
|
||||
if err := outputs.Register(pipeline.DefaultOutputModule, func() (contracts.OutputEncoder, error) {
|
||||
@@ -154,10 +155,14 @@ func seriatimRunnerRegistries(t *testing.T, extractor contracts.LegacyRawExtract
|
||||
t.Fatalf("register output: %v", err)
|
||||
}
|
||||
|
||||
codecs := pipeline.NewArtifactCodecRegistry()
|
||||
if err := pipeline.RegisterArtifactCodec(codecs, seriatimArtifactCodec{}); err != nil {
|
||||
t.Fatalf("register artifact codec: %v", err)
|
||||
}
|
||||
return pipeline.Registries{
|
||||
Inputs: inputs,
|
||||
Chunkers: chunkers,
|
||||
ArtifactCodecs: pipeline.NewArtifactCodecRegistry(),
|
||||
ArtifactCodecs: codecs,
|
||||
Extractors: extractors,
|
||||
Mergers: mergers,
|
||||
Normalizers: normalizers,
|
||||
@@ -203,36 +208,33 @@ func (e *runnerSeriatimExtractor) ReferenceSlots() []contracts.ReferenceSlot {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (e *runnerSeriatimExtractor) Extract(ctx context.Context, req contracts.ExtractionRequest) (contracts.ExtractionResult, error) {
|
||||
func (e *runnerSeriatimExtractor) Extract(ctx context.Context, req contracts.TypedExtractionRequest) (contracts.TypedExtractionResult[seriatimArtifact], error) {
|
||||
e.calls++
|
||||
if req.Source == nil {
|
||||
return contracts.ExtractionResult{}, fmt.Errorf("source must not be nil")
|
||||
return contracts.TypedExtractionResult[seriatimArtifact]{}, fmt.Errorf("source must not be nil")
|
||||
}
|
||||
if req.Chunk == nil {
|
||||
return contracts.ExtractionResult{}, fmt.Errorf("chunk must not be nil")
|
||||
return contracts.TypedExtractionResult[seriatimArtifact]{}, fmt.Errorf("chunk must not be nil")
|
||||
}
|
||||
if got := unitIDs(req.Source.Units); !equalInts(got, []int{1, 2}) {
|
||||
return contracts.ExtractionResult{}, fmt.Errorf("source unit IDs = %#v, want Seriatim segment IDs", got)
|
||||
return contracts.TypedExtractionResult[seriatimArtifact]{}, fmt.Errorf("source unit IDs = %#v, want Seriatim segment IDs", got)
|
||||
}
|
||||
if got := unitIDs(req.Chunk.Units); !equalInts(got, []int{1, 2}) {
|
||||
return contracts.ExtractionResult{}, fmt.Errorf("chunk unit IDs = %#v, want Seriatim segment IDs", got)
|
||||
return contracts.TypedExtractionResult[seriatimArtifact]{}, fmt.Errorf("chunk unit IDs = %#v, want Seriatim segment IDs", got)
|
||||
}
|
||||
for _, unit := range req.Chunk.Units {
|
||||
if speaker, ok := Speaker(unit); !ok || speaker == "" {
|
||||
return contracts.ExtractionResult{}, fmt.Errorf("unit %d missing speaker metadata", unit.ID)
|
||||
return contracts.TypedExtractionResult[seriatimArtifact]{}, fmt.Errorf("unit %d missing speaker metadata", unit.ID)
|
||||
}
|
||||
if _, ok := Start(unit); !ok {
|
||||
return contracts.ExtractionResult{}, fmt.Errorf("unit %d missing start metadata", unit.ID)
|
||||
return contracts.TypedExtractionResult[seriatimArtifact]{}, fmt.Errorf("unit %d missing start metadata", unit.ID)
|
||||
}
|
||||
if _, ok := End(unit); !ok {
|
||||
return contracts.ExtractionResult{}, fmt.Errorf("unit %d missing end metadata", unit.ID)
|
||||
return contracts.TypedExtractionResult[seriatimArtifact]{}, fmt.Errorf("unit %d missing end metadata", unit.ID)
|
||||
}
|
||||
}
|
||||
|
||||
payload, err := json.Marshal(struct {
|
||||
Value string `json:"value"`
|
||||
SourceRefs []source.SourceRef `json:"source_refs"`
|
||||
}{
|
||||
return contracts.TypedExtractionResult[seriatimArtifact]{Value: seriatimArtifact{
|
||||
Value: "seriatim-source-ref",
|
||||
SourceRefs: []source.SourceRef{
|
||||
{
|
||||
@@ -241,20 +243,7 @@ func (e *runnerSeriatimExtractor) Extract(ctx context.Context, req contracts.Ext
|
||||
EndUnitID: req.Chunk.Units[len(req.Chunk.Units)-1].ID,
|
||||
},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
return contracts.ExtractionResult{}, err
|
||||
}
|
||||
|
||||
return contracts.ExtractionResult{
|
||||
Output: contracts.ExtractOutput{
|
||||
Schema: contracts.ResponseSchema{ID: "fake.event", Name: "fake_event", Version: "v1"},
|
||||
Payload: contracts.RawPayload{
|
||||
Content: payload,
|
||||
MediaType: "application/json",
|
||||
},
|
||||
},
|
||||
}, nil
|
||||
}}, nil
|
||||
}
|
||||
|
||||
type runnerSeriatimOutput struct{}
|
||||
@@ -292,7 +281,7 @@ func equalInts(a, b []int) bool {
|
||||
}
|
||||
|
||||
var (
|
||||
_ contracts.Chunker = runnerSeriatimChunker{}
|
||||
_ contracts.LegacyRawExtractor = (*runnerSeriatimExtractor)(nil)
|
||||
_ contracts.OutputEncoder = runnerSeriatimOutput{}
|
||||
_ contracts.Chunker = runnerSeriatimChunker{}
|
||||
_ contracts.Extractor[seriatimArtifact] = (*runnerSeriatimExtractor)(nil)
|
||||
_ contracts.OutputEncoder = runnerSeriatimOutput{}
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user