Remove legacy raw pipeline contracts

This commit is contained in:
2026-07-17 08:13:08 +00:00
parent 814fcdc6ba
commit adfe3825ee
68 changed files with 823 additions and 7919 deletions

View File

@@ -52,7 +52,7 @@ func (c *Codec) Encode(value dnd.SpellList) ([]byte, error) {
}
// EncodeCandidate provides the same stable representation before typed
// validators have approved a value on the temporary raw downstream path.
// validators have approved a value.
func (c *Codec) EncodeCandidate(value dnd.SpellList) ([]byte, error) {
content, err := json.Marshal(value)
if err != nil {
@@ -73,7 +73,7 @@ func (c *Codec) Decode(content []byte) (dnd.SpellList, error) {
}
// DecodeCandidate reads the durable representation before semantic validators
// have approved it on the temporary raw runner path.
// have approved it.
func (c *Codec) DecodeCandidate(content []byte) (dnd.SpellList, error) {
decoder := json.NewDecoder(bytes.NewReader(content))
decoder.DisallowUnknownFields()

View File

@@ -39,18 +39,6 @@ type Extractor struct {
llm contracts.StructuredLLMClient
}
type rawAdapter struct {
extractor *Extractor
codec RawAdapterCodec
}
type RawAdapterCodec interface {
contracts.ArtifactCodec[dnd.SpellList]
EncodeCandidate(dnd.SpellList) ([]byte, error)
}
var _ contracts.LegacyRawExtractor = (*rawAdapter)(nil)
func New(llmClient contracts.StructuredLLMClient, _ Options) (*Extractor, error) {
if llmClient == nil {
return nil, extractorErrorf("LLM client must not be nil")
@@ -169,66 +157,6 @@ func Register(registry *pipeline.ExtractorRegistry) error {
})
}
// RegisterWithRawAdapter keeps existing raw downstream implementations usable
// while the extractor itself produces the canonical typed artifact.
func RegisterWithRawAdapter(registry *pipeline.ExtractorRegistry, codec RawAdapterCodec) error {
if codec == nil {
return extractorErrorf("artifact codec must not be nil")
}
build := func(request pipeline.BuildRequest) (*Extractor, error) {
options, err := DecodeOptions(request.Options)
if err != nil {
return nil, err
}
return New(request.Dependencies.LLM, options)
}
return pipeline.RegisterExtractorBuilderWithRawAdapter(registry, ModuleSpec(), validateOptions,
func(request pipeline.BuildRequest) (contracts.Extractor[dnd.SpellList], error) {
return build(request)
},
func(request pipeline.BuildRequest) (contracts.LegacyRawExtractor, error) {
extractor, err := build(request)
if err != nil {
return nil, err
}
return &rawAdapter{extractor: extractor, codec: codec}, nil
},
)
}
func (adapter *rawAdapter) Key() string { return Key }
func (adapter *rawAdapter) ReferenceSlots() []contracts.ReferenceSlot {
return adapter.extractor.ReferenceSlots()
}
func (adapter *rawAdapter) ManifestMetadata() map[string]any {
return adapter.extractor.ManifestMetadata()
}
func (adapter *rawAdapter) Extract(ctx context.Context, req contracts.ExtractionRequest) (contracts.ExtractionResult, error) {
result, err := adapter.extractor.Extract(ctx, contracts.TypedExtractionRequest{
Source: req.Source, Chunk: req.Chunk, AmbientContext: req.AmbientContext,
SourceInput: req.SourceInput, SessionID: req.SessionID, References: req.References,
LLMProfile: req.LLMProfile, Metadata: req.Metadata,
})
if err != nil {
return contracts.ExtractionResult{}, err
}
content, err := adapter.codec.EncodeCandidate(result.Value)
if err != nil {
return contracts.ExtractionResult{}, extractorErrorf("encode canonical output: %w", err)
}
schema := adapter.codec.Schema()
return contracts.ExtractionResult{
Output: contracts.ExtractOutput{
Schema: contracts.ResponseSchema{ID: schema.ID, Name: schema.Name, Version: schema.Version, JSONSchema: append([]byte(nil), schema.JSONSchema...)},
Payload: contracts.RawPayload{Content: content, MediaType: adapter.codec.MediaType(), Metadata: map[string]any{"spell_cast_count": len(result.Value.SpellCasts)}},
},
Warnings: result.Warnings,
}, nil
}
func validateOptions(options map[string]any) error {
_, err := DecodeOptions(options)
return err

View File

@@ -76,9 +76,6 @@ func TestRegisterMakesExtractorBuildable(t *testing.T) {
t.Fatalf("Register() error = %v, want nil", err)
}
if _, err := registry.BuildLegacyRaw(Key); err == nil || !strings.Contains(err.Error(), "legacy raw") {
t.Fatalf("BuildLegacyRaw() error = %v, want typed registration error", err)
}
if _, err := DecodeOptions(map[string]any{"unexpected": true}); err == nil || !strings.Contains(err.Error(), "unknown option") {
t.Fatalf("DecodeOptions() error = %v, want unknown option error", err)
}

View File

@@ -15,10 +15,6 @@ const ReasonCode = "invalid_spell_shape"
type Options struct{}
type Validator struct{}
type legacyValidator struct{ codec decoder }
type decoder interface {
DecodeCandidate([]byte) (dnd.SpellList, error)
}
var _ contracts.TypedValidator[dnd.SpellList] = (*Validator)(nil)
@@ -58,18 +54,6 @@ func Validate(value dnd.SpellList) error {
return nil
}
func (v *legacyValidator) Name() string { return Key }
func (v *legacyValidator) ExecutionClass() contracts.ExecutionClass {
return contracts.ExecutionClassDeterministic
}
func (v *legacyValidator) Validate(ctx context.Context, req contracts.ValidationRequest) (contracts.ValidationResult, error) {
value, err := v.codec.DecodeCandidate(req.Payload.Content)
if err != nil {
return rejection(err.Error()), nil
}
return New(Options{}).Validate(ctx, contracts.TypedValidationRequest[dnd.SpellList]{Value: value})
}
func Spec() pipeline.ValidatorSpec {
return pipeline.ValidatorSpec{Key: Key, ExecutionClass: contracts.ExecutionClassDeterministic}
}
@@ -82,17 +66,6 @@ func Register(registry *pipeline.ValidatorRegistry) error {
return New(options), nil
})
}
func RegisterLegacy(registry *pipeline.ValidatorRegistry, codec decoder) error {
if codec == nil {
return fmt.Errorf("spell shape validator codec must not be nil")
}
return registry.RegisterLegacyRawBuilderWithSpec(Spec(), validateOptions, func(request pipeline.BuildRequest) (contracts.LegacyRawValidator, error) {
if _, err := DecodeOptions(request.Options); err != nil {
return nil, err
}
return &legacyValidator{codec: codec}, nil
})
}
func DecodeOptions(options map[string]any) (Options, error) {
if err := pipeline.RejectUnknownOptions(options); err != nil {
return Options{}, err

View File

@@ -16,10 +16,6 @@ const ReasonCode = "invalid_source_refs"
type Options struct{}
type Validator struct{}
type legacyValidator struct{ codec decoder }
type decoder interface {
DecodeCandidate([]byte) (dnd.SpellList, error)
}
var _ contracts.TypedValidator[dnd.SpellList] = (*Validator)(nil)
@@ -41,20 +37,6 @@ func (v *Validator) Validate(_ context.Context, req contracts.TypedValidationReq
}
return contracts.ValidationResult{Approved: true}, nil
}
func (v *legacyValidator) Name() string { return Key }
func (v *legacyValidator) ExecutionClass() contracts.ExecutionClass {
return contracts.ExecutionClassDeterministic
}
func (v *legacyValidator) Validate(ctx context.Context, req contracts.ValidationRequest) (contracts.ValidationResult, error) {
value, err := v.codec.DecodeCandidate(req.Payload.Content)
if err != nil {
return rejection(err.Error()), nil
}
if err := spellshape.Validate(value); err != nil {
return rejection(err.Error()), nil
}
return New(Options{}).Validate(ctx, contracts.TypedValidationRequest[dnd.SpellList]{Source: req.Source, Value: value})
}
func Spec() pipeline.ValidatorSpec {
return pipeline.ValidatorSpec{Key: Key, ExecutionClass: contracts.ExecutionClassDeterministic}
}
@@ -67,17 +49,6 @@ func Register(registry *pipeline.ValidatorRegistry) error {
return New(options), nil
})
}
func RegisterLegacy(registry *pipeline.ValidatorRegistry, codec decoder) error {
if codec == nil {
return fmt.Errorf("spell source references validator codec must not be nil")
}
return registry.RegisterLegacyRawBuilderWithSpec(Spec(), validateOptions, func(request pipeline.BuildRequest) (contracts.LegacyRawValidator, error) {
if _, err := DecodeOptions(request.Options); err != nil {
return nil, err
}
return &legacyValidator{codec: codec}, nil
})
}
func DecodeOptions(options map[string]any) (Options, error) {
if err := pipeline.RejectUnknownOptions(options); err != nil {
return Options{}, err

View File

@@ -17,10 +17,6 @@ const WarningReasonCode = "spell_not_near_source"
type Options struct{}
type Validator struct{}
type legacyValidator struct{ codec decoder }
type decoder interface {
DecodeCandidate([]byte) (dnd.SpellList, error)
}
var _ contracts.TypedValidator[dnd.SpellList] = (*Validator)(nil)
@@ -41,20 +37,6 @@ func (v *Validator) Validate(_ context.Context, req contracts.TypedValidationReq
}
return contracts.ValidationResult{Approved: true, Warnings: warnings}, nil
}
func (v *legacyValidator) Name() string { return Key }
func (v *legacyValidator) ExecutionClass() contracts.ExecutionClass {
return contracts.ExecutionClassDeterministic
}
func (v *legacyValidator) Validate(ctx context.Context, req contracts.ValidationRequest) (contracts.ValidationResult, error) {
value, err := v.codec.DecodeCandidate(req.Payload.Content)
if err != nil {
return contracts.ValidationResult{Approved: true}, nil
}
if err := spellshape.Validate(value); err != nil {
return contracts.ValidationResult{Approved: true}, nil
}
return New(Options{}).Validate(ctx, contracts.TypedValidationRequest[dnd.SpellList]{Source: req.Source, Value: value})
}
func spellAppearsInCitedText(doc *source.SourceDocument, spell dnd.SpellCast) bool {
name := strings.ToLower(strings.TrimSpace(spell.Spell))
if name == "" {
@@ -100,17 +82,6 @@ func Register(registry *pipeline.ValidatorRegistry) error {
return New(options), nil
})
}
func RegisterLegacy(registry *pipeline.ValidatorRegistry, codec decoder) error {
if codec == nil {
return fmt.Errorf("spell source relatedness validator codec must not be nil")
}
return registry.RegisterLegacyRawBuilderWithSpec(Spec(), validateOptions, func(request pipeline.BuildRequest) (contracts.LegacyRawValidator, error) {
if _, err := DecodeOptions(request.Options); err != nil {
return nil, err
}
return &legacyValidator{codec: codec}, nil
})
}
func DecodeOptions(options map[string]any) (Options, error) {
if err := pipeline.RejectUnknownOptions(options); err != nil {
return Options{}, err

View File

@@ -1,234 +1,15 @@
package appendorder
import (
"context"
"encoding/json"
"fmt"
"mime"
"sort"
"strings"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
)
const Key = "appendorder"
var _ contracts.LegacyRawMerger = (*Merger)(nil)
type Merger struct{}
func New() *Merger {
return &Merger{}
}
func (m *Merger) Key() string {
return Key
}
func (m *Merger) Merge(ctx context.Context, req contracts.MergeRequest) (contracts.MergeResult, error) {
if m == nil {
return contracts.MergeResult{}, mergerErrorf("merger must not be nil")
}
if ctx == nil {
return contracts.MergeResult{}, mergerErrorf("context must not be nil")
}
if err := ctx.Err(); err != nil {
return contracts.MergeResult{}, mergerErrorf("context error before merge: %w", err)
}
outputs, err := orderedOutputs(req.ExtractOutputs)
if err != nil {
return contracts.MergeResult{}, err
}
if len(outputs) == 1 {
payload := cloneRawPayload(outputs[0].Payload)
return contracts.MergeResult{
Output: contracts.MergeOutput{
LaneID: req.LaneID,
MergerKey: Key,
SourceID: outputs[0].SourceID,
Schema: outputs[0].Schema,
Payload: payload,
},
}, nil
}
content, err := mergedContent(outputs)
if err != nil {
return contracts.MergeResult{}, err
}
return contracts.MergeResult{
Output: contracts.MergeOutput{
LaneID: req.LaneID,
MergerKey: Key,
SourceID: sourceID(outputs),
Schema: commonSchema(outputs),
Payload: contracts.RawPayload{
Content: content,
MediaType: "application/json",
},
},
}, nil
}
func ModuleSpec() pipeline.ModuleSpec {
return pipeline.ModuleSpec{
Key: Key,
Stage: pipeline.StageMerge,
Provides: []string{"merged"},
}
}
func Register(registry *pipeline.MergerRegistry) error {
return registry.RegisterLegacyRawWithSpec(ModuleSpec(), func() (contracts.LegacyRawMerger, error) {
return New(), nil
})
}
func orderedOutputs(outputs []contracts.ExtractOutput) ([]contracts.ExtractOutput, error) {
ordered := make([]contracts.ExtractOutput, 0, len(outputs))
for _, output := range outputs {
if !isJSONMediaType(output.Payload.MediaType) {
return nil, mergerErrorf("extract output for chunk %q has unsupported media type %q", output.ChunkID, output.Payload.MediaType)
}
if !json.Valid(output.Payload.Content) {
return nil, mergerErrorf("extract output for chunk %q contains invalid JSON", output.ChunkID)
}
ordered = append(ordered, cloneExtractOutput(output))
}
sort.SliceStable(ordered, func(i, j int) bool {
return ordered[i].ChunkIndex < ordered[j].ChunkIndex
})
return ordered, nil
}
func mergedContent(outputs []contracts.ExtractOutput) ([]byte, error) {
values := make([]any, 0, len(outputs))
objects := make([]map[string]any, 0, len(outputs))
for _, output := range outputs {
var value any
if err := json.Unmarshal(output.Payload.Content, &value); err != nil {
return nil, mergerErrorf("decode extract output for chunk %q: %w", output.ChunkID, err)
}
values = append(values, value)
object, ok := value.(map[string]any)
if !ok {
continue
}
objects = append(objects, object)
}
if len(objects) == len(outputs) {
if field, ok := commonArrayField(objects); ok {
merged := make([]any, 0)
for _, object := range objects {
items := object[field].([]any)
merged = append(merged, items...)
}
return marshalMerged(map[string]any{field: merged})
}
}
return marshalMerged(values)
}
func commonArrayField(objects []map[string]any) (string, bool) {
if len(objects) == 0 {
return "", false
}
candidates := map[string]struct{}{}
for key, value := range objects[0] {
if _, ok := value.([]any); ok {
candidates[key] = struct{}{}
}
}
for _, object := range objects[1:] {
for key := range candidates {
if _, ok := object[key].([]any); !ok {
delete(candidates, key)
}
}
}
if len(candidates) != 1 {
return "", false
}
for key := range candidates {
return key, true
}
return "", false
}
func marshalMerged(value any) ([]byte, error) {
content, err := json.Marshal(value)
if err != nil {
return nil, mergerErrorf("encode merged output: %w", err)
}
return content, nil
}
func isJSONMediaType(mediaType string) bool {
base, _, err := mime.ParseMediaType(strings.TrimSpace(mediaType))
if err != nil {
base = strings.TrimSpace(mediaType)
}
return strings.EqualFold(base, "application/json")
}
func sourceID(outputs []contracts.ExtractOutput) string {
for _, output := range outputs {
if output.SourceID != "" {
return output.SourceID
}
}
return ""
}
func commonSchema(outputs []contracts.ExtractOutput) contracts.ResponseSchema {
if len(outputs) == 0 {
return contracts.ResponseSchema{}
}
schema := outputs[0].Schema
for _, output := range outputs[1:] {
if !sameResponseSchema(output.Schema, schema) {
return contracts.ResponseSchema{}
}
}
return schema
}
func sameResponseSchema(left contracts.ResponseSchema, right contracts.ResponseSchema) bool {
return left.ID == right.ID && left.Name == right.Name && left.Version == right.Version && string(left.JSONSchema) == string(right.JSONSchema)
}
func cloneExtractOutput(output contracts.ExtractOutput) contracts.ExtractOutput {
output.Schema = cloneResponseSchema(output.Schema)
output.Payload = cloneRawPayload(output.Payload)
return output
}
func cloneResponseSchema(schema contracts.ResponseSchema) contracts.ResponseSchema {
schema.JSONSchema = append([]byte(nil), schema.JSONSchema...)
return schema
}
func cloneRawPayload(payload contracts.RawPayload) contracts.RawPayload {
return contracts.RawPayload{
Content: append([]byte(nil), payload.Content...),
MediaType: payload.MediaType,
Metadata: cloneMetadata(payload.Metadata),
Warnings: append([]contracts.Warning(nil), payload.Warnings...),
}
}
func cloneMetadata(metadata map[string]any) map[string]any {
if len(metadata) == 0 {
return nil
}
out := make(map[string]any, len(metadata))
for key, value := range metadata {
out[key] = value
}
return out
return pipeline.ModuleSpec{Key: Key, Stage: pipeline.StageMerge, Provides: []string{"merged"}}
}
func mergerErrorf(format string, args ...any) error {

View File

@@ -1,215 +0,0 @@
package appendorder
import (
"context"
"encoding/json"
"reflect"
"strings"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
)
func TestModuleSpecAndRegister(t *testing.T) {
want := pipeline.ModuleSpec{
Key: Key,
Stage: pipeline.StageMerge,
Provides: []string{"merged"},
}
if got := ModuleSpec(); !reflect.DeepEqual(got, want) {
t.Fatalf("ModuleSpec() = %#v, want %#v", got, want)
}
registry := pipeline.NewMergerRegistry()
if err := Register(registry); err != nil {
t.Fatalf("Register() error = %v, want nil", err)
}
spec, ok := registry.Spec(Key)
if !ok {
t.Fatalf("Spec(%q) ok = false, want true", Key)
}
if !reflect.DeepEqual(spec, want) {
t.Fatalf("registered spec = %#v, want %#v", spec, want)
}
}
func TestMergePassesThroughSingleExtractOutput(t *testing.T) {
input := extractOutput("chunk-0", 0, `{"name":"original"}`)
result, err := New().Merge(context.Background(), contracts.MergeRequest{
LaneID: "events",
ExtractOutputs: []contracts.ExtractOutput{input},
})
if err != nil {
t.Fatalf("Merge() error = %v, want nil", err)
}
if result.Output.LaneID != "events" || result.Output.MergerKey != Key {
t.Fatalf("output provenance = %#v, want lane and merger", result.Output)
}
if string(result.Output.Payload.Content) != `{"name":"original"}` {
t.Fatalf("content = %s, want original content", result.Output.Payload.Content)
}
if result.Output.Payload.Metadata["name"] != "chunk-0" {
t.Fatalf("metadata = %#v, want original metadata", result.Output.Payload.Metadata)
}
}
func TestMergeDefensivelyCopiesRawPayload(t *testing.T) {
input := extractOutput("chunk-0", 0, `{"name":"original"}`)
result, err := New().Merge(context.Background(), contracts.MergeRequest{
LaneID: "events",
ExtractOutputs: []contracts.ExtractOutput{input},
})
if err != nil {
t.Fatalf("Merge() error = %v, want nil", err)
}
input.Payload.Content[0] = '['
input.Payload.Metadata["name"] = "changed"
if string(result.Output.Payload.Content) != `{"name":"original"}` {
t.Fatalf("content changed after input mutation: %s", result.Output.Payload.Content)
}
if result.Output.Payload.Metadata["name"] != "chunk-0" {
t.Fatalf("metadata changed after input mutation: %#v", result.Output.Payload.Metadata)
}
}
func TestMergeConcatenatesCommonTopLevelArrayFieldInChunkOrder(t *testing.T) {
result, err := New().Merge(context.Background(), contracts.MergeRequest{
LaneID: "events",
ExtractOutputs: []contracts.ExtractOutput{
extractOutput("chunk-1", 1, `{"events":[{"name":"second"}]}`),
extractOutput("chunk-0", 0, `{"events":[{"name":"first"}]}`),
},
})
if err != nil {
t.Fatalf("Merge() error = %v, want nil", err)
}
if result.Output.Payload.MediaType != "application/json" {
t.Fatalf("MediaType = %q, want application/json", result.Output.Payload.MediaType)
}
var decoded struct {
Events []struct {
Name string `json:"name"`
} `json:"events"`
}
if err := json.Unmarshal(result.Output.Payload.Content, &decoded); err != nil {
t.Fatalf("Unmarshal() error = %v, want nil", err)
}
if len(decoded.Events) != 2 || decoded.Events[0].Name != "first" || decoded.Events[1].Name != "second" {
t.Fatalf("events = %#v, want concatenated chunk order", decoded.Events)
}
if result.Output.Schema.ID != "schema-id" {
t.Fatalf("schema = %#v, want common extract schema", result.Output.Schema)
}
}
func TestMergeFallsBackToOrderedJSONValueArrayWhenShapesDiffer(t *testing.T) {
result, err := New().Merge(context.Background(), contracts.MergeRequest{
LaneID: "events",
ExtractOutputs: []contracts.ExtractOutput{
extractOutput("chunk-1", 1, `{"notes":["second"]}`),
extractOutput("chunk-0", 0, `{"events":[{"name":"first"}]}`),
},
})
if err != nil {
t.Fatalf("Merge() error = %v, want nil", err)
}
var decoded []map[string]any
if err := json.Unmarshal(result.Output.Payload.Content, &decoded); err != nil {
t.Fatalf("Unmarshal() error = %v, want nil", err)
}
if len(decoded) != 2 {
t.Fatalf("len(decoded) = %d, want 2", len(decoded))
}
if _, ok := decoded[0]["events"]; !ok {
t.Fatalf("decoded[0] = %#v, want first chunk value", decoded[0])
}
if _, ok := decoded[1]["notes"]; !ok {
t.Fatalf("decoded[1] = %#v, want second chunk value", decoded[1])
}
}
func TestMergeRejectsInvalidJSONAndNonJSONMediaTypes(t *testing.T) {
tests := []struct {
name string
output contracts.ExtractOutput
want string
}{
{
name: "invalid JSON",
output: extractOutput("chunk-0", 0, `{"events":[`),
want: "invalid JSON",
},
{
name: "non JSON media type",
output: func() contracts.ExtractOutput {
output := extractOutput("chunk-0", 0, `{"events":[]}`)
output.Payload.MediaType = "text/plain"
return output
}(),
want: "unsupported media type",
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
_, err := New().Merge(context.Background(), contracts.MergeRequest{
LaneID: "events",
ExtractOutputs: []contracts.ExtractOutput{test.output},
})
if err == nil {
t.Fatal("Merge() error = nil, want error")
}
if !strings.Contains(err.Error(), test.want) {
t.Fatalf("Merge() error = %q, want %q", err.Error(), test.want)
}
})
}
}
func TestTypedMergeUsesRequestOrderForReusableValueType(t *testing.T) {
type notes struct{ Values []string }
merger, err := NewTyped(func(values []notes) (notes, error) {
var combined notes
for _, value := range values {
combined.Values = append(combined.Values, value.Values...)
}
return combined, nil
})
if err != nil {
t.Fatalf("NewTyped() error = %v", err)
}
result, err := merger.Merge(context.Background(), contracts.TypedMergeRequest[notes]{ExtractOutputs: []contracts.ExtractArtifact[notes]{
{ChunkIndex: 4, Value: notes{Values: []string{"first"}}},
{ChunkIndex: 1, Value: notes{Values: []string{"second"}}},
}})
if err != nil {
t.Fatalf("Merge() error = %v", err)
}
if got := result.Value.Values; !reflect.DeepEqual(got, []string{"first", "second"}) {
t.Fatalf("Values = %#v", got)
}
}
func extractOutput(chunkID string, chunkIndex int, content string) contracts.ExtractOutput {
return contracts.ExtractOutput{
LaneID: "events",
ExtractorKey: "extract",
SourceID: "source-1",
ChunkID: chunkID,
ChunkIndex: chunkIndex,
Schema: contracts.ResponseSchema{ID: "schema-id", Name: "schema-name", Version: "v1"},
Payload: contracts.RawPayload{
Content: []byte(content),
MediaType: "application/json",
Metadata: map[string]any{"name": chunkID},
},
}
}

View File

@@ -0,0 +1,25 @@
package appendorder
import (
"context"
"reflect"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
)
func TestTypedMergerCombinesValuesInFrameworkOrder(t *testing.T) {
merger, err := NewTyped(func(values []string) (string, error) {
if !reflect.DeepEqual(values, []string{"first", "second"}) {
t.Fatalf("values=%#v", values)
}
return values[0] + values[1], nil
})
if err != nil {
t.Fatal(err)
}
result, err := merger.Merge(context.Background(), contracts.TypedMergeRequest[string]{ExtractOutputs: []contracts.ExtractArtifact[string]{{ChunkIndex: 0, Value: "first"}, {ChunkIndex: 1, Value: "second"}}})
if err != nil || result.Value != "firstsecond" {
t.Fatalf("result=%#v err=%v", result, err)
}
}

View File

@@ -1,85 +1,15 @@
package noop
import (
"context"
"fmt"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
)
const Key = "noop"
var _ contracts.LegacyRawNormalizer = (*Normalizer)(nil)
type Normalizer struct{}
func New() *Normalizer {
return &Normalizer{}
}
func (n *Normalizer) Key() string {
return Key
}
func (n *Normalizer) ReferenceSlots() []contracts.ReferenceSlot {
return nil
}
func (n *Normalizer) Normalize(ctx context.Context, req contracts.NormalizeRequest) (contracts.NormalizeResult, error) {
if n == nil {
return contracts.NormalizeResult{}, normalizerErrorf("normalizer must not be nil")
}
if ctx == nil {
return contracts.NormalizeResult{}, normalizerErrorf("context must not be nil")
}
if err := ctx.Err(); err != nil {
return contracts.NormalizeResult{}, normalizerErrorf("context error before normalize: %w", err)
}
return contracts.NormalizeResult{
Output: contracts.NormalizeOutput{
LaneID: req.LaneID,
NormalizerKey: Key,
SourceID: req.MergeOutput.SourceID,
Schema: req.MergeOutput.Schema,
Payload: cloneRawPayload(req.MergeOutput.Payload),
},
}, nil
}
func ModuleSpec() pipeline.ModuleSpec {
return pipeline.ModuleSpec{
Key: Key,
Stage: pipeline.StageNormalize,
Requires: []string{"merged"},
Provides: []string{"normalized"},
}
}
func Register(registry *pipeline.NormalizerRegistry) error {
return registry.RegisterLegacyRawWithSpec(ModuleSpec(), func() (contracts.LegacyRawNormalizer, error) {
return New(), nil
})
}
func cloneRawPayload(payload contracts.RawPayload) contracts.RawPayload {
return contracts.RawPayload{
Content: append([]byte(nil), payload.Content...),
MediaType: payload.MediaType,
Metadata: cloneMetadata(payload.Metadata),
Warnings: append([]contracts.Warning(nil), payload.Warnings...),
}
}
func cloneMetadata(metadata map[string]any) map[string]any {
if len(metadata) == 0 {
return nil
}
out := make(map[string]any, len(metadata))
for key, value := range metadata {
out[key] = value
}
return out
return pipeline.ModuleSpec{Key: Key, Stage: pipeline.StageNormalize, Requires: []string{"merged"}, Provides: []string{"normalized"}}
}
func normalizerErrorf(format string, args ...any) error {

View File

@@ -1,112 +0,0 @@
package noop
import (
"context"
"reflect"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
)
func TestModuleSpecAndRegister(t *testing.T) {
want := pipeline.ModuleSpec{
Key: Key,
Stage: pipeline.StageNormalize,
Requires: []string{"merged"},
Provides: []string{"normalized"},
}
if got := ModuleSpec(); !reflect.DeepEqual(got, want) {
t.Fatalf("ModuleSpec() = %#v, want %#v", got, want)
}
registry := pipeline.NewNormalizerRegistry()
if err := Register(registry); err != nil {
t.Fatalf("Register() error = %v, want nil", err)
}
spec, ok := registry.Spec(Key)
if !ok {
t.Fatalf("Spec(%q) ok = false, want true", Key)
}
if !reflect.DeepEqual(spec, want) {
t.Fatalf("registered spec = %#v, want %#v", spec, want)
}
normalizer, err := registry.BuildLegacyRaw(Key)
if err != nil {
t.Fatalf("Build(%q) error = %v, want nil", Key, err)
}
if slots := normalizer.ReferenceSlots(); len(slots) != 0 {
t.Fatalf("ReferenceSlots() = %#v, want none", slots)
}
}
func TestNormalizePassesThroughMergeOutput(t *testing.T) {
input := mergeOutput(`{"name":"original"}`)
result, err := New().Normalize(context.Background(), contracts.NormalizeRequest{
LaneID: "events",
MergeOutput: input,
})
if err != nil {
t.Fatalf("Normalize() error = %v, want nil", err)
}
if result.Output.LaneID != "events" || result.Output.NormalizerKey != Key {
t.Fatalf("output provenance = %#v, want lane and normalizer", result.Output)
}
if string(result.Output.Payload.Content) != `{"name":"original"}` {
t.Fatalf("content = %s, want original content", result.Output.Payload.Content)
}
if result.Output.Payload.Metadata["name"] != "original" {
t.Fatalf("metadata = %#v, want original metadata", result.Output.Payload.Metadata)
}
}
func TestNormalizeDefensivelyCopiesRawPayload(t *testing.T) {
input := mergeOutput(`{"name":"original"}`)
result, err := New().Normalize(context.Background(), contracts.NormalizeRequest{
LaneID: "events",
MergeOutput: input,
})
if err != nil {
t.Fatalf("Normalize() error = %v, want nil", err)
}
input.Payload.Content[0] = '['
input.Payload.Metadata["name"] = "changed"
if string(result.Output.Payload.Content) != `{"name":"original"}` {
t.Fatalf("content changed after input mutation: %s", result.Output.Payload.Content)
}
if result.Output.Payload.Metadata["name"] != "original" {
t.Fatalf("metadata changed after input mutation: %#v", result.Output.Payload.Metadata)
}
}
func TestTypedNormalizePassesThroughReusableValueType(t *testing.T) {
type score struct{ Value int }
result, err := NewTyped[score]().Normalize(context.Background(), contracts.TypedNormalizeRequest[score]{
MergeOutput: contracts.MergeArtifact[score]{Value: score{Value: 7}},
})
if err != nil {
t.Fatalf("Normalize() error = %v", err)
}
if result.Value.Value != 7 {
t.Fatalf("Value = %d, want 7", result.Value.Value)
}
}
func mergeOutput(content string) contracts.MergeOutput {
return contracts.MergeOutput{
LaneID: "events",
MergerKey: "merge",
SourceID: "source-1",
Schema: contracts.ResponseSchema{ID: "schema-id", Name: "schema-name", Version: "v1"},
Payload: contracts.RawPayload{
Content: []byte(content),
MediaType: "application/json",
Metadata: map[string]any{"name": "original"},
},
}
}

View File

@@ -0,0 +1,16 @@
package noop
import (
"context"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
)
func TestTypedNormalizerPreservesValue(t *testing.T) {
normalizer := NewTyped[string]()
result, err := normalizer.Normalize(context.Background(), contracts.TypedNormalizeRequest[string]{MergeOutput: contracts.MergeArtifact[string]{Value: "value"}})
if err != nil || result.Value != "value" {
t.Fatalf("result=%#v err=%v", result, err)
}
}

View File

@@ -253,15 +253,6 @@ func cloneNormalizeOutputs(outputs []contracts.SerializedOutput) []contracts.Ser
return out
}
func cloneRawPayload(payload contracts.RawPayload) contracts.RawPayload {
return contracts.RawPayload{
Content: append([]byte(nil), payload.Content...),
MediaType: payload.MediaType,
Metadata: cloneMetadata(payload.Metadata),
Warnings: append([]contracts.Warning(nil), payload.Warnings...),
}
}
func cloneRejected(rejected []contracts.RejectedOutput) []contracts.RejectedOutput {
if len(rejected) == 0 {
return []contracts.RejectedOutput{}

View File

@@ -7,8 +7,6 @@ import (
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
"gitea.maximumdirect.net/eric/notarius/internal/modules/generic/chunk/units"
"gitea.maximumdirect.net/eric/notarius/internal/modules/generic/merge/appendorder"
"gitea.maximumdirect.net/eric/notarius/internal/modules/generic/normalize/noop"
jsonoutput "gitea.maximumdirect.net/eric/notarius/internal/modules/generic/output/json"
alwaysaccept "gitea.maximumdirect.net/eric/notarius/internal/modules/generic/validate/always_accept"
alwaysreject "gitea.maximumdirect.net/eric/notarius/internal/modules/generic/validate/always_reject"
@@ -27,8 +25,6 @@ func Register(registries pipeline.Registries, assets *llm.AssetRegistry) error {
register func() error
}{
{name: "generic chunker", register: func() error { return units.Register(registries.Chunkers) }},
{name: "appendorder merger", register: func() error { return appendorder.Register(registries.Mergers) }},
{name: "noop normalizer", register: func() error { return noop.Register(registries.Normalizers) }},
{name: "always accept validator", register: func() error { return alwaysaccept.Register(registries.Validators) }},
{name: "always reject validator", register: func() error { return alwaysreject.Register(registries.Validators) }},
{name: "valid json validator", register: func() error { return validjson.Register(registries.Validators) }},

View File

@@ -14,8 +14,8 @@ func TestRegisterAddsGenericFamily(t *testing.T) {
t.Fatalf("Register() error = %v, want nil", err)
}
assertKeys(t, "chunkers", registries.Chunkers.RegisteredKeys(), []string{"generic"})
assertKeys(t, "mergers", registries.Mergers.RegisteredKeys(), []string{"appendorder"})
assertKeys(t, "normalizers", registries.Normalizers.RegisteredKeys(), []string{"noop"})
assertKeys(t, "mergers", registries.Mergers.RegisteredKeys(), nil)
assertKeys(t, "normalizers", registries.Normalizers.RegisteredKeys(), nil)
assertKeys(t, "validators", registries.Validators.RegisteredKeys(), []string{
"generic/always_accept",
"generic/always_reject",

View File

@@ -12,7 +12,6 @@ const Key = "generic/always_accept"
type Options struct{}
type ChunkValidator struct{}
type TypedValidator[T any] struct{}
type legacyValidator struct{}
var _ contracts.ChunkValidator = (*ChunkValidator)(nil)
@@ -35,33 +34,17 @@ func (v *TypedValidator[T]) Validate(context.Context, contracts.TypedValidationR
return contracts.ValidationResult{Approved: true}, nil
}
func (v *legacyValidator) Name() string { return Key }
func (v *legacyValidator) ExecutionClass() contracts.ExecutionClass {
return contracts.ExecutionClassDeterministic
}
func (v *legacyValidator) Validate(context.Context, contracts.ValidationRequest) (contracts.ValidationResult, error) {
return contracts.ValidationResult{Approved: true}, nil
}
func Spec() pipeline.ValidatorSpec {
return pipeline.ValidatorSpec{Key: Key, ExecutionClass: contracts.ExecutionClassDeterministic}
}
func Register(registry *pipeline.ValidatorRegistry) error {
if err := pipeline.RegisterChunkValidatorBuilder(registry, Spec(), validateOptions, func(request pipeline.BuildRequest) (contracts.ChunkValidator, error) {
return pipeline.RegisterChunkValidatorBuilder(registry, Spec(), validateOptions, func(request pipeline.BuildRequest) (contracts.ChunkValidator, error) {
options, err := DecodeOptions(request.Options)
if err != nil {
return nil, err
}
return NewChunk(options), nil
}); err != nil {
return err
}
return registry.RegisterLegacyRawBuilderWithSpec(Spec(), validateOptions, func(request pipeline.BuildRequest) (contracts.LegacyRawValidator, error) {
if _, err := DecodeOptions(request.Options); err != nil {
return nil, err
}
return &legacyValidator{}, nil
})
}

View File

@@ -30,11 +30,7 @@ func TestSpecAndRegister(t *testing.T) {
if err := Register(registry); err != nil {
t.Fatalf("Register() error = %v, want nil", err)
}
validator, err := registry.BuildLegacyRaw(Key)
if err != nil {
t.Fatalf("Build(%q) error = %v, want nil", Key, err)
}
if validator.Name() != Key {
t.Fatalf("Name() = %q, want %q", validator.Name(), Key)
if registered, ok := registry.Spec(Key); !ok || registered.Key != Key {
t.Fatalf("Spec(%q) = %#v, %v", Key, registered, ok)
}
}

View File

@@ -13,7 +13,6 @@ const ReasonCode = "always_reject"
type Options struct{}
type ChunkValidator struct{}
type TypedValidator[T any] struct{}
type legacyValidator struct{}
func NewChunk(Options) *ChunkValidator { return &ChunkValidator{} }
func NewTyped[T any](Options) *TypedValidator[T] { return &TypedValidator[T]{} }
@@ -35,32 +34,16 @@ func (v *TypedValidator[T]) ExecutionClass() contracts.ExecutionClass {
func (v *TypedValidator[T]) Validate(context.Context, contracts.TypedValidationRequest[T]) (contracts.ValidationResult, error) {
return rejection(), nil
}
func (v *legacyValidator) Name() string { return Key }
func (v *legacyValidator) ExecutionClass() contracts.ExecutionClass {
return contracts.ExecutionClassDeterministic
}
func (v *legacyValidator) Validate(context.Context, contracts.ValidationRequest) (contracts.ValidationResult, error) {
return rejection(), nil
}
func Spec() pipeline.ValidatorSpec {
return pipeline.ValidatorSpec{Key: Key, ExecutionClass: contracts.ExecutionClassDeterministic}
}
func Register(registry *pipeline.ValidatorRegistry) error {
if err := pipeline.RegisterChunkValidatorBuilder(registry, Spec(), validateOptions, func(request pipeline.BuildRequest) (contracts.ChunkValidator, error) {
return pipeline.RegisterChunkValidatorBuilder(registry, Spec(), validateOptions, func(request pipeline.BuildRequest) (contracts.ChunkValidator, error) {
options, err := DecodeOptions(request.Options)
if err != nil {
return nil, err
}
return NewChunk(options), nil
}); err != nil {
return err
}
return registry.RegisterLegacyRawBuilderWithSpec(Spec(), validateOptions, func(request pipeline.BuildRequest) (contracts.LegacyRawValidator, error) {
if _, err := DecodeOptions(request.Options); err != nil {
return nil, err
}
return &legacyValidator{}, nil
})
}
func RegisterTyped[T any](registry *pipeline.ValidatorRegistry, kind contracts.ArtifactKind) error {

View File

@@ -33,11 +33,7 @@ func TestSpecAndRegister(t *testing.T) {
if err := Register(registry); err != nil {
t.Fatalf("Register() error = %v, want nil", err)
}
validator, err := registry.BuildLegacyRaw(Key)
if err != nil {
t.Fatalf("Build(%q) error = %v, want nil", Key, err)
}
if validator.Name() != Key {
t.Fatalf("Name() = %q, want %q", validator.Name(), Key)
if registered, ok := registry.Spec(Key); !ok || registered.Key != Key {
t.Fatalf("Spec(%q) = %#v, %v", Key, registered, ok)
}
}

View File

@@ -15,10 +15,7 @@ type Options struct{}
type Validator struct{}
type legacyValidator struct{}
var _ contracts.SerializedValidator = (*Validator)(nil)
var _ contracts.LegacyRawValidator = (*legacyValidator)(nil)
func New(Options) *Validator { return &Validator{} }
@@ -32,16 +29,6 @@ func (v *Validator) Validate(_ context.Context, req contracts.SerializedValidati
return validate(req.Content), nil
}
func (v *legacyValidator) Name() string { return Key }
func (v *legacyValidator) ExecutionClass() contracts.ExecutionClass {
return contracts.ExecutionClassDeterministic
}
func (v *legacyValidator) Validate(_ context.Context, req contracts.ValidationRequest) (contracts.ValidationResult, error) {
return validate(req.Payload.Content), nil
}
func validate(content []byte) contracts.ValidationResult {
if !json.Valid(content) {
return contracts.ValidationResult{Approved: false, ReasonCode: ReasonCodeInvalidJSON, Message: "payload is not valid JSON"}
@@ -54,7 +41,7 @@ func Spec() pipeline.ValidatorSpec {
}
func Register(registry *pipeline.ValidatorRegistry) error {
if err := pipeline.RegisterSerializedValidatorBuilder(registry, pipeline.SerializedValidatorSpec{
return pipeline.RegisterSerializedValidatorBuilder(registry, pipeline.SerializedValidatorSpec{
ValidatorSpec: Spec(), SupportsChunks: true, SupportsArtifacts: true,
}, validateOptions, func(request pipeline.BuildRequest) (contracts.SerializedValidator, error) {
options, err := DecodeOptions(request.Options)
@@ -62,14 +49,6 @@ func Register(registry *pipeline.ValidatorRegistry) error {
return nil, err
}
return New(options), nil
}); err != nil {
return err
}
return registry.RegisterLegacyRawBuilderWithSpec(Spec(), validateOptions, func(request pipeline.BuildRequest) (contracts.LegacyRawValidator, error) {
if _, err := DecodeOptions(request.Options); err != nil {
return nil, err
}
return &legacyValidator{}, nil
})
}

View File

@@ -47,12 +47,8 @@ func TestSpecAndRegister(t *testing.T) {
if err := Register(registry); err != nil {
t.Fatalf("Register() error = %v, want nil", err)
}
validator, err := registry.BuildLegacyRaw(Key)
if err != nil {
t.Fatalf("Build(%q) error = %v, want nil", Key, err)
}
if validator.Name() != Key {
t.Fatalf("Name() = %q, want %q", validator.Name(), Key)
if registered, ok := registry.Spec(Key); !ok || registered.Key != Key {
t.Fatalf("Spec(%q) = %#v, %v", Key, registered, ok)
}
}

View File

@@ -17,10 +17,8 @@ const ReasonCodeSchemaInvalid = "json_schema_invalid"
type Options struct{}
type Validator struct{}
type legacyValidator struct{}
var _ contracts.SerializedValidator = (*Validator)(nil)
var _ contracts.LegacyRawValidator = (*legacyValidator)(nil)
func New(Options) *Validator { return &Validator{} }
func (v *Validator) Name() string { return Key }
@@ -32,14 +30,6 @@ func (v *Validator) Validate(_ context.Context, req contracts.SerializedValidati
return validate(req.Content, req.Schema.JSONSchema)
}
func (v *legacyValidator) Name() string { return Key }
func (v *legacyValidator) ExecutionClass() contracts.ExecutionClass {
return contracts.ExecutionClassDeterministic
}
func (v *legacyValidator) Validate(_ context.Context, req contracts.ValidationRequest) (contracts.ValidationResult, error) {
return validate(req.Payload.Content, req.Schema.JSONSchema)
}
func validate(content, schemaContent []byte) (contracts.ValidationResult, error) {
if len(schemaContent) == 0 {
return contracts.ValidationResult{}, fmt.Errorf("response schema content is not available")
@@ -71,7 +61,7 @@ func Spec() pipeline.ValidatorSpec {
}
func Register(registry *pipeline.ValidatorRegistry) error {
if err := pipeline.RegisterSerializedValidatorBuilder(registry, pipeline.SerializedValidatorSpec{
return pipeline.RegisterSerializedValidatorBuilder(registry, pipeline.SerializedValidatorSpec{
ValidatorSpec: Spec(), SupportsChunks: true, SupportsArtifacts: true,
}, validateOptions, func(request pipeline.BuildRequest) (contracts.SerializedValidator, error) {
options, err := DecodeOptions(request.Options)
@@ -79,14 +69,6 @@ func Register(registry *pipeline.ValidatorRegistry) error {
return nil, err
}
return New(options), nil
}); err != nil {
return err
}
return registry.RegisterLegacyRawBuilderWithSpec(Spec(), validateOptions, func(request pipeline.BuildRequest) (contracts.LegacyRawValidator, error) {
if _, err := DecodeOptions(request.Options); err != nil {
return nil, err
}
return &legacyValidator{}, nil
})
}

View File

@@ -74,12 +74,8 @@ func TestSpecAndRegister(t *testing.T) {
if err := Register(registry); err != nil {
t.Fatalf("Register() error = %v, want nil", err)
}
validator, err := registry.BuildLegacyRaw(Key)
if err != nil {
t.Fatalf("Build(%q) error = %v, want nil", Key, err)
}
if validator.Name() != Key {
t.Fatalf("Name() = %q, want %q", validator.Name(), Key)
if registered, ok := registry.Spec(Key); !ok || registered.Key != Key {
t.Fatalf("Spec(%q) = %#v, %v", Key, registered, ok)
}
}

View File

@@ -10,6 +10,7 @@ import (
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
spellcodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/spells"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/spells"
"gitea.maximumdirect.net/eric/notarius/internal/modules/generic/merge/appendorder"
@@ -178,37 +179,39 @@ func dndSpellsTestCatalog(t *testing.T, specs dndSpellsCatalogSpecs) pipeline.Mo
t.Fatalf("register chunker: %v", err)
}
codec := spellcodec.New()
if err := pipeline.RegisterArtifactCodec(codecs, codec); err != nil {
t.Fatalf("register dnd spells codec: %v", err)
}
if specs.extractor.Key == "" {
codec := spellcodec.New()
if err := pipeline.RegisterArtifactCodec(codecs, codec); err != nil {
t.Fatalf("register dnd spells codec: %v", err)
}
if err := spells.RegisterWithRawAdapter(extractors, codec); err != nil {
if err := spells.Register(extractors); err != nil {
t.Fatalf("register dnd spells extractor: %v", err)
}
} else {
specs.extractor.ArtifactKind = ""
if err := extractors.RegisterLegacyRawWithSpec(specs.extractor, func() (contracts.LegacyRawExtractor, error) {
return configLegacyExtractor{key: specs.extractor.Key}, nil
specs.extractor.ArtifactKind = dnd.SpellListKind
if err := pipeline.RegisterExtractor[dnd.SpellList](extractors, specs.extractor, func() (contracts.Extractor[dnd.SpellList], error) {
return configExtractor{key: specs.extractor.Key}, nil
}); err != nil {
t.Fatalf("register dnd spells extractor override: %v", err)
}
}
if err := mergers.RegisterLegacyRawWithSpec(pipeline.ModuleSpec{
Key: pipeline.DefaultMergeModule,
Stage: pipeline.StageMerge,
Requires: []string{"dnd.spell_casts"},
}, func() (contracts.LegacyRawMerger, error) {
return appendorder.New(), nil
if err := pipeline.RegisterMerger[dnd.SpellList](mergers, pipeline.ModuleSpec{
Key: pipeline.DefaultMergeModule,
Stage: pipeline.StageMerge,
ArtifactKind: dnd.SpellListKind,
Requires: []string{"dnd.spell_casts"},
}, func() (contracts.Merger[dnd.SpellList], error) {
return appendorder.NewTyped(appendSpellLists)
}); err != nil {
t.Fatalf("register merger: %v", err)
}
if err := normalizers.RegisterLegacyRawWithSpec(pipeline.ModuleSpec{
Key: pipeline.DefaultNormalizeModule,
Stage: pipeline.StageNormalize,
}, func() (contracts.LegacyRawNormalizer, error) {
return noop.New(), nil
if err := pipeline.RegisterNormalizer[dnd.SpellList](normalizers, pipeline.ModuleSpec{
Key: pipeline.DefaultNormalizeModule,
Stage: pipeline.StageNormalize,
ArtifactKind: dnd.SpellListKind,
}, func() (contracts.Normalizer[dnd.SpellList], error) {
return noop.NewTyped[dnd.SpellList](), nil
}); err != nil {
t.Fatalf("register normalizer: %v", err)
}
@@ -233,12 +236,20 @@ func dndSpellsTestCatalog(t *testing.T, specs dndSpellsCatalogSpecs) pipeline.Mo
}
}
type configLegacyExtractor struct{ key string }
type configExtractor struct{ key string }
func (extractor configLegacyExtractor) Key() string { return extractor.key }
func (configLegacyExtractor) ReferenceSlots() []contracts.ReferenceSlot { return nil }
func (configLegacyExtractor) Extract(context.Context, contracts.ExtractionRequest) (contracts.ExtractionResult, error) {
return contracts.ExtractionResult{}, nil
func (extractor configExtractor) Key() string { return extractor.key }
func (configExtractor) ReferenceSlots() []contracts.ReferenceSlot { return nil }
func (configExtractor) Extract(context.Context, contracts.TypedExtractionRequest) (contracts.TypedExtractionResult[dnd.SpellList], error) {
return contracts.TypedExtractionResult[dnd.SpellList]{}, nil
}
func appendSpellLists(values []dnd.SpellList) (dnd.SpellList, error) {
combined := dnd.SpellList{SpellCasts: []dnd.SpellCast{}}
for _, value := range values {
combined.SpellCasts = append(combined.SpellCasts, value.SpellCasts...)
}
return combined, nil
}
func dndSpellsChunkerSpec() pipeline.ModuleSpec {

View File

@@ -60,11 +60,11 @@ func TestRunnerProcessesSeriatimInputWithDNDSpellsExtractor(t *testing.T) {
if len(output.NormalizeOutputs) != 1 {
t.Fatalf("len(NormalizeOutputs) = %d, want 1", len(output.NormalizeOutputs))
}
rawOutput := output.NormalizeOutputs[0]
if rawOutput.LaneID != "spells" || rawOutput.Artifact.Schema.ID != spells.ResponseSchemaID || rawOutput.Artifact.Schema.Version != spells.SchemaVersion {
t.Fatalf("raw output envelope = %#v, want dnd spells schema on spells lane", rawOutput)
serializedOutput := output.NormalizeOutputs[0]
if serializedOutput.LaneID != "spells" || serializedOutput.Artifact.Schema.ID != spells.ResponseSchemaID || serializedOutput.Artifact.Schema.Version != spells.SchemaVersion {
t.Fatalf("serialized output envelope = %#v, want dnd spells schema on spells lane", serializedOutput)
}
response := decodeRunnerSpellResponse(t, rawOutput.Artifact.Content)
response := decodeRunnerSpellResponse(t, serializedOutput.Artifact.Content)
if len(response.SpellCasts) != 2 {
t.Fatalf("len(spell_casts) = %d, want 2", len(response.SpellCasts))
}
@@ -204,7 +204,7 @@ func TestRunnerDoesNotExtractSpellMentionedOnlyInPartyReference(t *testing.T) {
}
}
func TestRunnerCarriesDNDSpellCastWithInvalidSourceRefAsRawOutput(t *testing.T) {
func TestRunnerCarriesDNDSpellCastWithInvalidSourceRefToSerializedOutput(t *testing.T) {
raw := readDNDSpellsFixture(t)
resolved := resolveDNDSpellsPipeline(t)
llmClient := &fakeSpellsLLMClient{
@@ -235,7 +235,7 @@ func TestRunnerCarriesDNDSpellCastWithInvalidSourceRefAsRawOutput(t *testing.T)
t.Fatalf("len(spell_casts) = %d, want 1", len(response.SpellCasts))
}
if response.SpellCasts[0].SourceRefs[0].SourceID != "spell-session" {
t.Fatalf("SourceID = %q, want raw invalid source ref preserved", response.SpellCasts[0].SourceRefs[0].SourceID)
t.Fatalf("SourceID = %q, want invalid source ref preserved", response.SpellCasts[0].SourceRefs[0].SourceID)
}
if len(output.Rejected) != 0 {
t.Fatalf("len(Rejected) = %d, want 0", len(output.Rejected))
@@ -282,7 +282,7 @@ func dndSpellsReferenceSet(party string, glossary string) contracts.ReferenceSet
return contracts.ReferenceSet{Slots: slots}
}
func TestRunnerCarriesMalformedDNDSpellsExtractorOutput(t *testing.T) {
func TestRunnerRejectsMalformedDNDSpellsArtifactAtSerializationBoundary(t *testing.T) {
raw := readDNDSpellsFixture(t)
resolved := resolveDNDSpellsPipeline(t)
llmClient := &fakeSpellsLLMClient{response: extractionResponse{}}
@@ -290,17 +290,11 @@ func TestRunnerCarriesMalformedDNDSpellsExtractorOutput(t *testing.T) {
output, err := runPreparedPipeline(t, dndSpellsRunnerRegistries(t), resolved.ResolvedPipeline, llmClient, pipeline.RunInput{
RawInput: raw,
})
if err != nil {
t.Fatalf("Run() error = %v, want nil", err)
if err == nil || !strings.Contains(err.Error(), "spell_casts must be present") {
t.Fatalf("Run() error = %v, want invalid spell-list serialization error", err)
}
if len(output.NormalizeOutputs) != 1 {
t.Fatalf("len(NormalizeOutputs) = %d, want raw output", len(output.NormalizeOutputs))
}
if string(output.NormalizeOutputs[0].Artifact.Content) != `{"spell_casts":null}` {
t.Fatalf("content = %s, want canonical structured output", output.NormalizeOutputs[0].Artifact.Content)
}
if output.Manifest.ValidationStatus != "approved" {
t.Fatalf("ValidationStatus = %q, want approved", output.Manifest.ValidationStatus)
if len(output.NormalizeOutputs) != 0 {
t.Fatalf("len(NormalizeOutputs) = %d, want no serialized malformed artifact", len(output.NormalizeOutputs))
}
}

View File

@@ -2,12 +2,14 @@ package transcript
import (
"context"
"encoding/json"
"os"
"reflect"
"strings"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
"gitea.maximumdirect.net/eric/notarius/internal/modules/generic/merge/appendorder"
@@ -131,29 +133,36 @@ func seriatimTestCatalog(t *testing.T, inputSpec pipeline.ModuleSpec) pipeline.M
Provides: []string{"chunks"},
})
mustRegisterExtractor(t, extractors, pipeline.ModuleSpec{
Key: "fake/extract",
Stage: pipeline.StageExtract,
Requires: []string{"chunks", "transcript.speaker", "transcript.timestamps"},
Provides: []string{"fake.artifacts"},
Key: "fake/extract",
Stage: pipeline.StageExtract,
ArtifactKind: seriatimArtifactKind,
Requires: []string{"chunks", "transcript.speaker", "transcript.timestamps"},
Provides: []string{"fake.artifacts"},
})
mustRegisterMerger(t, mergers, pipeline.ModuleSpec{
Key: pipeline.DefaultMergeModule,
Stage: pipeline.StageMerge,
Requires: []string{"fake.artifacts"},
Key: pipeline.DefaultMergeModule,
Stage: pipeline.StageMerge,
ArtifactKind: seriatimArtifactKind,
Requires: []string{"fake.artifacts"},
})
mustRegisterNormalizer(t, normalizers, pipeline.ModuleSpec{
Key: pipeline.DefaultNormalizeModule,
Stage: pipeline.StageNormalize,
Key: pipeline.DefaultNormalizeModule,
Stage: pipeline.StageNormalize,
ArtifactKind: seriatimArtifactKind,
})
mustRegisterOutput(t, outputs, pipeline.ModuleSpec{
Key: pipeline.DefaultOutputModule,
Stage: pipeline.StageOutput,
})
codecs := pipeline.NewArtifactCodecRegistry()
if err := pipeline.RegisterArtifactCodec(codecs, seriatimArtifactCodec{}); err != nil {
t.Fatalf("register artifact codec: %v", err)
}
return pipeline.ModuleCatalog{
Inputs: inputs,
Chunkers: chunkers,
ArtifactCodecs: pipeline.NewArtifactCodecRegistry(),
ArtifactCodecs: codecs,
Extractors: extractors,
Mergers: mergers,
Normalizers: normalizers,
@@ -173,7 +182,7 @@ func mustRegisterChunker(t *testing.T, registry *pipeline.ChunkerRegistry, spec
func mustRegisterExtractor(t *testing.T, registry *pipeline.ExtractorRegistry, spec pipeline.ModuleSpec) {
t.Helper()
if err := registry.RegisterLegacyRawWithSpec(spec, func() (contracts.LegacyRawExtractor, error) {
if err := pipeline.RegisterExtractor[seriatimArtifact](registry, spec, func() (contracts.Extractor[seriatimArtifact], error) {
return fakeExtractor{}, nil
}); err != nil {
t.Fatalf("register extractor: %v", err)
@@ -182,8 +191,13 @@ func mustRegisterExtractor(t *testing.T, registry *pipeline.ExtractorRegistry, s
func mustRegisterMerger(t *testing.T, registry *pipeline.MergerRegistry, spec pipeline.ModuleSpec) {
t.Helper()
if err := registry.RegisterLegacyRawWithSpec(spec, func() (contracts.LegacyRawMerger, error) {
return appendorder.New(), nil
if err := pipeline.RegisterMerger[seriatimArtifact](registry, spec, func() (contracts.Merger[seriatimArtifact], error) {
return appendorder.NewTyped(func(values []seriatimArtifact) (seriatimArtifact, error) {
if len(values) == 0 {
return seriatimArtifact{}, nil
}
return values[0], nil
})
}); err != nil {
t.Fatalf("register merger: %v", err)
}
@@ -191,8 +205,8 @@ func mustRegisterMerger(t *testing.T, registry *pipeline.MergerRegistry, spec pi
func mustRegisterNormalizer(t *testing.T, registry *pipeline.NormalizerRegistry, spec pipeline.ModuleSpec) {
t.Helper()
if err := registry.RegisterLegacyRawWithSpec(spec, func() (contracts.LegacyRawNormalizer, error) {
return noop.New(), nil
if err := pipeline.RegisterNormalizer[seriatimArtifact](registry, spec, func() (contracts.Normalizer[seriatimArtifact], error) {
return noop.NewTyped[seriatimArtifact](), nil
}); err != nil {
t.Fatalf("register normalizer: %v", err)
}
@@ -223,8 +237,8 @@ func (fakeExtractor) Key() string { return "fake/extract" }
func (fakeExtractor) ReferenceSlots() []contracts.ReferenceSlot { return nil }
func (fakeExtractor) Extract(ctx context.Context, req contracts.ExtractionRequest) (contracts.ExtractionResult, error) {
return contracts.ExtractionResult{}, nil
func (fakeExtractor) Extract(ctx context.Context, req contracts.TypedExtractionRequest) (contracts.TypedExtractionResult[seriatimArtifact], error) {
return contracts.TypedExtractionResult[seriatimArtifact]{}, nil
}
type fakeOutput struct{}
@@ -246,7 +260,30 @@ func withoutCapability(capabilities []string, capability string) []string {
}
var (
_ contracts.Chunker = fakeChunker{}
_ contracts.LegacyRawExtractor = fakeExtractor{}
_ contracts.OutputEncoder = fakeOutput{}
_ contracts.Chunker = fakeChunker{}
_ contracts.Extractor[seriatimArtifact] = fakeExtractor{}
_ contracts.OutputEncoder = fakeOutput{}
)
const seriatimArtifactKind contracts.ArtifactKind = "test/seriatim-event"
type seriatimArtifact struct {
Value string `json:"value"`
SourceRefs []source.SourceRef `json:"source_refs"`
}
type seriatimArtifactCodec struct{}
func (seriatimArtifactCodec) Kind() contracts.ArtifactKind { return seriatimArtifactKind }
func (seriatimArtifactCodec) Schema() contracts.ArtifactSchema {
return contracts.ArtifactSchema{ID: "fake.event", Name: "fake_event", Version: "v1", JSONSchema: []byte(`{"type":"object"}`)}
}
func (seriatimArtifactCodec) MediaType() string { return "application/json" }
func (seriatimArtifactCodec) Encode(value seriatimArtifact) ([]byte, error) {
return json.Marshal(value)
}
func (seriatimArtifactCodec) Decode(content []byte) (seriatimArtifact, error) {
var value seriatimArtifact
err := json.Unmarshal(content, &value)
return value, err
}

View File

@@ -55,16 +55,16 @@ func TestRunnerProcessesSeriatimInputWithFakeModules(t *testing.T) {
t.Fatalf("len(NormalizeOutputs) = %d, want 1", len(output.NormalizeOutputs))
}
rawOutput := output.NormalizeOutputs[0]
if rawOutput.LaneID != "events" || rawOutput.NormalizerKey != pipeline.DefaultNormalizeModule || rawOutput.Artifact.Schema.ID != "fake.event" || rawOutput.Artifact.Schema.Version != "v1" {
t.Fatalf("raw output envelope = %#v, want fake extractor envelope", rawOutput)
serializedOutput := output.NormalizeOutputs[0]
if serializedOutput.LaneID != "events" || serializedOutput.NormalizerKey != pipeline.DefaultNormalizeModule || serializedOutput.Artifact.Schema.ID != "fake.event" || serializedOutput.Artifact.Schema.Version != "v1" {
t.Fatalf("serialized output envelope = %#v, want fake extractor envelope", serializedOutput)
}
var payload struct {
Value string `json:"value"`
SourceRefs []source.SourceRef `json:"source_refs"`
}
if err := json.Unmarshal(rawOutput.Artifact.Content, &payload); err != nil {
t.Fatalf("Unmarshal(raw output) error = %v, want nil", err)
if err := json.Unmarshal(serializedOutput.Artifact.Content, &payload); err != nil {
t.Fatalf("Unmarshal(serialized output) error = %v, want nil", err)
}
if len(payload.SourceRefs) != 1 {
t.Fatalf("len(SourceRefs) = %d, want 1", len(payload.SourceRefs))
@@ -115,7 +115,7 @@ func configResolveInput(t *testing.T) config.ResolveInput {
}
}
func seriatimRunnerRegistries(t *testing.T, extractor contracts.LegacyRawExtractor) pipeline.Registries {
func seriatimRunnerRegistries(t *testing.T, extractor contracts.Extractor[seriatimArtifact]) pipeline.Registries {
t.Helper()
inputs := pipeline.NewInputAdapterRegistry()
@@ -133,19 +133,20 @@ func seriatimRunnerRegistries(t *testing.T, extractor contracts.LegacyRawExtract
}); err != nil {
t.Fatalf("register chunker: %v", err)
}
if err := extractors.RegisterLegacyRaw("fake/extract", func() (contracts.LegacyRawExtractor, error) {
if err := pipeline.RegisterExtractor[seriatimArtifact](extractors, pipeline.ModuleSpec{Key: "fake/extract", Stage: pipeline.StageExtract, ArtifactKind: seriatimArtifactKind, Requires: []string{"chunks", "transcript.speaker", "transcript.timestamps"}, Provides: []string{"fake.artifacts"}}, func() (contracts.Extractor[seriatimArtifact], error) {
return extractor, nil
}); err != nil {
t.Fatalf("register extractor: %v", err)
}
if err := mergers.RegisterLegacyRaw(pipeline.DefaultMergeModule, func() (contracts.LegacyRawMerger, error) {
return appendorder.New(), nil
if err := appendorder.RegisterTyped(mergers, seriatimArtifactKind, func(values []seriatimArtifact) (seriatimArtifact, error) {
if len(values) == 0 {
return seriatimArtifact{}, nil
}
return values[0], nil
}); err != nil {
t.Fatalf("register merger: %v", err)
}
if err := normalizers.RegisterLegacyRaw(pipeline.DefaultNormalizeModule, func() (contracts.LegacyRawNormalizer, error) {
return noop.New(), nil
}); err != nil {
if err := noop.RegisterTyped[seriatimArtifact](normalizers, seriatimArtifactKind); err != nil {
t.Fatalf("register normalizer: %v", err)
}
if err := outputs.Register(pipeline.DefaultOutputModule, func() (contracts.OutputEncoder, error) {
@@ -154,10 +155,14 @@ func seriatimRunnerRegistries(t *testing.T, extractor contracts.LegacyRawExtract
t.Fatalf("register output: %v", err)
}
codecs := pipeline.NewArtifactCodecRegistry()
if err := pipeline.RegisterArtifactCodec(codecs, seriatimArtifactCodec{}); err != nil {
t.Fatalf("register artifact codec: %v", err)
}
return pipeline.Registries{
Inputs: inputs,
Chunkers: chunkers,
ArtifactCodecs: pipeline.NewArtifactCodecRegistry(),
ArtifactCodecs: codecs,
Extractors: extractors,
Mergers: mergers,
Normalizers: normalizers,
@@ -203,36 +208,33 @@ func (e *runnerSeriatimExtractor) ReferenceSlots() []contracts.ReferenceSlot {
return nil
}
func (e *runnerSeriatimExtractor) Extract(ctx context.Context, req contracts.ExtractionRequest) (contracts.ExtractionResult, error) {
func (e *runnerSeriatimExtractor) Extract(ctx context.Context, req contracts.TypedExtractionRequest) (contracts.TypedExtractionResult[seriatimArtifact], error) {
e.calls++
if req.Source == nil {
return contracts.ExtractionResult{}, fmt.Errorf("source must not be nil")
return contracts.TypedExtractionResult[seriatimArtifact]{}, fmt.Errorf("source must not be nil")
}
if req.Chunk == nil {
return contracts.ExtractionResult{}, fmt.Errorf("chunk must not be nil")
return contracts.TypedExtractionResult[seriatimArtifact]{}, fmt.Errorf("chunk must not be nil")
}
if got := unitIDs(req.Source.Units); !equalInts(got, []int{1, 2}) {
return contracts.ExtractionResult{}, fmt.Errorf("source unit IDs = %#v, want Seriatim segment IDs", got)
return contracts.TypedExtractionResult[seriatimArtifact]{}, fmt.Errorf("source unit IDs = %#v, want Seriatim segment IDs", got)
}
if got := unitIDs(req.Chunk.Units); !equalInts(got, []int{1, 2}) {
return contracts.ExtractionResult{}, fmt.Errorf("chunk unit IDs = %#v, want Seriatim segment IDs", got)
return contracts.TypedExtractionResult[seriatimArtifact]{}, fmt.Errorf("chunk unit IDs = %#v, want Seriatim segment IDs", got)
}
for _, unit := range req.Chunk.Units {
if speaker, ok := Speaker(unit); !ok || speaker == "" {
return contracts.ExtractionResult{}, fmt.Errorf("unit %d missing speaker metadata", unit.ID)
return contracts.TypedExtractionResult[seriatimArtifact]{}, fmt.Errorf("unit %d missing speaker metadata", unit.ID)
}
if _, ok := Start(unit); !ok {
return contracts.ExtractionResult{}, fmt.Errorf("unit %d missing start metadata", unit.ID)
return contracts.TypedExtractionResult[seriatimArtifact]{}, fmt.Errorf("unit %d missing start metadata", unit.ID)
}
if _, ok := End(unit); !ok {
return contracts.ExtractionResult{}, fmt.Errorf("unit %d missing end metadata", unit.ID)
return contracts.TypedExtractionResult[seriatimArtifact]{}, fmt.Errorf("unit %d missing end metadata", unit.ID)
}
}
payload, err := json.Marshal(struct {
Value string `json:"value"`
SourceRefs []source.SourceRef `json:"source_refs"`
}{
return contracts.TypedExtractionResult[seriatimArtifact]{Value: seriatimArtifact{
Value: "seriatim-source-ref",
SourceRefs: []source.SourceRef{
{
@@ -241,20 +243,7 @@ func (e *runnerSeriatimExtractor) Extract(ctx context.Context, req contracts.Ext
EndUnitID: req.Chunk.Units[len(req.Chunk.Units)-1].ID,
},
},
})
if err != nil {
return contracts.ExtractionResult{}, err
}
return contracts.ExtractionResult{
Output: contracts.ExtractOutput{
Schema: contracts.ResponseSchema{ID: "fake.event", Name: "fake_event", Version: "v1"},
Payload: contracts.RawPayload{
Content: payload,
MediaType: "application/json",
},
},
}, nil
}}, nil
}
type runnerSeriatimOutput struct{}
@@ -292,7 +281,7 @@ func equalInts(a, b []int) bool {
}
var (
_ contracts.Chunker = runnerSeriatimChunker{}
_ contracts.LegacyRawExtractor = (*runnerSeriatimExtractor)(nil)
_ contracts.OutputEncoder = runnerSeriatimOutput{}
_ contracts.Chunker = runnerSeriatimChunker{}
_ contracts.Extractor[seriatimArtifact] = (*runnerSeriatimExtractor)(nil)
_ contracts.OutputEncoder = runnerSeriatimOutput{}
)