refactor!: introduce generic processors registry and remove normalize registry adapter

- add new `processors` package with canonical `Processor` interface
- add `processors.Registry` with Register/Build/BuildChain factory model
- switch `pipeline.Pipeline` to `[]processors.Processor`
- replace `normalize.Registry` + registry adapter with direct `normalize.Processor`
- remove `normalize/registry.go`
- update root docs to position normalize as one optional processing stage
- add tests for processors registry, normalize processor behavior, and pipeline flow

BREAKING CHANGE:
- `pipeline.Processor` removed; use `processors.Processor`
- `normalize.Registry` and old normalize processor adapter APIs removed
- downstream daemons must update processor wiring to new `processors.Registry`
  and `normalize.NewProcessor(...)`
This commit is contained in:
2026-03-16 13:14:24 -05:00
parent 6c5f95ad26
commit 96039f6530
12 changed files with 543 additions and 162 deletions

View File

@@ -5,15 +5,11 @@ import (
"fmt"
"gitea.maximumdirect.net/ejr/feedkit/event"
"gitea.maximumdirect.net/ejr/feedkit/processors"
)
// Processor can mutate/drop events (dedupe, rate-limit, normalization tweaks).
type Processor interface {
Process(ctx context.Context, in event.Event) (out *event.Event, err error)
}
type Pipeline struct {
Processors []Processor
Processors []processors.Processor
}
func (p *Pipeline) Process(ctx context.Context, e event.Event) (*event.Event, error) {

115
pipeline/pipeline_test.go Normal file
View File

@@ -0,0 +1,115 @@
package pipeline
import (
"context"
"fmt"
"strings"
"testing"
"time"
"gitea.maximumdirect.net/ejr/feedkit/event"
"gitea.maximumdirect.net/ejr/feedkit/processors"
)
type procFunc func(context.Context, event.Event) (*event.Event, error)
func (f procFunc) Process(ctx context.Context, in event.Event) (*event.Event, error) {
return f(ctx, in)
}
func TestPipelineProcessSequentialOrder(t *testing.T) {
var gotOrder []string
p := &Pipeline{
Processors: []processors.Processor{
procFunc(func(_ context.Context, in event.Event) (*event.Event, error) {
gotOrder = append(gotOrder, "first")
out := in
out.Schema = "stage.one.v1"
return &out, nil
}),
procFunc(func(_ context.Context, in event.Event) (*event.Event, error) {
gotOrder = append(gotOrder, "second")
if in.Schema != "stage.one.v1" {
return nil, fmt.Errorf("expected schema from first stage, got %q", in.Schema)
}
out := in
out.Schema = "stage.two.v1"
return &out, nil
}),
},
}
out, err := p.Process(context.Background(), validEvent())
if err != nil {
t.Fatalf("Process error: %v", err)
}
if out == nil {
t.Fatalf("expected output event, got nil")
}
if out.Schema != "stage.two.v1" {
t.Fatalf("unexpected output schema: %q", out.Schema)
}
if strings.Join(gotOrder, ",") != "first,second" {
t.Fatalf("unexpected processor order: %v", gotOrder)
}
}
func TestPipelineProcessInvalidInput(t *testing.T) {
p := &Pipeline{}
_, err := p.Process(context.Background(), event.Event{})
if err == nil {
t.Fatalf("expected input validation error")
}
if !strings.Contains(err.Error(), "invalid input event") {
t.Fatalf("unexpected error: %v", err)
}
}
func TestPipelineProcessDrop(t *testing.T) {
p := &Pipeline{
Processors: []processors.Processor{
procFunc(func(context.Context, event.Event) (*event.Event, error) {
return nil, nil
}),
},
}
out, err := p.Process(context.Background(), validEvent())
if err != nil {
t.Fatalf("Process error: %v", err)
}
if out != nil {
t.Fatalf("expected nil output for dropped event, got %#v", out)
}
}
func TestPipelineProcessInvalidOutput(t *testing.T) {
p := &Pipeline{
Processors: []processors.Processor{
procFunc(func(_ context.Context, in event.Event) (*event.Event, error) {
out := in
out.Payload = nil
return &out, nil
}),
},
}
_, err := p.Process(context.Background(), validEvent())
if err == nil {
t.Fatalf("expected output validation error")
}
if !strings.Contains(err.Error(), "invalid output event") {
t.Fatalf("unexpected error: %v", err)
}
}
func validEvent() event.Event {
return event.Event{
ID: "evt-1",
Kind: event.Kind("observation"),
Source: "source-1",
EmittedAt: time.Now().UTC(),
Payload: map[string]any{"ok": true},
}
}