refactor!: introduce generic processors registry and remove normalize registry adapter
- add new `processors` package with canonical `Processor` interface - add `processors.Registry` with Register/Build/BuildChain factory model - switch `pipeline.Pipeline` to `[]processors.Processor` - replace `normalize.Registry` + registry adapter with direct `normalize.Processor` - remove `normalize/registry.go` - update root docs to position normalize as one optional processing stage - add tests for processors registry, normalize processor behavior, and pipeline flow BREAKING CHANGE: - `pipeline.Processor` removed; use `processors.Processor` - `normalize.Registry` and old normalize processor adapter APIs removed - downstream daemons must update processor wiring to new `processors.Registry` and `normalize.NewProcessor(...)`
This commit is contained in:
@@ -1,4 +1,4 @@
|
||||
// Package normalize provides an OPTIONAL normalization hook for feedkit pipelines.
|
||||
// Package normalize provides a concrete normalization processor for feedkit pipelines.
|
||||
//
|
||||
// Motivation:
|
||||
// Many daemons have sources that:
|
||||
@@ -9,9 +9,9 @@
|
||||
// encourages duplication (unit conversions, common mapping helpers, etc.).
|
||||
//
|
||||
// This package lets a source emit a "raw" event (e.g., Schema="raw.openweather.current.v1",
|
||||
// Payload=json.RawMessage), and then a normalization processor can convert it into a
|
||||
// Payload=json.RawMessage), and then a normalize.Processor can convert it into a
|
||||
// normalized event (e.g., Schema="weather.observation.v1", Payload=WeatherObservation{}).
|
||||
//
|
||||
// Key property: normalization is optional.
|
||||
// If no registered Normalizer matches an event, it passes through unchanged.
|
||||
// If no Normalizer matches an event, Processor passes it through unchanged by default.
|
||||
package normalize
|
||||
|
||||
57
normalize/processor.go
Normal file
57
normalize/processor.go
Normal file
@@ -0,0 +1,57 @@
|
||||
package normalize
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"gitea.maximumdirect.net/ejr/feedkit/event"
|
||||
)
|
||||
|
||||
// Processor applies ordered normalization rules to pipeline events.
|
||||
//
|
||||
// Selection rule:
|
||||
// - iterate in Normalizers order
|
||||
// - the first Normalizer whose Match returns true is applied
|
||||
//
|
||||
// If no normalizer matches, the default behavior is pass-through.
|
||||
type Processor struct {
|
||||
Normalizers []Normalizer
|
||||
|
||||
// If true, events that do not match any normalizer cause an error.
|
||||
// Default is false (pass-through).
|
||||
RequireMatch bool
|
||||
}
|
||||
|
||||
// NewProcessor constructs a normalization processor from an ordered normalizer list.
|
||||
func NewProcessor(normalizers []Normalizer, requireMatch bool) Processor {
|
||||
return Processor{
|
||||
Normalizers: append([]Normalizer(nil), normalizers...),
|
||||
RequireMatch: requireMatch,
|
||||
}
|
||||
}
|
||||
|
||||
// Process implements processors.Processor.
|
||||
func (p Processor) Process(ctx context.Context, in event.Event) (*event.Event, error) {
|
||||
for _, n := range p.Normalizers {
|
||||
if n == nil {
|
||||
continue
|
||||
}
|
||||
if !n.Match(in) {
|
||||
continue
|
||||
}
|
||||
|
||||
out, err := n.Normalize(ctx, in)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("normalize: normalizer failed: %w", err)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
if p.RequireMatch {
|
||||
return nil, fmt.Errorf("normalize: no normalizer matched event (id=%s kind=%s source=%s schema=%q)",
|
||||
in.ID, in.Kind, in.Source, in.Schema)
|
||||
}
|
||||
|
||||
out := in
|
||||
return &out, nil
|
||||
}
|
||||
139
normalize/processor_test.go
Normal file
139
normalize/processor_test.go
Normal file
@@ -0,0 +1,139 @@
|
||||
package normalize
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/ejr/feedkit/event"
|
||||
)
|
||||
|
||||
func TestProcessorFirstMatchWins(t *testing.T) {
|
||||
var firstCalls, secondCalls int
|
||||
|
||||
p := NewProcessor([]Normalizer{
|
||||
Func{
|
||||
MatchFn: func(event.Event) bool { return true },
|
||||
NormalizeFn: func(_ context.Context, in event.Event) (*event.Event, error) {
|
||||
firstCalls++
|
||||
out := in
|
||||
out.Schema = "normalized.first.v1"
|
||||
return &out, nil
|
||||
},
|
||||
},
|
||||
Func{
|
||||
MatchFn: func(event.Event) bool { return true },
|
||||
NormalizeFn: func(_ context.Context, in event.Event) (*event.Event, error) {
|
||||
secondCalls++
|
||||
out := in
|
||||
out.Schema = "normalized.second.v1"
|
||||
return &out, nil
|
||||
},
|
||||
},
|
||||
}, false)
|
||||
|
||||
out, err := p.Process(context.Background(), testEvent())
|
||||
if err != nil {
|
||||
t.Fatalf("Process error: %v", err)
|
||||
}
|
||||
if out == nil {
|
||||
t.Fatalf("expected output event, got nil")
|
||||
}
|
||||
if out.Schema != "normalized.first.v1" {
|
||||
t.Fatalf("unexpected schema: %q", out.Schema)
|
||||
}
|
||||
if firstCalls != 1 {
|
||||
t.Fatalf("expected first normalizer called once, got %d", firstCalls)
|
||||
}
|
||||
if secondCalls != 0 {
|
||||
t.Fatalf("expected second normalizer skipped, got %d calls", secondCalls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessorNoMatchPassThroughAndRequireMatch(t *testing.T) {
|
||||
in := testEvent()
|
||||
in.Schema = "raw.schema.v1"
|
||||
|
||||
passThrough := NewProcessor([]Normalizer{
|
||||
Func{
|
||||
MatchFn: func(event.Event) bool { return false },
|
||||
NormalizeFn: func(_ context.Context, in event.Event) (*event.Event, error) {
|
||||
out := in
|
||||
out.Schema = "should.not.run"
|
||||
return &out, nil
|
||||
},
|
||||
},
|
||||
}, false)
|
||||
|
||||
out, err := passThrough.Process(context.Background(), in)
|
||||
if err != nil {
|
||||
t.Fatalf("pass-through Process error: %v", err)
|
||||
}
|
||||
if out == nil {
|
||||
t.Fatalf("expected pass-through output event, got nil")
|
||||
}
|
||||
if out.Schema != "raw.schema.v1" {
|
||||
t.Fatalf("expected unchanged schema, got %q", out.Schema)
|
||||
}
|
||||
|
||||
required := NewProcessor(nil, true)
|
||||
_, err = required.Process(context.Background(), in)
|
||||
if err == nil {
|
||||
t.Fatalf("expected require-match error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "no normalizer matched") {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessorDropAndErrorPropagation(t *testing.T) {
|
||||
t.Run("drop", func(t *testing.T) {
|
||||
p := NewProcessor([]Normalizer{
|
||||
Func{
|
||||
MatchFn: func(event.Event) bool { return true },
|
||||
NormalizeFn: func(context.Context, event.Event) (*event.Event, error) {
|
||||
return nil, nil
|
||||
},
|
||||
},
|
||||
}, false)
|
||||
|
||||
out, err := p.Process(context.Background(), testEvent())
|
||||
if err != nil {
|
||||
t.Fatalf("Process error: %v", err)
|
||||
}
|
||||
if out != nil {
|
||||
t.Fatalf("expected nil output for dropped event, got %#v", out)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("error", func(t *testing.T) {
|
||||
p := NewProcessor([]Normalizer{
|
||||
Func{
|
||||
MatchFn: func(event.Event) bool { return true },
|
||||
NormalizeFn: func(context.Context, event.Event) (*event.Event, error) {
|
||||
return nil, errors.New("map failed")
|
||||
},
|
||||
},
|
||||
}, false)
|
||||
|
||||
_, err := p.Process(context.Background(), testEvent())
|
||||
if err == nil {
|
||||
t.Fatalf("expected error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "normalizer failed") {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func testEvent() event.Event {
|
||||
return event.Event{
|
||||
ID: "evt-normalize-1",
|
||||
Kind: event.Kind("observation"),
|
||||
Source: "source-1",
|
||||
EmittedAt: time.Now().UTC(),
|
||||
Payload: map[string]any{"x": 1},
|
||||
}
|
||||
}
|
||||
@@ -1,140 +0,0 @@
|
||||
package normalize
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sync"
|
||||
|
||||
"gitea.maximumdirect.net/ejr/feedkit/event"
|
||||
)
|
||||
|
||||
// Registry holds a set of Normalizers and selects one for a given event.
|
||||
//
|
||||
// Selection rule (simple + predictable):
|
||||
// - iterate in registration order
|
||||
// - the FIRST Normalizer whose Match(e) returns true is used
|
||||
//
|
||||
// If none match, the event passes through unchanged.
|
||||
//
|
||||
// Why "first match wins"?
|
||||
// Normalization is usually a single mapping step from a raw schema/version into
|
||||
// a normalized schema/version. If you want multiple transformation steps,
|
||||
// model them as multiple pipeline processors (which feedkit already supports).
|
||||
type Registry struct {
|
||||
mu sync.RWMutex
|
||||
ns []Normalizer
|
||||
}
|
||||
|
||||
// Register adds a normalizer to the registry.
|
||||
//
|
||||
// Register panics if n is nil; this is a programmer error and should fail fast.
|
||||
func (r *Registry) Register(n Normalizer) {
|
||||
if n == nil {
|
||||
panic("normalize.Registry.Register: normalizer cannot be nil")
|
||||
}
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
r.ns = append(r.ns, n)
|
||||
}
|
||||
|
||||
// Normalize finds the first matching Normalizer and applies it.
|
||||
//
|
||||
// If no normalizer matches, it returns the input event unchanged.
|
||||
//
|
||||
// If a normalizer returns (nil, nil), the event is dropped.
|
||||
func (r *Registry) Normalize(ctx context.Context, in event.Event) (*event.Event, error) {
|
||||
if r == nil {
|
||||
// Nil registry is a valid "feature off" state.
|
||||
out := in
|
||||
return &out, nil
|
||||
}
|
||||
|
||||
r.mu.RLock()
|
||||
ns := append([]Normalizer(nil), r.ns...) // copy for safe iteration outside lock
|
||||
r.mu.RUnlock()
|
||||
|
||||
for _, n := range ns {
|
||||
if n == nil {
|
||||
// Shouldn't happen (Register panics), but guard anyway.
|
||||
continue
|
||||
}
|
||||
if !n.Match(in) {
|
||||
continue
|
||||
}
|
||||
|
||||
out, err := n.Normalize(ctx, in)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("normalize: normalizer failed: %w", err)
|
||||
}
|
||||
// out may be nil to signal "drop".
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// No match: pass through unchanged.
|
||||
out := in
|
||||
return &out, nil
|
||||
}
|
||||
|
||||
// Processor adapts a Registry into a pipeline Processor.
|
||||
//
|
||||
// It implements:
|
||||
//
|
||||
// Process(ctx context.Context, in event.Event) (*event.Event, error)
|
||||
//
|
||||
// which matches feedkit/pipeline.Processor.
|
||||
//
|
||||
// Optionality:
|
||||
// - If Registry is nil, Processor becomes a no-op pass-through.
|
||||
// - If Registry has no matching normalizer for an event, that event passes through unchanged.
|
||||
type Processor struct {
|
||||
Registry *Registry
|
||||
|
||||
// If true, events that do not match any normalizer cause an error.
|
||||
// Default is false (pass-through), which is the behavior you asked for.
|
||||
RequireMatch bool
|
||||
}
|
||||
|
||||
// Process implements the pipeline.Processor interface.
|
||||
func (p Processor) Process(ctx context.Context, in event.Event) (*event.Event, error) {
|
||||
// "Feature off": no registry means no normalization.
|
||||
if p.Registry == nil {
|
||||
out := in
|
||||
return &out, nil
|
||||
}
|
||||
|
||||
out, err := p.Registry.Normalize(ctx, in)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if out == nil {
|
||||
// Dropped by normalization policy.
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
if p.RequireMatch {
|
||||
// Detect "no-op pass-through due to no match" by checking whether a match existed.
|
||||
// We do this with a cheap second pass to avoid changing Normalize()'s signature.
|
||||
// (This is rare to enable; correctness/clarity > micro-optimization.)
|
||||
if !p.Registry.hasMatch(in) {
|
||||
return nil, fmt.Errorf("normalize: no normalizer matched event (id=%s kind=%s source=%s schema=%q)",
|
||||
in.ID, in.Kind, in.Source, in.Schema)
|
||||
}
|
||||
}
|
||||
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (r *Registry) hasMatch(in event.Event) bool {
|
||||
if r == nil {
|
||||
return false
|
||||
}
|
||||
r.mu.RLock()
|
||||
defer r.mu.RUnlock()
|
||||
for _, n := range r.ns {
|
||||
if n != nil && n.Match(in) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
Reference in New Issue
Block a user