Added a dedupe processor, and moved processor packages under processors/*

processors/normalize/doc.go

@@ -0,0 +1,17 @@
+// Package normalize provides a concrete normalization processor for feedkit pipelines.
+//
+// Motivation:
+// Many daemons have sources that:
+//  1. fetch raw upstream data (often JSON), and
+//  2. transform it into a domain's normalized payload format.
+//
+// Doing both steps inside Source.Poll works, but tends to make sources large and
+// encourages duplication (unit conversions, common mapping helpers, etc.).
+//
+// This package lets a source emit a "raw" event (e.g., Schema="raw.openweather.current.v1",
+// Payload=json.RawMessage), and then a normalize.Processor can convert it into a
+// normalized event (e.g., Schema="weather.observation.v1", Payload=WeatherObservation{}).
+//
+// Key property: normalization is optional.
+// If no Normalizer matches an event, Processor passes it through unchanged by default.
+package normalize

processors/normalize/normalize.go

@@ -0,0 +1,76 @@
+package normalize
+
+import (
+	"context"
+	"fmt"
+
+	"gitea.maximumdirect.net/ejr/feedkit/event"
+)
+
+// Normalizer converts one event shape into another.
+//
+// A Normalizer is typically domain-owned code (weatherfeeder/newsfeeder/...)
+// that knows how to interpret a specific upstream payload and produce a
+// normalized payload.
+//
+// Normalizers are selected via Match(). The matching strategy is intentionally
+// flexible: implementations may match on Schema, Kind, Source, or any other
+// Event fields.
+type Normalizer interface {
+	// Match reports whether this normalizer applies to the given event.
+	//
+	// Common patterns:
+	//   - match on e.Schema (recommended for versioning)
+	//   - match on e.Source (useful if Schema is empty)
+	//   - match on (e.Kind + e.Source), etc.
+	Match(e event.Event) bool
+
+	// Normalize transforms the incoming event into a new (or modified) event.
+	//
+	// Return values:
+	//   - (out, nil) where out != nil: emit the normalized event
+	//   - (nil, nil): drop the event (treat as policy drop)
+	//   - (nil, err): fail the pipeline
+	//
+	// Note: If you simply want to pass the event through unchanged, return &in.
+	Normalize(ctx context.Context, in event.Event) (*event.Event, error)
+}
+
+// Func is an ergonomic adapter that lets you define a Normalizer with functions.
+//
+// Example:
+//
+//	n := normalize.Func{
+//	  MatchFn: func(e event.Event) bool { return e.Schema == "raw.openweather.current.v1" },
+//	  NormalizeFn: func(ctx context.Context, in event.Event) (*event.Event, error) {
+//	    // ... map in.Payload -> normalized payload ...
+//	  },
+//	}
+type Func struct {
+	MatchFn     func(e event.Event) bool
+	NormalizeFn func(ctx context.Context, in event.Event) (*event.Event, error)
+
+	// Optional: helps produce nicer panic/error messages if something goes wrong.
+	Name string
+}
+
+func (f Func) Match(e event.Event) bool {
+	if f.MatchFn == nil {
+		return false
+	}
+	return f.MatchFn(e)
+}
+
+func (f Func) Normalize(ctx context.Context, in event.Event) (*event.Event, error) {
+	if f.NormalizeFn == nil {
+		return nil, fmt.Errorf("normalize.Func(%s): NormalizeFn is nil", f.safeName())
+	}
+	return f.NormalizeFn(ctx, in)
+}
+
+func (f Func) safeName() string {
+	if f.Name == "" {
+		return "<unnamed>"
+	}
+	return f.Name
+}

processors/normalize/processor.go

@@ -0,0 +1,57 @@
+package normalize
+
+import (
+	"context"
+	"fmt"
+
+	"gitea.maximumdirect.net/ejr/feedkit/event"
+)
+
+// Processor applies ordered normalization rules to pipeline events.
+//
+// Selection rule:
+//   - iterate in Normalizers order
+//   - the first Normalizer whose Match returns true is applied
+//
+// If no normalizer matches, the default behavior is pass-through.
+type Processor struct {
+	Normalizers []Normalizer
+
+	// If true, events that do not match any normalizer cause an error.
+	// Default is false (pass-through).
+	RequireMatch bool
+}
+
+// NewProcessor constructs a normalization processor from an ordered normalizer list.
+func NewProcessor(normalizers []Normalizer, requireMatch bool) Processor {
+	return Processor{
+		Normalizers:  append([]Normalizer(nil), normalizers...),
+		RequireMatch: requireMatch,
+	}
+}
+
+// Process implements processors.Processor.
+func (p Processor) Process(ctx context.Context, in event.Event) (*event.Event, error) {
+	for _, n := range p.Normalizers {
+		if n == nil {
+			continue
+		}
+		if !n.Match(in) {
+			continue
+		}
+
+		out, err := n.Normalize(ctx, in)
+		if err != nil {
+			return nil, fmt.Errorf("normalize: normalizer failed: %w", err)
+		}
+		return out, nil
+	}
+
+	if p.RequireMatch {
+		return nil, fmt.Errorf("normalize: no normalizer matched event (id=%s kind=%s source=%s schema=%q)",
+			in.ID, in.Kind, in.Source, in.Schema)
+	}
+
+	out := in
+	return &out, nil
+}

processors/normalize/processor_test.go

@@ -0,0 +1,139 @@
+package normalize
+
+import (
+	"context"
+	"errors"
+	"strings"
+	"testing"
+	"time"
+
+	"gitea.maximumdirect.net/ejr/feedkit/event"
+)
+
+func TestProcessorFirstMatchWins(t *testing.T) {
+	var firstCalls, secondCalls int
+
+	p := NewProcessor([]Normalizer{
+		Func{
+			MatchFn: func(event.Event) bool { return true },
+			NormalizeFn: func(_ context.Context, in event.Event) (*event.Event, error) {
+				firstCalls++
+				out := in
+				out.Schema = "normalized.first.v1"
+				return &out, nil
+			},
+		},
+		Func{
+			MatchFn: func(event.Event) bool { return true },
+			NormalizeFn: func(_ context.Context, in event.Event) (*event.Event, error) {
+				secondCalls++
+				out := in
+				out.Schema = "normalized.second.v1"
+				return &out, nil
+			},
+		},
+	}, false)
+
+	out, err := p.Process(context.Background(), testEvent())
+	if err != nil {
+		t.Fatalf("Process error: %v", err)
+	}
+	if out == nil {
+		t.Fatalf("expected output event, got nil")
+	}
+	if out.Schema != "normalized.first.v1" {
+		t.Fatalf("unexpected schema: %q", out.Schema)
+	}
+	if firstCalls != 1 {
+		t.Fatalf("expected first normalizer called once, got %d", firstCalls)
+	}
+	if secondCalls != 0 {
+		t.Fatalf("expected second normalizer skipped, got %d calls", secondCalls)
+	}
+}
+
+func TestProcessorNoMatchPassThroughAndRequireMatch(t *testing.T) {
+	in := testEvent()
+	in.Schema = "raw.schema.v1"
+
+	passThrough := NewProcessor([]Normalizer{
+		Func{
+			MatchFn: func(event.Event) bool { return false },
+			NormalizeFn: func(_ context.Context, in event.Event) (*event.Event, error) {
+				out := in
+				out.Schema = "should.not.run"
+				return &out, nil
+			},
+		},
+	}, false)
+
+	out, err := passThrough.Process(context.Background(), in)
+	if err != nil {
+		t.Fatalf("pass-through Process error: %v", err)
+	}
+	if out == nil {
+		t.Fatalf("expected pass-through output event, got nil")
+	}
+	if out.Schema != "raw.schema.v1" {
+		t.Fatalf("expected unchanged schema, got %q", out.Schema)
+	}
+
+	required := NewProcessor(nil, true)
+	_, err = required.Process(context.Background(), in)
+	if err == nil {
+		t.Fatalf("expected require-match error")
+	}
+	if !strings.Contains(err.Error(), "no normalizer matched") {
+		t.Fatalf("unexpected error: %v", err)
+	}
+}
+
+func TestProcessorDropAndErrorPropagation(t *testing.T) {
+	t.Run("drop", func(t *testing.T) {
+		p := NewProcessor([]Normalizer{
+			Func{
+				MatchFn: func(event.Event) bool { return true },
+				NormalizeFn: func(context.Context, event.Event) (*event.Event, error) {
+					return nil, nil
+				},
+			},
+		}, false)
+
+		out, err := p.Process(context.Background(), testEvent())
+		if err != nil {
+			t.Fatalf("Process error: %v", err)
+		}
+		if out != nil {
+			t.Fatalf("expected nil output for dropped event, got %#v", out)
+		}
+	})
+
+	t.Run("error", func(t *testing.T) {
+		p := NewProcessor([]Normalizer{
+			Func{
+				MatchFn: func(event.Event) bool { return true },
+				NormalizeFn: func(context.Context, event.Event) (*event.Event, error) {
+					return nil, errors.New("map failed")
+				},
+			},
+		}, false)
+
+		_, err := p.Process(context.Background(), testEvent())
+		if err == nil {
+			t.Fatalf("expected error")
+		}
+		if !strings.Contains(err.Error(), "normalizer failed") {
+			t.Fatalf("unexpected error: %v", err)
+		}
+	})
+}
+
+func testEvent() event.Event {
+	return event.Event{
+		ID:        "evt-normalize-1",
+		Kind:      event.Kind("observation"),
+		Source:    "source-1",
+		EmittedAt: time.Now().UTC(),
+		Payload:   map[string]any{"x": 1},
+	}
+}