Updates to track feedkit v0.7.2 and to add a dedupe processor

cmd/weatherfeeder/main.go

@@ -15,9 +15,10 @@ import (
 	"gitea.maximumdirect.net/ejr/feedkit/config"
 	fkdispatch "gitea.maximumdirect.net/ejr/feedkit/dispatch"
 	fkevent "gitea.maximumdirect.net/ejr/feedkit/event"
-	fknormalize "gitea.maximumdirect.net/ejr/feedkit/normalize"
 	fkpipeline "gitea.maximumdirect.net/ejr/feedkit/pipeline"
 	fkprocessors "gitea.maximumdirect.net/ejr/feedkit/processors"
+	fkdedupe "gitea.maximumdirect.net/ejr/feedkit/processors/dedupe"
+	fknormalize "gitea.maximumdirect.net/ejr/feedkit/processors/normalize"
 	fkscheduler "gitea.maximumdirect.net/ejr/feedkit/scheduler"
 	fksinks "gitea.maximumdirect.net/ejr/feedkit/sinks"
 	fksources "gitea.maximumdirect.net/ejr/feedkit/sources"
@@ -27,6 +28,8 @@ import (
 	wfsources "gitea.maximumdirect.net/ejr/weatherfeeder/internal/sources"
 )
 
+const dedupeMaxEntries = 2048
+
 func main() {
 	log.SetFlags(log.LstdFlags | log.Lmicroseconds)
 
@@ -103,9 +106,9 @@ func main() {
 
 	events := make(chan fkevent.Event, 256)
 
-	// --- Normalization (optional) ---
+	// --- Processors ---
 	//
-	// We install feedkit's normalize.Processor even before any normalizers exist.
+	// We install feedkit's processors/normalize.Processor even before any normalizers exist.
 	// With an empty normalizer list and RequireMatch=false, this is a no-op passthrough.
 	// It will begin transforming events as soon as:
 	//   1) sources emit raw schemas (raw.*), and
@@ -116,8 +119,9 @@ func main() {
 	procReg.Register("normalize", func() (fkprocessors.Processor, error) {
 		return fknormalize.NewProcessor(normalizers, false), nil
 	})
+	procReg.Register("dedupe", fkdedupe.Factory(dedupeMaxEntries))
 
-	chain, err := procReg.BuildChain([]string{"normalize"})
+	chain, err := procReg.BuildChain([]string{"normalize", "dedupe"})
 	if err != nil {
 		log.Fatalf("build processor chain failed: %v", err)
 	}