Added a dedupe processor, and moved processor packages under processors/*
This commit is contained in:
28
processors/dedupe/doc.go
Normal file
28
processors/dedupe/doc.go
Normal file
@@ -0,0 +1,28 @@
|
||||
// Package dedupe provides a default in-memory LRU deduplication processor.
|
||||
//
|
||||
// The processor keys strictly by event.Event.ID:
|
||||
// - first-seen IDs pass through
|
||||
// - repeated IDs are dropped
|
||||
//
|
||||
// The in-memory seen-ID set is bounded by a required maxEntries capacity.
|
||||
// When capacity is exceeded, the least recently used ID is evicted.
|
||||
//
|
||||
// Typical registry wiring:
|
||||
//
|
||||
// ```go
|
||||
// reg := processors.NewRegistry()
|
||||
// reg.Register("dedupe", dedupe.Factory(10_000))
|
||||
//
|
||||
// reg.Register("normalize", func() (processors.Processor, error) {
|
||||
// return normalize.NewProcessor(myNormalizers, false), nil
|
||||
// })
|
||||
//
|
||||
// chain, err := reg.BuildChain([]string{"dedupe", "normalize"})
|
||||
//
|
||||
// if err != nil {
|
||||
// // handle wiring error
|
||||
// }
|
||||
//
|
||||
// p := &pipeline.Pipeline{Processors: chain}
|
||||
// ```
|
||||
package dedupe
|
||||
Reference in New Issue
Block a user