29 lines
808 B
Go
29 lines
808 B
Go
// Package dedupe provides a default in-memory LRU deduplication processor.
|
|
//
|
|
// The processor keys strictly by event.Event.ID:
|
|
// - first-seen IDs pass through
|
|
// - repeated IDs are dropped
|
|
//
|
|
// The in-memory seen-ID set is bounded by a required maxEntries capacity.
|
|
// When capacity is exceeded, the least recently used ID is evicted.
|
|
//
|
|
// Typical registry wiring:
|
|
//
|
|
// ```go
|
|
// reg := processors.NewRegistry()
|
|
// reg.Register("dedupe", dedupe.Factory(10_000))
|
|
//
|
|
// reg.Register("normalize", func() (processors.Processor, error) {
|
|
// return normalize.NewProcessor(myNormalizers, false), nil
|
|
// })
|
|
//
|
|
// chain, err := reg.BuildChain([]string{"dedupe", "normalize"})
|
|
//
|
|
// if err != nil {
|
|
// // handle wiring error
|
|
// }
|
|
//
|
|
// p := &pipeline.Pipeline{Processors: chain}
|
|
// ```
|
|
package dedupe
|