sources: standardize Event.ID on Source:EffectiveAt; simplify raw event helper

- Adopt an opinionated Event.ID policy across sources:
  - use upstream-provided ID when available
  - otherwise derive a stable ID from Source:EffectiveAt (RFC3339Nano, UTC)
  - fall back to Source:EmittedAt when EffectiveAt is unavailable
- Add common/id helper to centralize ID selection logic and keep sources consistent
- Simplify common event construction by collapsing SingleRawEventAt/SingleRawEvent
  into a single explicit SingleRawEvent helper (emittedAt passed in)
- Update NWS/Open-Meteo/OpenWeather observation sources to:
  - compute EffectiveAt first
  - generate IDs via the shared helper
  - build envelopes via the unified SingleRawEvent helper
- Improve determinism and dedupe-friendliness without changing schemas or payloads
This commit is contained in:
2026-01-15 19:38:15 -06:00
parent d9474b5a5b
commit d8db58c004
5 changed files with 76 additions and 110 deletions

View File

@@ -4,8 +4,6 @@ package openmeteo
import (
"context"
"encoding/json"
"fmt"
"strings"
"time"
"gitea.maximumdirect.net/ejr/feedkit/config"
@@ -23,8 +21,6 @@ type ObservationSource struct {
func NewObservationSource(cfg config.SourceConfig) (*ObservationSource, error) {
const driver = "openmeteo_observation"
// We require params.user_agent for uniformity across sources (even though Open-Meteo
// itself does not strictly require a special User-Agent).
hs, err := common.NewHTTPSource(driver, cfg, "application/json")
if err != nil {
return nil, err
@@ -37,30 +33,27 @@ func (s *ObservationSource) Name() string { return s.http.Name }
func (s *ObservationSource) Kind() event.Kind { return event.Kind("observation") }
// Poll fetches Open-Meteo "current" and emits exactly one RAW Event.
func (s *ObservationSource) Poll(ctx context.Context) ([]event.Event, error) {
raw, meta, err := s.fetchRaw(ctx)
if err != nil {
return nil, err
}
eventID := buildEventID(s.http.Name, meta)
if strings.TrimSpace(eventID) == "" {
// Extremely defensive fallback: keep the envelope valid no matter what.
eventID = fmt.Sprintf("openmeteo:current:%s:%s", s.http.Name, time.Now().UTC().Format(time.RFC3339Nano))
}
var effectiveAt *time.Time
if !meta.ParsedTimestamp.IsZero() {
t := meta.ParsedTimestamp.UTC()
effectiveAt = &t
}
emittedAt := time.Now().UTC()
eventID := common.ChooseEventID("", s.http.Name, effectiveAt, emittedAt)
return common.SingleRawEvent(
s.Kind(),
s.http.Name,
standards.SchemaRawOpenMeteoCurrentV1,
eventID,
emittedAt,
effectiveAt,
raw,
)
@@ -69,10 +62,8 @@ func (s *ObservationSource) Poll(ctx context.Context) ([]event.Event, error) {
// ---- RAW fetch + minimal metadata decode ----
type openMeteoMeta struct {
Latitude float64 `json:"latitude"`
Longitude float64 `json:"longitude"`
Timezone string `json:"timezone"`
UTCOffsetSeconds int `json:"utc_offset_seconds"`
Timezone string `json:"timezone"`
UTCOffsetSeconds int `json:"utc_offset_seconds"`
Current struct {
Time string `json:"time"`
@@ -89,31 +80,13 @@ func (s *ObservationSource) fetchRaw(ctx context.Context) (json.RawMessage, open
var meta openMeteoMeta
if err := json.Unmarshal(raw, &meta); err != nil {
// If metadata decode fails, still return raw; envelope will fall back to computed ID without EffectiveAt.
// If metadata decode fails, still return raw; envelope will omit EffectiveAt.
return raw, openMeteoMeta{}, nil
}
// Best effort: compute a stable EffectiveAt + event ID component.
// If parsing fails, we simply omit EffectiveAt and fall back to time.Now() in buildEventID.
if t, err := openmeteo.ParseTime(meta.Current.Time, meta.Timezone, meta.UTCOffsetSeconds); err == nil {
meta.ParsedTimestamp = t.UTC()
}
return raw, meta, nil
}
func buildEventID(sourceName string, meta openMeteoMeta) string {
locKey := ""
if meta.Latitude != 0 || meta.Longitude != 0 {
locKey = fmt.Sprintf("coord:%.5f,%.5f", meta.Latitude, meta.Longitude)
} else {
locKey = "loc:unknown"
}
ts := meta.ParsedTimestamp
if ts.IsZero() {
ts = time.Now().UTC()
}
return fmt.Sprintf("openmeteo:current:%s:%s:%s", sourceName, locKey, ts.Format(time.RFC3339Nano))
}