ejr/weatherfeeder
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

sources: standardize HTTP source config + factor raw-event boilerplate

Browse Source 
- Require params.user_agent for all HTTP sources (uniform config across providers)
- Add common.RequireHTTPSourceConfig() to validate name/url/user_agent in one call
- Add common.NewHTTPClient() with DefaultHTTPTimeout for consistent client setup
- Add common.SingleRawEvent() to centralize event envelope construction + validation
- Refactor NWS/Open-Meteo/OpenWeather observation sources to use new helpers
This commit is contained in:
Eric Rakestraw
2026-01-15 09:43:22 -06:00
parent e28ff49201
commit 59111a1c82
6 changed files with 212 additions and 177 deletions
Download Patch File Download Diff File Expand all files Collapse all files

89
internal/sources/openweather/observation.go
View File

@@ -18,14 +18,6 @@ import (
// ObservationSource polls the OpenWeatherMap "Current weather" endpoint and emits a RAW observation Event.
//
// Refactor (mirrors NWS):
// - Source responsibility: fetch bytes + emit a valid event envelope.
// - Normalizer responsibility: decode JSON + map to canonical model.WeatherObservation.
//
// Typical URL shape (provided via config):
//
// https://api.openweathermap.org/data/2.5/weather?lat=...&lon=...&appid=...&units=metric
//
// IMPORTANT UNIT POLICY (weatherfeeder convention):
// OpenWeather changes units based on the `units` query parameter but does NOT include the unit
// system in the response body. To keep normalization deterministic, this driver *requires*
@@ -38,43 +30,29 @@ type ObservationSource struct {
}
func NewObservationSource(cfg config.SourceConfig) (*ObservationSource, error) {
if strings.TrimSpace(cfg.Name) == "" {
return nil, fmt.Errorf("openweather_observation: name is required")
}
if cfg.Params == nil {
return nil, fmt.Errorf("openweather_observation %q: params are required (need params.url)", cfg.Name)
const driver = "openweather_observation"
c, err := common.RequireHTTPSourceConfig(driver, cfg)
if err != nil {
return nil, err
}
rawURL, ok := cfg.ParamString("url", "URL")
if !ok {
return nil, fmt.Errorf("openweather_observation %q: params.url is required", cfg.Name)
if err := requireMetricUnits(c.URL); err != nil {
return nil, fmt.Errorf("openweather_observation %q: %w", c.Name, err)
}
// Fail fast: enforce deterministic unit system.
if err := requireMetricUnits(rawURL); err != nil {
return nil, fmt.Errorf("openweather_observation %q: %w", cfg.Name, err)
}
ua := cfg.ParamStringDefault("weatherfeeder (openweather client)", "user_agent", "userAgent")
return &ObservationSource{
name: cfg.Name,
url: rawURL,
userAgent: ua,
client: &http.Client{
Timeout: 10 * time.Second,
},
name: c.Name,
url: c.URL,
userAgent: c.UserAgent,
client: common.NewHTTPClient(common.DefaultHTTPTimeout),
}, nil
}
func (s *ObservationSource) Name() string { return s.name }
// Kind is used for routing/policy.
// We keep Kind canonical (observation) even for raw events; Schema differentiates raw vs canonical.
func (s *ObservationSource) Kind() event.Kind { return event.Kind("observation") }
// Poll fetches OpenWeather "current weather" and emits exactly one RAW Event.
// The RAW payload is json.RawMessage and Schema is standards.SchemaRawOpenWeatherCurrentV1.
func (s *ObservationSource) Poll(ctx context.Context) ([]event.Event, error) {
// Re-check policy defensively (in case the URL is mutated after construction).
if err := requireMetricUnits(s.url); err != nil {
@@ -88,7 +66,6 @@ func (s *ObservationSource) Poll(ctx context.Context) ([]event.Event, error) {
eventID := buildEventID(s.name, meta)
if strings.TrimSpace(eventID) == "" {
// Extremely defensive fallback: should never happen, but keep the envelope valid.
eventID = fmt.Sprintf("openweather:current:%s:%s", s.name, time.Now().UTC().Format(time.RFC3339Nano))
}
@@ -98,43 +75,29 @@ func (s *ObservationSource) Poll(ctx context.Context) ([]event.Event, error) {
effectiveAt = &t
}
e := event.Event{
ID: eventID,
Kind: s.Kind(),
Source: s.name,
EmittedAt: time.Now().UTC(),
EffectiveAt: effectiveAt,
// RAW schema (normalizer matches on this).
Schema: standards.SchemaRawOpenWeatherCurrentV1,
// Raw JSON; normalizer will decode and map to canonical model.WeatherObservation.
Payload: raw,
}
if err := e.Validate(); err != nil {
return nil, err
}
return []event.Event{e}, nil
return common.SingleRawEvent(
s.Kind(),
s.name,
standards.SchemaRawOpenWeatherCurrentV1,
eventID,
effectiveAt,
raw,
)
}
// ---- RAW fetch + minimal metadata decode ----
// openWeatherMeta is a *minimal* decode of the OpenWeather payload used only to build
// a stable Event.ID and a useful EffectiveAt for the envelope.
type openWeatherMeta struct {
Dt int64 `json:"dt"` // unix seconds, UTC
ID int64 `json:"id"` // city id (if present)
Name string `json:"name"` // city name (optional)
ID int64 `json:"id"`
Name string `json:"name"`
Coord struct {
Lon float64 `json:"lon"`
Lat float64 `json:"lat"`
} `json:"coord"`
// Convenience fields populated after decode.
ParsedTimestamp time.Time `json:"-"`
}
@@ -148,7 +111,6 @@ func (s *ObservationSource) fetchRaw(ctx context.Context) (json.RawMessage, open
var meta openWeatherMeta
if err := json.Unmarshal(b, &meta); err != nil {
// If metadata decode fails, still return raw; envelope will fall back to computed ID.
return raw, openWeatherMeta{}, nil
}
@@ -160,7 +122,6 @@ func (s *ObservationSource) fetchRaw(ctx context.Context) (json.RawMessage, open
}
func buildEventID(sourceName string, meta openWeatherMeta) string {
// Prefer provider city ID if present; otherwise fall back to lat/lon.
locKey := ""
if meta.ID != 0 {
locKey = fmt.Sprintf("city:%d", meta.ID)
@@ -172,19 +133,12 @@ func buildEventID(sourceName string, meta openWeatherMeta) string {
ts := meta.ParsedTimestamp
if ts.IsZero() {
// We prefer stable IDs, but if the payload didn't decode, use "now" so we still emit.
ts = time.Now().UTC()
}
// Example:
// openweather:current:<configured-source-name>:city:12345:2026-01-14T17:00:00.123Z
return fmt.Sprintf("openweather:current:%s:%s:%s", sourceName, locKey, ts.Format(time.RFC3339Nano))
}
// requireMetricUnits enforces weatherfeeder's OpenWeather unit policy.
//
// OpenWeather does not tell us the unit system in the response body. We therefore enforce that
// the request URL explicitly contains units=metric; otherwise normalization would be ambiguous.
func requireMetricUnits(rawURL string) error {
u, err := url.Parse(strings.TrimSpace(rawURL))
if err != nil {
@@ -193,7 +147,6 @@ func requireMetricUnits(rawURL string) error {
units := strings.ToLower(strings.TrimSpace(u.Query().Get("units")))
if units != "metric" {
// Treat missing units ("" -> standard) as non-compliant too.
if units == "" {
units = "(missing; defaults to standard)"
}