nws: refactored the NWS source files to relocate normalization logic to internal/normalizers.

This commit is contained in:
2026-01-14 11:18:21 -06:00
parent efc44e8c6a
commit 0ba2602bcc
11 changed files with 873 additions and 616 deletions

View File

@@ -0,0 +1,49 @@
// Package common contains cross-provider helper code used by weatherfeeder normalizers.
//
// Purpose
// -------
// Normalizers convert provider-specific RAW payloads into canonical internal/model types.
// Some small utilities are naturally reusable across multiple providers (unit conversions,
// payload extraction, common parsing, shared fallbacks). Those belong here.
//
// This package is intentionally "boring":
// - pure helpers (deterministic, no I/O)
// - minimal abstractions (prefer straightforward functions)
// - easy to unit test
//
// What belongs here
// -----------------
// Put code in internal/normalizers/common when it is:
//
// - potentially reusable by more than one provider
// - provider-agnostic (no NWS/OpenWeather/Open-Meteo specific assumptions)
// - stable, small, and readable
//
// Typical examples:
// - unit conversion helpers (°F <-> °C, m/s <-> km/h, hPa <-> Pa, etc.)
// - json.RawMessage payload extraction helpers (with good error messages)
// - shared parsing helpers (timestamps, simple numeric coercions)
// - generic fallbacks (e.g., mapping a human text description into a coarse canonical code),
// so long as the logic truly applies across providers
//
// What does NOT belong here
// -------------------------
// Do NOT put the following in this package:
//
// - Normalizer implementations (types that satisfy feedkit/normalize.Normalizer)
// - provider-specific JSON structs or mapping logic (put those under
// internal/normalizers/<provider>/)
// - network or filesystem I/O (sources fetch; normalizers transform)
// - code that depends on event.Source naming, config fields, or driver-specific params
//
// Style and API guidelines
// ------------------------
// - Prefer small, single-purpose functions.
// - Keep function names explicit (avoid clever generic “DoThing” helpers).
// - Return typed errors with context (include schema/field names where helpful).
// - Keep dependencies minimal: standard library + weatherfeeder packages only.
// - Add unit tests for any non-trivial logic (especially parsing and fallbacks).
//
// Keeping this clean matters: common is shared by all providers, so complexity here
// multiplies across the project.
package common

View File

@@ -0,0 +1,54 @@
// FILE: ./internal/normalizers/common/payload.go
package common
import (
"encoding/json"
"fmt"
"gitea.maximumdirect.net/ejr/feedkit/event"
)
// PayloadBytes extracts a JSON-ish payload into bytes suitable for json.Unmarshal.
//
// Supported payload shapes (weatherfeeder convention):
// - json.RawMessage (recommended for raw events)
// - []byte
// - string (assumed to contain JSON)
// - map[string]any (re-marshaled to JSON)
//
// If you add other raw representations later, extend this function.
func PayloadBytes(e event.Event) ([]byte, error) {
if e.Payload == nil {
return nil, fmt.Errorf("payload is nil")
}
switch v := e.Payload.(type) {
case json.RawMessage:
if len(v) == 0 {
return nil, fmt.Errorf("payload is empty json.RawMessage")
}
return []byte(v), nil
case []byte:
if len(v) == 0 {
return nil, fmt.Errorf("payload is empty []byte")
}
return v, nil
case string:
if v == "" {
return nil, fmt.Errorf("payload is empty string")
}
return []byte(v), nil
case map[string]any:
b, err := json.Marshal(v)
if err != nil {
return nil, fmt.Errorf("marshal map payload: %w", err)
}
return b, nil
default:
return nil, fmt.Errorf("unsupported payload type %T", e.Payload)
}
}

View File

@@ -0,0 +1,15 @@
// FILE: ./internal/normalizers/common/units.go
package common
// Common unit conversions used across providers.
//
// These helpers are intentionally small and “obvious” and are meant to remove
// duplication across normalizers (and eventually across sources, once refactored).
func TempCFromF(f float64) float64 { return (f - 32.0) * 5.0 / 9.0 }
func TempCFromK(k float64) float64 { return k - 273.15 }
func SpeedKmhFromMps(ms float64) float64 { return ms * 3.6 }
func SpeedKmhFromMph(mph float64) float64 { return mph * 1.609344 }
func PressurePaFromHPa(hpa float64) float64 { return hpa * 100.0 }

View File

@@ -0,0 +1,129 @@
// FILE: ./internal/normalizers/common/wmo_text.go
package common
import (
"strings"
"gitea.maximumdirect.net/ejr/weatherfeeder/internal/model"
)
// WMOFromTextDescription is a cross-provider fallback that tries to infer a WMO code
// from a human-readable condition string.
//
// This is intentionally “coarse” and conservative. It is useful when a provider:
//
// - does not provide a condition code, or
// - provides it inconsistently / null, or
// - provides only textual conditions for some endpoints.
//
// Providers may still choose to override this with richer signals
// (METAR phenomena, explicit numeric codes, etc.).
func WMOFromTextDescription(desc string) model.WMOCode {
s := strings.ToLower(strings.TrimSpace(desc))
if s == "" {
return model.WMOUnknown
}
// Thunder / hail
if strings.Contains(s, "thunder") {
if strings.Contains(s, "hail") {
return 99
}
return 95
}
// Freezing hazards
if strings.Contains(s, "freezing rain") {
if strings.Contains(s, "light") {
return 66
}
return 67
}
if strings.Contains(s, "freezing drizzle") {
if strings.Contains(s, "light") {
return 56
}
return 57
}
// Drizzle
if strings.Contains(s, "drizzle") {
if strings.Contains(s, "heavy") || strings.Contains(s, "dense") {
return 55
}
if strings.Contains(s, "light") {
return 51
}
return 53
}
// Showers
if strings.Contains(s, "showers") {
if strings.Contains(s, "heavy") {
return 82
}
if strings.Contains(s, "light") {
return 80
}
return 81
}
// Rain
if strings.Contains(s, "rain") {
if strings.Contains(s, "heavy") {
return 65
}
if strings.Contains(s, "light") {
return 61
}
return 63
}
// Snow (check snow showers first)
if strings.Contains(s, "snow showers") {
if strings.Contains(s, "light") {
return 85
}
return 86
}
if strings.Contains(s, "snow grains") {
return 77
}
if strings.Contains(s, "snow") {
if strings.Contains(s, "heavy") {
return 75
}
if strings.Contains(s, "light") {
return 71
}
return 73
}
// Fog / mist
if strings.Contains(s, "rime fog") {
return 48
}
if strings.Contains(s, "fog") || strings.Contains(s, "mist") {
return 45
}
// Sky-only
if strings.Contains(s, "overcast") {
return 3
}
if strings.Contains(s, "cloudy") {
return 3
}
if strings.Contains(s, "partly cloudy") {
return 2
}
if strings.Contains(s, "mostly sunny") || strings.Contains(s, "mostly clear") ||
strings.Contains(s, "mainly sunny") || strings.Contains(s, "mainly clear") {
return 1
}
if strings.Contains(s, "clear") || strings.Contains(s, "sunny") {
return 0
}
return model.WMOUnknown
}

View File

@@ -19,7 +19,8 @@
// Example:
//
// internal/normalizers/nws/observation.go
// internal/normalizers/nws/common.go
// internal/normalizers/nws/types.go
// internal/normalizers/nws/wmo_map.go
// internal/normalizers/openweather/observation.go
// internal/normalizers/openmeteo/observation.go
// internal/normalizers/common/units.go
@@ -27,14 +28,30 @@
// Rules:
//
// 1. One normalizer per file.
// Each file contains exactly one Normalizer implementation (one type).
// Each file contains exactly one Normalizer implementation (one type that
// satisfies feedkit/normalize.Normalizer).
// Helper files are encouraged (types.go, common.go, mapping.go, etc.) as long
// as they do not define additional Normalizer types.
//
// 2. Provider-level shared helpers live in:
// internal/normalizers/<provider>/common.go
// 2. Provider-level shared helpers live under the provider directory:
// internal/normalizers/<provider>/
//
// You may use multiple helper files (recommended) when it improves clarity:
// - types.go (provider JSON structs)
// - common.go (provider-shared helpers)
// - mapping.go (provider mapping logic)
// Use common.go only when you truly have “shared across multiple normalizers
// within this provider” helpers.
//
// 3. Cross-provider helpers live in:
// internal/normalizers/common/
//
// Prefer extracting small, pure helpers here when they are reused by ≥2 providers.
// Keep these helpers:
// - deterministic (no I/O)
// - side-effect free
// - easy to read (avoid clever abstractions)
//
// 4. Matching is standardized on Event.Schema.
// (Do not match on event.Source or event.Kind in weatherfeeder normalizers.)
//
@@ -82,10 +99,13 @@
// Every normalizer type must have a doc comment that states:
//
// - what it converts (e.g., “OpenWeather current -> WeatherObservation”)
// - which raw schema it matches (constant name + value)
// - which canonical schema it produces (constant name + value)
// - which raw schema it matches (constant identifier from internal/standards)
// - which canonical schema it produces (constant identifier from internal/standards)
// - any special caveats (units, day/night inference, missing fields, etc.)
//
// Including literal schema string values is optional,
// but the constant identifiers are required.
//
// Event field handling (strong defaults)
// --------------------------------------
// Normalizers should treat the incoming event envelope as stable identity and
@@ -126,6 +146,22 @@
//
// which calls each providers Register() in a stable order.
//
// Registry ordering
// -----------------------------
// feedkit normalization uses a match-driven registry (“first match wins”).
// Therefore order matters:
//
// - Register more specific normalizers before more general ones.
// - Avoid “catch-all” Match() implementations.
// - Keep Match() cheap and deterministic (Schema equality checks are ideal).
//
// Reuse guidance (strong recommendation)
// --------------------------------------
// Before adding provider-specific logic, check internal/normalizers/common for an
// existing helper (payload extraction, unit conversions, text fallbacks, etc.).
// If you discover logic that could potentially apply to another provider, prefer extracting
// it into internal/normalizers/common as appropriate.
//
// Testing guidance (recommended)
// ------------------------------
// Add a unit test per normalizer:

View File

@@ -0,0 +1,51 @@
// FILE: ./internal/normalizers/nws/metar.go
package nws
import (
"encoding/json"
"strings"
)
// metarPhenomenon is a typed view of NWS presentWeather objects.
type metarPhenomenon struct {
Intensity *string `json:"intensity"` // "light", "heavy", or null
Modifier *string `json:"modifier"` // "freezing", "showers", etc., or null
Weather string `json:"weather"` // e.g. "rain", "snow", "fog_mist", ...
RawString string `json:"rawString"`
InVicinity *bool `json:"inVicinity"`
}
func decodeMetarPhenomena(raw []map[string]any) []metarPhenomenon {
if len(raw) == 0 {
return nil
}
out := make([]metarPhenomenon, 0, len(raw))
for _, m := range raw {
b, err := json.Marshal(m)
if err != nil {
continue
}
var p metarPhenomenon
if err := json.Unmarshal(b, &p); err != nil {
continue
}
p.Weather = strings.ToLower(strings.TrimSpace(p.Weather))
p.RawString = strings.TrimSpace(p.RawString)
out = append(out, p)
}
return out
}
func containsWeather(phenomena []metarPhenomenon, weather string) bool {
weather = strings.ToLower(strings.TrimSpace(weather))
for _, p := range phenomena {
if p.Weather == weather {
return true
}
}
return false
}

View File

@@ -0,0 +1,145 @@
// FILE: ./internal/normalizers/nws/observation.go
package nws
import (
"context"
"encoding/json"
"fmt"
"strings"
"time"
"gitea.maximumdirect.net/ejr/feedkit/event"
"gitea.maximumdirect.net/ejr/weatherfeeder/internal/model"
normcommon "gitea.maximumdirect.net/ejr/weatherfeeder/internal/normalizers/common"
"gitea.maximumdirect.net/ejr/weatherfeeder/internal/standards"
)
// ObservationNormalizer converts:
//
// standards.SchemaRawNWSObservationV1 -> standards.SchemaWeatherObservationV1
//
// It interprets NWS GeoJSON station observations and maps them into the
// canonical model.WeatherObservation representation.
//
// Precedence for determining ConditionCode (WMO):
// 1. presentWeather (METAR phenomena objects) — strongest signal
// 2. textDescription keyword fallback — reusable across providers
// 3. cloudLayers sky-only fallback — NWS/METAR-specific
type ObservationNormalizer struct{}
func (ObservationNormalizer) Match(e event.Event) bool {
return strings.TrimSpace(e.Schema) == standards.SchemaRawNWSObservationV1
}
func (ObservationNormalizer) Normalize(ctx context.Context, in event.Event) (*event.Event, error) {
_ = ctx // normalization is pure/CPU; keep ctx for future expensive steps
rawBytes, err := normcommon.PayloadBytes(in)
if err != nil {
return nil, fmt.Errorf("nws observation normalize: %w", err)
}
var parsed nwsObservationResponse
if err := json.Unmarshal(rawBytes, &parsed); err != nil {
return nil, fmt.Errorf("nws observation normalize: decode raw payload: %w", err)
}
obs, effectiveAt, err := buildObservation(parsed)
if err != nil {
return nil, err
}
out := in
out.Schema = standards.SchemaWeatherObservationV1
out.Payload = obs
// EffectiveAt is optional; for observations it is naturally the observation timestamp.
if !effectiveAt.IsZero() {
t := effectiveAt.UTC()
out.EffectiveAt = &t
}
if err := out.Validate(); err != nil {
return nil, err
}
return &out, nil
}
// buildObservation contains the domain mapping logic (provider -> canonical model).
func buildObservation(parsed nwsObservationResponse) (model.WeatherObservation, time.Time, error) {
// Timestamp (RFC3339)
var ts time.Time
if s := strings.TrimSpace(parsed.Properties.Timestamp); s != "" {
t, err := time.Parse(time.RFC3339, s)
if err != nil {
return model.WeatherObservation{}, time.Time{}, fmt.Errorf("nws observation normalize: invalid timestamp %q: %w", s, err)
}
ts = t
}
cloudLayers := make([]model.CloudLayer, 0, len(parsed.Properties.CloudLayers))
for _, cl := range parsed.Properties.CloudLayers {
cloudLayers = append(cloudLayers, model.CloudLayer{
BaseMeters: cl.Base.Value,
Amount: cl.Amount,
})
}
// Preserve raw presentWeather objects (for troubleshooting / drift analysis).
present := make([]model.PresentWeather, 0, len(parsed.Properties.PresentWeather))
for _, pw := range parsed.Properties.PresentWeather {
present = append(present, model.PresentWeather{Raw: pw})
}
// Decode presentWeather into typed METAR phenomena for mapping.
phenomena := decodeMetarPhenomena(parsed.Properties.PresentWeather)
providerDesc := strings.TrimSpace(parsed.Properties.TextDescription)
// Determine canonical WMO condition code.
wmo := mapNWSToWMO(providerDesc, cloudLayers, phenomena)
// Canonical condition text comes from our WMO table.
// NWS observation responses typically do not include a day/night flag -> nil.
canonicalText := standards.WMOText(wmo, nil)
obs := model.WeatherObservation{
StationID: parsed.Properties.StationID,
StationName: parsed.Properties.StationName,
Timestamp: ts,
ConditionCode: wmo,
ConditionText: canonicalText,
IsDay: nil,
ProviderRawDescription: providerDesc,
// Transitional / human-facing:
// keep output consistent by populating TextDescription from canonical text.
TextDescription: canonicalText,
IconURL: parsed.Properties.Icon,
TemperatureC: parsed.Properties.Temperature.Value,
DewpointC: parsed.Properties.Dewpoint.Value,
WindDirectionDegrees: parsed.Properties.WindDirection.Value,
WindSpeedKmh: parsed.Properties.WindSpeed.Value,
WindGustKmh: parsed.Properties.WindGust.Value,
BarometricPressurePa: parsed.Properties.BarometricPressure.Value,
SeaLevelPressurePa: parsed.Properties.SeaLevelPressure.Value,
VisibilityMeters: parsed.Properties.Visibility.Value,
RelativeHumidityPercent: parsed.Properties.RelativeHumidity.Value,
WindChillC: parsed.Properties.WindChill.Value,
HeatIndexC: parsed.Properties.HeatIndex.Value,
ElevationMeters: parsed.Properties.Elevation.Value,
RawMessage: parsed.Properties.RawMessage,
PresentWeather: present,
CloudLayers: cloudLayers,
}
return obs, ts, nil
}

View File

@@ -1,3 +1,4 @@
// FILE: ./internal/normalizers/nws/register.go
package nws
import (
@@ -5,17 +6,11 @@ import (
)
// Register registers NWS normalizers into the provided registry.
//
// This is intentionally empty as a stub. As normalizers are implemented,
// register them here, e.g.:
//
// reg.Register(ObservationNormalizer{})
// reg.Register(ForecastNormalizer{})
// reg.Register(AlertsNormalizer{})
func Register(reg *fknormalize.Registry) {
if reg == nil {
return
}
// TODO: register NWS normalizers here.
// Observations
reg.Register(ObservationNormalizer{})
}

View File

@@ -0,0 +1,89 @@
// FILE: ./internal/normalizers/nws/types.go
package nws
// nwsObservationResponse is a minimal-but-sufficient representation of the NWS
// station observation GeoJSON payload needed for mapping into model.WeatherObservation.
type nwsObservationResponse struct {
ID string `json:"id"`
Properties struct {
StationID string `json:"stationId"`
StationName string `json:"stationName"`
Timestamp string `json:"timestamp"`
TextDescription string `json:"textDescription"`
Icon string `json:"icon"`
RawMessage string `json:"rawMessage"`
Elevation struct {
UnitCode string `json:"unitCode"`
Value *float64 `json:"value"`
} `json:"elevation"`
Temperature struct {
UnitCode string `json:"unitCode"`
Value *float64 `json:"value"`
} `json:"temperature"`
Dewpoint struct {
UnitCode string `json:"unitCode"`
Value *float64 `json:"value"`
} `json:"dewpoint"`
WindDirection struct {
UnitCode string `json:"unitCode"`
Value *float64 `json:"value"`
} `json:"windDirection"`
WindSpeed struct {
UnitCode string `json:"unitCode"`
Value *float64 `json:"value"`
} `json:"windSpeed"`
WindGust struct {
UnitCode string `json:"unitCode"`
Value *float64 `json:"value"`
} `json:"windGust"`
BarometricPressure struct {
UnitCode string `json:"unitCode"`
Value *float64 `json:"value"`
} `json:"barometricPressure"`
SeaLevelPressure struct {
UnitCode string `json:"unitCode"`
Value *float64 `json:"value"`
} `json:"seaLevelPressure"`
Visibility struct {
UnitCode string `json:"unitCode"`
Value *float64 `json:"value"`
} `json:"visibility"`
RelativeHumidity struct {
UnitCode string `json:"unitCode"`
Value *float64 `json:"value"`
} `json:"relativeHumidity"`
WindChill struct {
UnitCode string `json:"unitCode"`
Value *float64 `json:"value"`
} `json:"windChill"`
HeatIndex struct {
UnitCode string `json:"unitCode"`
Value *float64 `json:"value"`
} `json:"heatIndex"`
// NWS returns "presentWeather" as decoded METAR phenomena objects.
// We decode these as generic maps, then optionally interpret them in metar.go.
PresentWeather []map[string]any `json:"presentWeather"`
CloudLayers []struct {
Base struct {
UnitCode string `json:"unitCode"`
Value *float64 `json:"value"`
} `json:"base"`
Amount string `json:"amount"`
} `json:"cloudLayers"`
} `json:"properties"`
}

View File

@@ -0,0 +1,223 @@
// FILE: ./internal/normalizers/nws/wmo_map.go
package nws
import (
"strings"
"gitea.maximumdirect.net/ejr/weatherfeeder/internal/model"
normcommon "gitea.maximumdirect.net/ejr/weatherfeeder/internal/normalizers/common"
)
// mapNWSToWMO maps NWS signals into a canonical WMO code.
//
// Precedence:
// 1. METAR phenomena (presentWeather) — most reliable for precip/hazards
// 2. textDescription keywords — weaker, but reusable across providers
// 3. cloud layers fallback — only for sky-only conditions
func mapNWSToWMO(providerDesc string, cloudLayers []model.CloudLayer, phenomena []metarPhenomenon) model.WMOCode {
// 1) Prefer METAR phenomena if present.
if code := wmoFromPhenomena(phenomena); code != model.WMOUnknown {
return code
}
// 2) Reusable fallback: infer WMO from human text description.
if code := normcommon.WMOFromTextDescription(providerDesc); code != model.WMOUnknown {
return code
}
// 3) NWS/METAR-specific sky fallback.
if code := wmoFromCloudLayers(cloudLayers); code != model.WMOUnknown {
return code
}
return model.WMOUnknown
}
func wmoFromPhenomena(phenomena []metarPhenomenon) model.WMOCode {
if len(phenomena) == 0 {
return model.WMOUnknown
}
intensityOf := func(p metarPhenomenon) string {
if p.Intensity == nil {
return ""
}
return strings.ToLower(strings.TrimSpace(*p.Intensity))
}
modifierOf := func(p metarPhenomenon) string {
if p.Modifier == nil {
return ""
}
return strings.ToLower(strings.TrimSpace(*p.Modifier))
}
// Pass 1: thunder + hail overrides everything (hazard).
hasThunder := false
hailIntensity := ""
for _, p := range phenomena {
switch p.Weather {
case "thunderstorms":
hasThunder = true
case "hail":
if hailIntensity == "" {
hailIntensity = intensityOf(p)
}
}
}
if hasThunder {
if hailIntensity != "" || containsWeather(phenomena, "hail") {
if hailIntensity == "heavy" {
return 99
}
return 96
}
return 95
}
// Pass 2: freezing hazards.
for _, p := range phenomena {
if modifierOf(p) != "freezing" {
continue
}
switch p.Weather {
case "rain":
if intensityOf(p) == "light" {
return 66
}
return 67
case "drizzle":
if intensityOf(p) == "light" {
return 56
}
return 57
case "fog", "fog_mist":
return 48
}
}
// Pass 3: fog / obscuration.
for _, p := range phenomena {
switch p.Weather {
case "fog", "fog_mist":
return 45
case "haze", "smoke", "dust", "sand", "spray", "volcanic_ash":
return 45
}
}
// Pass 4: precip families.
for _, p := range phenomena {
inten := intensityOf(p)
mod := modifierOf(p)
if mod == "showers" {
switch p.Weather {
case "rain":
if inten == "light" {
return 80
}
if inten == "heavy" {
return 82
}
return 81
case "snow":
if inten == "light" {
return 85
}
return 86
}
}
switch p.Weather {
case "drizzle":
if inten == "heavy" {
return 55
}
if inten == "light" {
return 51
}
return 53
case "rain":
if inten == "heavy" {
return 65
}
if inten == "light" {
return 61
}
return 63
case "snow":
if inten == "heavy" {
return 75
}
if inten == "light" {
return 71
}
return 73
case "snow_grains":
return 77
case "ice_pellets", "snow_pellets":
return 73
}
}
return model.WMOUnknown
}
func wmoFromCloudLayers(cloudLayers []model.CloudLayer) model.WMOCode {
// NWS cloud layer amount values commonly include:
// OVC, BKN, SCT, FEW, SKC, CLR, VV (vertical visibility / obscured sky)
//
// Conservative mapping within our current WMO subset:
// - OVC / BKN / VV => Cloudy (3)
// - SCT => Partly Cloudy (2)
// - FEW => Mainly Sunny/Clear (1)
// - CLR / SKC => Sunny/Clear (0)
//
// Multiple layers: bias toward “most cloudy”.
mostCloudy := ""
for _, cl := range cloudLayers {
a := strings.ToUpper(strings.TrimSpace(cl.Amount))
if a == "" {
continue
}
switch a {
case "OVC":
return 3
case "BKN", "VV":
if mostCloudy != "OVC" {
mostCloudy = a
}
case "SCT":
if mostCloudy == "" {
mostCloudy = "SCT"
}
case "FEW":
if mostCloudy == "" {
mostCloudy = "FEW"
}
case "CLR", "SKC":
if mostCloudy == "" {
mostCloudy = "CLR"
}
}
}
switch mostCloudy {
case "BKN", "VV":
return 3
case "SCT":
return 2
case "FEW":
return 1
case "CLR":
return 0
default:
return model.WMOUnknown
}
}