feat(nws, normalizers): add NWS hourly forecast normalization and enforce canonical float rounding
- Implement full NWS hourly forecast normalizer (raw.nws.hourly.forecast.v1 → weather.forecast.v1) - Add GeoJSON forecast types and helpers for NWS gridpoint hourly payloads - Normalize temperatures, winds, humidity, PoP, and infer WMO condition codes from forecast text/icons - Treat forecast IssuedAt as EffectiveAt for stable, dedupe-friendly event IDs - Introduce project-wide float rounding at normalization finalization - Round all float values in canonical payloads to 2 decimal places - Apply consistently across pointers, slices, maps, and nested structs - Preserve opaque structs (e.g., time.Time) unchanged - Add SchemaRawNWSHourlyForecastV1 and align schema matching/comments - Clean up NWS helper organization and comments - Update documentation to reflect numeric wire-format and normalization policies This establishes a complete, deterministic hourly forecast pipeline for NWS and improves JSON output stability across all canonical weather schemas.
This commit is contained in:
159
internal/normalizers/nws/forecast.go
Normal file
159
internal/normalizers/nws/forecast.go
Normal file
@@ -0,0 +1,159 @@
|
||||
// FILE: internal/normalizers/nws/forecast.go
|
||||
package nws
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/ejr/feedkit/event"
|
||||
"gitea.maximumdirect.net/ejr/weatherfeeder/internal/model"
|
||||
normcommon "gitea.maximumdirect.net/ejr/weatherfeeder/internal/normalizers/common"
|
||||
nwscommon "gitea.maximumdirect.net/ejr/weatherfeeder/internal/providers/nws"
|
||||
"gitea.maximumdirect.net/ejr/weatherfeeder/internal/standards"
|
||||
)
|
||||
|
||||
// ForecastNormalizer converts:
|
||||
//
|
||||
// standards.SchemaRawNWSHourlyForecastV1 -> standards.SchemaWeatherForecastV1
|
||||
//
|
||||
// It interprets NWS GeoJSON gridpoint *hourly* forecast responses and maps them into
|
||||
// the canonical model.WeatherForecastRun representation.
|
||||
//
|
||||
// Caveats / policy:
|
||||
// 1. NWS forecast periods do not include METAR presentWeather phenomena, so ConditionCode
|
||||
// is inferred from period.shortForecast (with a conservative icon-based fallback).
|
||||
// 2. Temperature is converted to °C when NWS supplies °F.
|
||||
// 3. WindSpeed is parsed from strings like "9 mph" / "10 to 15 mph" and converted to km/h.
|
||||
type ForecastNormalizer struct{}
|
||||
|
||||
func (ForecastNormalizer) Match(e event.Event) bool {
|
||||
s := strings.TrimSpace(e.Schema)
|
||||
return s == standards.SchemaRawNWSHourlyForecastV1
|
||||
}
|
||||
|
||||
func (ForecastNormalizer) Normalize(ctx context.Context, in event.Event) (*event.Event, error) {
|
||||
_ = ctx // normalization is pure/CPU; keep ctx for future expensive steps
|
||||
|
||||
return normcommon.NormalizeJSON(
|
||||
in,
|
||||
"nws hourly forecast",
|
||||
standards.SchemaWeatherForecastV1,
|
||||
buildForecast,
|
||||
)
|
||||
}
|
||||
|
||||
// buildForecast contains the domain mapping logic (provider -> canonical model).
|
||||
func buildForecast(parsed nwsForecastResponse) (model.WeatherForecastRun, time.Time, error) {
|
||||
// IssuedAt is required by the canonical model.
|
||||
issuedStr := strings.TrimSpace(parsed.Properties.GeneratedAt)
|
||||
if issuedStr == "" {
|
||||
return model.WeatherForecastRun{}, time.Time{}, fmt.Errorf("missing properties.generatedAt")
|
||||
}
|
||||
issuedAt, err := nwscommon.ParseTime(issuedStr)
|
||||
if err != nil {
|
||||
return model.WeatherForecastRun{}, time.Time{}, fmt.Errorf("invalid properties.generatedAt %q: %w", issuedStr, err)
|
||||
}
|
||||
issuedAt = issuedAt.UTC()
|
||||
|
||||
// UpdatedAt is optional.
|
||||
var updatedAt *time.Time
|
||||
if s := strings.TrimSpace(parsed.Properties.UpdateTime); s != "" {
|
||||
if t, err := nwscommon.ParseTime(s); err == nil {
|
||||
tt := t.UTC()
|
||||
updatedAt = &tt
|
||||
}
|
||||
}
|
||||
|
||||
// Best-effort location centroid from the GeoJSON polygon (optional).
|
||||
lat, lon := centroidLatLon(parsed.Geometry.Coordinates)
|
||||
|
||||
// Schema is explicitly hourly, so product is not a heuristic.
|
||||
run := model.WeatherForecastRun{
|
||||
LocationID: "",
|
||||
LocationName: "",
|
||||
|
||||
IssuedAt: issuedAt,
|
||||
UpdatedAt: updatedAt,
|
||||
Product: model.ForecastProductHourly,
|
||||
|
||||
Latitude: lat,
|
||||
Longitude: lon,
|
||||
ElevationMeters: parsed.Properties.Elevation.Value,
|
||||
|
||||
Periods: nil,
|
||||
}
|
||||
|
||||
periods := make([]model.WeatherForecastPeriod, 0, len(parsed.Properties.Periods))
|
||||
for i, p := range parsed.Properties.Periods {
|
||||
startStr := strings.TrimSpace(p.StartTime)
|
||||
endStr := strings.TrimSpace(p.EndTime)
|
||||
|
||||
if startStr == "" || endStr == "" {
|
||||
return model.WeatherForecastRun{}, time.Time{}, fmt.Errorf("periods[%d]: missing startTime/endTime", i)
|
||||
}
|
||||
|
||||
start, err := nwscommon.ParseTime(startStr)
|
||||
if err != nil {
|
||||
return model.WeatherForecastRun{}, time.Time{}, fmt.Errorf("periods[%d].startTime invalid %q: %w", i, startStr, err)
|
||||
}
|
||||
end, err := nwscommon.ParseTime(endStr)
|
||||
if err != nil {
|
||||
return model.WeatherForecastRun{}, time.Time{}, fmt.Errorf("periods[%d].endTime invalid %q: %w", i, endStr, err)
|
||||
}
|
||||
start = start.UTC()
|
||||
end = end.UTC()
|
||||
|
||||
// NWS hourly supplies isDaytime; make it a pointer to match the canonical model.
|
||||
var isDay *bool
|
||||
if p.IsDaytime != nil {
|
||||
b := *p.IsDaytime
|
||||
isDay = &b
|
||||
}
|
||||
|
||||
tempC := tempCFromNWS(p.Temperature, p.TemperatureUnit)
|
||||
|
||||
// Infer WMO from shortForecast (and fall back to icon token).
|
||||
providerDesc := strings.TrimSpace(p.ShortForecast)
|
||||
wmo := wmoFromNWSForecast(providerDesc, p.Icon, tempC)
|
||||
|
||||
canonicalText := standards.WMOText(wmo, isDay)
|
||||
|
||||
period := model.WeatherForecastPeriod{
|
||||
StartTime: start,
|
||||
EndTime: end,
|
||||
|
||||
Name: strings.TrimSpace(p.Name),
|
||||
IsDay: isDay,
|
||||
|
||||
ConditionCode: wmo,
|
||||
ConditionText: canonicalText,
|
||||
|
||||
ProviderRawDescription: providerDesc,
|
||||
|
||||
// For forecasts, keep provider text as the human-facing description.
|
||||
TextDescription: strings.TrimSpace(p.ShortForecast),
|
||||
DetailedText: strings.TrimSpace(p.DetailedForecast),
|
||||
|
||||
IconURL: strings.TrimSpace(p.Icon),
|
||||
|
||||
TemperatureC: tempC,
|
||||
|
||||
DewpointC: p.Dewpoint.Value,
|
||||
RelativeHumidityPercent: p.RelativeHumidity.Value,
|
||||
|
||||
WindDirectionDegrees: parseNWSWindDirectionDegrees(p.WindDirection),
|
||||
WindSpeedKmh: parseNWSWindSpeedKmh(p.WindSpeed),
|
||||
|
||||
ProbabilityOfPrecipitationPercent: p.ProbabilityOfPrecipitation.Value,
|
||||
}
|
||||
|
||||
periods = append(periods, period)
|
||||
}
|
||||
|
||||
run.Periods = periods
|
||||
|
||||
// EffectiveAt policy for forecasts: treat IssuedAt as the effective time (dedupe-friendly).
|
||||
return run, issuedAt, nil
|
||||
}
|
||||
235
internal/normalizers/nws/helpers.go
Normal file
235
internal/normalizers/nws/helpers.go
Normal file
@@ -0,0 +1,235 @@
|
||||
// FILE: internal/normalizers/nws/helpers.go
|
||||
package nws
|
||||
|
||||
import (
|
||||
"strconv"
|
||||
"strings"
|
||||
"unicode"
|
||||
|
||||
"gitea.maximumdirect.net/ejr/weatherfeeder/internal/model"
|
||||
normcommon "gitea.maximumdirect.net/ejr/weatherfeeder/internal/normalizers/common"
|
||||
)
|
||||
|
||||
// centroidLatLon returns a best-effort centroid (lat, lon) from a GeoJSON polygon.
|
||||
// If geometry is missing or malformed, returns (nil, nil).
|
||||
func centroidLatLon(coords [][][]float64) (lat *float64, lon *float64) {
|
||||
if len(coords) == 0 || len(coords[0]) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
var sumLon, sumLat float64
|
||||
var n float64
|
||||
|
||||
for _, pt := range coords[0] {
|
||||
if len(pt) < 2 {
|
||||
continue
|
||||
}
|
||||
sumLon += pt[0]
|
||||
sumLat += pt[1]
|
||||
n++
|
||||
}
|
||||
|
||||
if n == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
avgLon := sumLon / n
|
||||
avgLat := sumLat / n
|
||||
|
||||
return &avgLat, &avgLon
|
||||
}
|
||||
|
||||
func tempCFromNWS(v *float64, unit string) *float64 {
|
||||
if v == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
u := strings.ToUpper(strings.TrimSpace(unit))
|
||||
switch u {
|
||||
case "F":
|
||||
c := normcommon.TempCFromF(*v)
|
||||
return &c
|
||||
case "C":
|
||||
c := *v
|
||||
return &c
|
||||
default:
|
||||
// Unknown unit; be conservative.
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// wmoFromNWSForecast infers a canonical WMO code for a forecast period.
|
||||
//
|
||||
// Strategy:
|
||||
// 1. Try to infer from shortForecast using the cross-provider fallback.
|
||||
// 2. Special-case mixed rain+snow using temperature when available (since our WMO table
|
||||
// does not include a “mixed precip” code).
|
||||
// 3. Fall back to an icon token (e.g., "rain", "snow", "ovc", "bkn", "sct", ...).
|
||||
func wmoFromNWSForecast(shortForecast, iconURL string, tempC *float64) model.WMOCode {
|
||||
sf := strings.TrimSpace(shortForecast)
|
||||
s := strings.ToLower(sf)
|
||||
|
||||
// Mixed precip heuristic: choose rain vs snow based on temperature.
|
||||
if strings.Contains(s, "rain") && strings.Contains(s, "snow") {
|
||||
if tempC != nil && *tempC <= 0.0 {
|
||||
return 73 // Snow
|
||||
}
|
||||
return 63 // Rain
|
||||
}
|
||||
|
||||
if code := normcommon.WMOFromTextDescription(sf); code != model.WMOUnknown {
|
||||
return code
|
||||
}
|
||||
|
||||
// Icon fallback: token is usually the last path segment (before any comma/query).
|
||||
if token := nwsIconToken(iconURL); token != "" {
|
||||
// Try the general text fallback first (works for "rain", "snow", etc.).
|
||||
if code := normcommon.WMOFromTextDescription(token); code != model.WMOUnknown {
|
||||
return code
|
||||
}
|
||||
|
||||
// Sky-condition icon tokens are common; map conservatively.
|
||||
switch token {
|
||||
case "ovc", "bkn", "cloudy", "ovcast":
|
||||
return 3
|
||||
case "sct", "bkn-sct":
|
||||
return 2
|
||||
case "few":
|
||||
return 1
|
||||
case "skc", "clr", "clear":
|
||||
return 0
|
||||
}
|
||||
}
|
||||
|
||||
return model.WMOUnknown
|
||||
}
|
||||
|
||||
func nwsIconToken(iconURL string) string {
|
||||
u := strings.TrimSpace(iconURL)
|
||||
if u == "" {
|
||||
return ""
|
||||
}
|
||||
|
||||
// Drop query string.
|
||||
base := strings.SplitN(u, "?", 2)[0]
|
||||
|
||||
// Take last path segment.
|
||||
parts := strings.Split(base, "/")
|
||||
if len(parts) == 0 {
|
||||
return ""
|
||||
}
|
||||
last := parts[len(parts)-1]
|
||||
if last == "" && len(parts) > 1 {
|
||||
last = parts[len(parts)-2]
|
||||
}
|
||||
|
||||
// Some icons look like "rain,30" or "snow,20".
|
||||
last = strings.SplitN(last, ",", 2)[0]
|
||||
last = strings.ToLower(strings.TrimSpace(last))
|
||||
|
||||
return last
|
||||
}
|
||||
|
||||
// parseNWSWindSpeedKmh parses NWS wind speed strings like:
|
||||
// - "9 mph"
|
||||
// - "10 to 15 mph"
|
||||
//
|
||||
// and converts to km/h.
|
||||
//
|
||||
// Policy: if a range is present, we use the midpoint (best effort).
|
||||
func parseNWSWindSpeedKmh(s string) *float64 {
|
||||
raw := strings.ToLower(strings.TrimSpace(s))
|
||||
if raw == "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
nums := extractFloats(raw)
|
||||
if len(nums) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
val := nums[0]
|
||||
if len(nums) >= 2 && (strings.Contains(raw, " to ") || strings.Contains(raw, "-")) {
|
||||
val = (nums[0] + nums[1]) / 2.0
|
||||
}
|
||||
|
||||
switch {
|
||||
case strings.Contains(raw, "mph"):
|
||||
k := normcommon.SpeedKmhFromMph(val)
|
||||
return &k
|
||||
|
||||
case strings.Contains(raw, "km/h") || strings.Contains(raw, "kph"):
|
||||
k := val
|
||||
return &k
|
||||
|
||||
case strings.Contains(raw, "kt") || strings.Contains(raw, "kts") || strings.Contains(raw, "knot"):
|
||||
// 1 knot = 1.852 km/h
|
||||
k := val * 1.852
|
||||
return &k
|
||||
|
||||
default:
|
||||
// Unknown unit; be conservative.
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// parseNWSWindDirectionDegrees maps compass directions to degrees.
|
||||
// Returns nil if direction is empty/unknown.
|
||||
func parseNWSWindDirectionDegrees(dir string) *float64 {
|
||||
d := strings.ToUpper(strings.TrimSpace(dir))
|
||||
if d == "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
// 16-wind compass.
|
||||
m := map[string]float64{
|
||||
"N": 0,
|
||||
"NNE": 22.5,
|
||||
"NE": 45,
|
||||
"ENE": 67.5,
|
||||
"E": 90,
|
||||
"ESE": 112.5,
|
||||
"SE": 135,
|
||||
"SSE": 157.5,
|
||||
"S": 180,
|
||||
"SSW": 202.5,
|
||||
"SW": 225,
|
||||
"WSW": 247.5,
|
||||
"W": 270,
|
||||
"WNW": 292.5,
|
||||
"NW": 315,
|
||||
"NNW": 337.5,
|
||||
}
|
||||
|
||||
if deg, ok := m[d]; ok {
|
||||
return °
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func extractFloats(s string) []float64 {
|
||||
var out []float64
|
||||
var buf strings.Builder
|
||||
|
||||
flush := func() {
|
||||
if buf.Len() == 0 {
|
||||
return
|
||||
}
|
||||
v, err := strconv.ParseFloat(buf.String(), 64)
|
||||
if err == nil {
|
||||
out = append(out, v)
|
||||
}
|
||||
buf.Reset()
|
||||
}
|
||||
|
||||
for _, r := range s {
|
||||
if unicode.IsDigit(r) || r == '.' {
|
||||
buf.WriteRune(r)
|
||||
continue
|
||||
}
|
||||
flush()
|
||||
}
|
||||
flush()
|
||||
|
||||
return out
|
||||
}
|
||||
@@ -13,4 +13,7 @@ func Register(reg *fknormalize.Registry) {
|
||||
|
||||
// Observations
|
||||
reg.Register(ObservationNormalizer{})
|
||||
|
||||
// Forecasts
|
||||
reg.Register(ForecastNormalizer{})
|
||||
}
|
||||
|
||||
@@ -87,3 +87,66 @@ type nwsObservationResponse struct {
|
||||
} `json:"cloudLayers"`
|
||||
} `json:"properties"`
|
||||
}
|
||||
|
||||
// nwsForecastResponse is a minimal-but-sufficient representation of the NWS
|
||||
// gridpoint forecast GeoJSON payload needed for mapping into model.WeatherForecastRun.
|
||||
//
|
||||
// This is currently designed to support the hourly forecast endpoint; revisions may be needed
|
||||
// to accommodate other forecast endpoints in the future.
|
||||
type nwsForecastResponse struct {
|
||||
Geometry struct {
|
||||
Type string `json:"type"`
|
||||
Coordinates [][][]float64 `json:"coordinates"` // GeoJSON polygon: [ring][point][lon,lat]
|
||||
} `json:"geometry"`
|
||||
|
||||
Properties struct {
|
||||
Units string `json:"units"` // "us" or "si" (often "us" for hourly)
|
||||
ForecastGenerator string `json:"forecastGenerator"` // e.g. "HourlyForecastGenerator"
|
||||
|
||||
GeneratedAt string `json:"generatedAt"` // RFC3339-ish
|
||||
UpdateTime string `json:"updateTime"` // RFC3339-ish
|
||||
ValidTimes string `json:"validTimes"`
|
||||
|
||||
Elevation struct {
|
||||
UnitCode string `json:"unitCode"`
|
||||
Value *float64 `json:"value"`
|
||||
} `json:"elevation"`
|
||||
|
||||
Periods []nwsForecastPeriod `json:"periods"`
|
||||
} `json:"properties"`
|
||||
}
|
||||
|
||||
type nwsForecastPeriod struct {
|
||||
Number int `json:"number"`
|
||||
Name string `json:"name"`
|
||||
StartTime string `json:"startTime"`
|
||||
EndTime string `json:"endTime"`
|
||||
|
||||
IsDaytime *bool `json:"isDaytime"`
|
||||
|
||||
Temperature *float64 `json:"temperature"`
|
||||
TemperatureUnit string `json:"temperatureUnit"` // "F" or "C"
|
||||
TemperatureTrend any `json:"temperatureTrend"`
|
||||
|
||||
ProbabilityOfPrecipitation struct {
|
||||
UnitCode string `json:"unitCode"`
|
||||
Value *float64 `json:"value"`
|
||||
} `json:"probabilityOfPrecipitation"`
|
||||
|
||||
Dewpoint struct {
|
||||
UnitCode string `json:"unitCode"`
|
||||
Value *float64 `json:"value"`
|
||||
} `json:"dewpoint"`
|
||||
|
||||
RelativeHumidity struct {
|
||||
UnitCode string `json:"unitCode"`
|
||||
Value *float64 `json:"value"`
|
||||
} `json:"relativeHumidity"`
|
||||
|
||||
WindSpeed string `json:"windSpeed"` // e.g. "9 mph", "10 to 15 mph"
|
||||
WindDirection string `json:"windDirection"` // e.g. "W", "NW"
|
||||
|
||||
Icon string `json:"icon"`
|
||||
ShortForecast string `json:"shortForecast"`
|
||||
DetailedForecast string `json:"detailedForecast"`
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user