From 356c3be6487d6afb7bd799d3851dce07f17e83da Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Fri, 27 Mar 2026 16:07:12 -0500 Subject: [PATCH] Feature addition to support narrative forecast updates from the NWS --- README.md | 2 +- cmd/weatherfeeder/config.yml | 9 ++ internal/normalizers/nws/forecast.go | 96 ++++++++++++++- internal/normalizers/nws/forecast_test.go | 121 +++++++++++++++++++ internal/normalizers/nws/types.go | 52 ++++++++- internal/sources/builtins.go | 3 + internal/sources/builtins_test.go | 13 +++ internal/sources/nws/forecast_narrative.go | 129 +++++++++++++++++++++ standards/schema.go | 1 + 9 files changed, 421 insertions(+), 5 deletions(-) create mode 100644 internal/sources/nws/forecast_narrative.go diff --git a/README.md b/README.md index 81ce516..32a35de 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ For the complete wire contract (event envelope + payload schemas, fields, units, ## Upstream providers (current MVP) -- NWS: observations, hourly forecasts, alerts +- NWS: observations, hourly forecasts, narrative forecasts, alerts - Open-Meteo: observations, hourly forecasts - OpenWeather: observations diff --git a/cmd/weatherfeeder/config.yml b/cmd/weatherfeeder/config.yml index 41f6900..c20434c 100644 --- a/cmd/weatherfeeder/config.yml +++ b/cmd/weatherfeeder/config.yml @@ -54,6 +54,15 @@ sources: url: "https://api.weather.gov/gridpoints/LSX/90,74/forecast/hourly" user_agent: "HomeOps (eric@maximumdirect.net)" + - name: NWSNarrativeForecastSTL + mode: poll + kinds: ["forecast"] + driver: nws_forecast_narrative + every: 45m + params: + url: "https://api.weather.gov/gridpoints/LSX/90,74/forecast?units=us" + user_agent: "HomeOps (eric@maximumdirect.net)" + - name: OpenMeteoHourlyForecastSTL mode: poll kinds: ["forecast"] diff --git a/internal/normalizers/nws/forecast.go b/internal/normalizers/nws/forecast.go index fb892e6..a029341 100644 --- a/internal/normalizers/nws/forecast.go +++ b/internal/normalizers/nws/forecast.go @@ -16,10 +16,11 @@ import ( // ForecastNormalizer converts: // -// standards.SchemaRawNWSHourlyForecastV1 -> standards.SchemaWeatherForecastV1 +// standards.SchemaRawNWSHourlyForecastV1 -> standards.SchemaWeatherForecastV1 +// standards.SchemaRawNWSNarrativeForecastV1 -> standards.SchemaWeatherForecastV1 // // It keeps one NWS forecast normalization entrypoint and dispatches to product-specific -// builders by raw schema. Today only hourly is implemented. +// builders by raw schema. // // Caveats / policy: // 1. NWS forecast periods do not include METAR presentWeather phenomena, so ConditionCode @@ -32,6 +33,8 @@ func (ForecastNormalizer) Match(e event.Event) bool { switch strings.TrimSpace(e.Schema) { case standards.SchemaRawNWSHourlyForecastV1: return true + case standards.SchemaRawNWSNarrativeForecastV1: + return true default: return false } @@ -47,6 +50,8 @@ func normalizeForecastEventBySchema(in event.Event) (*event.Event, error) { switch strings.TrimSpace(in.Schema) { case standards.SchemaRawNWSHourlyForecastV1: return normalizeHourlyForecastEvent(in) + case standards.SchemaRawNWSNarrativeForecastV1: + return normalizeNarrativeForecastEvent(in) default: return nil, fmt.Errorf("unsupported nws forecast schema %q", strings.TrimSpace(in.Schema)) } @@ -61,6 +66,15 @@ func normalizeHourlyForecastEvent(in event.Event) (*event.Event, error) { ) } +func normalizeNarrativeForecastEvent(in event.Event) (*event.Event, error) { + return normcommon.NormalizeJSON( + in, + "nws narrative forecast", + standards.SchemaWeatherForecastV1, + buildNarrativeForecast, + ) +} + // buildHourlyForecast contains hourly forecast mapping logic (provider -> canonical model). func buildHourlyForecast(parsed nwsHourlyForecastResponse) (model.WeatherForecastRun, time.Time, error) { issuedAt, updatedAt, err := parseForecastRunTimes(parsed.Properties.GeneratedAt, parsed.Properties.UpdateTime) @@ -95,6 +109,40 @@ func buildHourlyForecast(parsed nwsHourlyForecastResponse) (model.WeatherForecas return run, issuedAt, nil } +// buildNarrativeForecast contains narrative forecast mapping logic (provider -> canonical model). +func buildNarrativeForecast(parsed nwsNarrativeForecastResponse) (model.WeatherForecastRun, time.Time, error) { + issuedAt, updatedAt, err := parseForecastRunTimes(parsed.Properties.GeneratedAt, parsed.Properties.UpdateTime) + if err != nil { + return model.WeatherForecastRun{}, time.Time{}, err + } + + // Best-effort location centroid from the GeoJSON polygon (optional). + lat, lon := centroidLatLon(parsed.Geometry.Coordinates) + + run := newForecastRunBase( + issuedAt, + updatedAt, + model.ForecastProductNarrative, + lat, + lon, + parsed.Properties.Elevation.Value, + ) + + periods := make([]model.WeatherForecastPeriod, 0, len(parsed.Properties.Periods)) + for i, p := range parsed.Properties.Periods { + period, err := mapNarrativeForecastPeriod(i, p) + if err != nil { + return model.WeatherForecastRun{}, time.Time{}, err + } + periods = append(periods, period) + } + + run.Periods = periods + + // EffectiveAt policy for forecasts: treat IssuedAt as the effective time (dedupe-friendly). + return run, issuedAt, nil +} + func parseForecastRunTimes(generatedAt, updateTime string) (time.Time, *time.Time, error) { issuedStr := strings.TrimSpace(generatedAt) if issuedStr == "" { @@ -199,3 +247,47 @@ func mapHourlyForecastPeriod(idx int, p nwsHourlyForecastPeriod) (model.WeatherF ProbabilityOfPrecipitationPercent: p.ProbabilityOfPrecipitation.Value, }, nil } + +func mapNarrativeForecastPeriod(idx int, p nwsNarrativeForecastPeriod) (model.WeatherForecastPeriod, error) { + start, end, err := parseForecastPeriodWindow(p.StartTime, p.EndTime, idx) + if err != nil { + return model.WeatherForecastPeriod{}, err + } + + // NWS narrative supplies isDaytime; make it a pointer to match the canonical model. + var isDay *bool + if p.IsDaytime != nil { + b := *p.IsDaytime + isDay = &b + } + + tempC := tempCFromNWS(p.Temperature, p.TemperatureUnit) + + // Infer WMO from shortForecast (and fall back to icon token). + shortForecast := strings.TrimSpace(p.ShortForecast) + wmo := wmoFromNWSForecast(shortForecast, p.Icon, tempC) + + textDescription := strings.TrimSpace(p.DetailedForecast) + if textDescription == "" { + textDescription = shortForecast + } + + return model.WeatherForecastPeriod{ + StartTime: start, + EndTime: end, + + Name: strings.TrimSpace(p.Name), + IsDay: isDay, + + ConditionCode: wmo, + + TextDescription: textDescription, + + TemperatureC: tempC, + + WindDirectionDegrees: parseNWSWindDirectionDegrees(p.WindDirection), + WindSpeedKmh: parseNWSWindSpeedKmh(p.WindSpeed), + + ProbabilityOfPrecipitationPercent: p.ProbabilityOfPrecipitation.Value, + }, nil +} diff --git a/internal/normalizers/nws/forecast_test.go b/internal/normalizers/nws/forecast_test.go index 5210163..4e5d71c 100644 --- a/internal/normalizers/nws/forecast_test.go +++ b/internal/normalizers/nws/forecast_test.go @@ -2,11 +2,13 @@ package nws import ( "encoding/json" + "math" "strings" "testing" "time" "gitea.maximumdirect.net/ejr/feedkit/event" + "gitea.maximumdirect.net/ejr/weatherfeeder/model" "gitea.maximumdirect.net/ejr/weatherfeeder/standards" ) @@ -70,6 +72,118 @@ func TestNormalizeForecastEventBySchemaRoutesHourly(t *testing.T) { } } +func TestNormalizeForecastEventBySchemaRoutesNarrative(t *testing.T) { + _, err := normalizeForecastEventBySchema(event.Event{ + Schema: standards.SchemaRawNWSNarrativeForecastV1, + Payload: map[string]any{"properties": map[string]any{}}, + }) + if err == nil { + t.Fatalf("normalizeForecastEventBySchema() expected build error for missing generatedAt") + } + if !strings.Contains(err.Error(), "missing properties.generatedAt") { + t.Fatalf("error = %q, want missing properties.generatedAt", err) + } +} + +func TestBuildNarrativeForecastMapsExpectedFields(t *testing.T) { + parsed := nwsNarrativeForecastResponse{} + parsed.Properties.GeneratedAt = "2026-03-27T15:17:01Z" + isDay := true + tempF := 53.0 + pop := 20.0 + + parsed.Properties.Periods = []nwsNarrativeForecastPeriod{ + { + Name: "Today", + StartTime: "2026-03-27T10:00:00-05:00", + EndTime: "2026-03-27T18:00:00-05:00", + IsDaytime: &isDay, + Temperature: &tempF, + TemperatureUnit: "F", + WindSpeed: "10 to 14 mph", + WindDirection: "SW", + ShortForecast: "Partly Sunny", + DetailedForecast: " Partly sunny, with a high near 53. ", + ProbabilityOfPrecipitation: struct { + UnitCode string `json:"unitCode"` + Value *float64 `json:"value"` + }{ + UnitCode: "wmoUnit:percent", + Value: &pop, + }, + Icon: "https://api.weather.gov/icons/land/day/bkn?size=medium", + }, + } + + run, effectiveAt, err := buildNarrativeForecast(parsed) + if err != nil { + t.Fatalf("buildNarrativeForecast() error = %v", err) + } + if got, want := run.Product, model.ForecastProductNarrative; got != want { + t.Fatalf("Product = %q, want %q", got, want) + } + if len(run.Periods) != 1 { + t.Fatalf("periods len = %d, want 1", len(run.Periods)) + } + + p := run.Periods[0] + if got, want := p.TextDescription, "Partly sunny, with a high near 53."; got != want { + t.Fatalf("TextDescription = %q, want %q", got, want) + } + if p.TemperatureC == nil { + t.Fatalf("TemperatureC is nil, want converted value") + } + if math.Abs(*p.TemperatureC-11.6666666667) > 0.0001 { + t.Fatalf("TemperatureC = %.6f, want ~11.6667", *p.TemperatureC) + } + if p.IsDay == nil || !*p.IsDay { + t.Fatalf("IsDay = %v, want true", p.IsDay) + } + if p.WindDirectionDegrees == nil || *p.WindDirectionDegrees != 225 { + t.Fatalf("WindDirectionDegrees = %v, want 225", p.WindDirectionDegrees) + } + if p.WindSpeedKmh == nil || math.Abs(*p.WindSpeedKmh-19.3128) > 0.001 { + t.Fatalf("WindSpeedKmh = %.6f, want ~19.3128", derefOrZero(p.WindSpeedKmh)) + } + if p.ProbabilityOfPrecipitationPercent == nil || *p.ProbabilityOfPrecipitationPercent != 20 { + t.Fatalf("ProbabilityOfPrecipitationPercent = %v, want 20", p.ProbabilityOfPrecipitationPercent) + } + + wantIssued := time.Date(2026, 3, 27, 15, 17, 1, 0, time.UTC) + if !run.IssuedAt.Equal(wantIssued) { + t.Fatalf("IssuedAt = %s, want %s", run.IssuedAt.Format(time.RFC3339), wantIssued.Format(time.RFC3339)) + } + if !effectiveAt.Equal(wantIssued) { + t.Fatalf("effectiveAt = %s, want %s", effectiveAt.Format(time.RFC3339), wantIssued.Format(time.RFC3339)) + } + + assertNoLegacyForecastDescriptionKeys(t, p) +} + +func TestBuildNarrativeForecastFallsBackToShortForecastDescription(t *testing.T) { + parsed := nwsNarrativeForecastResponse{} + parsed.Properties.GeneratedAt = "2026-03-27T15:17:01Z" + parsed.Properties.Periods = []nwsNarrativeForecastPeriod{ + { + StartTime: "2026-03-27T18:00:00-05:00", + EndTime: "2026-03-28T06:00:00-05:00", + ShortForecast: " Mostly Clear ", + DetailedForecast: " ", + }, + } + + run, _, err := buildNarrativeForecast(parsed) + if err != nil { + t.Fatalf("buildNarrativeForecast() error = %v", err) + } + if len(run.Periods) != 1 { + t.Fatalf("periods len = %d, want 1", len(run.Periods)) + } + if got, want := run.Periods[0].TextDescription, "Mostly Clear"; got != want { + t.Fatalf("TextDescription = %q, want %q", got, want) + } +} + func assertNoLegacyForecastDescriptionKeys(t *testing.T, period any) { t.Helper() @@ -88,3 +202,10 @@ func assertNoLegacyForecastDescriptionKeys(t *testing.T, period any) { } } } + +func derefOrZero(v *float64) float64 { + if v == nil { + return 0 + } + return *v +} diff --git a/internal/normalizers/nws/types.go b/internal/normalizers/nws/types.go index e4c3a30..af132f9 100644 --- a/internal/normalizers/nws/types.go +++ b/internal/normalizers/nws/types.go @@ -100,8 +100,6 @@ type nwsCloudLayer struct { // nwsHourlyForecastResponse is a minimal-but-sufficient representation of the NWS // gridpoint hourly forecast GeoJSON payload needed for mapping into model.WeatherForecastRun. -// -// Daily and narrative variants should be added as distinct structs in follow-up work. type nwsHourlyForecastResponse struct { Geometry struct { Type string `json:"type"` @@ -160,6 +158,56 @@ type nwsHourlyForecastPeriod struct { DetailedForecast string `json:"detailedForecast"` } +// nwsNarrativeForecastResponse is a minimal-but-sufficient representation of the NWS +// gridpoint narrative forecast GeoJSON payload needed for mapping into model.WeatherForecastRun. +type nwsNarrativeForecastResponse struct { + Geometry struct { + Type string `json:"type"` + Coordinates [][][]float64 `json:"coordinates"` // GeoJSON polygon: [ring][point][lon,lat] + } `json:"geometry"` + + Properties struct { + Units string `json:"units"` // "us" or "si" (often "us" for narrative) + ForecastGenerator string `json:"forecastGenerator"` // e.g. "BaselineForecastGenerator" + + GeneratedAt string `json:"generatedAt"` // RFC3339-ish + UpdateTime string `json:"updateTime"` // RFC3339-ish + ValidTimes string `json:"validTimes"` + + Elevation struct { + UnitCode string `json:"unitCode"` + Value *float64 `json:"value"` + } `json:"elevation"` + + Periods []nwsNarrativeForecastPeriod `json:"periods"` + } `json:"properties"` +} + +type nwsNarrativeForecastPeriod struct { + Number int `json:"number"` + Name string `json:"name"` + StartTime string `json:"startTime"` + EndTime string `json:"endTime"` + + IsDaytime *bool `json:"isDaytime"` + + Temperature *float64 `json:"temperature"` + TemperatureUnit string `json:"temperatureUnit"` // "F" or "C" + TemperatureTrend any `json:"temperatureTrend"` + + ProbabilityOfPrecipitation struct { + UnitCode string `json:"unitCode"` + Value *float64 `json:"value"` + } `json:"probabilityOfPrecipitation"` + + WindSpeed string `json:"windSpeed"` // e.g. "9 mph", "10 to 15 mph" + WindDirection string `json:"windDirection"` // e.g. "W", "NW" + + Icon string `json:"icon"` + ShortForecast string `json:"shortForecast"` + DetailedForecast string `json:"detailedForecast"` +} + // nwsAlertsResponse is a minimal-but-sufficient representation of the NWS /alerts // FeatureCollection payload needed for mapping into model.WeatherAlertRun. type nwsAlertsResponse struct { diff --git a/internal/sources/builtins.go b/internal/sources/builtins.go index 0011a87..805916f 100644 --- a/internal/sources/builtins.go +++ b/internal/sources/builtins.go @@ -22,6 +22,9 @@ func RegisterBuiltins(r *fksource.Registry) { r.RegisterPoll("nws_forecast_hourly", func(cfg config.SourceConfig) (fksource.PollSource, error) { return nws.NewHourlyForecastSource(cfg) }) + r.RegisterPoll("nws_forecast_narrative", func(cfg config.SourceConfig) (fksource.PollSource, error) { + return nws.NewNarrativeForecastSource(cfg) + }) // Open-Meteo drivers r.RegisterPoll("openmeteo_observation", func(cfg config.SourceConfig) (fksource.PollSource, error) { diff --git a/internal/sources/builtins_test.go b/internal/sources/builtins_test.go index 9ef013d..d5cce92 100644 --- a/internal/sources/builtins_test.go +++ b/internal/sources/builtins_test.go @@ -21,6 +21,19 @@ func TestRegisterBuiltinsRegistersNWSHourlyForecastDriver(t *testing.T) { } } +func TestRegisterBuiltinsRegistersNWSNarrativeForecastDriver(t *testing.T) { + reg := fksource.NewRegistry() + RegisterBuiltins(reg) + + in, err := reg.BuildInput(sourceConfigForDriver("nws_forecast_narrative")) + if err != nil { + t.Fatalf("BuildInput(nws_forecast_narrative) error = %v", err) + } + if _, ok := in.(fksource.PollSource); !ok { + t.Fatalf("BuildInput(nws_forecast_narrative) type = %T, want PollSource", in) + } +} + func TestRegisterBuiltinsDoesNotRegisterLegacyNWSForecastDriver(t *testing.T) { reg := fksource.NewRegistry() RegisterBuiltins(reg) diff --git a/internal/sources/nws/forecast_narrative.go b/internal/sources/nws/forecast_narrative.go new file mode 100644 index 0000000..d112629 --- /dev/null +++ b/internal/sources/nws/forecast_narrative.go @@ -0,0 +1,129 @@ +// FILE: internal/sources/nws/forecast_narrative.go +package nws + +import ( + "context" + "encoding/json" + "strings" + "time" + + "gitea.maximumdirect.net/ejr/feedkit/config" + "gitea.maximumdirect.net/ejr/feedkit/event" + nwscommon "gitea.maximumdirect.net/ejr/weatherfeeder/internal/providers/nws" + "gitea.maximumdirect.net/ejr/weatherfeeder/internal/sources/common" + "gitea.maximumdirect.net/ejr/weatherfeeder/standards" +) + +// NarrativeForecastSource polls an NWS narrative forecast endpoint and emits a RAW forecast Event. +// +// It intentionally emits the *entire* upstream payload as json.RawMessage and only decodes +// minimal metadata for Event.EffectiveAt and Event.ID. +// +// Output schema: +// - standards.SchemaRawNWSNarrativeForecastV1 +type NarrativeForecastSource struct { + http *common.HTTPSource +} + +func NewNarrativeForecastSource(cfg config.SourceConfig) (*NarrativeForecastSource, error) { + const driver = "nws_forecast_narrative" + + // NWS forecast endpoints are GeoJSON (and sometimes also advertise json-ld/json). + hs, err := common.NewHTTPSource(driver, cfg, "application/geo+json, application/json") + if err != nil { + return nil, err + } + + return &NarrativeForecastSource{http: hs}, nil +} + +func (s *NarrativeForecastSource) Name() string { return s.http.Name } + +// Kind is used for routing/policy. +func (s *NarrativeForecastSource) Kind() event.Kind { return event.Kind("forecast") } + +func (s *NarrativeForecastSource) Poll(ctx context.Context) ([]event.Event, error) { + raw, meta, err := s.fetchRaw(ctx) + if err != nil { + return nil, err + } + + // EffectiveAt is optional; for forecasts it’s most naturally the run “issued” time. + // NWS gridpoint forecasts expose generatedAt (preferred) and updateTime/updated. + var effectiveAt *time.Time + switch { + case !meta.ParsedGeneratedAt.IsZero(): + t := meta.ParsedGeneratedAt.UTC() + effectiveAt = &t + case !meta.ParsedUpdateTime.IsZero(): + t := meta.ParsedUpdateTime.UTC() + effectiveAt = &t + } + + emittedAt := time.Now().UTC() + + // NWS gridpoint forecast GeoJSON commonly has a stable "id" equal to the endpoint URL. + // That is *not* unique per issued run, so we intentionally do not use it for Event.ID. + // Instead we rely on Source:EffectiveAt (or Source:EmittedAt fallback). + eventID := common.ChooseEventID("", s.http.Name, effectiveAt, emittedAt) + + return common.SingleRawEvent( + s.Kind(), + s.http.Name, + standards.SchemaRawNWSNarrativeForecastV1, + eventID, + emittedAt, + effectiveAt, + raw, + ) +} + +// ---- RAW fetch + minimal metadata decode ---- + +type narrativeForecastMeta struct { + // Present for GeoJSON Feature responses, but often stable (endpoint URL). + ID string `json:"id"` + + Properties struct { + GeneratedAt string `json:"generatedAt"` // preferred “issued/run generated” time + UpdateTime string `json:"updateTime"` // last update time of underlying data + Updated string `json:"updated"` // deprecated alias for updateTime + } `json:"properties"` + + ParsedGeneratedAt time.Time `json:"-"` + ParsedUpdateTime time.Time `json:"-"` +} + +func (s *NarrativeForecastSource) fetchRaw(ctx context.Context) (json.RawMessage, narrativeForecastMeta, error) { + raw, err := s.http.FetchJSON(ctx) + if err != nil { + return nil, narrativeForecastMeta{}, err + } + + var meta narrativeForecastMeta + if err := json.Unmarshal(raw, &meta); err != nil { + // If metadata decode fails, still return raw; Poll will fall back to Source:EmittedAt. + return raw, narrativeForecastMeta{}, nil + } + + // generatedAt (preferred) + genStr := strings.TrimSpace(meta.Properties.GeneratedAt) + if genStr != "" { + if t, err := nwscommon.ParseTime(genStr); err == nil { + meta.ParsedGeneratedAt = t.UTC() + } + } + + // updateTime, with fallback to deprecated "updated" + updStr := strings.TrimSpace(meta.Properties.UpdateTime) + if updStr == "" { + updStr = strings.TrimSpace(meta.Properties.Updated) + } + if updStr != "" { + if t, err := nwscommon.ParseTime(updStr); err == nil { + meta.ParsedUpdateTime = t.UTC() + } + } + + return raw, meta, nil +} diff --git a/standards/schema.go b/standards/schema.go index e2a15f4..eca717d 100644 --- a/standards/schema.go +++ b/standards/schema.go @@ -16,6 +16,7 @@ const ( SchemaRawOpenWeatherCurrentV1 = "raw.openweather.current.v1" SchemaRawNWSHourlyForecastV1 = "raw.nws.hourly.forecast.v1" + SchemaRawNWSNarrativeForecastV1 = "raw.nws.narrative.forecast.v1" SchemaRawOpenMeteoHourlyForecastV1 = "raw.openmeteo.hourly.forecast.v1" SchemaRawOpenWeatherHourlyForecastV1 = "raw.openweather.hourly.forecast.v1"