Feature addition to support narrative forecast updates from the NWS

2026-03-27 16:07:12 -05:00
parent dbaebbbd7a
commit 356c3be648
9 changed files with 421 additions and 5 deletions
--- a/README.md
+++ b/README.md
@@ -22,7 +22,7 @@ For the complete wire contract (event envelope + payload schemas, fields, units,

 ## Upstream providers (current MVP)

- NWS: observations, hourly forecasts, alerts
+- NWS: observations, hourly forecasts, narrative forecasts, alerts
 - Open-Meteo: observations, hourly forecasts
 - OpenWeather: observations

--- a/cmd/weatherfeeder/config.yml
+++ b/cmd/weatherfeeder/config.yml
@@ -54,6 +54,15 @@ sources:
      url: "https://api.weather.gov/gridpoints/LSX/90,74/forecast/hourly"
      user_agent: "HomeOps (eric@maximumdirect.net)"

+  - name: NWSNarrativeForecastSTL
+    mode: poll
+    kinds: ["forecast"]
+    driver: nws_forecast_narrative
+    every: 45m
+    params:
+      url: "https://api.weather.gov/gridpoints/LSX/90,74/forecast?units=us"
+      user_agent: "HomeOps (eric@maximumdirect.net)"
+
  - name: OpenMeteoHourlyForecastSTL
    mode: poll
    kinds: ["forecast"]
--- a/internal/normalizers/nws/forecast.go
+++ b/internal/normalizers/nws/forecast.go
@@ -16,10 +16,11 @@ import (

 // ForecastNormalizer converts:
 //
-//	standards.SchemaRawNWSHourlyForecastV1 -> standards.SchemaWeatherForecastV1
+//	standards.SchemaRawNWSHourlyForecastV1    -> standards.SchemaWeatherForecastV1
+//	standards.SchemaRawNWSNarrativeForecastV1 -> standards.SchemaWeatherForecastV1
 //
 // It keeps one NWS forecast normalization entrypoint and dispatches to product-specific
-// builders by raw schema. Today only hourly is implemented.
+// builders by raw schema.
 //
 // Caveats / policy:
 //  1. NWS forecast periods do not include METAR presentWeather phenomena, so ConditionCode
@@ -32,6 +33,8 @@ func (ForecastNormalizer) Match(e event.Event) bool {
 	switch strings.TrimSpace(e.Schema) {
 	case standards.SchemaRawNWSHourlyForecastV1:
 		return true
+	case standards.SchemaRawNWSNarrativeForecastV1:
+		return true
 	default:
 		return false
 	}
@@ -47,6 +50,8 @@ func normalizeForecastEventBySchema(in event.Event) (*event.Event, error) {
 	switch strings.TrimSpace(in.Schema) {
 	case standards.SchemaRawNWSHourlyForecastV1:
 		return normalizeHourlyForecastEvent(in)
+	case standards.SchemaRawNWSNarrativeForecastV1:
+		return normalizeNarrativeForecastEvent(in)
 	default:
 		return nil, fmt.Errorf("unsupported nws forecast schema %q", strings.TrimSpace(in.Schema))
 	}
@@ -61,6 +66,15 @@ func normalizeHourlyForecastEvent(in event.Event) (*event.Event, error) {
 	)
 }

+func normalizeNarrativeForecastEvent(in event.Event) (*event.Event, error) {
+	return normcommon.NormalizeJSON(
+		in,
+		"nws narrative forecast",
+		standards.SchemaWeatherForecastV1,
+		buildNarrativeForecast,
+	)
+}
+
 // buildHourlyForecast contains hourly forecast mapping logic (provider -> canonical model).
 func buildHourlyForecast(parsed nwsHourlyForecastResponse) (model.WeatherForecastRun, time.Time, error) {
 	issuedAt, updatedAt, err := parseForecastRunTimes(parsed.Properties.GeneratedAt, parsed.Properties.UpdateTime)
@@ -95,6 +109,40 @@ func buildHourlyForecast(parsed nwsHourlyForecastResponse) (model.WeatherForecas
 	return run, issuedAt, nil
 }

+// buildNarrativeForecast contains narrative forecast mapping logic (provider -> canonical model).
+func buildNarrativeForecast(parsed nwsNarrativeForecastResponse) (model.WeatherForecastRun, time.Time, error) {
+	issuedAt, updatedAt, err := parseForecastRunTimes(parsed.Properties.GeneratedAt, parsed.Properties.UpdateTime)
+	if err != nil {
+		return model.WeatherForecastRun{}, time.Time{}, err
+	}
+
+	// Best-effort location centroid from the GeoJSON polygon (optional).
+	lat, lon := centroidLatLon(parsed.Geometry.Coordinates)
+
+	run := newForecastRunBase(
+		issuedAt,
+		updatedAt,
+		model.ForecastProductNarrative,
+		lat,
+		lon,
+		parsed.Properties.Elevation.Value,
+	)
+
+	periods := make([]model.WeatherForecastPeriod, 0, len(parsed.Properties.Periods))
+	for i, p := range parsed.Properties.Periods {
+		period, err := mapNarrativeForecastPeriod(i, p)
+		if err != nil {
+			return model.WeatherForecastRun{}, time.Time{}, err
+		}
+		periods = append(periods, period)
+	}
+
+	run.Periods = periods
+
+	// EffectiveAt policy for forecasts: treat IssuedAt as the effective time (dedupe-friendly).
+	return run, issuedAt, nil
+}
+
 func parseForecastRunTimes(generatedAt, updateTime string) (time.Time, *time.Time, error) {
 	issuedStr := strings.TrimSpace(generatedAt)
 	if issuedStr == "" {
@@ -199,3 +247,47 @@ func mapHourlyForecastPeriod(idx int, p nwsHourlyForecastPeriod) (model.WeatherF
 		ProbabilityOfPrecipitationPercent: p.ProbabilityOfPrecipitation.Value,
 	}, nil
 }
+
+func mapNarrativeForecastPeriod(idx int, p nwsNarrativeForecastPeriod) (model.WeatherForecastPeriod, error) {
+	start, end, err := parseForecastPeriodWindow(p.StartTime, p.EndTime, idx)
+	if err != nil {
+		return model.WeatherForecastPeriod{}, err
+	}
+
+	// NWS narrative supplies isDaytime; make it a pointer to match the canonical model.
+	var isDay *bool
+	if p.IsDaytime != nil {
+		b := *p.IsDaytime
+		isDay = &b
+	}
+
+	tempC := tempCFromNWS(p.Temperature, p.TemperatureUnit)
+
+	// Infer WMO from shortForecast (and fall back to icon token).
+	shortForecast := strings.TrimSpace(p.ShortForecast)
+	wmo := wmoFromNWSForecast(shortForecast, p.Icon, tempC)
+
+	textDescription := strings.TrimSpace(p.DetailedForecast)
+	if textDescription == "" {
+		textDescription = shortForecast
+	}
+
+	return model.WeatherForecastPeriod{
+		StartTime: start,
+		EndTime:   end,
+
+		Name:  strings.TrimSpace(p.Name),
+		IsDay: isDay,
+
+		ConditionCode: wmo,
+
+		TextDescription: textDescription,
+
+		TemperatureC: tempC,
+
+		WindDirectionDegrees: parseNWSWindDirectionDegrees(p.WindDirection),
+		WindSpeedKmh:         parseNWSWindSpeedKmh(p.WindSpeed),
+
+		ProbabilityOfPrecipitationPercent: p.ProbabilityOfPrecipitation.Value,
+	}, nil
+}
--- a/internal/normalizers/nws/forecast_test.go
+++ b/internal/normalizers/nws/forecast_test.go
@@ -2,11 +2,13 @@ package nws

 import (
 	"encoding/json"
+	"math"
 	"strings"
 	"testing"
 	"time"

 	"gitea.maximumdirect.net/ejr/feedkit/event"
+	"gitea.maximumdirect.net/ejr/weatherfeeder/model"
 	"gitea.maximumdirect.net/ejr/weatherfeeder/standards"
 )

@@ -70,6 +72,118 @@ func TestNormalizeForecastEventBySchemaRoutesHourly(t *testing.T) {
 	}
 }

+func TestNormalizeForecastEventBySchemaRoutesNarrative(t *testing.T) {
+	_, err := normalizeForecastEventBySchema(event.Event{
+		Schema:  standards.SchemaRawNWSNarrativeForecastV1,
+		Payload: map[string]any{"properties": map[string]any{}},
+	})
+	if err == nil {
+		t.Fatalf("normalizeForecastEventBySchema() expected build error for missing generatedAt")
+	}
+	if !strings.Contains(err.Error(), "missing properties.generatedAt") {
+		t.Fatalf("error = %q, want missing properties.generatedAt", err)
+	}
+}
+
+func TestBuildNarrativeForecastMapsExpectedFields(t *testing.T) {
+	parsed := nwsNarrativeForecastResponse{}
+	parsed.Properties.GeneratedAt = "2026-03-27T15:17:01Z"
+	isDay := true
+	tempF := 53.0
+	pop := 20.0
+
+	parsed.Properties.Periods = []nwsNarrativeForecastPeriod{
+		{
+			Name:             "Today",
+			StartTime:        "2026-03-27T10:00:00-05:00",
+			EndTime:          "2026-03-27T18:00:00-05:00",
+			IsDaytime:        &isDay,
+			Temperature:      &tempF,
+			TemperatureUnit:  "F",
+			WindSpeed:        "10 to 14 mph",
+			WindDirection:    "SW",
+			ShortForecast:    "Partly Sunny",
+			DetailedForecast: "  Partly sunny, with a high near 53.  ",
+			ProbabilityOfPrecipitation: struct {
+				UnitCode string   `json:"unitCode"`
+				Value    *float64 `json:"value"`
+			}{
+				UnitCode: "wmoUnit:percent",
+				Value:    &pop,
+			},
+			Icon: "https://api.weather.gov/icons/land/day/bkn?size=medium",
+		},
+	}
+
+	run, effectiveAt, err := buildNarrativeForecast(parsed)
+	if err != nil {
+		t.Fatalf("buildNarrativeForecast() error = %v", err)
+	}
+	if got, want := run.Product, model.ForecastProductNarrative; got != want {
+		t.Fatalf("Product = %q, want %q", got, want)
+	}
+	if len(run.Periods) != 1 {
+		t.Fatalf("periods len = %d, want 1", len(run.Periods))
+	}
+
+	p := run.Periods[0]
+	if got, want := p.TextDescription, "Partly sunny, with a high near 53."; got != want {
+		t.Fatalf("TextDescription = %q, want %q", got, want)
+	}
+	if p.TemperatureC == nil {
+		t.Fatalf("TemperatureC is nil, want converted value")
+	}
+	if math.Abs(*p.TemperatureC-11.6666666667) > 0.0001 {
+		t.Fatalf("TemperatureC = %.6f, want ~11.6667", *p.TemperatureC)
+	}
+	if p.IsDay == nil || !*p.IsDay {
+		t.Fatalf("IsDay = %v, want true", p.IsDay)
+	}
+	if p.WindDirectionDegrees == nil || *p.WindDirectionDegrees != 225 {
+		t.Fatalf("WindDirectionDegrees = %v, want 225", p.WindDirectionDegrees)
+	}
+	if p.WindSpeedKmh == nil || math.Abs(*p.WindSpeedKmh-19.3128) > 0.001 {
+		t.Fatalf("WindSpeedKmh = %.6f, want ~19.3128", derefOrZero(p.WindSpeedKmh))
+	}
+	if p.ProbabilityOfPrecipitationPercent == nil || *p.ProbabilityOfPrecipitationPercent != 20 {
+		t.Fatalf("ProbabilityOfPrecipitationPercent = %v, want 20", p.ProbabilityOfPrecipitationPercent)
+	}
+
+	wantIssued := time.Date(2026, 3, 27, 15, 17, 1, 0, time.UTC)
+	if !run.IssuedAt.Equal(wantIssued) {
+		t.Fatalf("IssuedAt = %s, want %s", run.IssuedAt.Format(time.RFC3339), wantIssued.Format(time.RFC3339))
+	}
+	if !effectiveAt.Equal(wantIssued) {
+		t.Fatalf("effectiveAt = %s, want %s", effectiveAt.Format(time.RFC3339), wantIssued.Format(time.RFC3339))
+	}
+
+	assertNoLegacyForecastDescriptionKeys(t, p)
+}
+
+func TestBuildNarrativeForecastFallsBackToShortForecastDescription(t *testing.T) {
+	parsed := nwsNarrativeForecastResponse{}
+	parsed.Properties.GeneratedAt = "2026-03-27T15:17:01Z"
+	parsed.Properties.Periods = []nwsNarrativeForecastPeriod{
+		{
+			StartTime:        "2026-03-27T18:00:00-05:00",
+			EndTime:          "2026-03-28T06:00:00-05:00",
+			ShortForecast:    "  Mostly Clear  ",
+			DetailedForecast: "   ",
+		},
+	}
+
+	run, _, err := buildNarrativeForecast(parsed)
+	if err != nil {
+		t.Fatalf("buildNarrativeForecast() error = %v", err)
+	}
+	if len(run.Periods) != 1 {
+		t.Fatalf("periods len = %d, want 1", len(run.Periods))
+	}
+	if got, want := run.Periods[0].TextDescription, "Mostly Clear"; got != want {
+		t.Fatalf("TextDescription = %q, want %q", got, want)
+	}
+}
+
 func assertNoLegacyForecastDescriptionKeys(t *testing.T, period any) {
 	t.Helper()

@@ -88,3 +202,10 @@ func assertNoLegacyForecastDescriptionKeys(t *testing.T, period any) {
 		}
 	}
 }
+
+func derefOrZero(v *float64) float64 {
+	if v == nil {
+		return 0
+	}
+	return *v
+}
--- a/internal/normalizers/nws/types.go
+++ b/internal/normalizers/nws/types.go
@@ -100,8 +100,6 @@ type nwsCloudLayer struct {

 // nwsHourlyForecastResponse is a minimal-but-sufficient representation of the NWS
 // gridpoint hourly forecast GeoJSON payload needed for mapping into model.WeatherForecastRun.
-//
-// Daily and narrative variants should be added as distinct structs in follow-up work.
 type nwsHourlyForecastResponse struct {
 	Geometry struct {
 		Type        string        `json:"type"`
@@ -160,6 +158,56 @@ type nwsHourlyForecastPeriod struct {
 	DetailedForecast string `json:"detailedForecast"`
 }

+// nwsNarrativeForecastResponse is a minimal-but-sufficient representation of the NWS
+// gridpoint narrative forecast GeoJSON payload needed for mapping into model.WeatherForecastRun.
+type nwsNarrativeForecastResponse struct {
+	Geometry struct {
+		Type        string        `json:"type"`
+		Coordinates [][][]float64 `json:"coordinates"` // GeoJSON polygon: [ring][point][lon,lat]
+	} `json:"geometry"`
+
+	Properties struct {
+		Units             string `json:"units"`             // "us" or "si" (often "us" for narrative)
+		ForecastGenerator string `json:"forecastGenerator"` // e.g. "BaselineForecastGenerator"
+
+		GeneratedAt string `json:"generatedAt"` // RFC3339-ish
+		UpdateTime  string `json:"updateTime"`  // RFC3339-ish
+		ValidTimes  string `json:"validTimes"`
+
+		Elevation struct {
+			UnitCode string   `json:"unitCode"`
+			Value    *float64 `json:"value"`
+		} `json:"elevation"`
+
+		Periods []nwsNarrativeForecastPeriod `json:"periods"`
+	} `json:"properties"`
+}
+
+type nwsNarrativeForecastPeriod struct {
+	Number    int    `json:"number"`
+	Name      string `json:"name"`
+	StartTime string `json:"startTime"`
+	EndTime   string `json:"endTime"`
+
+	IsDaytime *bool `json:"isDaytime"`
+
+	Temperature      *float64 `json:"temperature"`
+	TemperatureUnit  string   `json:"temperatureUnit"` // "F" or "C"
+	TemperatureTrend any      `json:"temperatureTrend"`
+
+	ProbabilityOfPrecipitation struct {
+		UnitCode string   `json:"unitCode"`
+		Value    *float64 `json:"value"`
+	} `json:"probabilityOfPrecipitation"`
+
+	WindSpeed     string `json:"windSpeed"`     // e.g. "9 mph", "10 to 15 mph"
+	WindDirection string `json:"windDirection"` // e.g. "W", "NW"
+
+	Icon             string `json:"icon"`
+	ShortForecast    string `json:"shortForecast"`
+	DetailedForecast string `json:"detailedForecast"`
+}
+
 // nwsAlertsResponse is a minimal-but-sufficient representation of the NWS /alerts
 // FeatureCollection payload needed for mapping into model.WeatherAlertRun.
 type nwsAlertsResponse struct {
--- a/internal/sources/builtins.go
+++ b/internal/sources/builtins.go
@@ -22,6 +22,9 @@ func RegisterBuiltins(r *fksource.Registry) {
 	r.RegisterPoll("nws_forecast_hourly", func(cfg config.SourceConfig) (fksource.PollSource, error) {
 		return nws.NewHourlyForecastSource(cfg)
 	})
+	r.RegisterPoll("nws_forecast_narrative", func(cfg config.SourceConfig) (fksource.PollSource, error) {
+		return nws.NewNarrativeForecastSource(cfg)
+	})

 	// Open-Meteo drivers
 	r.RegisterPoll("openmeteo_observation", func(cfg config.SourceConfig) (fksource.PollSource, error) {
--- a/internal/sources/builtins_test.go
+++ b/internal/sources/builtins_test.go
@@ -21,6 +21,19 @@ func TestRegisterBuiltinsRegistersNWSHourlyForecastDriver(t *testing.T) {
 	}
 }

+func TestRegisterBuiltinsRegistersNWSNarrativeForecastDriver(t *testing.T) {
+	reg := fksource.NewRegistry()
+	RegisterBuiltins(reg)
+
+	in, err := reg.BuildInput(sourceConfigForDriver("nws_forecast_narrative"))
+	if err != nil {
+		t.Fatalf("BuildInput(nws_forecast_narrative) error = %v", err)
+	}
+	if _, ok := in.(fksource.PollSource); !ok {
+		t.Fatalf("BuildInput(nws_forecast_narrative) type = %T, want PollSource", in)
+	}
+}
+
 func TestRegisterBuiltinsDoesNotRegisterLegacyNWSForecastDriver(t *testing.T) {
 	reg := fksource.NewRegistry()
 	RegisterBuiltins(reg)
--- a/internal/sources/nws/forecast_narrative.go
+++ b/internal/sources/nws/forecast_narrative.go
@@ -0,0 +1,129 @@
+// FILE: internal/sources/nws/forecast_narrative.go
+package nws
+
+import (
+	"context"
+	"encoding/json"
+	"strings"
+	"time"
+
+	"gitea.maximumdirect.net/ejr/feedkit/config"
+	"gitea.maximumdirect.net/ejr/feedkit/event"
+	nwscommon "gitea.maximumdirect.net/ejr/weatherfeeder/internal/providers/nws"
+	"gitea.maximumdirect.net/ejr/weatherfeeder/internal/sources/common"
+	"gitea.maximumdirect.net/ejr/weatherfeeder/standards"
+)
+
+// NarrativeForecastSource polls an NWS narrative forecast endpoint and emits a RAW forecast Event.
+//
+// It intentionally emits the *entire* upstream payload as json.RawMessage and only decodes
+// minimal metadata for Event.EffectiveAt and Event.ID.
+//
+// Output schema:
+//   - standards.SchemaRawNWSNarrativeForecastV1
+type NarrativeForecastSource struct {
+	http *common.HTTPSource
+}
+
+func NewNarrativeForecastSource(cfg config.SourceConfig) (*NarrativeForecastSource, error) {
+	const driver = "nws_forecast_narrative"
+
+	// NWS forecast endpoints are GeoJSON (and sometimes also advertise json-ld/json).
+	hs, err := common.NewHTTPSource(driver, cfg, "application/geo+json, application/json")
+	if err != nil {
+		return nil, err
+	}
+
+	return &NarrativeForecastSource{http: hs}, nil
+}
+
+func (s *NarrativeForecastSource) Name() string { return s.http.Name }
+
+// Kind is used for routing/policy.
+func (s *NarrativeForecastSource) Kind() event.Kind { return event.Kind("forecast") }
+
+func (s *NarrativeForecastSource) Poll(ctx context.Context) ([]event.Event, error) {
+	raw, meta, err := s.fetchRaw(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	// EffectiveAt is optional; for forecasts it’s most naturally the run “issued” time.
+	// NWS gridpoint forecasts expose generatedAt (preferred) and updateTime/updated.
+	var effectiveAt *time.Time
+	switch {
+	case !meta.ParsedGeneratedAt.IsZero():
+		t := meta.ParsedGeneratedAt.UTC()
+		effectiveAt = &t
+	case !meta.ParsedUpdateTime.IsZero():
+		t := meta.ParsedUpdateTime.UTC()
+		effectiveAt = &t
+	}
+
+	emittedAt := time.Now().UTC()
+
+	// NWS gridpoint forecast GeoJSON commonly has a stable "id" equal to the endpoint URL.
+	// That is *not* unique per issued run, so we intentionally do not use it for Event.ID.
+	// Instead we rely on Source:EffectiveAt (or Source:EmittedAt fallback).
+	eventID := common.ChooseEventID("", s.http.Name, effectiveAt, emittedAt)
+
+	return common.SingleRawEvent(
+		s.Kind(),
+		s.http.Name,
+		standards.SchemaRawNWSNarrativeForecastV1,
+		eventID,
+		emittedAt,
+		effectiveAt,
+		raw,
+	)
+}
+
+// ---- RAW fetch + minimal metadata decode ----
+
+type narrativeForecastMeta struct {
+	// Present for GeoJSON Feature responses, but often stable (endpoint URL).
+	ID string `json:"id"`
+
+	Properties struct {
+		GeneratedAt string `json:"generatedAt"` // preferred “issued/run generated” time
+		UpdateTime  string `json:"updateTime"`  // last update time of underlying data
+		Updated     string `json:"updated"`     // deprecated alias for updateTime
+	} `json:"properties"`
+
+	ParsedGeneratedAt time.Time `json:"-"`
+	ParsedUpdateTime  time.Time `json:"-"`
+}
+
+func (s *NarrativeForecastSource) fetchRaw(ctx context.Context) (json.RawMessage, narrativeForecastMeta, error) {
+	raw, err := s.http.FetchJSON(ctx)
+	if err != nil {
+		return nil, narrativeForecastMeta{}, err
+	}
+
+	var meta narrativeForecastMeta
+	if err := json.Unmarshal(raw, &meta); err != nil {
+		// If metadata decode fails, still return raw; Poll will fall back to Source:EmittedAt.
+		return raw, narrativeForecastMeta{}, nil
+	}
+
+	// generatedAt (preferred)
+	genStr := strings.TrimSpace(meta.Properties.GeneratedAt)
+	if genStr != "" {
+		if t, err := nwscommon.ParseTime(genStr); err == nil {
+			meta.ParsedGeneratedAt = t.UTC()
+		}
+	}
+
+	// updateTime, with fallback to deprecated "updated"
+	updStr := strings.TrimSpace(meta.Properties.UpdateTime)
+	if updStr == "" {
+		updStr = strings.TrimSpace(meta.Properties.Updated)
+	}
+	if updStr != "" {
+		if t, err := nwscommon.ParseTime(updStr); err == nil {
+			meta.ParsedUpdateTime = t.UTC()
+		}
+	}
+
+	return raw, meta, nil
+}
--- a/standards/schema.go
+++ b/standards/schema.go
@@ -16,6 +16,7 @@ const (
 	SchemaRawOpenWeatherCurrentV1 = "raw.openweather.current.v1"

 	SchemaRawNWSHourlyForecastV1         = "raw.nws.hourly.forecast.v1"
+	SchemaRawNWSNarrativeForecastV1      = "raw.nws.narrative.forecast.v1"
 	SchemaRawOpenMeteoHourlyForecastV1   = "raw.openmeteo.hourly.forecast.v1"
 	SchemaRawOpenWeatherHourlyForecastV1 = "raw.openweather.hourly.forecast.v1"