Add SPC outlook normalization
This commit is contained in:
@@ -7,12 +7,14 @@ import (
|
|||||||
"gitea.maximumdirect.net/ejr/weatherfeeder/internal/normalizers/nws"
|
"gitea.maximumdirect.net/ejr/weatherfeeder/internal/normalizers/nws"
|
||||||
"gitea.maximumdirect.net/ejr/weatherfeeder/internal/normalizers/openmeteo"
|
"gitea.maximumdirect.net/ejr/weatherfeeder/internal/normalizers/openmeteo"
|
||||||
"gitea.maximumdirect.net/ejr/weatherfeeder/internal/normalizers/openweather"
|
"gitea.maximumdirect.net/ejr/weatherfeeder/internal/normalizers/openweather"
|
||||||
|
"gitea.maximumdirect.net/ejr/weatherfeeder/internal/normalizers/spc"
|
||||||
)
|
)
|
||||||
|
|
||||||
var builtinRegistrations = []func([]fknormalize.Normalizer) []fknormalize.Normalizer{
|
var builtinRegistrations = []func([]fknormalize.Normalizer) []fknormalize.Normalizer{
|
||||||
nws.Register,
|
nws.Register,
|
||||||
openmeteo.Register,
|
openmeteo.Register,
|
||||||
openweather.Register,
|
openweather.Register,
|
||||||
|
spc.Register,
|
||||||
}
|
}
|
||||||
|
|
||||||
// RegisterBuiltins registers all normalizers shipped with this binary.
|
// RegisterBuiltins registers all normalizers shipped with this binary.
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ import (
|
|||||||
"gitea.maximumdirect.net/ejr/weatherfeeder/internal/normalizers/nws"
|
"gitea.maximumdirect.net/ejr/weatherfeeder/internal/normalizers/nws"
|
||||||
"gitea.maximumdirect.net/ejr/weatherfeeder/internal/normalizers/openmeteo"
|
"gitea.maximumdirect.net/ejr/weatherfeeder/internal/normalizers/openmeteo"
|
||||||
"gitea.maximumdirect.net/ejr/weatherfeeder/internal/normalizers/openweather"
|
"gitea.maximumdirect.net/ejr/weatherfeeder/internal/normalizers/openweather"
|
||||||
|
"gitea.maximumdirect.net/ejr/weatherfeeder/internal/normalizers/spc"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestRegisterBuiltinsOrder(t *testing.T) {
|
func TestRegisterBuiltinsOrder(t *testing.T) {
|
||||||
@@ -25,6 +26,7 @@ func TestRegisterBuiltinsOrder(t *testing.T) {
|
|||||||
openmeteo.ObservationNormalizer{},
|
openmeteo.ObservationNormalizer{},
|
||||||
openmeteo.ForecastNormalizer{},
|
openmeteo.ForecastNormalizer{},
|
||||||
openweather.ObservationNormalizer{},
|
openweather.ObservationNormalizer{},
|
||||||
|
spc.ConvectiveOutlookNormalizer{},
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(got) != len(want) {
|
if len(got) != len(want) {
|
||||||
|
|||||||
314
internal/normalizers/spc/convective_outlook.go
Normal file
314
internal/normalizers/spc/convective_outlook.go
Normal file
@@ -0,0 +1,314 @@
|
|||||||
|
package spc
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"math"
|
||||||
|
"regexp"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/ejr/feedkit/event"
|
||||||
|
"gitea.maximumdirect.net/ejr/weatherfeeder/internal/geo"
|
||||||
|
normcommon "gitea.maximumdirect.net/ejr/weatherfeeder/internal/normalizers/common"
|
||||||
|
spcprovider "gitea.maximumdirect.net/ejr/weatherfeeder/internal/providers/spc"
|
||||||
|
"gitea.maximumdirect.net/ejr/weatherfeeder/model"
|
||||||
|
"gitea.maximumdirect.net/ejr/weatherfeeder/standards"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
providerSPC = "spc"
|
||||||
|
productConvective = "convective"
|
||||||
|
outlookNormalizer = "spc convective outlook"
|
||||||
|
outlookKind = "outlook"
|
||||||
|
outlookTypeUnknown = 99
|
||||||
|
)
|
||||||
|
|
||||||
|
var idTokenRE = regexp.MustCompile(`[^a-z0-9]+`)
|
||||||
|
|
||||||
|
// ConvectiveOutlookNormalizer converts:
|
||||||
|
//
|
||||||
|
// standards.SchemaRawSPCConvectiveOutlookV1 -> standards.SchemaWeatherOutlookV1
|
||||||
|
//
|
||||||
|
// It maps SPC GeoJSON outlook features into canonical outlook polygons and
|
||||||
|
// enriches each day with the matching required print-page discussion.
|
||||||
|
type ConvectiveOutlookNormalizer struct{}
|
||||||
|
|
||||||
|
func (ConvectiveOutlookNormalizer) Match(e event.Event) bool {
|
||||||
|
return strings.TrimSpace(e.Schema) == standards.SchemaRawSPCConvectiveOutlookV1
|
||||||
|
}
|
||||||
|
|
||||||
|
func (ConvectiveOutlookNormalizer) Normalize(ctx context.Context, in event.Event) (*event.Event, error) {
|
||||||
|
_ = ctx
|
||||||
|
|
||||||
|
fallbackAsOf := in.EmittedAt.UTC()
|
||||||
|
if in.EffectiveAt != nil && !in.EffectiveAt.IsZero() {
|
||||||
|
fallbackAsOf = in.EffectiveAt.UTC()
|
||||||
|
}
|
||||||
|
|
||||||
|
return normcommon.NormalizeJSON(
|
||||||
|
in,
|
||||||
|
outlookNormalizer,
|
||||||
|
standards.SchemaWeatherOutlookV1,
|
||||||
|
func(parsed spcprovider.RawConvectiveOutlookBundle) (model.WeatherOutlookRun, time.Time, error) {
|
||||||
|
return buildConvectiveOutlook(parsed, fallbackAsOf)
|
||||||
|
},
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
func buildConvectiveOutlook(bundle spcprovider.RawConvectiveOutlookBundle, fallbackAsOf time.Time) (model.WeatherOutlookRun, time.Time, error) {
|
||||||
|
if err := validateCoordinates(bundle.Latitude, bundle.Longitude); err != nil {
|
||||||
|
return model.WeatherOutlookRun{}, time.Time{}, err
|
||||||
|
}
|
||||||
|
|
||||||
|
discussions, latestDiscussionUpdated, err := parseDiscussions(bundle.Discussions)
|
||||||
|
if err != nil {
|
||||||
|
return model.WeatherOutlookRun{}, time.Time{}, err
|
||||||
|
}
|
||||||
|
|
||||||
|
products := orderedProducts(bundle.Products)
|
||||||
|
point := geo.Point{Latitude: bundle.Latitude, Longitude: bundle.Longitude}
|
||||||
|
outlooks := make([]model.WeatherOutlook, 0)
|
||||||
|
var latestIssue time.Time
|
||||||
|
|
||||||
|
for _, product := range products {
|
||||||
|
if err := validateProductMetadata(product); err != nil {
|
||||||
|
return model.WeatherOutlookRun{}, time.Time{}, err
|
||||||
|
}
|
||||||
|
discussion, ok := discussions[product.Day]
|
||||||
|
if !ok {
|
||||||
|
return model.WeatherOutlookRun{}, time.Time{}, fmt.Errorf("product %s: discussion for day %d is required", product.Key, product.Day)
|
||||||
|
}
|
||||||
|
|
||||||
|
collection, err := spcprovider.DecodeGeoJSON(product.Body)
|
||||||
|
if err != nil {
|
||||||
|
return model.WeatherOutlookRun{}, time.Time{}, fmt.Errorf("product %s: %w", product.Key, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
for i, feature := range collection.Features {
|
||||||
|
outlook, err := mapFeature(product, feature, i, point, discussion)
|
||||||
|
if err != nil {
|
||||||
|
return model.WeatherOutlookRun{}, time.Time{}, err
|
||||||
|
}
|
||||||
|
if latestIssue.IsZero() || outlook.IssuedAt.After(latestIssue) {
|
||||||
|
latestIssue = outlook.IssuedAt
|
||||||
|
}
|
||||||
|
outlooks = append(outlooks, outlook)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
asOf := latestIssue
|
||||||
|
if asOf.IsZero() {
|
||||||
|
asOf = latestDiscussionUpdated
|
||||||
|
}
|
||||||
|
if asOf.IsZero() {
|
||||||
|
asOf = fallbackAsOf.UTC()
|
||||||
|
}
|
||||||
|
|
||||||
|
var issuedAt *time.Time
|
||||||
|
if !latestIssue.IsZero() {
|
||||||
|
t := latestIssue.UTC()
|
||||||
|
issuedAt = &t
|
||||||
|
}
|
||||||
|
|
||||||
|
lat := bundle.Latitude
|
||||||
|
lon := bundle.Longitude
|
||||||
|
run := model.WeatherOutlookRun{
|
||||||
|
LocationID: strings.TrimSpace(bundle.LocationID),
|
||||||
|
LocationName: strings.TrimSpace(bundle.LocationName),
|
||||||
|
Latitude: &lat,
|
||||||
|
Longitude: &lon,
|
||||||
|
AsOf: asOf.UTC(),
|
||||||
|
IssuedAt: issuedAt,
|
||||||
|
Outlooks: outlooks,
|
||||||
|
}
|
||||||
|
return run, run.AsOf, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
type parsedDiscussion struct {
|
||||||
|
Headline string
|
||||||
|
Summary string
|
||||||
|
Discussion string
|
||||||
|
UpdatedAt *time.Time
|
||||||
|
}
|
||||||
|
|
||||||
|
func parseDiscussions(pages []spcprovider.RawDiscussionPage) (map[int]parsedDiscussion, time.Time, error) {
|
||||||
|
out := map[int]parsedDiscussion{}
|
||||||
|
var latestUpdated time.Time
|
||||||
|
for _, page := range pages {
|
||||||
|
text, err := spcprovider.ExtractProductText(page.Body)
|
||||||
|
if err != nil {
|
||||||
|
return nil, time.Time{}, fmt.Errorf("discussion %s: %w", page.Key, err)
|
||||||
|
}
|
||||||
|
parsed := spcprovider.ParseDiscussionText(text)
|
||||||
|
day := page.Day
|
||||||
|
if day == 0 {
|
||||||
|
if meta, ok := spcprovider.DiscussionProductByKey(page.Key); ok {
|
||||||
|
day = meta.Day
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if day < 1 || day > 3 {
|
||||||
|
return nil, time.Time{}, fmt.Errorf("discussion %s: day must be 1, 2, or 3, got %d", page.Key, page.Day)
|
||||||
|
}
|
||||||
|
disc := parsedDiscussion{
|
||||||
|
Headline: strings.TrimSpace(parsed.Headline),
|
||||||
|
Summary: strings.TrimSpace(parsed.Summary),
|
||||||
|
Discussion: strings.TrimSpace(parsed.Discussion),
|
||||||
|
UpdatedAt: parsed.UpdatedAt,
|
||||||
|
}
|
||||||
|
out[day] = disc
|
||||||
|
if parsed.UpdatedAt != nil && (latestUpdated.IsZero() || parsed.UpdatedAt.After(latestUpdated)) {
|
||||||
|
latestUpdated = parsed.UpdatedAt.UTC()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out, latestUpdated, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func orderedProducts(products []spcprovider.RawOutlookProduct) []spcprovider.RawOutlookProduct {
|
||||||
|
out := make([]spcprovider.RawOutlookProduct, len(products))
|
||||||
|
copy(out, products)
|
||||||
|
sort.SliceStable(out, func(i, j int) bool {
|
||||||
|
if out[i].Day != out[j].Day {
|
||||||
|
return out[i].Day < out[j].Day
|
||||||
|
}
|
||||||
|
left := outlookTypeOrder(out[i].OutlookType)
|
||||||
|
right := outlookTypeOrder(out[j].OutlookType)
|
||||||
|
if left != right {
|
||||||
|
return left < right
|
||||||
|
}
|
||||||
|
return out[i].Key < out[j].Key
|
||||||
|
})
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func outlookTypeOrder(outlookType string) int {
|
||||||
|
switch strings.TrimSpace(outlookType) {
|
||||||
|
case spcprovider.OutlookTypeCategorical:
|
||||||
|
return 0
|
||||||
|
case spcprovider.OutlookTypeTornado:
|
||||||
|
return 1
|
||||||
|
case spcprovider.OutlookTypeHail:
|
||||||
|
return 2
|
||||||
|
case spcprovider.OutlookTypeWind:
|
||||||
|
return 3
|
||||||
|
default:
|
||||||
|
return outlookTypeUnknown
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func validateProductMetadata(product spcprovider.RawOutlookProduct) error {
|
||||||
|
if product.Day < 1 || product.Day > 3 {
|
||||||
|
return fmt.Errorf("product %s: day must be 1, 2, or 3, got %d", product.Key, product.Day)
|
||||||
|
}
|
||||||
|
switch strings.TrimSpace(product.OutlookType) {
|
||||||
|
case spcprovider.OutlookTypeCategorical, spcprovider.OutlookTypeTornado, spcprovider.OutlookTypeHail, spcprovider.OutlookTypeWind:
|
||||||
|
return nil
|
||||||
|
default:
|
||||||
|
return fmt.Errorf("product %s: unsupported outlook type %q", product.Key, product.OutlookType)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func mapFeature(product spcprovider.RawOutlookProduct, feature spcprovider.GeoJSONFeature, index int, point geo.Point, discussion parsedDiscussion) (model.WeatherOutlook, error) {
|
||||||
|
fieldPrefix := fmt.Sprintf("product %s feature %d", product.Key, index)
|
||||||
|
props := feature.Properties
|
||||||
|
|
||||||
|
validFrom, err := parseRequiredSPCTime(props.ValidISO, fieldPrefix+".VALID_ISO")
|
||||||
|
if err != nil {
|
||||||
|
return model.WeatherOutlook{}, err
|
||||||
|
}
|
||||||
|
validTo, err := parseRequiredSPCTime(props.ExpireISO, fieldPrefix+".EXPIRE_ISO")
|
||||||
|
if err != nil {
|
||||||
|
return model.WeatherOutlook{}, err
|
||||||
|
}
|
||||||
|
issuedAt, err := parseRequiredSPCTime(props.IssueISO, fieldPrefix+".ISSUE_ISO")
|
||||||
|
if err != nil {
|
||||||
|
return model.WeatherOutlook{}, err
|
||||||
|
}
|
||||||
|
label := strings.TrimSpace(props.Label)
|
||||||
|
if label == "" {
|
||||||
|
return model.WeatherOutlook{}, fmt.Errorf("%s.LABEL is required", fieldPrefix)
|
||||||
|
}
|
||||||
|
if len(feature.Geometry) == 0 {
|
||||||
|
return model.WeatherOutlook{}, fmt.Errorf("%s.geometry is required", fieldPrefix)
|
||||||
|
}
|
||||||
|
containsLocation, err := geo.ContainsPoint(feature.Geometry, point)
|
||||||
|
if err != nil {
|
||||||
|
return model.WeatherOutlook{}, fmt.Errorf("%s.geometry: %w", fieldPrefix, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
geometry := make(json.RawMessage, len(feature.Geometry))
|
||||||
|
copy(geometry, feature.Geometry)
|
||||||
|
|
||||||
|
return model.WeatherOutlook{
|
||||||
|
ID: outlookID(product.Day, product.OutlookType, label, issuedAt, validFrom, index),
|
||||||
|
Provider: providerSPC,
|
||||||
|
Product: productConvective,
|
||||||
|
Day: product.Day,
|
||||||
|
OutlookType: strings.TrimSpace(product.OutlookType),
|
||||||
|
Label: label,
|
||||||
|
LabelText: strings.TrimSpace(props.Label2),
|
||||||
|
SeverityRank: props.DN,
|
||||||
|
ValidFrom: validFrom,
|
||||||
|
ValidTo: validTo,
|
||||||
|
IssuedAt: issuedAt,
|
||||||
|
ExpiresAt: validTo,
|
||||||
|
Forecaster: strings.TrimSpace(props.Forecaster),
|
||||||
|
Headline: discussion.Headline,
|
||||||
|
Summary: discussion.Summary,
|
||||||
|
Discussion: discussion.Discussion,
|
||||||
|
SourceURL: strings.TrimSpace(product.URL),
|
||||||
|
ImageURL: "",
|
||||||
|
ContainsLocation: containsLocation,
|
||||||
|
Geometry: geometry,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func parseRequiredSPCTime(value, field string) (time.Time, error) {
|
||||||
|
if strings.TrimSpace(value) == "" {
|
||||||
|
return time.Time{}, fmt.Errorf("%s is required", field)
|
||||||
|
}
|
||||||
|
t, err := spcprovider.ParseISOTimestamp(value)
|
||||||
|
if err != nil {
|
||||||
|
return time.Time{}, fmt.Errorf("%s: %w", field, err)
|
||||||
|
}
|
||||||
|
return t.UTC(), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func outlookID(day int, outlookType, label string, issuedAt time.Time, validFrom time.Time, index int) string {
|
||||||
|
return fmt.Sprintf(
|
||||||
|
"spc-convective-day%d-%s-%s-%s-%s-%d",
|
||||||
|
day,
|
||||||
|
safeIDToken(outlookType),
|
||||||
|
safeIDToken(label),
|
||||||
|
issuedAt.UTC().Format(time.RFC3339),
|
||||||
|
validFrom.UTC().Format(time.RFC3339),
|
||||||
|
index,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
func safeIDToken(value string) string {
|
||||||
|
value = strings.ToLower(strings.TrimSpace(value))
|
||||||
|
value = idTokenRE.ReplaceAllString(value, "-")
|
||||||
|
value = strings.Trim(value, "-")
|
||||||
|
if value == "" {
|
||||||
|
return "unknown"
|
||||||
|
}
|
||||||
|
return value
|
||||||
|
}
|
||||||
|
|
||||||
|
func validateCoordinates(latitude, longitude float64) error {
|
||||||
|
switch {
|
||||||
|
case math.IsNaN(latitude) || math.IsInf(latitude, 0):
|
||||||
|
return fmt.Errorf("latitude must be finite")
|
||||||
|
case math.IsNaN(longitude) || math.IsInf(longitude, 0):
|
||||||
|
return fmt.Errorf("longitude must be finite")
|
||||||
|
case latitude < -90 || latitude > 90:
|
||||||
|
return fmt.Errorf("latitude must be between -90 and 90, got %v", latitude)
|
||||||
|
case longitude < -180 || longitude > 180:
|
||||||
|
return fmt.Errorf("longitude must be between -180 and 180, got %v", longitude)
|
||||||
|
default:
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
}
|
||||||
354
internal/normalizers/spc/convective_outlook_test.go
Normal file
354
internal/normalizers/spc/convective_outlook_test.go
Normal file
@@ -0,0 +1,354 @@
|
|||||||
|
package spc
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/ejr/feedkit/event"
|
||||||
|
spcprovider "gitea.maximumdirect.net/ejr/weatherfeeder/internal/providers/spc"
|
||||||
|
"gitea.maximumdirect.net/ejr/weatherfeeder/model"
|
||||||
|
"gitea.maximumdirect.net/ejr/weatherfeeder/standards"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestConvectiveOutlookNormalizerMatch(t *testing.T) {
|
||||||
|
n := ConvectiveOutlookNormalizer{}
|
||||||
|
if !n.Match(event.Event{Schema: standards.SchemaRawSPCConvectiveOutlookV1}) {
|
||||||
|
t.Fatalf("Match(raw SPC outlook) = false, want true")
|
||||||
|
}
|
||||||
|
if n.Match(event.Event{Schema: standards.SchemaRawNWSAlertsV1}) {
|
||||||
|
t.Fatalf("Match(raw NWS alerts) = true, want false")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestConvectiveOutlookNormalizerProducesCanonicalSchemaAndMapsSample(t *testing.T) {
|
||||||
|
out, err := (ConvectiveOutlookNormalizer{}).Normalize(nil, spcRawEvent(t, spcBundle(t, 38.5, -90.5)))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Normalize() error = %v", err)
|
||||||
|
}
|
||||||
|
if out.Schema != standards.SchemaWeatherOutlookV1 {
|
||||||
|
t.Fatalf("Schema = %q, want %q", out.Schema, standards.SchemaWeatherOutlookV1)
|
||||||
|
}
|
||||||
|
if out.Kind != event.Kind("outlook") {
|
||||||
|
t.Fatalf("Kind = %q, want outlook", out.Kind)
|
||||||
|
}
|
||||||
|
|
||||||
|
run, ok := out.Payload.(model.WeatherOutlookRun)
|
||||||
|
if !ok {
|
||||||
|
t.Fatalf("Payload type = %T, want model.WeatherOutlookRun", out.Payload)
|
||||||
|
}
|
||||||
|
wantAsOf := time.Date(2026, 6, 11, 19, 45, 0, 0, time.UTC)
|
||||||
|
if !run.AsOf.Equal(wantAsOf) {
|
||||||
|
t.Fatalf("AsOf = %s, want %s", run.AsOf, wantAsOf)
|
||||||
|
}
|
||||||
|
if run.IssuedAt == nil || !run.IssuedAt.Equal(wantAsOf) {
|
||||||
|
t.Fatalf("IssuedAt = %v, want %s", run.IssuedAt, wantAsOf)
|
||||||
|
}
|
||||||
|
if out.EffectiveAt == nil || !out.EffectiveAt.Equal(wantAsOf) {
|
||||||
|
t.Fatalf("EffectiveAt = %v, want %s", out.EffectiveAt, wantAsOf)
|
||||||
|
}
|
||||||
|
if run.LocationID != "stl" || run.LocationName != "St. Louis, MO" {
|
||||||
|
t.Fatalf("location metadata = %q/%q", run.LocationID, run.LocationName)
|
||||||
|
}
|
||||||
|
if run.Latitude == nil || *run.Latitude != 38.5 || run.Longitude == nil || *run.Longitude != -90.5 {
|
||||||
|
t.Fatalf("coordinates = %v,%v", run.Latitude, run.Longitude)
|
||||||
|
}
|
||||||
|
if len(run.Outlooks) != 12 {
|
||||||
|
t.Fatalf("Outlooks length = %d, want 12", len(run.Outlooks))
|
||||||
|
}
|
||||||
|
|
||||||
|
got := run.Outlooks[0]
|
||||||
|
if got.Provider != "spc" || got.Product != "convective" {
|
||||||
|
t.Fatalf("provider/product = %q/%q", got.Provider, got.Product)
|
||||||
|
}
|
||||||
|
if got.Day != 1 || got.OutlookType != spcprovider.OutlookTypeCategorical {
|
||||||
|
t.Fatalf("day/type = %d/%q", got.Day, got.OutlookType)
|
||||||
|
}
|
||||||
|
if got.Label != "SLGT" || got.LabelText != "Slight Risk" {
|
||||||
|
t.Fatalf("label fields = %q/%q", got.Label, got.LabelText)
|
||||||
|
}
|
||||||
|
if got.SeverityRank == nil || *got.SeverityRank != 3 {
|
||||||
|
t.Fatalf("SeverityRank = %v, want 3", got.SeverityRank)
|
||||||
|
}
|
||||||
|
assertTime(t, "ValidFrom", got.ValidFrom, 2026, 6, 11, 13, 0, 0)
|
||||||
|
assertTime(t, "ValidTo", got.ValidTo, 2026, 6, 12, 12, 0, 0)
|
||||||
|
assertTime(t, "IssuedAt", got.IssuedAt, 2026, 6, 11, 12, 34, 56)
|
||||||
|
assertTime(t, "ExpiresAt", got.ExpiresAt, 2026, 6, 12, 12, 0, 0)
|
||||||
|
if got.Forecaster != "SMITH" {
|
||||||
|
t.Fatalf("Forecaster = %q, want SMITH", got.Forecaster)
|
||||||
|
}
|
||||||
|
if got.SourceURL != "https://example.invalid/day1_categorical.geojson" {
|
||||||
|
t.Fatalf("SourceURL = %q", got.SourceURL)
|
||||||
|
}
|
||||||
|
wantGeometry := `{"type":"Polygon","coordinates":[[[-91.0,38.0],[-90.0,38.0],[-90.0,39.0],[-91.0,39.0],[-91.0,38.0]]]}`
|
||||||
|
if string(got.Geometry) != wantGeometry {
|
||||||
|
t.Fatalf("Geometry = %s, want %s", got.Geometry, wantGeometry)
|
||||||
|
}
|
||||||
|
if !got.ContainsLocation {
|
||||||
|
t.Fatalf("ContainsLocation = false, want true")
|
||||||
|
}
|
||||||
|
if got.Headline != "Day 1 Convective Outlook" {
|
||||||
|
t.Fatalf("Headline = %q", got.Headline)
|
||||||
|
}
|
||||||
|
if !strings.Contains(got.Summary, "central Plains") {
|
||||||
|
t.Fatalf("Summary = %q", got.Summary)
|
||||||
|
}
|
||||||
|
if !strings.Contains(got.Discussion, "...DISCUSSION...") {
|
||||||
|
t.Fatalf("Discussion missing product text: %q", got.Discussion)
|
||||||
|
}
|
||||||
|
if got.ID != "spc-convective-day1-categorical-slgt-2026-06-11T12:34:56Z-2026-06-11T13:00:00Z-0" {
|
||||||
|
t.Fatalf("ID = %q", got.ID)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestConvectiveOutlookNormalizerOrdersProductsByDayAndType(t *testing.T) {
|
||||||
|
bundle := spcBundle(t, 0, 0)
|
||||||
|
for i, j := 0, len(bundle.Products)-1; i < j; i, j = i+1, j-1 {
|
||||||
|
bundle.Products[i], bundle.Products[j] = bundle.Products[j], bundle.Products[i]
|
||||||
|
}
|
||||||
|
|
||||||
|
out, err := (ConvectiveOutlookNormalizer{}).Normalize(nil, spcRawEvent(t, bundle))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Normalize() error = %v", err)
|
||||||
|
}
|
||||||
|
run := out.Payload.(model.WeatherOutlookRun)
|
||||||
|
got := []string{
|
||||||
|
run.Outlooks[0].OutlookType,
|
||||||
|
run.Outlooks[1].OutlookType,
|
||||||
|
run.Outlooks[2].OutlookType,
|
||||||
|
run.Outlooks[3].OutlookType,
|
||||||
|
}
|
||||||
|
want := []string{
|
||||||
|
spcprovider.OutlookTypeCategorical,
|
||||||
|
spcprovider.OutlookTypeTornado,
|
||||||
|
spcprovider.OutlookTypeHail,
|
||||||
|
spcprovider.OutlookTypeWind,
|
||||||
|
}
|
||||||
|
for i := range want {
|
||||||
|
if got[i] != want[i] || run.Outlooks[i].Day != 1 {
|
||||||
|
t.Fatalf("outlook[%d] = day %d type %q, want day 1 type %q", i, run.Outlooks[i].Day, got[i], want[i])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestConvectiveOutlookNormalizerMapsProbabilisticOutlookTypes(t *testing.T) {
|
||||||
|
out, err := (ConvectiveOutlookNormalizer{}).Normalize(nil, spcRawEvent(t, spcBundle(t, 0, 0)))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Normalize() error = %v", err)
|
||||||
|
}
|
||||||
|
run := out.Payload.(model.WeatherOutlookRun)
|
||||||
|
for _, outlookType := range []string{
|
||||||
|
spcprovider.OutlookTypeTornado,
|
||||||
|
spcprovider.OutlookTypeHail,
|
||||||
|
spcprovider.OutlookTypeWind,
|
||||||
|
} {
|
||||||
|
if findOutlook(run.Outlooks, 1, outlookType) == nil {
|
||||||
|
t.Fatalf("missing day 1 outlook type %q", outlookType)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestConvectiveOutlookNormalizerContainsLocationFalseOutsidePolygon(t *testing.T) {
|
||||||
|
out, err := (ConvectiveOutlookNormalizer{}).Normalize(nil, spcRawEvent(t, spcBundle(t, 0, 0)))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Normalize() error = %v", err)
|
||||||
|
}
|
||||||
|
run := out.Payload.(model.WeatherOutlookRun)
|
||||||
|
if run.Outlooks[0].ContainsLocation {
|
||||||
|
t.Fatalf("ContainsLocation = true, want false")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestConvectiveOutlookNormalizerPreservesCorrectionMarker(t *testing.T) {
|
||||||
|
out, err := (ConvectiveOutlookNormalizer{}).Normalize(nil, spcRawEvent(t, spcBundle(t, 0, 0)))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Normalize() error = %v", err)
|
||||||
|
}
|
||||||
|
run := out.Payload.(model.WeatherOutlookRun)
|
||||||
|
got := findOutlook(run.Outlooks, 2, spcprovider.OutlookTypeTornado)
|
||||||
|
if got == nil {
|
||||||
|
t.Fatalf("missing day 2 tornado outlook")
|
||||||
|
}
|
||||||
|
if !strings.Contains(got.Headline, "CORR 1") {
|
||||||
|
t.Fatalf("Headline = %q, want correction marker", got.Headline)
|
||||||
|
}
|
||||||
|
if !strings.Contains(got.Discussion, "CORR 1") {
|
||||||
|
t.Fatalf("Discussion = %q, want correction marker", got.Discussion)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestConvectiveOutlookNormalizerMissingRSSNormalizes(t *testing.T) {
|
||||||
|
bundle := spcBundle(t, 0, 0)
|
||||||
|
bundle.RSS = nil
|
||||||
|
if _, err := (ConvectiveOutlookNormalizer{}).Normalize(nil, spcRawEvent(t, bundle)); err != nil {
|
||||||
|
t.Fatalf("Normalize() error = %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestConvectiveOutlookNormalizerInvalidRequiredTimestampFailsWithContext(t *testing.T) {
|
||||||
|
bundle := spcBundle(t, 0, 0)
|
||||||
|
bundle.Products[0].Body = json.RawMessage(strings.Replace(
|
||||||
|
string(bundle.Products[0].Body),
|
||||||
|
`"ISSUE_ISO": "2026-06-11T12:34:56Z"`,
|
||||||
|
`"ISSUE_ISO": "bad"`,
|
||||||
|
1,
|
||||||
|
))
|
||||||
|
|
||||||
|
_, err := (ConvectiveOutlookNormalizer{}).Normalize(nil, spcRawEvent(t, bundle))
|
||||||
|
if err == nil {
|
||||||
|
t.Fatalf("Normalize() error = nil, want error")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "product day1_categorical feature 0.ISSUE_ISO") {
|
||||||
|
t.Fatalf("error = %q, want product and feature context", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestConvectiveOutlookNormalizerInvalidGeometryFailsWithContext(t *testing.T) {
|
||||||
|
bundle := spcBundle(t, 0, 0)
|
||||||
|
bundle.Products[0].Body = json.RawMessage(strings.Replace(
|
||||||
|
string(bundle.Products[0].Body),
|
||||||
|
`"geometry": {`,
|
||||||
|
`"geometry": {"type":"LineString","coordinates":[[-91,38],[-90,39]]}, "oldGeometry": {`,
|
||||||
|
1,
|
||||||
|
))
|
||||||
|
|
||||||
|
_, err := (ConvectiveOutlookNormalizer{}).Normalize(nil, spcRawEvent(t, bundle))
|
||||||
|
if err == nil {
|
||||||
|
t.Fatalf("Normalize() error = nil, want error")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "product day1_categorical feature 0.geometry") {
|
||||||
|
t.Fatalf("error = %q, want product and feature context", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestConvectiveOutlookNormalizerRejectsMissingLabel(t *testing.T) {
|
||||||
|
bundle := spcBundle(t, 0, 0)
|
||||||
|
bundle.Products[0].Body = json.RawMessage(strings.Replace(
|
||||||
|
string(bundle.Products[0].Body),
|
||||||
|
`"LABEL": "SLGT"`,
|
||||||
|
`"LABEL": ""`,
|
||||||
|
1,
|
||||||
|
))
|
||||||
|
|
||||||
|
_, err := (ConvectiveOutlookNormalizer{}).Normalize(nil, spcRawEvent(t, bundle))
|
||||||
|
if err == nil {
|
||||||
|
t.Fatalf("Normalize() error = nil, want error")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "product day1_categorical feature 0.LABEL") {
|
||||||
|
t.Fatalf("error = %q, want label context", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestConvectiveOutlookNormalizerOutputJSONShape(t *testing.T) {
|
||||||
|
out, err := (ConvectiveOutlookNormalizer{}).Normalize(nil, spcRawEvent(t, spcBundle(t, 38.5, -90.5)))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Normalize() error = %v", err)
|
||||||
|
}
|
||||||
|
raw, err := json.Marshal(out.Payload)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Marshal(payload) error = %v", err)
|
||||||
|
}
|
||||||
|
got := string(raw)
|
||||||
|
for _, want := range []string{`"asOf"`, `"outlooks"`, `"containsLocation"`, `"geometry"`} {
|
||||||
|
if !strings.Contains(got, want) {
|
||||||
|
t.Fatalf("payload JSON missing %s: %s", want, got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, unwanted := range []string{`"products"`, `"discussions"`, `"fetchedAt"`, `"body"`} {
|
||||||
|
if strings.Contains(got, unwanted) {
|
||||||
|
t.Fatalf("payload JSON exposed raw key %s: %s", unwanted, got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func spcRawEvent(t *testing.T, bundle spcprovider.RawConvectiveOutlookBundle) event.Event {
|
||||||
|
t.Helper()
|
||||||
|
raw, err := json.Marshal(bundle)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Marshal(bundle) error = %v", err)
|
||||||
|
}
|
||||||
|
effectiveAt := time.Date(2026, 6, 11, 19, 45, 0, 0, time.UTC)
|
||||||
|
return event.Event{
|
||||||
|
ID: "evt-spc-outlook-1",
|
||||||
|
Kind: event.Kind("outlook"),
|
||||||
|
Source: "spc-test",
|
||||||
|
EmittedAt: time.Date(2026, 6, 11, 20, 5, 0, 0, time.UTC),
|
||||||
|
EffectiveAt: &effectiveAt,
|
||||||
|
Schema: standards.SchemaRawSPCConvectiveOutlookV1,
|
||||||
|
Payload: json.RawMessage(raw),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func spcBundle(t *testing.T, latitude, longitude float64) spcprovider.RawConvectiveOutlookBundle {
|
||||||
|
t.Helper()
|
||||||
|
fetchedAt := time.Date(2026, 6, 11, 20, 0, 0, 0, time.UTC)
|
||||||
|
products := make([]spcprovider.RawOutlookProduct, 0, len(spcprovider.GeoJSONProducts()))
|
||||||
|
for _, product := range spcprovider.GeoJSONProducts() {
|
||||||
|
products = append(products, spcprovider.RawOutlookProduct{
|
||||||
|
Key: product.Key,
|
||||||
|
Day: product.Day,
|
||||||
|
OutlookType: product.OutlookType,
|
||||||
|
URL: "https://example.invalid/" + product.Key + ".geojson",
|
||||||
|
FetchedAt: fetchedAt,
|
||||||
|
Body: json.RawMessage(geoJSONFixtureForProduct(t, product.Key)),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
return spcprovider.RawConvectiveOutlookBundle{
|
||||||
|
LocationID: "stl",
|
||||||
|
LocationName: "St. Louis, MO",
|
||||||
|
Latitude: latitude,
|
||||||
|
Longitude: longitude,
|
||||||
|
FetchedAt: fetchedAt,
|
||||||
|
Products: products,
|
||||||
|
Discussions: []spcprovider.RawDiscussionPage{
|
||||||
|
{Key: "day1", Day: 1, URL: "https://example.invalid/day1.html", FetchedAt: fetchedAt, Body: string(readSPCTestFixture(t, "day1_prt.html"))},
|
||||||
|
{Key: "day2", Day: 2, URL: "https://example.invalid/day2.html", FetchedAt: fetchedAt, Body: string(readSPCTestFixture(t, "day2_prt_corr.html"))},
|
||||||
|
{Key: "day3", Day: 3, URL: "https://example.invalid/day3.html", FetchedAt: fetchedAt, Body: string(readSPCTestFixture(t, "day3_prt.html"))},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func geoJSONFixtureForProduct(t *testing.T, key string) []byte {
|
||||||
|
t.Helper()
|
||||||
|
switch {
|
||||||
|
case strings.HasPrefix(key, "day1_"):
|
||||||
|
return readSPCTestFixture(t, "day1_cat.geojson")
|
||||||
|
case strings.HasPrefix(key, "day2_"):
|
||||||
|
return readSPCTestFixture(t, "day2_torn.geojson")
|
||||||
|
case strings.HasPrefix(key, "day3_"):
|
||||||
|
return readSPCTestFixture(t, "day3_wind.geojson")
|
||||||
|
default:
|
||||||
|
t.Fatalf("unknown product key %q", key)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func readSPCTestFixture(t *testing.T, name string) []byte {
|
||||||
|
t.Helper()
|
||||||
|
path := filepath.Join("..", "..", "providers", "spc", "testdata", name)
|
||||||
|
raw, err := os.ReadFile(path)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("read fixture %s: %v", path, err)
|
||||||
|
}
|
||||||
|
return raw
|
||||||
|
}
|
||||||
|
|
||||||
|
func findOutlook(outlooks []model.WeatherOutlook, day int, outlookType string) *model.WeatherOutlook {
|
||||||
|
for i := range outlooks {
|
||||||
|
if outlooks[i].Day == day && outlooks[i].OutlookType == outlookType {
|
||||||
|
return &outlooks[i]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func assertTime(t *testing.T, name string, got time.Time, year int, month time.Month, day int, hour int, minute int, second int) {
|
||||||
|
t.Helper()
|
||||||
|
want := time.Date(year, month, day, hour, minute, second, 0, time.UTC)
|
||||||
|
if !got.Equal(want) {
|
||||||
|
t.Fatalf("%s = %s, want %s", name, got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
14
internal/normalizers/spc/register.go
Normal file
14
internal/normalizers/spc/register.go
Normal file
@@ -0,0 +1,14 @@
|
|||||||
|
package spc
|
||||||
|
|
||||||
|
import (
|
||||||
|
fknormalize "gitea.maximumdirect.net/ejr/feedkit/processors/normalize"
|
||||||
|
)
|
||||||
|
|
||||||
|
var builtins = []fknormalize.Normalizer{
|
||||||
|
ConvectiveOutlookNormalizer{},
|
||||||
|
}
|
||||||
|
|
||||||
|
// Register appends SPC normalizers in stable order.
|
||||||
|
func Register(in []fknormalize.Normalizer) []fknormalize.Normalizer {
|
||||||
|
return append(in, builtins...)
|
||||||
|
}
|
||||||
42
model/outlook.go
Normal file
42
model/outlook.go
Normal file
@@ -0,0 +1,42 @@
|
|||||||
|
package model
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// WeatherOutlookRun is a snapshot of convective outlook polygons for a
|
||||||
|
// configured location as-of a provider issue time.
|
||||||
|
type WeatherOutlookRun struct {
|
||||||
|
LocationID string `json:"locationId,omitempty"`
|
||||||
|
LocationName string `json:"locationName,omitempty"`
|
||||||
|
Latitude *float64 `json:"latitude,omitempty"`
|
||||||
|
Longitude *float64 `json:"longitude,omitempty"`
|
||||||
|
AsOf time.Time `json:"asOf"`
|
||||||
|
IssuedAt *time.Time `json:"issuedAt,omitempty"`
|
||||||
|
Outlooks []WeatherOutlook `json:"outlooks"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// WeatherOutlook is a canonical representation of one outlook polygon.
|
||||||
|
type WeatherOutlook struct {
|
||||||
|
ID string `json:"id"`
|
||||||
|
Provider string `json:"provider"`
|
||||||
|
Product string `json:"product"`
|
||||||
|
Day int `json:"day"`
|
||||||
|
OutlookType string `json:"outlookType"`
|
||||||
|
Label string `json:"label"`
|
||||||
|
LabelText string `json:"labelText,omitempty"`
|
||||||
|
SeverityRank *int `json:"severityRank,omitempty"`
|
||||||
|
ValidFrom time.Time `json:"validFrom"`
|
||||||
|
ValidTo time.Time `json:"validTo"`
|
||||||
|
IssuedAt time.Time `json:"issuedAt"`
|
||||||
|
ExpiresAt time.Time `json:"expiresAt"`
|
||||||
|
Forecaster string `json:"forecaster,omitempty"`
|
||||||
|
Headline string `json:"headline,omitempty"`
|
||||||
|
Summary string `json:"summary,omitempty"`
|
||||||
|
Discussion string `json:"discussion,omitempty"`
|
||||||
|
SourceURL string `json:"sourceUrl,omitempty"`
|
||||||
|
ImageURL string `json:"imageUrl,omitempty"`
|
||||||
|
ContainsLocation bool `json:"containsLocation"`
|
||||||
|
Geometry json.RawMessage `json:"geometry"`
|
||||||
|
}
|
||||||
@@ -32,4 +32,5 @@ const (
|
|||||||
SchemaWeatherForecastDiscussionV1 = "weather.forecast_discussion.v1"
|
SchemaWeatherForecastDiscussionV1 = "weather.forecast_discussion.v1"
|
||||||
SchemaWeatherStoryV1 = "weather.weather_story.v1"
|
SchemaWeatherStoryV1 = "weather.weather_story.v1"
|
||||||
SchemaWeatherAlertV1 = "weather.alert.v1"
|
SchemaWeatherAlertV1 = "weather.alert.v1"
|
||||||
|
SchemaWeatherOutlookV1 = "weather.outlook.v1"
|
||||||
)
|
)
|
||||||
|
|||||||
Reference in New Issue
Block a user