diff --git a/internal/providers/spc/discussion.go b/internal/providers/spc/discussion.go new file mode 100644 index 0000000..ae79230 --- /dev/null +++ b/internal/providers/spc/discussion.go @@ -0,0 +1,158 @@ +package spc + +import ( + "fmt" + "html" + "regexp" + "strings" + "time" +) + +var ( + scriptBlockRE = regexp.MustCompile(`(?is)]*>.*?`) + preBlockRE = regexp.MustCompile(`(?is)]*>(.*?)`) + tagRE = regexp.MustCompile(`(?is)<[^>]+>`) + updatedRE = regexp.MustCompile(`(?im)^\s*Updated:\s*(.+?)\s*$`) + sectionRE = regexp.MustCompile(`^\s*\.\.\.[A-Z0-9 /-]+\.{3}\s*$`) +) + +// DiscussionText contains parsed text from an SPC print page. +type DiscussionText struct { + ProductTitle string + Headline string + Summary string + Discussion string + UpdatedAt *time.Time +} + +// ExtractProductText extracts and cleans the first useful preformatted SPC +// product text block from a print page. +func ExtractProductText(rawHTML string) (string, error) { + matches := preBlockRE.FindAllStringSubmatch(rawHTML, -1) + for _, match := range matches { + if len(match) < 2 { + continue + } + text := cleanHTMLText(match[1]) + if strings.TrimSpace(text) != "" { + return text, nil + } + } + return "", fmt.Errorf("no useful pre block found") +} + +// ParseDiscussionText extracts common SPC narrative metadata from cleaned +// product text. +func ParseDiscussionText(text string) DiscussionText { + text = trimBlankLines(normalizeNewlines(text)) + title := ParseProductTitle(text) + return DiscussionText{ + ProductTitle: title, + Headline: title, + Summary: ExtractSummary(text), + Discussion: text, + UpdatedAt: ParseUpdatedTimestamp(text), + } +} + +// ParseUpdatedTimestamp parses an SPC print-page Updated line when present. +func ParseUpdatedTimestamp(text string) *time.Time { + match := updatedRE.FindStringSubmatch(normalizeNewlines(text)) + if len(match) != 2 { + return nil + } + return parseUpdatedValue(match[1]) +} + +// ParseProductTitle returns the first non-empty product line from cleaned text. +func ParseProductTitle(text string) string { + for _, line := range strings.Split(normalizeNewlines(text), "\n") { + line = strings.TrimSpace(line) + if line == "" || strings.HasPrefix(line, "Updated:") { + continue + } + return line + } + return "" +} + +// ParseHeadline returns the human-facing headline from cleaned text. +func ParseHeadline(text string) string { + return ParseProductTitle(text) +} + +// ExtractSummary returns text under the ...SUMMARY... section through the next +// SPC section heading. +func ExtractSummary(text string) string { + lines := strings.Split(normalizeNewlines(text), "\n") + start := -1 + for i, line := range lines { + if strings.EqualFold(strings.TrimSpace(line), "...SUMMARY...") { + start = i + 1 + break + } + } + if start < 0 { + return "" + } + + var out []string + for _, line := range lines[start:] { + if sectionRE.MatchString(line) { + break + } + out = append(out, line) + } + return trimBlankLines(strings.Join(out, "\n")) +} + +func cleanHTMLText(raw string) string { + raw = scriptBlockRE.ReplaceAllString(raw, "") + raw = tagRE.ReplaceAllString(raw, "") + raw = html.UnescapeString(raw) + raw = normalizeNewlines(raw) + return trimBlankLines(raw) +} + +func normalizeNewlines(text string) string { + text = strings.ReplaceAll(text, "\r\n", "\n") + text = strings.ReplaceAll(text, "\r", "\n") + return text +} + +func trimBlankLines(text string) string { + lines := strings.Split(normalizeNewlines(text), "\n") + start := 0 + for start < len(lines) && strings.TrimSpace(lines[start]) == "" { + start++ + } + end := len(lines) + for end > start && strings.TrimSpace(lines[end-1]) == "" { + end-- + } + return strings.Join(lines[start:end], "\n") +} + +func parseUpdatedValue(value string) *time.Time { + value = strings.TrimSpace(value) + if value == "" { + return nil + } + if t := parseOptionalISOTimestamp(value); t != nil { + return t + } + for _, layout := range []string{ + "1504 UTC Mon Jan 2 2006", + "1504Z Mon Jan 2 2006", + "3:04 PM UTC Mon Jan 2 2006", + time.RFC1123, + time.RFC1123Z, + } { + t, err := time.Parse(layout, value) + if err == nil { + tt := t.UTC() + return &tt + } + } + return nil +} diff --git a/internal/providers/spc/discussion_test.go b/internal/providers/spc/discussion_test.go new file mode 100644 index 0000000..1640da8 --- /dev/null +++ b/internal/providers/spc/discussion_test.go @@ -0,0 +1,90 @@ +package spc + +import ( + "strings" + "testing" + "time" +) + +func TestExtractProductTextCleansPreBlock(t *testing.T) { + raw := string(readTestFile(t, "day1_prt.html")) + + got, err := ExtractProductText(raw) + if err != nil { + t.Fatalf("ExtractProductText() error = %v", err) + } + if strings.Contains(got, " 3 { + return fmt.Errorf("day must be 1, 2, or 3, got %d", day) + } + return nil +} diff --git a/internal/providers/spc/product_test.go b/internal/providers/spc/product_test.go new file mode 100644 index 0000000..a4f3a04 --- /dev/null +++ b/internal/providers/spc/product_test.go @@ -0,0 +1,56 @@ +package spc + +import "testing" + +func TestGeoJSONProductsStableOrder(t *testing.T) { + got := GeoJSONProducts() + if len(got) != 12 { + t.Fatalf("GeoJSONProducts() length = %d, want 12", len(got)) + } + + wantKeys := []string{ + "day1_categorical", + "day1_tornado", + "day1_hail", + "day1_wind", + "day2_categorical", + "day2_tornado", + "day2_hail", + "day2_wind", + "day3_categorical", + "day3_tornado", + "day3_hail", + "day3_wind", + } + for i, want := range wantKeys { + if got[i].Key != want { + t.Fatalf("GeoJSONProducts()[%d].Key = %q, want %q", i, got[i].Key, want) + } + if err := validateProductDay(got[i].Day); err != nil { + t.Fatalf("GeoJSONProducts()[%d].Day invalid: %v", i, err) + } + if got[i].URL == "" { + t.Fatalf("GeoJSONProducts()[%d].URL is empty", i) + } + } +} + +func TestDiscussionProductsStableOrder(t *testing.T) { + got := DiscussionProducts() + if len(got) != 3 { + t.Fatalf("DiscussionProducts() length = %d, want 3", len(got)) + } + + wantKeys := []string{"day1", "day2", "day3"} + for i, want := range wantKeys { + if got[i].Key != want { + t.Fatalf("DiscussionProducts()[%d].Key = %q, want %q", i, got[i].Key, want) + } + if got[i].Day != i+1 { + t.Fatalf("DiscussionProducts()[%d].Day = %d, want %d", i, got[i].Day, i+1) + } + if got[i].URL == "" { + t.Fatalf("DiscussionProducts()[%d].URL is empty", i) + } + } +} diff --git a/internal/providers/spc/raw.go b/internal/providers/spc/raw.go new file mode 100644 index 0000000..fa27bb1 --- /dev/null +++ b/internal/providers/spc/raw.go @@ -0,0 +1,45 @@ +package spc + +import ( + "encoding/json" + "time" +) + +// RawConvectiveOutlookBundle is the provider payload shape for SPC convective +// outlook fetch bundles. +type RawConvectiveOutlookBundle struct { + LocationID string `json:"locationId,omitempty"` + LocationName string `json:"locationName,omitempty"` + Latitude float64 `json:"latitude"` + Longitude float64 `json:"longitude"` + FetchedAt time.Time `json:"fetchedAt"` + Products []RawOutlookProduct `json:"products"` + Discussions []RawDiscussionPage `json:"discussions"` + RSS *RawRSSFeed `json:"rss,omitempty"` +} + +// RawOutlookProduct contains one fetched SPC GeoJSON product. +type RawOutlookProduct struct { + Key string `json:"key"` + Day int `json:"day"` + OutlookType string `json:"outlookType"` + URL string `json:"url"` + FetchedAt time.Time `json:"fetchedAt"` + Body json.RawMessage `json:"body"` +} + +// RawDiscussionPage contains one fetched SPC print page. +type RawDiscussionPage struct { + Key string `json:"key"` + Day int `json:"day"` + URL string `json:"url"` + FetchedAt time.Time `json:"fetchedAt"` + Body string `json:"body"` +} + +// RawRSSFeed contains optional fetched SPC RSS metadata. +type RawRSSFeed struct { + URL string `json:"url"` + FetchedAt time.Time `json:"fetchedAt"` + Body string `json:"body"` +} diff --git a/internal/providers/spc/raw_test.go b/internal/providers/spc/raw_test.go new file mode 100644 index 0000000..5bf01bf --- /dev/null +++ b/internal/providers/spc/raw_test.go @@ -0,0 +1,51 @@ +package spc + +import ( + "encoding/json" + "testing" + "time" +) + +func TestRawConvectiveOutlookBundleJSONShape(t *testing.T) { + fetchedAt := time.Date(2026, 6, 11, 20, 0, 0, 0, time.UTC) + bundle := RawConvectiveOutlookBundle{ + LocationID: "stl", + LocationName: "St. Louis, MO", + Latitude: 38.6239, + Longitude: -90.3571, + FetchedAt: fetchedAt, + Products: []RawOutlookProduct{{ + Key: "day1_categorical", + Day: 1, + OutlookType: OutlookTypeCategorical, + URL: "https://example.invalid/day1.geojson", + FetchedAt: fetchedAt, + Body: json.RawMessage(`{"type":"FeatureCollection","features":[]}`), + }}, + Discussions: []RawDiscussionPage{{ + Key: "day1", + Day: 1, + URL: "https://example.invalid/day1.html", + FetchedAt: fetchedAt, + Body: "Day 1 Convective Outlook", + }}, + } + + raw, err := json.Marshal(bundle) + if err != nil { + t.Fatalf("Marshal() error = %v", err) + } + + var got map[string]any + if err := json.Unmarshal(raw, &got); err != nil { + t.Fatalf("Unmarshal() error = %v", err) + } + for _, key := range []string{"locationId", "locationName", "latitude", "longitude", "fetchedAt", "products", "discussions"} { + if _, ok := got[key]; !ok { + t.Fatalf("marshaled bundle missing key %q in %s", key, raw) + } + } + if _, ok := got["rss"]; ok { + t.Fatalf("marshaled bundle included empty rss: %s", raw) + } +} diff --git a/internal/providers/spc/rss.go b/internal/providers/spc/rss.go new file mode 100644 index 0000000..244b331 --- /dev/null +++ b/internal/providers/spc/rss.go @@ -0,0 +1,81 @@ +package spc + +import ( + "encoding/xml" + "fmt" + "strings" + "time" +) + +// RSSFeed is a minimal view of the optional SPC RSS feed. +type RSSFeed struct { + Title string + Link string + Description string + LastBuildDate *time.Time + Items []RSSItem +} + +// RSSItem is a minimal view of one optional SPC RSS item. +type RSSItem struct { + Title string + Link string + Description string + PubDate string + GUID string +} + +// ParseRSSFeed decodes supplemental SPC RSS metadata. +func ParseRSSFeed(raw string) (RSSFeed, error) { + var doc struct { + Channel struct { + Title string `xml:"title"` + Link string `xml:"link"` + Description string `xml:"description"` + LastBuildDate string `xml:"lastBuildDate"` + Items []struct { + Title string `xml:"title"` + Link string `xml:"link"` + Description string `xml:"description"` + PubDate string `xml:"pubDate"` + GUID string `xml:"guid"` + } `xml:"item"` + } `xml:"channel"` + } + if err := xml.Unmarshal([]byte(raw), &doc); err != nil { + return RSSFeed{}, fmt.Errorf("decode rss: %w", err) + } + + feed := RSSFeed{ + Title: strings.TrimSpace(doc.Channel.Title), + Link: strings.TrimSpace(doc.Channel.Link), + Description: strings.TrimSpace(doc.Channel.Description), + LastBuildDate: parseRSSDate(doc.Channel.LastBuildDate), + Items: make([]RSSItem, 0, len(doc.Channel.Items)), + } + for _, item := range doc.Channel.Items { + feed.Items = append(feed.Items, RSSItem{ + Title: strings.TrimSpace(item.Title), + Link: strings.TrimSpace(item.Link), + Description: strings.TrimSpace(item.Description), + PubDate: strings.TrimSpace(item.PubDate), + GUID: strings.TrimSpace(item.GUID), + }) + } + return feed, nil +} + +func parseRSSDate(value string) *time.Time { + value = strings.TrimSpace(value) + if value == "" { + return nil + } + for _, layout := range []string{time.RFC1123Z, time.RFC1123} { + t, err := time.Parse(layout, value) + if err == nil { + tt := t.UTC() + return &tt + } + } + return nil +} diff --git a/internal/providers/spc/rss_test.go b/internal/providers/spc/rss_test.go new file mode 100644 index 0000000..0c8bf26 --- /dev/null +++ b/internal/providers/spc/rss_test.go @@ -0,0 +1,43 @@ +package spc + +import ( + "testing" + "time" +) + +func TestParseRSSFeed(t *testing.T) { + const raw = ` + + + SPC AC RSS + https://www.spc.noaa.gov/products/ + SPC products + Thu, 11 Jun 2026 19:00:00 +0000 + + Day 1 Convective Outlook + https://www.spc.noaa.gov/products/outlook/day1otlk.html + Outlook text + Thu, 11 Jun 2026 18:55:00 +0000 + day1 + + +` + + got, err := ParseRSSFeed(raw) + if err != nil { + t.Fatalf("ParseRSSFeed() error = %v", err) + } + if got.Title != "SPC AC RSS" { + t.Fatalf("Title = %q", got.Title) + } + wantBuild := time.Date(2026, 6, 11, 19, 0, 0, 0, time.UTC) + if got.LastBuildDate == nil || !got.LastBuildDate.Equal(wantBuild) { + t.Fatalf("LastBuildDate = %v, want %s", got.LastBuildDate, wantBuild) + } + if len(got.Items) != 1 { + t.Fatalf("Items length = %d, want 1", len(got.Items)) + } + if got.Items[0].GUID != "day1" { + t.Fatalf("Item GUID = %q", got.Items[0].GUID) + } +} diff --git a/internal/providers/spc/testdata/day1_cat.geojson b/internal/providers/spc/testdata/day1_cat.geojson new file mode 100644 index 0000000..22e440a --- /dev/null +++ b/internal/providers/spc/testdata/day1_cat.geojson @@ -0,0 +1,29 @@ +{ + "type": "FeatureCollection", + "features": [ + { + "type": "Feature", + "properties": { + "VALID_ISO": "2026-06-11T13:00:00Z", + "EXPIRE_ISO": "2026-06-12T12:00:00Z", + "ISSUE_ISO": "2026-06-11T12:34:56Z", + "FORECASTER": "SMITH", + "LABEL": "SLGT", + "LABEL2": "Slight Risk", + "DN": 3 + }, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [-91.0, 38.0], + [-90.0, 38.0], + [-90.0, 39.0], + [-91.0, 39.0], + [-91.0, 38.0] + ] + ] + } + } + ] +} diff --git a/internal/providers/spc/testdata/day1_prt.html b/internal/providers/spc/testdata/day1_prt.html new file mode 100644 index 0000000..954a232 --- /dev/null +++ b/internal/providers/spc/testdata/day1_prt.html @@ -0,0 +1,19 @@ + + +Day 1 Convective Outlook + +
+
+Day 1 Convective Outlook
+NWS Storm Prediction Center Norman OK
+Updated: 2026-06-11T12:45:00Z
+
+...SUMMARY...
+Severe thunderstorms are possible across parts of the central Plains
+and mid Mississippi Valley this afternoon and evening.
+
+...DISCUSSION...
+The primary threats will be damaging wind and large hail.
+
+ + diff --git a/internal/providers/spc/testdata/day2_prt_corr.html b/internal/providers/spc/testdata/day2_prt_corr.html new file mode 100644 index 0000000..fe9ec52 --- /dev/null +++ b/internal/providers/spc/testdata/day2_prt_corr.html @@ -0,0 +1,15 @@ + + + +
+Day 2 Convective Outlook CORR 1
+NWS Storm Prediction Center Norman OK
+
+...SUMMARY...
+Scattered severe thunderstorms remain possible across the southern Plains.
+
+...DISCUSSION...
+Corrected outlook text remains otherwise unchanged.
+
+ + diff --git a/internal/providers/spc/testdata/day2_torn.geojson b/internal/providers/spc/testdata/day2_torn.geojson new file mode 100644 index 0000000..f73ec27 --- /dev/null +++ b/internal/providers/spc/testdata/day2_torn.geojson @@ -0,0 +1,29 @@ +{ + "type": "FeatureCollection", + "features": [ + { + "type": "Feature", + "properties": { + "VALID_ISO": "2026-06-12T12:00:00Z", + "EXPIRE_ISO": "2026-06-13T12:00:00Z", + "ISSUE_ISO": "2026-06-11T17:30:00Z", + "FORECASTER": "DOE", + "LABEL": "5", + "LABEL2": "5% Tornado", + "DN": "5" + }, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [-100.0, 35.0], + [-98.0, 35.0], + [-98.0, 37.0], + [-100.0, 37.0], + [-100.0, 35.0] + ] + ] + } + } + ] +} diff --git a/internal/providers/spc/testdata/day3_prt.html b/internal/providers/spc/testdata/day3_prt.html new file mode 100644 index 0000000..20b2051 --- /dev/null +++ b/internal/providers/spc/testdata/day3_prt.html @@ -0,0 +1,16 @@ + + + +
+Day 3 Convective Outlook
+NWS Storm Prediction Center Norman OK
+Updated: 2026-06-11T20:00:00Z
+
+...SUMMARY...
+A corridor of strong to severe storms may develop near a frontal zone.
+
+...DISCUSSION...
+Confidence remains moderate for organized storms.
+
+ + diff --git a/internal/providers/spc/testdata/day3_wind.geojson b/internal/providers/spc/testdata/day3_wind.geojson new file mode 100644 index 0000000..3e40d09 --- /dev/null +++ b/internal/providers/spc/testdata/day3_wind.geojson @@ -0,0 +1,31 @@ +{ + "type": "FeatureCollection", + "features": [ + { + "type": "Feature", + "properties": { + "VALID_ISO": "2026-06-13T12:00:00Z", + "EXPIRE_ISO": "2026-06-14T12:00:00Z", + "ISSUE_ISO": "2026-06-11T19:45:00Z", + "FORECASTER": "LEE", + "LABEL": "15", + "LABEL2": "15% Wind", + "DN": 15 + }, + "geometry": { + "type": "MultiPolygon", + "coordinates": [ + [ + [ + [-103.0, 34.0], + [-101.0, 34.0], + [-101.0, 36.0], + [-103.0, 36.0], + [-103.0, 34.0] + ] + ] + ] + } + } + ] +} diff --git a/internal/providers/spc/time.go b/internal/providers/spc/time.go new file mode 100644 index 0000000..c3fa2d0 --- /dev/null +++ b/internal/providers/spc/time.go @@ -0,0 +1,24 @@ +package spc + +import ( + "strings" + "time" +) + +// ParseISOTimestamp parses SPC ISO timestamps from GeoJSON properties. +func ParseISOTimestamp(value string) (time.Time, error) { + return time.Parse(time.RFC3339, strings.TrimSpace(value)) +} + +func parseOptionalISOTimestamp(value string) *time.Time { + value = strings.TrimSpace(value) + if value == "" { + return nil + } + t, err := ParseISOTimestamp(value) + if err != nil { + return nil + } + tt := t.UTC() + return &tt +}