ejr/weatherfeeder
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Generalize NWS forecast discussion heading parsing

Browse Source
This commit is contained in:
Eric Rakestraw
2026-08-02 22:53:00 +00:00
parent 29c65971eb
commit 2b19a121fa
4 changed files with 618 additions and 331 deletions
Download Patch File Download Diff File Expand all files Collapse all files

142
internal/providers/nws/forecast_discussion.go
View File

@@ -27,13 +27,12 @@ type ForecastDiscussionSection struct {
Text string
}
const (
forecastDiscussionSectionKeyMessages = "KEY MESSAGES"
forecastDiscussionSectionShortTerm = "SHORT TERM"
forecastDiscussionSectionLongTerm = "LONG TERM"
forecastDiscussionSectionAviation = "AVIATION"
type forecastDiscussionSectionRole uint8
forecastDiscussionSectionIdentityPattern = "KEY MESSAGES|SHORT TERM|LONG TERM|AVIATION"
const (
forecastDiscussionSectionRoleKeyMessages forecastDiscussionSectionRole = iota
forecastDiscussionSectionRoleShortTerm
forecastDiscussionSectionRoleLongTerm
)
type forecastDiscussionSectionHeading struct {
@@ -47,11 +46,14 @@ type forecastDiscussionSectionBlock struct {
}
var (
forecastDiscussionEllipsisHeadingRE = regexp.MustCompile(`^\.(` + forecastDiscussionSectionIdentityPattern + `)\.\.\.(.*)$`)
forecastDiscussionSlashHeadingRE = regexp.MustCompile(`^\.(` + forecastDiscussionSectionIdentityPattern + `)[ \t]+/([^/]+)/\.\.\.$`)
forecastDiscussionAFDRE = regexp.MustCompile(`^AFD([A-Z]{3})$`)
forecastDiscussionWMORE = regexp.MustCompile(`\bK([A-Z]{3})\b`)
forecastDiscussionSigRE = regexp.MustCompile(`^[A-Z]{2,6}$`)
forecastDiscussionSectionRoles = map[string]forecastDiscussionSectionRole{
"KEY MESSAGES": forecastDiscussionSectionRoleKeyMessages,
"SHORT TERM": forecastDiscussionSectionRoleShortTerm,
"LONG TERM": forecastDiscussionSectionRoleLongTerm,
}
forecastDiscussionAFDRE = regexp.MustCompile(`^AFD([A-Z]{3})$`)
forecastDiscussionWMORE = regexp.MustCompile(`\bK([A-Z]{3})\b`)
forecastDiscussionSigRE = regexp.MustCompile(`^[A-Z]{2,6}$`)
)
func ParseForecastDiscussionHTML(raw string) (ForecastDiscussion, error) {
@@ -119,20 +121,20 @@ func ParseForecastDiscussionText(text string) (ForecastDiscussion, error) {
IssuedAt: issuedAt.UTC(),
}
if block, ok := extractForecastDiscussionSection(lines, forecastDiscussionSectionKeyMessages); ok {
if block, ok := extractForecastDiscussionSection(lines, forecastDiscussionSectionForRole(forecastDiscussionSectionRoleKeyMessages)); ok {
out.KeyMessages = parseForecastDiscussionKeyMessages(block.body)
}
if block, ok := extractForecastDiscussionSection(lines, forecastDiscussionSectionShortTerm); ok {
if block, ok := extractForecastDiscussionSection(lines, forecastDiscussionSectionForRole(forecastDiscussionSectionRoleShortTerm)); ok {
section, err := parseForecastDiscussionTextSection(block)
if err != nil {
return ForecastDiscussion{}, fmt.Errorf("parse %s: %w", forecastDiscussionSectionShortTerm, err)
return ForecastDiscussion{}, fmt.Errorf("parse %s: %w", block.heading.section, err)
}
out.ShortTerm = &section
}
if block, ok := extractForecastDiscussionSection(lines, forecastDiscussionSectionLongTerm); ok {
if block, ok := extractForecastDiscussionSection(lines, forecastDiscussionSectionForRole(forecastDiscussionSectionRoleLongTerm)); ok {
section, err := parseForecastDiscussionTextSection(block)
if err != nil {
return ForecastDiscussion{}, fmt.Errorf("parse %s: %w", forecastDiscussionSectionLongTerm, err)
return ForecastDiscussion{}, fmt.Errorf("parse %s: %w", block.heading.section, err)
}
out.LongTerm = &section
}
@@ -408,23 +410,103 @@ func forecastDiscussionLocation(abbrev string) (*time.Location, error) {
func parseForecastDiscussionSectionHeading(line string) (forecastDiscussionSectionHeading, bool) {
line = strings.TrimSpace(line)
if m := forecastDiscussionEllipsisHeadingRE.FindStringSubmatch(line); len(m) == 3 {
return forecastDiscussionSectionHeading{
section: m[1],
qualifier: strings.TrimSpace(m[2]),
}, true
if len(line) < 2 || line[0] != '.' {
return forecastDiscussionSectionHeading{}, false
}
if m := forecastDiscussionSlashHeadingRE.FindStringSubmatch(line); len(m) == 3 {
qualifier := strings.TrimSpace(m[2])
if qualifier == "" {
return forecastDiscussionSectionHeading{}, false
if strings.HasSuffix(line, "/...") {
return parseForecastDiscussionSlashQualifiedHeading(line)
}
return parseForecastDiscussionEllipsisHeading(line)
}
func parseForecastDiscussionSlashQualifiedHeading(line string) (forecastDiscussionSectionHeading, bool) {
content := strings.TrimSuffix(line[1:], "/...")
separator := -1
for i := 1; i < len(content); i++ {
if content[i] == '/' && isForecastDiscussionHorizontalWhitespace(content[i-1]) {
separator = i
break
}
return forecastDiscussionSectionHeading{
section: m[1],
qualifier: qualifier,
}, true
}
return forecastDiscussionSectionHeading{}, false
if separator < 0 {
return forecastDiscussionSectionHeading{}, false
}
section, ok := normalizeForecastDiscussionSectionIdentity(content[:separator])
if !ok {
return forecastDiscussionSectionHeading{}, false
}
qualifier := strings.TrimSpace(content[separator+1:])
if qualifier == "" {
return forecastDiscussionSectionHeading{}, false
}
return forecastDiscussionSectionHeading{section: section, qualifier: qualifier}, true
}
func parseForecastDiscussionEllipsisHeading(line string) (forecastDiscussionSectionHeading, bool) {
content := line[1:]
delimiter := strings.Index(content, "...")
if delimiter < 0 {
return forecastDiscussionSectionHeading{}, false
}
section, ok := normalizeForecastDiscussionSectionIdentity(content[:delimiter])
if !ok {
return forecastDiscussionSectionHeading{}, false
}
return forecastDiscussionSectionHeading{
section: section,
qualifier: strings.TrimSpace(content[delimiter+3:]),
}, true
}
func normalizeForecastDiscussionSectionIdentity(raw string) (string, bool) {
var normalized strings.Builder
pendingSpace := false
hasLetterOrDigit := false
for i := 0; i < len(raw); i++ {
b := raw[i]
switch {
case isForecastDiscussionIdentityLetterOrDigit(b):
hasLetterOrDigit = true
case b == ' ' || b == '\t':
pendingSpace = normalized.Len() > 0
continue
case b == '/' && i > 0 && isForecastDiscussionHorizontalWhitespace(raw[i-1]):
return "", false
case b != '/' && b != '&' && b != '\'' && b != '-':
return "", false
}
if pendingSpace {
normalized.WriteByte(' ')
pendingSpace = false
}
normalized.WriteByte(b)
}
if !hasLetterOrDigit {
return "", false
}
return normalized.String(), true
}
func isForecastDiscussionIdentityLetterOrDigit(b byte) bool {
return b >= 'A' && b <= 'Z' || b >= '0' && b <= '9'
}
func isForecastDiscussionHorizontalWhitespace(b byte) bool {
return b == ' ' || b == '\t'
}
func forecastDiscussionSectionForRole(role forecastDiscussionSectionRole) string {
for section, registeredRole := range forecastDiscussionSectionRoles {
if registeredRole == role {
return section
}
}
return ""
}
func extractForecastDiscussionSection(lines []string, section string) (forecastDiscussionSectionBlock, bool) {

170
internal/providers/nws/forecast_discussion_test.go
View File

@@ -20,88 +20,150 @@ func TestParseForecastDiscussionSectionHeading(t *testing.T) {
{
name: "ellipsis-first key messages",
line: ".KEY MESSAGES... (Through Tonight)",
wantSection: forecastDiscussionSectionKeyMessages,
wantSection: "KEY MESSAGES",
wantQualifier: "(Through Tonight)",
wantOK: true,
},
{
name: "ellipsis-first short term",
line: ".SHORT TERM... (Sunday)",
wantSection: forecastDiscussionSectionShortTerm,
wantSection: "SHORT TERM",
wantQualifier: "(Sunday)",
wantOK: true,
},
{
name: "ellipsis-first long term",
line: ".LONG TERM... (Monday Through Friday)",
wantSection: forecastDiscussionSectionLongTerm,
wantSection: "LONG TERM",
wantQualifier: "(Monday Through Friday)",
wantOK: true,
},
{
name: "ellipsis-first aviation without qualifier",
line: ".AVIATION...",
wantSection: forecastDiscussionSectionAviation,
wantSection: "AVIATION",
wantOK: true,
},
{
name: "slash-qualified key messages",
line: ".KEY MESSAGES /Tonight/...",
wantSection: forecastDiscussionSectionKeyMessages,
wantSection: "KEY MESSAGES",
wantQualifier: "Tonight",
wantOK: true,
},
{
name: "slash-qualified short term with outer whitespace",
line: " .SHORT TERM /Through Late Sunday Night/... ",
wantSection: forecastDiscussionSectionShortTerm,
wantSection: "SHORT TERM",
wantQualifier: "Through Late Sunday Night",
wantOK: true,
},
{
name: "slash-qualified long term",
line: ".LONG TERM /Monday through Next Saturday/...",
wantSection: forecastDiscussionSectionLongTerm,
wantSection: "LONG TERM",
wantQualifier: "Monday through Next Saturday",
wantOK: true,
},
{
name: "slash-qualified aviation",
line: ".AVIATION /18Z TAFS/...",
wantSection: forecastDiscussionSectionAviation,
wantSection: "AVIATION",
wantQualifier: "18Z TAFS",
wantOK: true,
},
{
name: "unknown identity",
line: ".SYNOPSIS... Overview",
name: "generic synopsis",
line: ".SYNOPSIS... Overview",
wantSection: "SYNOPSIS",
wantQualifier: "Overview",
wantOK: true,
},
{
name: "generic update",
line: ".UPDATE...",
wantSection: "UPDATE",
wantOK: true,
},
{
name: "generic marine",
line: ".MARINE...",
wantSection: "MARINE",
wantOK: true,
},
{
name: "generic hydrology",
line: ".HYDROLOGY...",
wantSection: "HYDROLOGY",
wantOK: true,
},
{
name: "office watch advisory heading",
line: ".LSX WATCHES/WARNINGS/ADVISORIES...",
wantSection: "LSX WATCHES/WARNINGS/ADVISORIES",
wantOK: true,
},
{
name: "preliminary point temperatures heading",
line: ".PRELIMINARY POINT TEMPS/POPS ...",
wantSection: "PRELIMINARY POINT TEMPS/POPS",
wantOK: true,
},
{
name: "generic identity punctuation and digits",
line: ".DAY 1 FIRE-WEATHER & HYDROLOGY'S/OUTLOOK...",
wantSection: "DAY 1 FIRE-WEATHER & HYDROLOGY'S/OUTLOOK",
wantOK: true,
},
{
name: "slash qualifier contains slash",
line: ".LONG TERM /Tonight/Sunday/...",
wantSection: "LONG TERM",
wantQualifier: "Tonight/Sunday",
wantOK: true,
},
{
name: "normalized identity whitespace",
line: " .SHORT\t\tTERM \t... (Tonight) ",
wantSection: "SHORT TERM",
wantQualifier: "(Tonight)",
wantOK: true,
},
{
name: "lowercase prose",
line: ".This is ordinary prose...",
wantOK: false,
},
{
name: "slash qualifier missing opening slash",
name: "empty identity",
line: "....",
wantOK: false,
},
{
name: "punctuation only identity",
line: ".-/&'...",
wantOK: false,
},
{
name: "slash qualifier missing opening separator",
line: ".SHORT TERM Through Tonight/...",
wantOK: false,
},
{
name: "slash qualifier missing closing slash",
line: ".SHORT TERM /Through Tonight...",
wantOK: false,
},
{
name: "slash qualifier contains embedded slash",
line: ".SHORT TERM /Tonight/Sunday/...",
wantOK: false,
},
{
name: "slash qualifier whitespace only",
line: ".SHORT TERM / /...",
wantOK: false,
},
{
name: "slash qualifier missing required whitespace",
line: ".SHORT TERM/Through Tonight/...",
wantOK: false,
},
{
name: "slash qualifier missing closing terminal",
line: ".SHORT TERM /Through Tonight...",
wantOK: false,
},
{
name: "slash qualifier whitespace only",
line: ".SHORT TERM / /...",
wantOK: false,
},
{
name: "slash qualifier missing final ellipsis",
line: ".SHORT TERM /Through Tonight/..",
@@ -113,8 +175,23 @@ func TestParseForecastDiscussionSectionHeading(t *testing.T) {
wantOK: false,
},
{
name: "lowercase identity",
line: ".short term... (Tonight)",
name: "missing ellipsis",
line: ".SHORT TERM",
wantOK: false,
},
{
name: "unsupported identity punctuation",
line: ".SHORT TERM:...",
wantOK: false,
},
{
name: "unicode ellipsis",
line: ".SHORT TERM…",
wantOK: false,
},
{
name: "mixed case identity",
line: ".Short Term... (Tonight)",
wantOK: false,
},
}
@@ -138,13 +215,34 @@ func TestParseForecastDiscussionSectionHeading(t *testing.T) {
}
}
func TestForecastDiscussionSectionRoleHeadingsParse(t *testing.T) {
if len(forecastDiscussionSectionRoles) != 3 {
t.Fatalf("role registry has %d entries, want 3", len(forecastDiscussionSectionRoles))
}
if _, ok := forecastDiscussionSectionRoles["AVIATION"]; ok {
t.Fatalf("AVIATION must remain boundary-only")
}
for section := range forecastDiscussionSectionRoles {
t.Run(section, func(t *testing.T) {
got, ok := parseForecastDiscussionSectionHeading("." + section + "...")
if !ok {
t.Fatalf("parseForecastDiscussionSectionHeading() did not recognize %q", section)
}
if got.section != section {
t.Fatalf("section = %q, want %q", got.section, section)
}
})
}
}
func TestExtractForecastDiscussionSectionStopsAtSlashQualifiedHeading(t *testing.T) {
block, ok := extractForecastDiscussionSection([]string{
".LONG TERM /Monday through Next Saturday/...",
"Long-term prose.",
".AVIATION /18Z TAFS/...",
"Aviation prose.",
}, forecastDiscussionSectionLongTerm)
}, forecastDiscussionSectionForRole(forecastDiscussionSectionRoleLongTerm))
if !ok {
t.Fatalf("extractForecastDiscussionSection() found no LONG TERM block")
}
@@ -162,7 +260,7 @@ func TestExtractForecastDiscussionSectionAllowsEmptyBodyBeforeHeading(t *testing
".SHORT TERM... (Tonight)",
".LONG TERM... (Tomorrow)",
"Long-term prose.",
}, forecastDiscussionSectionShortTerm)
}, forecastDiscussionSectionForRole(forecastDiscussionSectionRoleShortTerm))
if !ok {
t.Fatalf("extractForecastDiscussionSection() found no SHORT TERM block")
}
@@ -171,23 +269,23 @@ func TestExtractForecastDiscussionSectionAllowsEmptyBodyBeforeHeading(t *testing
}
}
func TestExtractForecastDiscussionSectionKeepsUnrecognizedHeadingLikeLines(t *testing.T) {
func TestExtractForecastDiscussionSectionKeepsMalformedHeadingLikeLines(t *testing.T) {
block, ok := extractForecastDiscussionSection([]string{
".SHORT TERM... (Tonight)",
".SYNOPSIS... This unknown section stays in the body.",
".short term... Lowercase prose stays in the body.",
".LONG TERM /Tomorrow/..",
".LONG TERM /Tomorrow/Next Week/...",
".LONG TERM /Tomorrow/... more",
"Expected short-term prose.",
".LONG TERM... (Tomorrow)",
"Long-term prose.",
}, forecastDiscussionSectionShortTerm)
}, forecastDiscussionSectionForRole(forecastDiscussionSectionRoleShortTerm))
if !ok {
t.Fatalf("extractForecastDiscussionSection() found no SHORT TERM block")
}
wantBody := []string{
".SYNOPSIS... This unknown section stays in the body.",
".short term... Lowercase prose stays in the body.",
".LONG TERM /Tomorrow/..",
".LONG TERM /Tomorrow/Next Week/...",
".LONG TERM /Tomorrow/... more",
"Expected short-term prose.",
}
if !reflect.DeepEqual(block.body, wantBody) {