ejr/weatherfeeder
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Harden NWS key message parsing

Browse Source
This commit is contained in:
Eric Rakestraw
2026-08-03 00:19:48 +00:00
parent 9a09454621
commit 6943a5ebc9
2 changed files with 259 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

92
internal/providers/nws/forecast_discussion.go
View File

@@ -594,6 +594,18 @@ func parseForecastDiscussionKeyMessages(body []string) []string {
if len(body) == 0 {
return nil
}
hasMarkers := false
for _, raw := range body {
line := strings.TrimSpace(raw)
if line == "" {
continue
}
if _, ok := stripForecastDiscussionKeyMessageMarker(line); ok {
hasMarkers = true
break
}
}
var messages []string
var current strings.Builder
@@ -605,15 +617,21 @@ func parseForecastDiscussionKeyMessages(body []string) []string {
current.Reset()
}
seenMarker := false
for _, raw := range body {
line := strings.TrimSpace(raw)
if line == "" {
if !hasMarkers || !seenMarker {
flush()
}
continue
}
if strings.HasPrefix(line, "-") {
if stripped, ok := stripForecastDiscussionKeyMessageMarker(line); ok {
flush()
line = strings.TrimSpace(strings.TrimPrefix(line, "-"))
current.WriteString(line)
seenMarker = true
line = stripped
}
if line == "" {
continue
}
if current.Len() > 0 {
@@ -690,11 +708,75 @@ func isForecastDiscussionIssuedAtLine(line string) bool {
func isForecastDiscussionKeyMessageMetadataLine(line string) bool {
line = strings.TrimSpace(line)
return hasForecastDiscussionASCIIPrefix(line, "Issued at") || hasForecastDiscussionASCIIPrefix(line, "Updated at")
for _, label := range []string{"Issued at", "Updated at"} {
if !hasForecastDiscussionASCIIPrefix(line, label) || len(line) == len(label) || !isForecastDiscussionHorizontalWhitespace(line[len(label)]) {
continue
}
if _, err := parseForecastDiscussionIssueTime(strings.TrimSpace(line[len(label):])); err == nil {
return true
}
}
return false
}
func stripForecastDiscussionKeyMessageMarker(line string) (string, bool) {
if line == "" {
return "", false
}
if line[0] == '-' || line[0] == '*' {
content := line[1:]
hadWhitespace := len(content) > 0 && isForecastDiscussionHorizontalWhitespace(content[0])
content = strings.TrimLeft(content, " \t")
if hadWhitespace {
if stripped, ok := stripForecastDiscussionKeyMessageNumericMarker(content); ok {
content = stripped
}
}
return content, true
}
return stripForecastDiscussionKeyMessageNumericMarker(line)
}
func stripForecastDiscussionKeyMessageNumericMarker(line string) (string, bool) {
digitEnd := 0
for digitEnd < len(line) && line[digitEnd] >= '0' && line[digitEnd] <= '9' {
digitEnd++
}
if digitEnd == 0 || digitEnd == len(line) || (line[digitEnd] != ')' && line[digitEnd] != '.') {
return "", false
}
markerEnd := digitEnd + 1
if markerEnd < len(line) && !isForecastDiscussionHorizontalWhitespace(line[markerEnd]) {
return "", false
}
value, err := strconv.ParseUint(line[:digitEnd], 10, 0)
if err != nil || value == 0 {
return "", false
}
return strings.TrimLeft(line[markerEnd:], " \t"), true
}
func hasForecastDiscussionASCIIPrefix(line, prefix string) bool {
return len(line) >= len(prefix) && strings.EqualFold(line[:len(prefix)], prefix)
if len(line) < len(prefix) {
return false
}
for i := range prefix {
actual := line[i]
if actual >= 'A' && actual <= 'Z' {
actual += 'a' - 'A'
}
expected := prefix[i]
if expected >= 'A' && expected <= 'Z' {
expected += 'a' - 'A'
}
if actual != expected {
return false
}
}
return true
}
func trimBlankLines(lines []string) []string {

172
internal/providers/nws/forecast_discussion_test.go
View File

@@ -740,6 +740,178 @@ func TestParseForecastDiscussionTextSectionKeepsArbitraryDashedProse(t *testing.
}
}
func TestIsForecastDiscussionKeyMessageMetadataLine(t *testing.T) {
tests := []struct {
name string
line string
want bool
}{
{
name: "issued at",
line: "Issued at 300 PM CDT Sat Mar 28 2026",
want: true,
},
{
name: "updated at ASCII case insensitive",
line: "uPdAtEd At 300 PM CDT Sat Mar 28 2026",
want: true,
},
{
name: "label without whitespace boundary",
line: "Updated at300 PM CDT Sat Mar 28 2026",
want: false,
},
{
name: "invalid timestamp",
line: "Updated at not a timestamp",
want: false,
},
{
name: "nonmetadata prose",
line: "Updated atmospheric conditions remain unsettled.",
want: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := isForecastDiscussionKeyMessageMetadataLine(tt.line); got != tt.want {
t.Fatalf("isForecastDiscussionKeyMessageMetadataLine(%q) = %t, want %t", tt.line, got, tt.want)
}
})
}
}
func TestStripForecastDiscussionKeyMessageMarker(t *testing.T) {
tests := []struct {
name string
line string
want string
ok bool
}{
{name: "hyphen", line: "- First message.", want: "First message.", ok: true},
{name: "hyphen without whitespace", line: "-First message.", want: "First message.", ok: true},
{name: "asterisk", line: "* First message.", want: "First message.", ok: true},
{name: "numeric parenthesis", line: "1) First message.", want: "First message.", ok: true},
{name: "numeric dot", line: "2. Second message.", want: "Second message.", ok: true},
{name: "composite hyphen", line: "- 1. First message.", want: "First message.", ok: true},
{name: "composite asterisk", line: "* 1) First message.", want: "First message.", ok: true},
{name: "multi digit numeric", line: "12. Twelfth message.", want: "Twelfth message.", ok: true},
{name: "hyphen only", line: "-", want: "", ok: true},
{name: "numeric marker only", line: "1)", want: "", ok: true},
{name: "zero numeric marker", line: "0) Not a marker.", want: "", ok: false},
{name: "overflow numeric marker", line: "999999999999999999999999999999. Too big.", want: "", ok: false},
{name: "alphanumeric numeric prefix", line: "1x Not a marker.", want: "", ok: false},
{name: "numeric marker without boundary", line: "1)Not a marker.", want: "", ok: false},
{name: "numeric marker with unsupported punctuation", line: "1, Not a marker.", want: "", ok: false},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, ok := stripForecastDiscussionKeyMessageMarker(tt.line)
if ok != tt.ok || got != tt.want {
t.Fatalf("stripForecastDiscussionKeyMessageMarker(%q) = (%q, %t), want (%q, %t)", tt.line, got, ok, tt.want, tt.ok)
}
})
}
}
func TestParseForecastDiscussionKeyMessagesClassifiesListsAndMetadata(t *testing.T) {
tests := []struct {
name string
body []string
want []string
}{
{
name: "numbered list with continuation",
body: []string{
"1) First numbered message.",
"Wrapped continuation.",
"2. Second numbered message.",
},
want: []string{"First numbered message. Wrapped continuation.", "Second numbered message."},
},
{
name: "asterisk list",
body: []string{
"* First asterisk message.",
"*Second asterisk message.",
},
want: []string{"First asterisk message.", "Second asterisk message."},
},
{
name: "unmarked paragraphs",
body: []string{
"First paragraph.",
"Wrapped continuation.",
"",
"",
"Second paragraph.",
},
want: []string{"First paragraph. Wrapped continuation.", "Second paragraph."},
},
{
name: "introductory prose before marked items",
body: []string{
"Introductory prose.",
"",
"- First marked message.",
"Wrapped continuation.",
"- Second marked message.",
},
want: []string{"Introductory prose.", "First marked message. Wrapped continuation.", "Second marked message."},
},
{
name: "leading issued metadata",
body: []string{
"iSsUeD At 300 PM CDT Sat Mar 28 2026",
"- First message.",
},
want: []string{"First message."},
},
{
name: "leading updated metadata",
body: []string{
"Updated at 300 PM CDT Sat Mar 28 2026",
"- First message.",
},
want: []string{"First message."},
},
{
name: "updated atmospheric prose remains content",
body: []string{
"Updated atmospheric conditions remain unsettled.",
"Additional detail.",
},
want: []string{"Updated atmospheric conditions remain unsettled. Additional detail."},
},
{
name: "invalid updated metadata remains content",
body: []string{
"Updated at not a timestamp",
"Additional detail.",
},
want: []string{"Updated at not a timestamp Additional detail."},
},
{
name: "later valid metadata remains content",
body: []string{
"- First message.",
"Updated at 300 PM CDT Sat Mar 28 2026",
},
want: []string{"First message. Updated at 300 PM CDT Sat Mar 28 2026"},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := parseForecastDiscussionKeyMessages(tt.body); !reflect.DeepEqual(got, tt.want) {
t.Fatalf("KeyMessages = %#v, want %#v", got, tt.want)
}
})
}
}
func TestParseForecastDiscussionHTMLParsesCrossOfficeLayout(t *testing.T) {
got, err := ParseForecastDiscussionHTML(loadForecastDiscussionBOUSampleHTML(t))
if err != nil {