From 882059014ca3e22aab5062fd285fe2dbdf74289a Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Sun, 2 Aug 2026 22:58:48 +0000 Subject: [PATCH] Normalize NWS forecast discussion preambles --- internal/providers/nws/forecast_discussion.go | 67 ++++++- .../providers/nws/forecast_discussion_test.go | 184 ++++++++++++++++++ 2 files changed, 242 insertions(+), 9 deletions(-) diff --git a/internal/providers/nws/forecast_discussion.go b/internal/providers/nws/forecast_discussion.go index e6901d8..7a8ac92 100644 --- a/internal/providers/nws/forecast_discussion.go +++ b/internal/providers/nws/forecast_discussion.go @@ -315,8 +315,9 @@ func parseForecastDiscussionHeader(lines []string) (string, time.Time, error) { func parseForecastDiscussionIssueTime(line string) (time.Time, error) { line = strings.TrimSpace(line) - line = strings.TrimPrefix(line, "Issued at ") - line = strings.TrimSpace(line) + if isForecastDiscussionIssuedAtLine(line) { + line = strings.TrimSpace(line[len("Issued at"):]) + } parts := strings.Fields(line) if len(parts) != 7 { @@ -547,11 +548,14 @@ func parseForecastDiscussionSectionBlocks(lines []string) []forecastDiscussionSe } func parseForecastDiscussionKeyMessages(body []string) []string { + body = removeForecastDiscussionPresentationMarkers(body) + body = trimBlankLines(body) + if len(body) > 0 && isForecastDiscussionKeyMessageMetadataLine(body[0]) { + body = trimBlankLines(body[1:]) + } if len(body) == 0 { return nil } - - body = trimBlankLines(body) var messages []string var current strings.Builder @@ -589,16 +593,19 @@ func parseForecastDiscussionTextSection(block forecastDiscussionSectionBlock) (F Qualifier: block.heading.qualifier, } - body := trimBlankLines(block.body) + body := trimBlankLines(removeForecastDiscussionPresentationMarkers(block.body)) + if section.Qualifier == "" && len(body) > 0 && isForecastDiscussionStandaloneParenthetical(body[0]) { + section.Qualifier = strings.TrimSpace(body[0]) + body = trimBlankLines(body[1:]) + } if len(body) == 0 { return section, nil } - first := strings.TrimSpace(body[0]) - if strings.HasPrefix(first, "Issued at ") { - issuedAt, err := parseForecastDiscussionIssueTime(first) + if isForecastDiscussionIssuedAtLine(body[0]) { + issuedAt, err := parseForecastDiscussionIssueTime(body[0]) if err != nil { - return ForecastDiscussionSection{}, fmt.Errorf("parse section issuedAt %q: %w", first, err) + return ForecastDiscussionSection{}, fmt.Errorf("parse section issuedAt %q: %w", strings.TrimSpace(body[0]), err) } tt := issuedAt.UTC() section.IssuedAt = &tt @@ -610,6 +617,48 @@ func parseForecastDiscussionTextSection(block forecastDiscussionSectionBlock) (F return section, nil } +func isForecastDiscussionPresentationMarker(line string) bool { + switch { + case strings.EqualFold(strings.TrimSpace(line), "-- Changed Discussion --"): + return true + case strings.EqualFold(strings.TrimSpace(line), "-- End Changed Discussion --"): + return true + default: + return false + } +} + +func removeForecastDiscussionPresentationMarkers(lines []string) []string { + body := make([]string, 0, len(lines)) + for _, line := range lines { + if !isForecastDiscussionPresentationMarker(line) { + body = append(body, line) + } + } + return body +} + +func isForecastDiscussionStandaloneParenthetical(line string) bool { + line = strings.TrimSpace(line) + return len(line) > 2 && line[0] == '(' && line[len(line)-1] == ')' && strings.TrimSpace(line[1:len(line)-1]) != "" +} + +func isForecastDiscussionIssuedAtLine(line string) bool { + line = strings.TrimSpace(line) + return len(line) > len("Issued at") && + strings.EqualFold(line[:len("Issued at")], "Issued at") && + isForecastDiscussionHorizontalWhitespace(line[len("Issued at")]) +} + +func isForecastDiscussionKeyMessageMetadataLine(line string) bool { + line = strings.TrimSpace(line) + return hasForecastDiscussionASCIIPrefix(line, "Issued at") || hasForecastDiscussionASCIIPrefix(line, "Updated at") +} + +func hasForecastDiscussionASCIIPrefix(line, prefix string) bool { + return len(line) >= len(prefix) && strings.EqualFold(line[:len(prefix)], prefix) +} + func trimBlankLines(lines []string) []string { start := 0 for start < len(lines) && strings.TrimSpace(lines[start]) == "" { diff --git a/internal/providers/nws/forecast_discussion_test.go b/internal/providers/nws/forecast_discussion_test.go index 7d42e28..16b48e6 100644 --- a/internal/providers/nws/forecast_discussion_test.go +++ b/internal/providers/nws/forecast_discussion_test.go @@ -379,6 +379,180 @@ func TestParseForecastDiscussionTextKeepsFirstMappedSections(t *testing.T) { } } +func TestParseForecastDiscussionTextSectionKeepsHeadingQualifiers(t *testing.T) { + tests := []struct { + name string + heading string + wantQualifier string + }{ + { + name: "legacy heading", + heading: ".SHORT TERM... (Tonight)", + wantQualifier: "(Tonight)", + }, + { + name: "slash-qualified heading", + heading: ".SHORT TERM /Tonight/...", + wantQualifier: "Tonight", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + section, err := parseForecastDiscussionTextSection(forecastDiscussionSectionBlock{ + heading: mustParseForecastDiscussionSectionHeading(t, tt.heading), + body: []string{"Section prose."}, + }) + if err != nil { + t.Fatalf("parseForecastDiscussionTextSection() error = %v", err) + } + if section.Qualifier != tt.wantQualifier { + t.Fatalf("Qualifier = %q, want %q", section.Qualifier, tt.wantQualifier) + } + if section.Text != "Section prose." { + t.Fatalf("Text = %q", section.Text) + } + }) + } +} + +func TestParseForecastDiscussionTextSectionReadsNextLineQualifierAndIssuedAt(t *testing.T) { + section, err := parseForecastDiscussionTextSection(forecastDiscussionSectionBlock{ + heading: mustParseForecastDiscussionSectionHeading(t, ".SHORT TERM..."), + body: []string{ + "(Tonight)", + "Issued at 219 PM CDT Sat Mar 28 2026", + "Section prose.", + }, + }) + if err != nil { + t.Fatalf("parseForecastDiscussionTextSection() error = %v", err) + } + if section.Qualifier != "(Tonight)" { + t.Fatalf("Qualifier = %q", section.Qualifier) + } + wantIssuedAt := time.Date(2026, 3, 28, 19, 19, 0, 0, time.UTC) + if section.IssuedAt == nil || !section.IssuedAt.Equal(wantIssuedAt) { + t.Fatalf("IssuedAt = %v, want %s", section.IssuedAt, wantIssuedAt.Format(time.RFC3339)) + } + if section.Text != "Section prose." { + t.Fatalf("Text = %q", section.Text) + } +} + +func TestParseForecastDiscussionTextSectionRecognizesUppercaseIssuedAt(t *testing.T) { + section, err := parseForecastDiscussionTextSection(forecastDiscussionSectionBlock{ + heading: mustParseForecastDiscussionSectionHeading(t, ".SHORT TERM..."), + body: []string{ + "ISSUED AT 219 PM CDT Sat Mar 28 2026", + "Section prose.", + }, + }) + if err != nil { + t.Fatalf("parseForecastDiscussionTextSection() error = %v", err) + } + wantIssuedAt := time.Date(2026, 3, 28, 19, 19, 0, 0, time.UTC) + if section.IssuedAt == nil || !section.IssuedAt.Equal(wantIssuedAt) { + t.Fatalf("IssuedAt = %v, want %s", section.IssuedAt, wantIssuedAt.Format(time.RFC3339)) + } +} + +func TestParseForecastDiscussionTextSectionKeepsHeadingQualifierBeforeParentheticalProse(t *testing.T) { + section, err := parseForecastDiscussionTextSection(forecastDiscussionSectionBlock{ + heading: mustParseForecastDiscussionSectionHeading(t, ".SHORT TERM... (From Heading)"), + body: []string{"(Parenthetical prose.)"}, + }) + if err != nil { + t.Fatalf("parseForecastDiscussionTextSection() error = %v", err) + } + if section.Qualifier != "(From Heading)" { + t.Fatalf("Qualifier = %q", section.Qualifier) + } + if section.Text != "(Parenthetical prose.)" { + t.Fatalf("Text = %q", section.Text) + } +} + +func TestParseForecastDiscussionTextSectionHandlesEmptyAndInvalidPreambles(t *testing.T) { + t.Run("empty section", func(t *testing.T) { + section, err := parseForecastDiscussionTextSection(forecastDiscussionSectionBlock{ + heading: mustParseForecastDiscussionSectionHeading(t, ".SHORT TERM..."), + }) + if err != nil { + t.Fatalf("parseForecastDiscussionTextSection() error = %v", err) + } + if section.Qualifier != "" || section.IssuedAt != nil || section.Text != "" { + t.Fatalf("section = %#v, want empty section", section) + } + }) + + t.Run("invalid issue time", func(t *testing.T) { + _, err := parseForecastDiscussionTextSection(forecastDiscussionSectionBlock{ + heading: mustParseForecastDiscussionSectionHeading(t, ".SHORT TERM..."), + body: []string{"ISSUED AT not a timestamp"}, + }) + if err == nil { + t.Fatalf("parseForecastDiscussionTextSection() error = nil") + } + if !strings.Contains(err.Error(), "parse section issuedAt") { + t.Fatalf("error = %q, want section issue-time context", err) + } + }) +} + +func TestParseForecastDiscussionTextSectionRemovesPresentationMarkers(t *testing.T) { + section, err := parseForecastDiscussionTextSection(forecastDiscussionSectionBlock{ + heading: mustParseForecastDiscussionSectionHeading(t, ".SHORT TERM..."), + body: []string{ + "-- Changed Discussion --", + "(Tonight)", + "Issued at 219 PM CDT Sat Mar 28 2026", + "Section prose.", + "-- END CHANGED DISCUSSION --", + }, + }) + if err != nil { + t.Fatalf("parseForecastDiscussionTextSection() error = %v", err) + } + if section.Qualifier != "(Tonight)" || section.Text != "Section prose." { + t.Fatalf("section = %#v", section) + } + if strings.Contains(section.Text, "Changed Discussion") { + t.Fatalf("Text contains presentation marker: %q", section.Text) + } +} + +func TestParseForecastDiscussionKeyMessagesRemovesMarkersAndLeadingMetadata(t *testing.T) { + got := parseForecastDiscussionKeyMessages([]string{ + "-- changed discussion --", + "Updated at 300 PM CDT Sat Mar 28 2026", + "- First message.", + "Updated at after the first message.", + "-- End Changed Discussion --", + "- Second message.", + }) + want := []string{ + "First message. Updated at after the first message.", + "Second message.", + } + if !reflect.DeepEqual(got, want) { + t.Fatalf("KeyMessages = %#v, want %#v", got, want) + } +} + +func TestParseForecastDiscussionTextSectionKeepsArbitraryDashedProse(t *testing.T) { + section, err := parseForecastDiscussionTextSection(forecastDiscussionSectionBlock{ + heading: mustParseForecastDiscussionSectionHeading(t, ".SHORT TERM..."), + body: []string{"-- Changed Discussion -- extra", "Section prose."}, + }) + if err != nil { + t.Fatalf("parseForecastDiscussionTextSection() error = %v", err) + } + if section.Text != "-- Changed Discussion -- extra Section prose." { + t.Fatalf("Text = %q", section.Text) + } +} + func TestParseForecastDiscussionHTMLParsesExpectedFields(t *testing.T) { raw := loadForecastDiscussionSampleHTML(t) @@ -563,3 +737,13 @@ func loadMixedFormatForecastDiscussionSampleHTML(t *testing.T) string { return raw } + +func mustParseForecastDiscussionSectionHeading(t *testing.T, line string) forecastDiscussionSectionHeading { + t.Helper() + + heading, ok := parseForecastDiscussionSectionHeading(line) + if !ok { + t.Fatalf("parseForecastDiscussionSectionHeading(%q) did not recognize heading", line) + } + return heading +}