Normalize NWS forecast discussion preambles

2026-08-02 22:58:48 +00:00
parent 0a3e52d0e5
commit 882059014c
2 changed files with 242 additions and 9 deletions
--- a/internal/providers/nws/forecast_discussion.go
+++ b/internal/providers/nws/forecast_discussion.go
@@ -315,8 +315,9 @@ func parseForecastDiscussionHeader(lines []string) (string, time.Time, error) {

 func parseForecastDiscussionIssueTime(line string) (time.Time, error) {
 	line = strings.TrimSpace(line)
-	line = strings.TrimPrefix(line, "Issued at ")
-	line = strings.TrimSpace(line)
+	if isForecastDiscussionIssuedAtLine(line) {
+		line = strings.TrimSpace(line[len("Issued at"):])
+	}

 	parts := strings.Fields(line)
 	if len(parts) != 7 {
@@ -547,11 +548,14 @@ func parseForecastDiscussionSectionBlocks(lines []string) []forecastDiscussionSe
 }

 func parseForecastDiscussionKeyMessages(body []string) []string {
+	body = removeForecastDiscussionPresentationMarkers(body)
+	body = trimBlankLines(body)
+	if len(body) > 0 && isForecastDiscussionKeyMessageMetadataLine(body[0]) {
+		body = trimBlankLines(body[1:])
+	}
 	if len(body) == 0 {
 		return nil
 	}
-
-	body = trimBlankLines(body)
 	var messages []string
 	var current strings.Builder

@@ -589,16 +593,19 @@ func parseForecastDiscussionTextSection(block forecastDiscussionSectionBlock) (F
 		Qualifier: block.heading.qualifier,
 	}

-	body := trimBlankLines(block.body)
+	body := trimBlankLines(removeForecastDiscussionPresentationMarkers(block.body))
+	if section.Qualifier == "" && len(body) > 0 && isForecastDiscussionStandaloneParenthetical(body[0]) {
+		section.Qualifier = strings.TrimSpace(body[0])
+		body = trimBlankLines(body[1:])
+	}
 	if len(body) == 0 {
 		return section, nil
 	}

-	first := strings.TrimSpace(body[0])
-	if strings.HasPrefix(first, "Issued at ") {
-		issuedAt, err := parseForecastDiscussionIssueTime(first)
+	if isForecastDiscussionIssuedAtLine(body[0]) {
+		issuedAt, err := parseForecastDiscussionIssueTime(body[0])
 		if err != nil {
-			return ForecastDiscussionSection{}, fmt.Errorf("parse section issuedAt %q: %w", first, err)
+			return ForecastDiscussionSection{}, fmt.Errorf("parse section issuedAt %q: %w", strings.TrimSpace(body[0]), err)
 		}
 		tt := issuedAt.UTC()
 		section.IssuedAt = &tt
@@ -610,6 +617,48 @@ func parseForecastDiscussionTextSection(block forecastDiscussionSectionBlock) (F
 	return section, nil
 }

+func isForecastDiscussionPresentationMarker(line string) bool {
+	switch {
+	case strings.EqualFold(strings.TrimSpace(line), "-- Changed Discussion --"):
+		return true
+	case strings.EqualFold(strings.TrimSpace(line), "-- End Changed Discussion --"):
+		return true
+	default:
+		return false
+	}
+}
+
+func removeForecastDiscussionPresentationMarkers(lines []string) []string {
+	body := make([]string, 0, len(lines))
+	for _, line := range lines {
+		if !isForecastDiscussionPresentationMarker(line) {
+			body = append(body, line)
+		}
+	}
+	return body
+}
+
+func isForecastDiscussionStandaloneParenthetical(line string) bool {
+	line = strings.TrimSpace(line)
+	return len(line) > 2 && line[0] == '(' && line[len(line)-1] == ')' && strings.TrimSpace(line[1:len(line)-1]) != ""
+}
+
+func isForecastDiscussionIssuedAtLine(line string) bool {
+	line = strings.TrimSpace(line)
+	return len(line) > len("Issued at") &&
+		strings.EqualFold(line[:len("Issued at")], "Issued at") &&
+		isForecastDiscussionHorizontalWhitespace(line[len("Issued at")])
+}
+
+func isForecastDiscussionKeyMessageMetadataLine(line string) bool {
+	line = strings.TrimSpace(line)
+	return hasForecastDiscussionASCIIPrefix(line, "Issued at") || hasForecastDiscussionASCIIPrefix(line, "Updated at")
+}
+
+func hasForecastDiscussionASCIIPrefix(line, prefix string) bool {
+	return len(line) >= len(prefix) && strings.EqualFold(line[:len(prefix)], prefix)
+}
+
 func trimBlankLines(lines []string) []string {
 	start := 0
 	for start < len(lines) && strings.TrimSpace(lines[start]) == "" {
--- a/internal/providers/nws/forecast_discussion_test.go
+++ b/internal/providers/nws/forecast_discussion_test.go
@@ -379,6 +379,180 @@ func TestParseForecastDiscussionTextKeepsFirstMappedSections(t *testing.T) {
 	}
 }

+func TestParseForecastDiscussionTextSectionKeepsHeadingQualifiers(t *testing.T) {
+	tests := []struct {
+		name          string
+		heading       string
+		wantQualifier string
+	}{
+		{
+			name:          "legacy heading",
+			heading:       ".SHORT TERM... (Tonight)",
+			wantQualifier: "(Tonight)",
+		},
+		{
+			name:          "slash-qualified heading",
+			heading:       ".SHORT TERM /Tonight/...",
+			wantQualifier: "Tonight",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			section, err := parseForecastDiscussionTextSection(forecastDiscussionSectionBlock{
+				heading: mustParseForecastDiscussionSectionHeading(t, tt.heading),
+				body:    []string{"Section prose."},
+			})
+			if err != nil {
+				t.Fatalf("parseForecastDiscussionTextSection() error = %v", err)
+			}
+			if section.Qualifier != tt.wantQualifier {
+				t.Fatalf("Qualifier = %q, want %q", section.Qualifier, tt.wantQualifier)
+			}
+			if section.Text != "Section prose." {
+				t.Fatalf("Text = %q", section.Text)
+			}
+		})
+	}
+}
+
+func TestParseForecastDiscussionTextSectionReadsNextLineQualifierAndIssuedAt(t *testing.T) {
+	section, err := parseForecastDiscussionTextSection(forecastDiscussionSectionBlock{
+		heading: mustParseForecastDiscussionSectionHeading(t, ".SHORT TERM..."),
+		body: []string{
+			"(Tonight)",
+			"Issued at 219 PM CDT Sat Mar 28 2026",
+			"Section prose.",
+		},
+	})
+	if err != nil {
+		t.Fatalf("parseForecastDiscussionTextSection() error = %v", err)
+	}
+	if section.Qualifier != "(Tonight)" {
+		t.Fatalf("Qualifier = %q", section.Qualifier)
+	}
+	wantIssuedAt := time.Date(2026, 3, 28, 19, 19, 0, 0, time.UTC)
+	if section.IssuedAt == nil || !section.IssuedAt.Equal(wantIssuedAt) {
+		t.Fatalf("IssuedAt = %v, want %s", section.IssuedAt, wantIssuedAt.Format(time.RFC3339))
+	}
+	if section.Text != "Section prose." {
+		t.Fatalf("Text = %q", section.Text)
+	}
+}
+
+func TestParseForecastDiscussionTextSectionRecognizesUppercaseIssuedAt(t *testing.T) {
+	section, err := parseForecastDiscussionTextSection(forecastDiscussionSectionBlock{
+		heading: mustParseForecastDiscussionSectionHeading(t, ".SHORT TERM..."),
+		body: []string{
+			"ISSUED AT 219 PM CDT Sat Mar 28 2026",
+			"Section prose.",
+		},
+	})
+	if err != nil {
+		t.Fatalf("parseForecastDiscussionTextSection() error = %v", err)
+	}
+	wantIssuedAt := time.Date(2026, 3, 28, 19, 19, 0, 0, time.UTC)
+	if section.IssuedAt == nil || !section.IssuedAt.Equal(wantIssuedAt) {
+		t.Fatalf("IssuedAt = %v, want %s", section.IssuedAt, wantIssuedAt.Format(time.RFC3339))
+	}
+}
+
+func TestParseForecastDiscussionTextSectionKeepsHeadingQualifierBeforeParentheticalProse(t *testing.T) {
+	section, err := parseForecastDiscussionTextSection(forecastDiscussionSectionBlock{
+		heading: mustParseForecastDiscussionSectionHeading(t, ".SHORT TERM... (From Heading)"),
+		body:    []string{"(Parenthetical prose.)"},
+	})
+	if err != nil {
+		t.Fatalf("parseForecastDiscussionTextSection() error = %v", err)
+	}
+	if section.Qualifier != "(From Heading)" {
+		t.Fatalf("Qualifier = %q", section.Qualifier)
+	}
+	if section.Text != "(Parenthetical prose.)" {
+		t.Fatalf("Text = %q", section.Text)
+	}
+}
+
+func TestParseForecastDiscussionTextSectionHandlesEmptyAndInvalidPreambles(t *testing.T) {
+	t.Run("empty section", func(t *testing.T) {
+		section, err := parseForecastDiscussionTextSection(forecastDiscussionSectionBlock{
+			heading: mustParseForecastDiscussionSectionHeading(t, ".SHORT TERM..."),
+		})
+		if err != nil {
+			t.Fatalf("parseForecastDiscussionTextSection() error = %v", err)
+		}
+		if section.Qualifier != "" || section.IssuedAt != nil || section.Text != "" {
+			t.Fatalf("section = %#v, want empty section", section)
+		}
+	})
+
+	t.Run("invalid issue time", func(t *testing.T) {
+		_, err := parseForecastDiscussionTextSection(forecastDiscussionSectionBlock{
+			heading: mustParseForecastDiscussionSectionHeading(t, ".SHORT TERM..."),
+			body:    []string{"ISSUED AT not a timestamp"},
+		})
+		if err == nil {
+			t.Fatalf("parseForecastDiscussionTextSection() error = nil")
+		}
+		if !strings.Contains(err.Error(), "parse section issuedAt") {
+			t.Fatalf("error = %q, want section issue-time context", err)
+		}
+	})
+}
+
+func TestParseForecastDiscussionTextSectionRemovesPresentationMarkers(t *testing.T) {
+	section, err := parseForecastDiscussionTextSection(forecastDiscussionSectionBlock{
+		heading: mustParseForecastDiscussionSectionHeading(t, ".SHORT TERM..."),
+		body: []string{
+			"-- Changed Discussion --",
+			"(Tonight)",
+			"Issued at 219 PM CDT Sat Mar 28 2026",
+			"Section prose.",
+			"-- END CHANGED DISCUSSION --",
+		},
+	})
+	if err != nil {
+		t.Fatalf("parseForecastDiscussionTextSection() error = %v", err)
+	}
+	if section.Qualifier != "(Tonight)" || section.Text != "Section prose." {
+		t.Fatalf("section = %#v", section)
+	}
+	if strings.Contains(section.Text, "Changed Discussion") {
+		t.Fatalf("Text contains presentation marker: %q", section.Text)
+	}
+}
+
+func TestParseForecastDiscussionKeyMessagesRemovesMarkersAndLeadingMetadata(t *testing.T) {
+	got := parseForecastDiscussionKeyMessages([]string{
+		"-- changed discussion --",
+		"Updated at 300 PM CDT Sat Mar 28 2026",
+		"- First message.",
+		"Updated at after the first message.",
+		"-- End Changed Discussion --",
+		"- Second message.",
+	})
+	want := []string{
+		"First message. Updated at after the first message.",
+		"Second message.",
+	}
+	if !reflect.DeepEqual(got, want) {
+		t.Fatalf("KeyMessages = %#v, want %#v", got, want)
+	}
+}
+
+func TestParseForecastDiscussionTextSectionKeepsArbitraryDashedProse(t *testing.T) {
+	section, err := parseForecastDiscussionTextSection(forecastDiscussionSectionBlock{
+		heading: mustParseForecastDiscussionSectionHeading(t, ".SHORT TERM..."),
+		body:    []string{"-- Changed Discussion -- extra", "Section prose."},
+	})
+	if err != nil {
+		t.Fatalf("parseForecastDiscussionTextSection() error = %v", err)
+	}
+	if section.Text != "-- Changed Discussion -- extra Section prose." {
+		t.Fatalf("Text = %q", section.Text)
+	}
+}
+
 func TestParseForecastDiscussionHTMLParsesExpectedFields(t *testing.T) {
 	raw := loadForecastDiscussionSampleHTML(t)

@@ -563,3 +737,13 @@ func loadMixedFormatForecastDiscussionSampleHTML(t *testing.T) string {

 	return raw
 }
+
+func mustParseForecastDiscussionSectionHeading(t *testing.T, line string) forecastDiscussionSectionHeading {
+	t.Helper()
+
+	heading, ok := parseForecastDiscussionSectionHeading(line)
+	if !ok {
+		t.Fatalf("parseForecastDiscussionSectionHeading(%q) did not recognize heading", line)
+	}
+	return heading
+}