diff --git a/internal/providers/spc/discussion.go b/internal/providers/spc/discussion.go new file mode 100644 index 0000000..ae79230 --- /dev/null +++ b/internal/providers/spc/discussion.go @@ -0,0 +1,158 @@ +package spc + +import ( + "fmt" + "html" + "regexp" + "strings" + "time" +) + +var ( + scriptBlockRE = regexp.MustCompile(`(?is)`) + preBlockRE = regexp.MustCompile(`(?is)
]*>(.*?)`) + tagRE = regexp.MustCompile(`(?is)<[^>]+>`) + updatedRE = regexp.MustCompile(`(?im)^\s*Updated:\s*(.+?)\s*$`) + sectionRE = regexp.MustCompile(`^\s*\.\.\.[A-Z0-9 /-]+\.{3}\s*$`) +) + +// DiscussionText contains parsed text from an SPC print page. +type DiscussionText struct { + ProductTitle string + Headline string + Summary string + Discussion string + UpdatedAt *time.Time +} + +// ExtractProductText extracts and cleans the first useful preformatted SPC +// product text block from a print page. +func ExtractProductText(rawHTML string) (string, error) { + matches := preBlockRE.FindAllStringSubmatch(rawHTML, -1) + for _, match := range matches { + if len(match) < 2 { + continue + } + text := cleanHTMLText(match[1]) + if strings.TrimSpace(text) != "" { + return text, nil + } + } + return "", fmt.Errorf("no useful pre block found") +} + +// ParseDiscussionText extracts common SPC narrative metadata from cleaned +// product text. +func ParseDiscussionText(text string) DiscussionText { + text = trimBlankLines(normalizeNewlines(text)) + title := ParseProductTitle(text) + return DiscussionText{ + ProductTitle: title, + Headline: title, + Summary: ExtractSummary(text), + Discussion: text, + UpdatedAt: ParseUpdatedTimestamp(text), + } +} + +// ParseUpdatedTimestamp parses an SPC print-page Updated line when present. +func ParseUpdatedTimestamp(text string) *time.Time { + match := updatedRE.FindStringSubmatch(normalizeNewlines(text)) + if len(match) != 2 { + return nil + } + return parseUpdatedValue(match[1]) +} + +// ParseProductTitle returns the first non-empty product line from cleaned text. +func ParseProductTitle(text string) string { + for _, line := range strings.Split(normalizeNewlines(text), "\n") { + line = strings.TrimSpace(line) + if line == "" || strings.HasPrefix(line, "Updated:") { + continue + } + return line + } + return "" +} + +// ParseHeadline returns the human-facing headline from cleaned text. +func ParseHeadline(text string) string { + return ParseProductTitle(text) +} + +// ExtractSummary returns text under the ...SUMMARY... section through the next +// SPC section heading. +func ExtractSummary(text string) string { + lines := strings.Split(normalizeNewlines(text), "\n") + start := -1 + for i, line := range lines { + if strings.EqualFold(strings.TrimSpace(line), "...SUMMARY...") { + start = i + 1 + break + } + } + if start < 0 { + return "" + } + + var out []string + for _, line := range lines[start:] { + if sectionRE.MatchString(line) { + break + } + out = append(out, line) + } + return trimBlankLines(strings.Join(out, "\n")) +} + +func cleanHTMLText(raw string) string { + raw = scriptBlockRE.ReplaceAllString(raw, "") + raw = tagRE.ReplaceAllString(raw, "") + raw = html.UnescapeString(raw) + raw = normalizeNewlines(raw) + return trimBlankLines(raw) +} + +func normalizeNewlines(text string) string { + text = strings.ReplaceAll(text, "\r\n", "\n") + text = strings.ReplaceAll(text, "\r", "\n") + return text +} + +func trimBlankLines(text string) string { + lines := strings.Split(normalizeNewlines(text), "\n") + start := 0 + for start < len(lines) && strings.TrimSpace(lines[start]) == "" { + start++ + } + end := len(lines) + for end > start && strings.TrimSpace(lines[end-1]) == "" { + end-- + } + return strings.Join(lines[start:end], "\n") +} + +func parseUpdatedValue(value string) *time.Time { + value = strings.TrimSpace(value) + if value == "" { + return nil + } + if t := parseOptionalISOTimestamp(value); t != nil { + return t + } + for _, layout := range []string{ + "1504 UTC Mon Jan 2 2006", + "1504Z Mon Jan 2 2006", + "3:04 PM UTC Mon Jan 2 2006", + time.RFC1123, + time.RFC1123Z, + } { + t, err := time.Parse(layout, value) + if err == nil { + tt := t.UTC() + return &tt + } + } + return nil +} diff --git a/internal/providers/spc/discussion_test.go b/internal/providers/spc/discussion_test.go new file mode 100644 index 0000000..1640da8 --- /dev/null +++ b/internal/providers/spc/discussion_test.go @@ -0,0 +1,90 @@ +package spc + +import ( + "strings" + "testing" + "time" +) + +func TestExtractProductTextCleansPreBlock(t *testing.T) { + raw := string(readTestFile(t, "day1_prt.html")) + + got, err := ExtractProductText(raw) + if err != nil { + t.Fatalf("ExtractProductText() error = %v", err) + } + if strings.Contains(got, " +Day 1 Convective Outlook +NWS Storm Prediction Center Norman OK +Updated: 2026-06-11T12:45:00Z + +...SUMMARY... +Severe thunderstorms are possible across parts of the central Plains +and mid Mississippi Valley this afternoon and evening. + +...DISCUSSION... +The primary threats will be damaging wind and large hail. + +