Add SPC provider parsing helpers

internal/providers/spc/discussion.go

@@ -0,0 +1,158 @@
+package spc
+
+import (
+	"fmt"
+	"html"
+	"regexp"
+	"strings"
+	"time"
+)
+
+var (
+	scriptBlockRE = regexp.MustCompile(`(?is)<script\b[^>]*>.*?</script>`)
+	preBlockRE    = regexp.MustCompile(`(?is)<pre\b[^>]*>(.*?)</pre>`)
+	tagRE         = regexp.MustCompile(`(?is)<[^>]+>`)
+	updatedRE     = regexp.MustCompile(`(?im)^\s*Updated:\s*(.+?)\s*$`)
+	sectionRE     = regexp.MustCompile(`^\s*\.\.\.[A-Z0-9 /-]+\.{3}\s*$`)
+)
+
+// DiscussionText contains parsed text from an SPC print page.
+type DiscussionText struct {
+	ProductTitle string
+	Headline     string
+	Summary      string
+	Discussion   string
+	UpdatedAt    *time.Time
+}
+
+// ExtractProductText extracts and cleans the first useful preformatted SPC
+// product text block from a print page.
+func ExtractProductText(rawHTML string) (string, error) {
+	matches := preBlockRE.FindAllStringSubmatch(rawHTML, -1)
+	for _, match := range matches {
+		if len(match) < 2 {
+			continue
+		}
+		text := cleanHTMLText(match[1])
+		if strings.TrimSpace(text) != "" {
+			return text, nil
+		}
+	}
+	return "", fmt.Errorf("no useful pre block found")
+}
+
+// ParseDiscussionText extracts common SPC narrative metadata from cleaned
+// product text.
+func ParseDiscussionText(text string) DiscussionText {
+	text = trimBlankLines(normalizeNewlines(text))
+	title := ParseProductTitle(text)
+	return DiscussionText{
+		ProductTitle: title,
+		Headline:     title,
+		Summary:      ExtractSummary(text),
+		Discussion:   text,
+		UpdatedAt:    ParseUpdatedTimestamp(text),
+	}
+}
+
+// ParseUpdatedTimestamp parses an SPC print-page Updated line when present.
+func ParseUpdatedTimestamp(text string) *time.Time {
+	match := updatedRE.FindStringSubmatch(normalizeNewlines(text))
+	if len(match) != 2 {
+		return nil
+	}
+	return parseUpdatedValue(match[1])
+}
+
+// ParseProductTitle returns the first non-empty product line from cleaned text.
+func ParseProductTitle(text string) string {
+	for _, line := range strings.Split(normalizeNewlines(text), "\n") {
+		line = strings.TrimSpace(line)
+		if line == "" || strings.HasPrefix(line, "Updated:") {
+			continue
+		}
+		return line
+	}
+	return ""
+}
+
+// ParseHeadline returns the human-facing headline from cleaned text.
+func ParseHeadline(text string) string {
+	return ParseProductTitle(text)
+}
+
+// ExtractSummary returns text under the ...SUMMARY... section through the next
+// SPC section heading.
+func ExtractSummary(text string) string {
+	lines := strings.Split(normalizeNewlines(text), "\n")
+	start := -1
+	for i, line := range lines {
+		if strings.EqualFold(strings.TrimSpace(line), "...SUMMARY...") {
+			start = i + 1
+			break
+		}
+	}
+	if start < 0 {
+		return ""
+	}
+
+	var out []string
+	for _, line := range lines[start:] {
+		if sectionRE.MatchString(line) {
+			break
+		}
+		out = append(out, line)
+	}
+	return trimBlankLines(strings.Join(out, "\n"))
+}
+
+func cleanHTMLText(raw string) string {
+	raw = scriptBlockRE.ReplaceAllString(raw, "")
+	raw = tagRE.ReplaceAllString(raw, "")
+	raw = html.UnescapeString(raw)
+	raw = normalizeNewlines(raw)
+	return trimBlankLines(raw)
+}
+
+func normalizeNewlines(text string) string {
+	text = strings.ReplaceAll(text, "\r\n", "\n")
+	text = strings.ReplaceAll(text, "\r", "\n")
+	return text
+}
+
+func trimBlankLines(text string) string {
+	lines := strings.Split(normalizeNewlines(text), "\n")
+	start := 0
+	for start < len(lines) && strings.TrimSpace(lines[start]) == "" {
+		start++
+	}
+	end := len(lines)
+	for end > start && strings.TrimSpace(lines[end-1]) == "" {
+		end--
+	}
+	return strings.Join(lines[start:end], "\n")
+}
+
+func parseUpdatedValue(value string) *time.Time {
+	value = strings.TrimSpace(value)
+	if value == "" {
+		return nil
+	}
+	if t := parseOptionalISOTimestamp(value); t != nil {
+		return t
+	}
+	for _, layout := range []string{
+		"1504 UTC Mon Jan 2 2006",
+		"1504Z Mon Jan 2 2006",
+		"3:04 PM UTC Mon Jan 2 2006",
+		time.RFC1123,
+		time.RFC1123Z,
+	} {
+		t, err := time.Parse(layout, value)
+		if err == nil {
+			tt := t.UTC()
+			return &tt
+		}
+	}
+	return nil
+}