package spc import ( "fmt" "html" "regexp" "strings" "time" ) var ( scriptBlockRE = regexp.MustCompile(`(?is)`) preBlockRE = regexp.MustCompile(`(?is)
]*>(.*?)`) tagRE = regexp.MustCompile(`(?is)<[^>]+>`) updatedRE = regexp.MustCompile(`(?im)^\s*Updated:\s*(.+?)\s*$`) pageUpdatedRE = regexp.MustCompile(`(?i)\bUpdated:\s*((?:\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z)|(?:[A-Z][a-z]{2}\s+[A-Z][a-z]{2}\s+\d{1,2}\s+\d{2}:\d{2}:\d{2}\s+UTC\s+\d{4})|(?:\d{4}\s+UTC\s+[A-Z][a-z]{2}\s+[A-Z][a-z]{2}\s+\d{1,2}\s+\d{4})|(?:\d{4}Z\s+[A-Z][a-z]{2}\s+[A-Z][a-z]{2}\s+\d{1,2}\s+\d{4}))`) productCodeRE = regexp.MustCompile(`(?i)^SPC\s+AC\s+\d+\s*$`) sectionRE = regexp.MustCompile(`^\s*\.\.\.[A-Z0-9 /-]+\.{3}\s*$`) ) // DiscussionText contains parsed text from an SPC print page. type DiscussionText struct { ProductTitle string Headline string Summary string Discussion string UpdatedAt *time.Time } // ExtractProductText extracts and cleans the first useful preformatted SPC // product text block from a print page. func ExtractProductText(rawHTML string) (string, error) { matches := preBlockRE.FindAllStringSubmatch(rawHTML, -1) for _, match := range matches { if len(match) < 2 { continue } text := cleanHTMLText(match[1]) if strings.TrimSpace(text) != "" { return text, nil } } return "", fmt.Errorf("no useful pre block found") } // ParseDiscussionHTML extracts SPC product text and page-level metadata from a // print-page HTML document. func ParseDiscussionHTML(rawHTML string) (DiscussionText, error) { text, err := ExtractProductText(rawHTML) if err != nil { return DiscussionText{}, err } parsed := ParseDiscussionText(text) if updatedAt := ParsePageUpdatedTimestamp(rawHTML); updatedAt != nil { parsed.UpdatedAt = updatedAt } return parsed, nil } // ParseDiscussionText extracts common SPC narrative metadata from cleaned // product text. func ParseDiscussionText(text string) DiscussionText { text = trimBlankLines(normalizeNewlines(text)) title := ParseProductTitle(text) return DiscussionText{ ProductTitle: title, Headline: title, Summary: ExtractSummary(text), Discussion: text, UpdatedAt: ParseUpdatedTimestamp(text), } } // ParsePageUpdatedTimestamp parses the page-level Updated row from an SPC print // page. SPC currently places this outside the product
block.
func ParsePageUpdatedTimestamp(rawHTML string) *time.Time {
text := cleanHTMLText(rawHTML)
text = strings.ReplaceAll(text, "\u00a0", " ")
text = strings.Join(strings.Fields(text), " ")
match := pageUpdatedRE.FindStringSubmatch(text)
if len(match) != 2 {
return nil
}
return parseUpdatedValue(match[1])
}
// ParseUpdatedTimestamp parses an SPC print-page Updated line when present.
func ParseUpdatedTimestamp(text string) *time.Time {
match := updatedRE.FindStringSubmatch(normalizeNewlines(text))
if len(match) != 2 {
return nil
}
return parseUpdatedValue(match[1])
}
// ParseProductTitle returns the first non-empty product line from cleaned text.
func ParseProductTitle(text string) string {
for _, line := range strings.Split(normalizeNewlines(text), "\n") {
line = strings.TrimSpace(line)
if line == "" || strings.HasPrefix(line, "Updated:") || productCodeRE.MatchString(line) {
continue
}
return line
}
return ""
}
// ParseHeadline returns the human-facing headline from cleaned text.
func ParseHeadline(text string) string {
return ParseProductTitle(text)
}
// ExtractSummary returns text under the ...SUMMARY... section through the next
// SPC section heading.
func ExtractSummary(text string) string {
lines := strings.Split(normalizeNewlines(text), "\n")
start := -1
for i, line := range lines {
if strings.EqualFold(strings.TrimSpace(line), "...SUMMARY...") {
start = i + 1
break
}
}
if start < 0 {
return ""
}
var out []string
for _, line := range lines[start:] {
if sectionRE.MatchString(line) {
break
}
out = append(out, line)
}
return trimBlankLines(strings.Join(out, "\n"))
}
func cleanHTMLText(raw string) string {
raw = scriptBlockRE.ReplaceAllString(raw, "")
raw = tagRE.ReplaceAllString(raw, "")
raw = html.UnescapeString(raw)
raw = normalizeNewlines(raw)
return trimBlankLines(raw)
}
func normalizeNewlines(text string) string {
text = strings.ReplaceAll(text, "\r\n", "\n")
text = strings.ReplaceAll(text, "\r", "\n")
return text
}
func trimBlankLines(text string) string {
lines := strings.Split(normalizeNewlines(text), "\n")
start := 0
for start < len(lines) && strings.TrimSpace(lines[start]) == "" {
start++
}
end := len(lines)
for end > start && strings.TrimSpace(lines[end-1]) == "" {
end--
}
return strings.Join(lines[start:end], "\n")
}
func parseUpdatedValue(value string) *time.Time {
value = strings.TrimSpace(value)
if value == "" {
return nil
}
if t := parseOptionalISOTimestamp(value); t != nil {
return t
}
for _, layout := range []string{
"Mon Jan 2 15:04:05 UTC 2006",
"1504 UTC Mon Jan 2 2006",
"1504Z Mon Jan 2 2006",
"3:04 PM UTC Mon Jan 2 2006",
time.RFC1123,
time.RFC1123Z,
} {
t, err := time.Parse(layout, value)
if err == nil {
tt := t.UTC()
return &tt
}
}
return nil
}