package spc import ( "fmt" "html" "regexp" "strings" "time" ) var ( scriptBlockRE = regexp.MustCompile(`(?is)`) preBlockRE = regexp.MustCompile(`(?is)
]*>(.*?)`) tagRE = regexp.MustCompile(`(?is)<[^>]+>`) updatedRE = regexp.MustCompile(`(?im)^\s*Updated:\s*(.+?)\s*$`) sectionRE = regexp.MustCompile(`^\s*\.\.\.[A-Z0-9 /-]+\.{3}\s*$`) ) // DiscussionText contains parsed text from an SPC print page. type DiscussionText struct { ProductTitle string Headline string Summary string Discussion string UpdatedAt *time.Time } // ExtractProductText extracts and cleans the first useful preformatted SPC // product text block from a print page. func ExtractProductText(rawHTML string) (string, error) { matches := preBlockRE.FindAllStringSubmatch(rawHTML, -1) for _, match := range matches { if len(match) < 2 { continue } text := cleanHTMLText(match[1]) if strings.TrimSpace(text) != "" { return text, nil } } return "", fmt.Errorf("no useful pre block found") } // ParseDiscussionText extracts common SPC narrative metadata from cleaned // product text. func ParseDiscussionText(text string) DiscussionText { text = trimBlankLines(normalizeNewlines(text)) title := ParseProductTitle(text) return DiscussionText{ ProductTitle: title, Headline: title, Summary: ExtractSummary(text), Discussion: text, UpdatedAt: ParseUpdatedTimestamp(text), } } // ParseUpdatedTimestamp parses an SPC print-page Updated line when present. func ParseUpdatedTimestamp(text string) *time.Time { match := updatedRE.FindStringSubmatch(normalizeNewlines(text)) if len(match) != 2 { return nil } return parseUpdatedValue(match[1]) } // ParseProductTitle returns the first non-empty product line from cleaned text. func ParseProductTitle(text string) string { for _, line := range strings.Split(normalizeNewlines(text), "\n") { line = strings.TrimSpace(line) if line == "" || strings.HasPrefix(line, "Updated:") { continue } return line } return "" } // ParseHeadline returns the human-facing headline from cleaned text. func ParseHeadline(text string) string { return ParseProductTitle(text) } // ExtractSummary returns text under the ...SUMMARY... section through the next // SPC section heading. func ExtractSummary(text string) string { lines := strings.Split(normalizeNewlines(text), "\n") start := -1 for i, line := range lines { if strings.EqualFold(strings.TrimSpace(line), "...SUMMARY...") { start = i + 1 break } } if start < 0 { return "" } var out []string for _, line := range lines[start:] { if sectionRE.MatchString(line) { break } out = append(out, line) } return trimBlankLines(strings.Join(out, "\n")) } func cleanHTMLText(raw string) string { raw = scriptBlockRE.ReplaceAllString(raw, "") raw = tagRE.ReplaceAllString(raw, "") raw = html.UnescapeString(raw) raw = normalizeNewlines(raw) return trimBlankLines(raw) } func normalizeNewlines(text string) string { text = strings.ReplaceAll(text, "\r\n", "\n") text = strings.ReplaceAll(text, "\r", "\n") return text } func trimBlankLines(text string) string { lines := strings.Split(normalizeNewlines(text), "\n") start := 0 for start < len(lines) && strings.TrimSpace(lines[start]) == "" { start++ } end := len(lines) for end > start && strings.TrimSpace(lines[end-1]) == "" { end-- } return strings.Join(lines[start:end], "\n") } func parseUpdatedValue(value string) *time.Time { value = strings.TrimSpace(value) if value == "" { return nil } if t := parseOptionalISOTimestamp(value); t != nil { return t } for _, layout := range []string{ "1504 UTC Mon Jan 2 2006", "1504Z Mon Jan 2 2006", "3:04 PM UTC Mon Jan 2 2006", time.RFC1123, time.RFC1123Z, } { t, err := time.Parse(layout, value) if err == nil { tt := t.UTC() return &tt } } return nil }