Add internal Markdown parsing
This commit is contained in:
171
internal/validation/markdown.go
Normal file
171
internal/validation/markdown.go
Normal file
@@ -0,0 +1,171 @@
|
||||
package validation
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"unicode"
|
||||
|
||||
"github.com/yuin/goldmark/v2/ast"
|
||||
"github.com/yuin/goldmark/v2/parser"
|
||||
)
|
||||
|
||||
type markdownLocation struct {
|
||||
Line int
|
||||
Column int
|
||||
}
|
||||
|
||||
type markdownLink struct {
|
||||
Destination string
|
||||
Title string
|
||||
Image bool
|
||||
Reference bool
|
||||
Location markdownLocation
|
||||
}
|
||||
|
||||
type markdownDocument struct {
|
||||
links []markdownLink
|
||||
anchors map[string]struct{}
|
||||
}
|
||||
|
||||
type markdownCache struct {
|
||||
tree *CandidateTree
|
||||
documents map[string]*markdownDocument
|
||||
}
|
||||
|
||||
func newMarkdownCache(tree *CandidateTree) *markdownCache {
|
||||
return &markdownCache{
|
||||
tree: tree,
|
||||
documents: make(map[string]*markdownDocument),
|
||||
}
|
||||
}
|
||||
|
||||
func (cache *markdownCache) document(ctx context.Context, path string) (*markdownDocument, error) {
|
||||
if document, ok := cache.documents[path]; ok {
|
||||
return document, nil
|
||||
}
|
||||
source, err := cache.tree.ReadFile(ctx, path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
document := parseMarkdown(source)
|
||||
cache.documents[path] = document
|
||||
return document, nil
|
||||
}
|
||||
|
||||
func parseMarkdown(source []byte) *markdownDocument {
|
||||
root := parser.New().Parse(source)
|
||||
document := &markdownDocument{
|
||||
anchors: make(map[string]struct{}),
|
||||
}
|
||||
lineStarts := markdownLineStarts(source)
|
||||
usedAnchors := make(map[string]struct{})
|
||||
nextSuffix := make(map[string]int)
|
||||
_ = ast.Walk(root, func(node ast.Node, entering bool) (ast.WalkStatus, error) {
|
||||
if !entering {
|
||||
return ast.WalkContinue, nil
|
||||
}
|
||||
switch node := node.(type) {
|
||||
case *ast.Link:
|
||||
document.links = append(document.links, markdownLink{
|
||||
Destination: node.Destination.Value(source),
|
||||
Title: node.Title.Value(source),
|
||||
Reference: node.Reference != nil,
|
||||
Location: markdownLocationAt(lineStarts, node.Pos()),
|
||||
})
|
||||
case *ast.Image:
|
||||
document.links = append(document.links, markdownLink{
|
||||
Destination: node.Destination.Value(source),
|
||||
Title: node.Title.Value(source),
|
||||
Image: true,
|
||||
Reference: node.Reference != nil,
|
||||
Location: markdownLocationAt(lineStarts, node.Pos()),
|
||||
})
|
||||
case *ast.AutoLink:
|
||||
document.links = append(document.links, markdownLink{
|
||||
Destination: node.Destination.Value(source),
|
||||
Location: markdownLocationAt(lineStarts, node.Pos()),
|
||||
})
|
||||
case *ast.Heading:
|
||||
base := markdownAnchor(markdownHeadingText(node, source))
|
||||
anchor := base
|
||||
if _, exists := usedAnchors[anchor]; exists {
|
||||
suffix, ok := nextSuffix[base]
|
||||
if !ok {
|
||||
suffix = 1
|
||||
}
|
||||
for {
|
||||
anchor = base + "-" + strconv.Itoa(suffix)
|
||||
suffix++
|
||||
if _, exists := usedAnchors[anchor]; !exists {
|
||||
break
|
||||
}
|
||||
}
|
||||
nextSuffix[base] = suffix
|
||||
}
|
||||
usedAnchors[anchor] = struct{}{}
|
||||
document.anchors[anchor] = struct{}{}
|
||||
}
|
||||
return ast.WalkContinue, nil
|
||||
})
|
||||
return document
|
||||
}
|
||||
|
||||
func markdownHeadingText(heading *ast.Heading, source []byte) string {
|
||||
var result strings.Builder
|
||||
var appendNode func(ast.Node)
|
||||
appendNode = func(node ast.Node) {
|
||||
switch node := node.(type) {
|
||||
case *ast.RawHTML:
|
||||
return
|
||||
case *ast.Text:
|
||||
result.WriteString(node.Value.Value(source))
|
||||
if node.SoftLineBreak() || node.HardLineBreak() {
|
||||
result.WriteByte(' ')
|
||||
}
|
||||
return
|
||||
case *ast.CodeSpan:
|
||||
result.WriteString(node.Value.Value(source))
|
||||
return
|
||||
}
|
||||
for child := node.FirstChild(); child != nil; child = child.NextSibling() {
|
||||
appendNode(child)
|
||||
}
|
||||
}
|
||||
for child := heading.FirstChild(); child != nil; child = child.NextSibling() {
|
||||
appendNode(child)
|
||||
}
|
||||
return result.String()
|
||||
}
|
||||
|
||||
func markdownAnchor(heading string) string {
|
||||
heading = strings.ToLower(heading)
|
||||
var result strings.Builder
|
||||
for _, r := range heading {
|
||||
if unicode.IsLetter(r) || unicode.IsNumber(r) || r == '_' || r == '-' || r == ' ' {
|
||||
result.WriteRune(r)
|
||||
}
|
||||
}
|
||||
return strings.ReplaceAll(result.String(), " ", "-")
|
||||
}
|
||||
|
||||
func markdownLineStarts(source []byte) []int {
|
||||
starts := []int{0}
|
||||
for offset, value := range source {
|
||||
if value == '\n' {
|
||||
starts = append(starts, offset+1)
|
||||
}
|
||||
}
|
||||
return starts
|
||||
}
|
||||
|
||||
func markdownLocationAt(lineStarts []int, offset int) markdownLocation {
|
||||
if offset < 0 {
|
||||
return markdownLocation{}
|
||||
}
|
||||
lineIndex := sort.Search(len(lineStarts), func(index int) bool {
|
||||
return lineStarts[index] > offset
|
||||
}) - 1
|
||||
return markdownLocation{Line: lineIndex + 1, Column: offset - lineStarts[lineIndex] + 1}
|
||||
}
|
||||
118
internal/validation/markdown_test.go
Normal file
118
internal/validation/markdown_test.go
Normal file
@@ -0,0 +1,118 @@
|
||||
package validation
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestParseMarkdownLinksAndSourceLocations(t *testing.T) {
|
||||
source := []byte("" +
|
||||
"é [inline](docs/a\\(b\\).md \"inline title\").\n" +
|
||||
"\n" +
|
||||
"[ref][target] then [ref][target].\n" +
|
||||
"[collapsed][] and [shortcut].\n" +
|
||||
"![logo][badge]\n" +
|
||||
"\n" +
|
||||
"[target]: docs/target.md \"reference title\"\n" +
|
||||
"[collapsed]: child.md\n" +
|
||||
"[shortcut]: short.md\n" +
|
||||
"[badge]: badge.png\n" +
|
||||
"Inline example: `[fake](fake.md)`.\n" +
|
||||
"```md\n[also fake](fenced.md)\n```\n" +
|
||||
"[undefined]\n" +
|
||||
"Autolink <https://example.invalid>.\n" +
|
||||
"<a href=\"raw.html\">HTML</a>\n")
|
||||
document := parseMarkdown(source)
|
||||
|
||||
want := []markdownLink{
|
||||
{Destination: "docs/a(b).md", Title: "inline title", Location: markdownLocation{Line: 1, Column: 4}},
|
||||
{Destination: "images/chart one.svg", Title: "image title", Image: true, Location: markdownLocation{Line: 2, Column: 1}},
|
||||
{Destination: "docs/target.md", Title: "reference title", Reference: true, Location: markdownLocation{Line: 3, Column: 1}},
|
||||
{Destination: "docs/target.md", Title: "reference title", Reference: true, Location: markdownLocation{Line: 3, Column: 20}},
|
||||
{Destination: "child.md", Reference: true, Location: markdownLocation{Line: 4, Column: 1}},
|
||||
{Destination: "short.md", Reference: true, Location: markdownLocation{Line: 4, Column: 19}},
|
||||
{Destination: "badge.png", Image: true, Reference: true, Location: markdownLocation{Line: 5, Column: 1}},
|
||||
{Destination: "https://example.invalid", Location: markdownLocation{Line: 16, Column: 10}},
|
||||
}
|
||||
if len(document.links) != len(want) {
|
||||
t.Fatalf("found %d links, want %d: %#v", len(document.links), len(want), document.links)
|
||||
}
|
||||
for index := range want {
|
||||
if got := document.links[index]; got != want[index] {
|
||||
t.Errorf("link %d = %#v, want %#v", index, got, want[index])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseMarkdownDecodesEntitiesInDestinations(t *testing.T) {
|
||||
document := parseMarkdown([]byte("[entity](<a&b.md>)\n"))
|
||||
if len(document.links) != 1 || document.links[0].Destination != "a&b.md" {
|
||||
t.Fatalf("parsed links = %#v, want decoded destination a&b.md", document.links)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseMarkdownHeadingAnchors(t *testing.T) {
|
||||
source := []byte("" +
|
||||
"# Café & 東京 `Code Here` <em>Bold</em>!\n" +
|
||||
"Setext **Heading**\n" +
|
||||
"---\n" +
|
||||
"# Foo\n" +
|
||||
"# Foo\n" +
|
||||
"# Foo-1\n" +
|
||||
"# Foo\n" +
|
||||
"# Foo-1\n")
|
||||
document := parseMarkdown(source)
|
||||
for _, anchor := range []string{
|
||||
"café--東京-code-here-bold",
|
||||
"setext-heading",
|
||||
"foo",
|
||||
"foo-1",
|
||||
"foo-1-1",
|
||||
"foo-2",
|
||||
"foo-1-2",
|
||||
} {
|
||||
if _, ok := document.anchors[anchor]; !ok {
|
||||
t.Errorf("missing anchor %q in %#v", anchor, document.anchors)
|
||||
}
|
||||
}
|
||||
if len(document.anchors) != 7 {
|
||||
t.Fatalf("found %d anchors, want 7: %#v", len(document.anchors), document.anchors)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMarkdownCacheKeepsParsedDocumentsPerCandidatePath(t *testing.T) {
|
||||
repository := newTreeRepository(t)
|
||||
writeTreeFile(t, repository, "guide.md", []byte("# Guide\n[other](other.md)\n"))
|
||||
writeTreeFile(t, repository, "other.md", []byte("# Other\n"))
|
||||
gitTree(t, repository, "add", "guide.md", "other.md")
|
||||
gitTree(t, repository, "commit", "-m", "add markdown files")
|
||||
|
||||
commit := strings.TrimSpace(gitTree(t, repository, "rev-parse", "HEAD"))
|
||||
tree, err := ReadCandidateTree(context.Background(), repository, commit)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cache := newMarkdownCache(tree)
|
||||
first, err := cache.document(context.Background(), "guide.md")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
second, err := cache.document(context.Background(), "guide.md")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if first != second {
|
||||
t.Fatal("repeated document lookup returned a different parsed document")
|
||||
}
|
||||
other, err := cache.document(context.Background(), "other.md")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if other == first {
|
||||
t.Fatal("different candidate paths shared a parsed document")
|
||||
}
|
||||
if _, ok := other.anchors["other"]; !ok {
|
||||
t.Fatalf("other document anchors = %#v", other.anchors)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user