Add internal Markdown parsing

2026-10-07 23:07:50 +00:00
parent f80970a9d4
commit b3b959ce77
9 changed files with 359 additions and 8 deletions
--- a/docs/internal/overview.md
+++ b/docs/internal/overview.md
@@ -25,9 +25,9 @@ Go declarations and GoDoc remain the owners of exact exported contracts.
 ## Module and dependencies

 The module is `gitea.maximumdirect.net/eric/ReleaseKit`, with Go language
-version `1.25.0`. `go.mod` has no required third-party modules; packages use the
-standard library and first-party ReleaseKit packages. `go list ./...` reports
-these implemented packages:
+version `1.25.0`. `go.mod` requires `golang.org/x/mod` and Goldmark for internal
+validation; public facades do not expose either dependency. `go list ./...`
+reports these implemented packages:

 | Package | Responsibility |
 | --- | --- |
@@ -46,7 +46,7 @@ these implemented packages:
 | `internal/tempdir` | Disposable system temporary storage outside source worktrees. |
 | `internal/process` | Cancellable child-process execution with per-command environment overlays and removals. |
 | `internal/preparation` | Coordination for expected-commit tag checkout and Go module dependency preparation. |
-| `internal/validation` | Shared Go validation with optional race, full-vet, and static package-compilation target selections, formatting, workspace-file and filesystem-replacement checks, per-blob size limits, structured failures, and lazy committed-file access for captured candidates; `internal/candidate` retains capture, callback sequencing, and local-state rechecks. |
+| `internal/validation` | Shared Go validation with optional race, full-vet, and static package-compilation target selections, formatting, workspace-file and filesystem-replacement checks, per-blob size limits, internal Markdown parsing and per-document anchor caching, structured failures, and lazy committed-file access for captured candidates; `internal/candidate` retains capture, callback sequencing, and local-state rechecks. |
 | `internal/publish` | Publication guards, one validation run, annotated tag creation, push, and recovery state. |
 | `internal/runtimeinfo` | Private version lookup used by the runtime facade. |
 | `internal/versioncontract` | Shared stable-version rules and linker-symbol identity. |
@@ -79,6 +79,7 @@ not a separate release matrix:
 | --- | --- | --- |
 | Runtime version lookup and injected application version | `buildinfo`, `internal/runtimeinfo`, `internal/versioncontract`, `internal/build` | `TestVersionInConsumerExecutable`, `TestVersion` |
 | Candidate capture, shared validation, public diagnostics, additional checks, and rechecks | `releasekit.Check`, `releasekit.Publish`, `internal/candidate`, `internal/validation`, `cmd/releasekit/internal/cli` | `TestRunUsesOneSuiteWithSelectedRaceSetting`, `TestRunReportsRacePrerequisiteFailureWithoutFallback`, `TestRunExecutesRaceEnabledSuiteOnHost`, `TestRunCompilesTargetsAfterTestAndVetInSelectionOrder`, `TestRunAttributesTargetCompileFailureAndStops`, `TestRunCompilesSourceOnlyAndExecutablePackagesWithoutArtifacts`, `TestCandidateRequestMapsGoValidation`, `TestLoadSettingsReadsGoValidation`, `TestCheckUsesSelectedRaceSuiteBeforeAdditionalChecks`, `TestCheckCapturesCandidateAndCallsAdditionalChecksOnce`, `TestCheckReturnsPublicValidationError`, `TestPublishPushesOnlyTheCapturedAnnotatedTag`, `TestPublishFailedStandardSuiteSkipsAdditionalChecksAndTagCreation`, `TestPublishAdditionalCheckFailurePreservesCauseAndPreventsTagCreation`, `TestCheckCommandBindsConfiguredChecksToCapturedCandidate`, `TestPublishCommandPublishesConfiguredCandidate`, `TestCandidateCommandsPresentValidationDiagnosticsOnce`, `TestDeliverPageOnlyCreatesAndReusesRelease` |
+| Internal Markdown syntax records, source use sites, heading anchors, and candidate-document cache | `internal/validation` | `TestParseMarkdownLinksAndSourceLocations`, `TestParseMarkdownDecodesEntitiesInDestinations`, `TestParseMarkdownHeadingAnchors`, `TestMarkdownCacheKeepsParsedDocumentsPerCandidatePath` |
 | Shared Go settings for validation and builds | `internal/gotool`, `internal/process`, `internal/candidate`, `internal/build` | `TestDefaultValidationDoesNotInheritGOFLAGSRunFilter`, `TestNativeAndStaticCommandsUseOwnedBaseline`, `TestPersistentSettingsPreserveCachesAndNativeCTools`, `TestBuildIgnoresAmbientAndPersistentArchitectureTuning` |
 | Authoritative endpoint and exact annotated tag publication | `releasekit.Publish`, `internal/publish`, `internal/git` | `TestPublishPushesOnlyTheCapturedAnnotatedTag`, `TestPublishRejectsUnexpectedPushURLBeforeValidation`, `TestPublishComparesExpectedURLWithResolvedInsteadOfDestination`, `TestPublishExpectedPushURLUsesPushURLWhenFetchAndPushDiffer`, `TestCheckIgnoresExpectedPushURL`, `TestResolveUsesPushURLAndQueriesWithoutMutation` |
 | Publication partial and uncertain recovery | `internal/publish`, `internal/git` | `TestPublishRejectedPushKeepsLocalTagAndReportsNotPublished`, `TestPublishLostAcknowledgmentConfirmsExactRemoteObject`, `TestPublishLostAcknowledgmentWithChangedEndpointRemainsUnknown`, `TestPublishUnavailableReconciliationRemainsUnknownAndPreservesPushError` |
--- a/docs/internal/validation.md
+++ b/docs/internal/validation.md
@@ -45,6 +45,15 @@ matching, then returns the first oversized path with `FileSizeMeasurement`.
 lazily, and resolves destinations without consulting the worktree or
 neighboring files.

+The internal Markdown parser uses Goldmark to record inline, reference, and
+autolinks, images, titles, and their source use sites, and to build heading
+anchors from visible AST text. It supports ATX and Setext headings, decodes
+Markdown text through the parser, and assigns collision-free numeric suffixes
+to repeated anchors. Link locations use one-based line and byte-column values.
+A per-path cache retains the parsed records and anchors for each candidate
+document. The parser remains internal; destination resolution and check selection
+are tracked in the [implementation plan](../roadmap/implementation.md).
+
 Committed-tree metadata and blob reads disable lazy fetching and Git transports
 for those child commands. Missing local objects produce inspection errors;
 source preparation retains its separate network-capable operations. See Git's
@@ -117,6 +126,6 @@ symlink expansion detects cycles, and traversal honors cancellation. Absolute
 destinations and symlink targets within the repository directory map back to
 candidate-relative paths; existence still comes from committed entries.

-The [Markdown roadmap](../roadmap/v0.2.0.md#8-offer-offline-markdown-link-validation)
-owns proposed link syntax, anchors, diagnostics, and boundary policy. This
-foundation provides committed-file and destination-resolution primitives.
+The [implementation plan](../roadmap/implementation.md) owns Markdown
+destination checking, diagnostics, and boundary policy. The committed-file and
+destination-resolution primitives are available for that later integration.
--- a/docs/oss/attribution.md
+++ b/docs/oss/attribution.md
@@ -12,6 +12,19 @@ License and notices: [BSD 3-Clause](licenses/BSD-3-Clause.txt) | [LICENSE](https

 [Source](https://go.googlesource.com/mod/+/d3398d06de5fa5c71083d3d1c26f2cda73508e0f).

+## Goldmark
+
+Selected module: `github.com/yuin/goldmark/v2` at `v2.1.6`.
+
+Copyright (c) 2019 Yusuke Inuzuka.
+
+SPDX-License-Identifier: MIT
+
+License and notices: [MIT](licenses/MIT.txt) |
+[LICENSE](https://github.com/yuin/goldmark/blob/a8d4ee5a0bd7d5493fdd1e339b4e75df92f8684c/LICENSE).
+
+[Source](https://github.com/yuin/goldmark/tree/a8d4ee5a0bd7d5493fdd1e339b4e75df92f8684c).
+
 ## Standard Repository Policies

 Material: policy documents under `docs/policy/`, the root `AGENTS.md` orientation,
--- a/docs/oss/licenses/MIT.txt
+++ b/docs/oss/licenses/MIT.txt
@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2019 Yusuke Inuzuka
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
--- a/docs/roadmap/implementation.md
+++ b/docs/roadmap/implementation.md
@@ -411,6 +411,11 @@ omitted submodule sizes. Historical and aggregate size do not become release gat

 ## Stage 9: Implement internal Markdown parsing and anchors

+**Completion:** Complete. Goldmark now records parsed links, images, reference
+uses, use-site locations, and collision-free heading anchors behind an internal
+per-candidate document cache. Its behavior remains unselected until the later
+validation-integration stage.
+
 **Outcome:** A parser with source positions and documented anchor semantics is ready.

 1. Add `github.com/yuin/goldmark/v2 v2.1.6`, prepared dependencies, and applicable
@@ -422,6 +427,14 @@ omitted submodule sizes. Historical and aggregate size do not become release gat
 4. Compare fixtures with PromptKit's starting examples. Record intended refinements
   in the roadmap until the public behavior lands, avoiding a second regex parser.

+**Parser refinements:** PromptKit's starting script scans inline links, images,
+and ATX headings with regular expressions. The Goldmark parser also records
+resolved full, collapsed, and shortcut reference uses at each source use site.
+Heading anchors use decoded visible AST text, include inline code,
+exclude raw HTML tags, and support Setext headings. Parser-recognized CommonMark
+autolinks are recorded alongside inline and reference links. Repeated-heading
+suffixes skip identifiers already used by another heading.
+
 **Verification:** Inline/reference forms and images, escaped/parenthesized
 and angle-bracket destinations, optional titles, code fences/inline code,
 Unicode/entity headings, ATX/Setext headings, repeated headings and suffix
--- a/go.mod
+++ b/go.mod
@@ -2,4 +2,7 @@ module gitea.maximumdirect.net/eric/ReleaseKit

 go 1.25.0

-require golang.org/x/mod v0.40.0
+require (
+	github.com/yuin/goldmark/v2 v2.1.6
+	golang.org/x/mod v0.40.0
+)
--- a/go.sum
+++ b/go.sum
@@ -1,2 +1,4 @@
+github.com/yuin/goldmark/v2 v2.1.6 h1:igy0q9JcqovoFXFx/XjtU8sHVhcUHavPj2sA3EoKJUE=
+github.com/yuin/goldmark/v2 v2.1.6/go.mod h1:G6M4/qOFtfn01/o14BU1UR2Lo5N3S9Qo7xCuz/sHjGQ=
 golang.org/x/mod v0.40.0 h1:hUv+3cXcdRHz08UmSiOob7sadHig73uo5bkXxQ/tvUs=
 golang.org/x/mod v0.40.0/go.mod h1:0/weTWkPWGBikyTWAX3dkjVztMmBA5hM0DH6BElSupE=
--- a/internal/validation/markdown.go
+++ b/internal/validation/markdown.go
@@ -0,0 +1,171 @@
+package validation
+
+import (
+	"context"
+	"sort"
+	"strconv"
+	"strings"
+	"unicode"
+
+	"github.com/yuin/goldmark/v2/ast"
+	"github.com/yuin/goldmark/v2/parser"
+)
+
+type markdownLocation struct {
+	Line   int
+	Column int
+}
+
+type markdownLink struct {
+	Destination string
+	Title       string
+	Image       bool
+	Reference   bool
+	Location    markdownLocation
+}
+
+type markdownDocument struct {
+	links   []markdownLink
+	anchors map[string]struct{}
+}
+
+type markdownCache struct {
+	tree      *CandidateTree
+	documents map[string]*markdownDocument
+}
+
+func newMarkdownCache(tree *CandidateTree) *markdownCache {
+	return &markdownCache{
+		tree:      tree,
+		documents: make(map[string]*markdownDocument),
+	}
+}
+
+func (cache *markdownCache) document(ctx context.Context, path string) (*markdownDocument, error) {
+	if document, ok := cache.documents[path]; ok {
+		return document, nil
+	}
+	source, err := cache.tree.ReadFile(ctx, path)
+	if err != nil {
+		return nil, err
+	}
+	document := parseMarkdown(source)
+	cache.documents[path] = document
+	return document, nil
+}
+
+func parseMarkdown(source []byte) *markdownDocument {
+	root := parser.New().Parse(source)
+	document := &markdownDocument{
+		anchors: make(map[string]struct{}),
+	}
+	lineStarts := markdownLineStarts(source)
+	usedAnchors := make(map[string]struct{})
+	nextSuffix := make(map[string]int)
+	_ = ast.Walk(root, func(node ast.Node, entering bool) (ast.WalkStatus, error) {
+		if !entering {
+			return ast.WalkContinue, nil
+		}
+		switch node := node.(type) {
+		case *ast.Link:
+			document.links = append(document.links, markdownLink{
+				Destination: node.Destination.Value(source),
+				Title:       node.Title.Value(source),
+				Reference:   node.Reference != nil,
+				Location:    markdownLocationAt(lineStarts, node.Pos()),
+			})
+		case *ast.Image:
+			document.links = append(document.links, markdownLink{
+				Destination: node.Destination.Value(source),
+				Title:       node.Title.Value(source),
+				Image:       true,
+				Reference:   node.Reference != nil,
+				Location:    markdownLocationAt(lineStarts, node.Pos()),
+			})
+		case *ast.AutoLink:
+			document.links = append(document.links, markdownLink{
+				Destination: node.Destination.Value(source),
+				Location:    markdownLocationAt(lineStarts, node.Pos()),
+			})
+		case *ast.Heading:
+			base := markdownAnchor(markdownHeadingText(node, source))
+			anchor := base
+			if _, exists := usedAnchors[anchor]; exists {
+				suffix, ok := nextSuffix[base]
+				if !ok {
+					suffix = 1
+				}
+				for {
+					anchor = base + "-" + strconv.Itoa(suffix)
+					suffix++
+					if _, exists := usedAnchors[anchor]; !exists {
+						break
+					}
+				}
+				nextSuffix[base] = suffix
+			}
+			usedAnchors[anchor] = struct{}{}
+			document.anchors[anchor] = struct{}{}
+		}
+		return ast.WalkContinue, nil
+	})
+	return document
+}
+
+func markdownHeadingText(heading *ast.Heading, source []byte) string {
+	var result strings.Builder
+	var appendNode func(ast.Node)
+	appendNode = func(node ast.Node) {
+		switch node := node.(type) {
+		case *ast.RawHTML:
+			return
+		case *ast.Text:
+			result.WriteString(node.Value.Value(source))
+			if node.SoftLineBreak() || node.HardLineBreak() {
+				result.WriteByte(' ')
+			}
+			return
+		case *ast.CodeSpan:
+			result.WriteString(node.Value.Value(source))
+			return
+		}
+		for child := node.FirstChild(); child != nil; child = child.NextSibling() {
+			appendNode(child)
+		}
+	}
+	for child := heading.FirstChild(); child != nil; child = child.NextSibling() {
+		appendNode(child)
+	}
+	return result.String()
+}
+
+func markdownAnchor(heading string) string {
+	heading = strings.ToLower(heading)
+	var result strings.Builder
+	for _, r := range heading {
+		if unicode.IsLetter(r) || unicode.IsNumber(r) || r == '_' || r == '-' || r == ' ' {
+			result.WriteRune(r)
+		}
+	}
+	return strings.ReplaceAll(result.String(), " ", "-")
+}
+
+func markdownLineStarts(source []byte) []int {
+	starts := []int{0}
+	for offset, value := range source {
+		if value == '\n' {
+			starts = append(starts, offset+1)
+		}
+	}
+	return starts
+}
+
+func markdownLocationAt(lineStarts []int, offset int) markdownLocation {
+	if offset < 0 {
+		return markdownLocation{}
+	}
+	lineIndex := sort.Search(len(lineStarts), func(index int) bool {
+		return lineStarts[index] > offset
+	}) - 1
+	return markdownLocation{Line: lineIndex + 1, Column: offset - lineStarts[lineIndex] + 1}
+}
--- a/internal/validation/markdown_test.go
+++ b/internal/validation/markdown_test.go
@@ -0,0 +1,118 @@
+package validation
+
+import (
+	"context"
+	"strings"
+	"testing"
+)
+
+func TestParseMarkdownLinksAndSourceLocations(t *testing.T) {
+	source := []byte("" +
+		"é [inline](docs/a\\(b\\).md \"inline title\").\n" +
+		"![chart](<images/chart one.svg> 'image title')\n" +
+		"[ref][target] then [ref][target].\n" +
+		"[collapsed][] and [shortcut].\n" +
+		"![logo][badge]\n" +
+		"\n" +
+		"[target]: docs/target.md \"reference title\"\n" +
+		"[collapsed]: child.md\n" +
+		"[shortcut]: short.md\n" +
+		"[badge]: badge.png\n" +
+		"Inline example: `[fake](fake.md)`.\n" +
+		"```md\n[also fake](fenced.md)\n```\n" +
+		"[undefined]\n" +
+		"Autolink <https://example.invalid>.\n" +
+		"<a href=\"raw.html\">HTML</a>\n")
+	document := parseMarkdown(source)
+
+	want := []markdownLink{
+		{Destination: "docs/a(b).md", Title: "inline title", Location: markdownLocation{Line: 1, Column: 4}},
+		{Destination: "images/chart one.svg", Title: "image title", Image: true, Location: markdownLocation{Line: 2, Column: 1}},
+		{Destination: "docs/target.md", Title: "reference title", Reference: true, Location: markdownLocation{Line: 3, Column: 1}},
+		{Destination: "docs/target.md", Title: "reference title", Reference: true, Location: markdownLocation{Line: 3, Column: 20}},
+		{Destination: "child.md", Reference: true, Location: markdownLocation{Line: 4, Column: 1}},
+		{Destination: "short.md", Reference: true, Location: markdownLocation{Line: 4, Column: 19}},
+		{Destination: "badge.png", Image: true, Reference: true, Location: markdownLocation{Line: 5, Column: 1}},
+		{Destination: "https://example.invalid", Location: markdownLocation{Line: 16, Column: 10}},
+	}
+	if len(document.links) != len(want) {
+		t.Fatalf("found %d links, want %d: %#v", len(document.links), len(want), document.links)
+	}
+	for index := range want {
+		if got := document.links[index]; got != want[index] {
+			t.Errorf("link %d = %#v, want %#v", index, got, want[index])
+		}
+	}
+}
+
+func TestParseMarkdownDecodesEntitiesInDestinations(t *testing.T) {
+	document := parseMarkdown([]byte("[entity](<a&amp;b.md>)\n"))
+	if len(document.links) != 1 || document.links[0].Destination != "a&b.md" {
+		t.Fatalf("parsed links = %#v, want decoded destination a&b.md", document.links)
+	}
+}
+
+func TestParseMarkdownHeadingAnchors(t *testing.T) {
+	source := []byte("" +
+		"# Caf&eacute; &amp; 東京 `Code Here` <em>Bold</em>!\n" +
+		"Setext **Heading**\n" +
+		"---\n" +
+		"# Foo\n" +
+		"# Foo\n" +
+		"# Foo-1\n" +
+		"# Foo\n" +
+		"# Foo-1\n")
+	document := parseMarkdown(source)
+	for _, anchor := range []string{
+		"café--東京-code-here-bold",
+		"setext-heading",
+		"foo",
+		"foo-1",
+		"foo-1-1",
+		"foo-2",
+		"foo-1-2",
+	} {
+		if _, ok := document.anchors[anchor]; !ok {
+			t.Errorf("missing anchor %q in %#v", anchor, document.anchors)
+		}
+	}
+	if len(document.anchors) != 7 {
+		t.Fatalf("found %d anchors, want 7: %#v", len(document.anchors), document.anchors)
+	}
+}
+
+func TestMarkdownCacheKeepsParsedDocumentsPerCandidatePath(t *testing.T) {
+	repository := newTreeRepository(t)
+	writeTreeFile(t, repository, "guide.md", []byte("# Guide\n[other](other.md)\n"))
+	writeTreeFile(t, repository, "other.md", []byte("# Other\n"))
+	gitTree(t, repository, "add", "guide.md", "other.md")
+	gitTree(t, repository, "commit", "-m", "add markdown files")
+
+	commit := strings.TrimSpace(gitTree(t, repository, "rev-parse", "HEAD"))
+	tree, err := ReadCandidateTree(context.Background(), repository, commit)
+	if err != nil {
+		t.Fatal(err)
+	}
+	cache := newMarkdownCache(tree)
+	first, err := cache.document(context.Background(), "guide.md")
+	if err != nil {
+		t.Fatal(err)
+	}
+	second, err := cache.document(context.Background(), "guide.md")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if first != second {
+		t.Fatal("repeated document lookup returned a different parsed document")
+	}
+	other, err := cache.document(context.Background(), "other.md")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if other == first {
+		t.Fatal("different candidate paths shared a parsed document")
+	}
+	if _, ok := other.anchors["other"]; !ok {
+		t.Fatalf("other document anchors = %#v", other.anchors)
+	}
+}