diff --git a/docs/internal/overview.md b/docs/internal/overview.md index 274761c..000a425 100644 --- a/docs/internal/overview.md +++ b/docs/internal/overview.md @@ -25,9 +25,9 @@ Go declarations and GoDoc remain the owners of exact exported contracts. ## Module and dependencies The module is `gitea.maximumdirect.net/eric/ReleaseKit`, with Go language -version `1.25.0`. `go.mod` has no required third-party modules; packages use the -standard library and first-party ReleaseKit packages. `go list ./...` reports -these implemented packages: +version `1.25.0`. `go.mod` requires `golang.org/x/mod` and Goldmark for internal +validation; public facades do not expose either dependency. `go list ./...` +reports these implemented packages: | Package | Responsibility | | --- | --- | @@ -46,7 +46,7 @@ these implemented packages: | `internal/tempdir` | Disposable system temporary storage outside source worktrees. | | `internal/process` | Cancellable child-process execution with per-command environment overlays and removals. | | `internal/preparation` | Coordination for expected-commit tag checkout and Go module dependency preparation. | -| `internal/validation` | Shared Go validation with optional race, full-vet, and static package-compilation target selections, formatting, workspace-file and filesystem-replacement checks, per-blob size limits, structured failures, and lazy committed-file access for captured candidates; `internal/candidate` retains capture, callback sequencing, and local-state rechecks. | +| `internal/validation` | Shared Go validation with optional race, full-vet, and static package-compilation target selections, formatting, workspace-file and filesystem-replacement checks, per-blob size limits, internal Markdown parsing and per-document anchor caching, structured failures, and lazy committed-file access for captured candidates; `internal/candidate` retains capture, callback sequencing, and local-state rechecks. | | `internal/publish` | Publication guards, one validation run, annotated tag creation, push, and recovery state. | | `internal/runtimeinfo` | Private version lookup used by the runtime facade. | | `internal/versioncontract` | Shared stable-version rules and linker-symbol identity. | @@ -79,6 +79,7 @@ not a separate release matrix: | --- | --- | --- | | Runtime version lookup and injected application version | `buildinfo`, `internal/runtimeinfo`, `internal/versioncontract`, `internal/build` | `TestVersionInConsumerExecutable`, `TestVersion` | | Candidate capture, shared validation, public diagnostics, additional checks, and rechecks | `releasekit.Check`, `releasekit.Publish`, `internal/candidate`, `internal/validation`, `cmd/releasekit/internal/cli` | `TestRunUsesOneSuiteWithSelectedRaceSetting`, `TestRunReportsRacePrerequisiteFailureWithoutFallback`, `TestRunExecutesRaceEnabledSuiteOnHost`, `TestRunCompilesTargetsAfterTestAndVetInSelectionOrder`, `TestRunAttributesTargetCompileFailureAndStops`, `TestRunCompilesSourceOnlyAndExecutablePackagesWithoutArtifacts`, `TestCandidateRequestMapsGoValidation`, `TestLoadSettingsReadsGoValidation`, `TestCheckUsesSelectedRaceSuiteBeforeAdditionalChecks`, `TestCheckCapturesCandidateAndCallsAdditionalChecksOnce`, `TestCheckReturnsPublicValidationError`, `TestPublishPushesOnlyTheCapturedAnnotatedTag`, `TestPublishFailedStandardSuiteSkipsAdditionalChecksAndTagCreation`, `TestPublishAdditionalCheckFailurePreservesCauseAndPreventsTagCreation`, `TestCheckCommandBindsConfiguredChecksToCapturedCandidate`, `TestPublishCommandPublishesConfiguredCandidate`, `TestCandidateCommandsPresentValidationDiagnosticsOnce`, `TestDeliverPageOnlyCreatesAndReusesRelease` | +| Internal Markdown syntax records, source use sites, heading anchors, and candidate-document cache | `internal/validation` | `TestParseMarkdownLinksAndSourceLocations`, `TestParseMarkdownDecodesEntitiesInDestinations`, `TestParseMarkdownHeadingAnchors`, `TestMarkdownCacheKeepsParsedDocumentsPerCandidatePath` | | Shared Go settings for validation and builds | `internal/gotool`, `internal/process`, `internal/candidate`, `internal/build` | `TestDefaultValidationDoesNotInheritGOFLAGSRunFilter`, `TestNativeAndStaticCommandsUseOwnedBaseline`, `TestPersistentSettingsPreserveCachesAndNativeCTools`, `TestBuildIgnoresAmbientAndPersistentArchitectureTuning` | | Authoritative endpoint and exact annotated tag publication | `releasekit.Publish`, `internal/publish`, `internal/git` | `TestPublishPushesOnlyTheCapturedAnnotatedTag`, `TestPublishRejectsUnexpectedPushURLBeforeValidation`, `TestPublishComparesExpectedURLWithResolvedInsteadOfDestination`, `TestPublishExpectedPushURLUsesPushURLWhenFetchAndPushDiffer`, `TestCheckIgnoresExpectedPushURL`, `TestResolveUsesPushURLAndQueriesWithoutMutation` | | Publication partial and uncertain recovery | `internal/publish`, `internal/git` | `TestPublishRejectedPushKeepsLocalTagAndReportsNotPublished`, `TestPublishLostAcknowledgmentConfirmsExactRemoteObject`, `TestPublishLostAcknowledgmentWithChangedEndpointRemainsUnknown`, `TestPublishUnavailableReconciliationRemainsUnknownAndPreservesPushError` | diff --git a/docs/internal/validation.md b/docs/internal/validation.md index 392a518..1a47b0f 100644 --- a/docs/internal/validation.md +++ b/docs/internal/validation.md @@ -45,6 +45,15 @@ matching, then returns the first oversized path with `FileSizeMeasurement`. lazily, and resolves destinations without consulting the worktree or neighboring files. +The internal Markdown parser uses Goldmark to record inline, reference, and +autolinks, images, titles, and their source use sites, and to build heading +anchors from visible AST text. It supports ATX and Setext headings, decodes +Markdown text through the parser, and assigns collision-free numeric suffixes +to repeated anchors. Link locations use one-based line and byte-column values. +A per-path cache retains the parsed records and anchors for each candidate +document. The parser remains internal; destination resolution and check selection +are tracked in the [implementation plan](../roadmap/implementation.md). + Committed-tree metadata and blob reads disable lazy fetching and Git transports for those child commands. Missing local objects produce inspection errors; source preparation retains its separate network-capable operations. See Git's @@ -117,6 +126,6 @@ symlink expansion detects cycles, and traversal honors cancellation. Absolute destinations and symlink targets within the repository directory map back to candidate-relative paths; existence still comes from committed entries. -The [Markdown roadmap](../roadmap/v0.2.0.md#8-offer-offline-markdown-link-validation) -owns proposed link syntax, anchors, diagnostics, and boundary policy. This -foundation provides committed-file and destination-resolution primitives. +The [implementation plan](../roadmap/implementation.md) owns Markdown +destination checking, diagnostics, and boundary policy. The committed-file and +destination-resolution primitives are available for that later integration. diff --git a/docs/oss/attribution.md b/docs/oss/attribution.md index 0abc051..0b17d9a 100644 --- a/docs/oss/attribution.md +++ b/docs/oss/attribution.md @@ -12,6 +12,19 @@ License and notices: [BSD 3-Clause](licenses/BSD-3-Clause.txt) | [LICENSE](https [Source](https://go.googlesource.com/mod/+/d3398d06de5fa5c71083d3d1c26f2cda73508e0f). +## Goldmark + +Selected module: `github.com/yuin/goldmark/v2` at `v2.1.6`. + +Copyright (c) 2019 Yusuke Inuzuka. + +SPDX-License-Identifier: MIT + +License and notices: [MIT](licenses/MIT.txt) | +[LICENSE](https://github.com/yuin/goldmark/blob/a8d4ee5a0bd7d5493fdd1e339b4e75df92f8684c/LICENSE). + +[Source](https://github.com/yuin/goldmark/tree/a8d4ee5a0bd7d5493fdd1e339b4e75df92f8684c). + ## Standard Repository Policies Material: policy documents under `docs/policy/`, the root `AGENTS.md` orientation, diff --git a/docs/oss/licenses/MIT.txt b/docs/oss/licenses/MIT.txt new file mode 100644 index 0000000..dc5b2a6 --- /dev/null +++ b/docs/oss/licenses/MIT.txt @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2019 Yusuke Inuzuka + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index b57b3b0..14c25e2 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -411,6 +411,11 @@ omitted submodule sizes. Historical and aggregate size do not become release gat ## Stage 9: Implement internal Markdown parsing and anchors +**Completion:** Complete. Goldmark now records parsed links, images, reference +uses, use-site locations, and collision-free heading anchors behind an internal +per-candidate document cache. Its behavior remains unselected until the later +validation-integration stage. + **Outcome:** A parser with source positions and documented anchor semantics is ready. 1. Add `github.com/yuin/goldmark/v2 v2.1.6`, prepared dependencies, and applicable @@ -422,6 +427,14 @@ omitted submodule sizes. Historical and aggregate size do not become release gat 4. Compare fixtures with PromptKit's starting examples. Record intended refinements in the roadmap until the public behavior lands, avoiding a second regex parser. +**Parser refinements:** PromptKit's starting script scans inline links, images, +and ATX headings with regular expressions. The Goldmark parser also records +resolved full, collapsed, and shortcut reference uses at each source use site. +Heading anchors use decoded visible AST text, include inline code, +exclude raw HTML tags, and support Setext headings. Parser-recognized CommonMark +autolinks are recorded alongside inline and reference links. Repeated-heading +suffixes skip identifiers already used by another heading. + **Verification:** Inline/reference forms and images, escaped/parenthesized and angle-bracket destinations, optional titles, code fences/inline code, Unicode/entity headings, ATX/Setext headings, repeated headings and suffix diff --git a/go.mod b/go.mod index e4b6572..f9dc2ad 100644 --- a/go.mod +++ b/go.mod @@ -2,4 +2,7 @@ module gitea.maximumdirect.net/eric/ReleaseKit go 1.25.0 -require golang.org/x/mod v0.40.0 +require ( + github.com/yuin/goldmark/v2 v2.1.6 + golang.org/x/mod v0.40.0 +) diff --git a/go.sum b/go.sum index e0d54fe..591797d 100644 --- a/go.sum +++ b/go.sum @@ -1,2 +1,4 @@ +github.com/yuin/goldmark/v2 v2.1.6 h1:igy0q9JcqovoFXFx/XjtU8sHVhcUHavPj2sA3EoKJUE= +github.com/yuin/goldmark/v2 v2.1.6/go.mod h1:G6M4/qOFtfn01/o14BU1UR2Lo5N3S9Qo7xCuz/sHjGQ= golang.org/x/mod v0.40.0 h1:hUv+3cXcdRHz08UmSiOob7sadHig73uo5bkXxQ/tvUs= golang.org/x/mod v0.40.0/go.mod h1:0/weTWkPWGBikyTWAX3dkjVztMmBA5hM0DH6BElSupE= diff --git a/internal/validation/markdown.go b/internal/validation/markdown.go new file mode 100644 index 0000000..ab62c9a --- /dev/null +++ b/internal/validation/markdown.go @@ -0,0 +1,171 @@ +package validation + +import ( + "context" + "sort" + "strconv" + "strings" + "unicode" + + "github.com/yuin/goldmark/v2/ast" + "github.com/yuin/goldmark/v2/parser" +) + +type markdownLocation struct { + Line int + Column int +} + +type markdownLink struct { + Destination string + Title string + Image bool + Reference bool + Location markdownLocation +} + +type markdownDocument struct { + links []markdownLink + anchors map[string]struct{} +} + +type markdownCache struct { + tree *CandidateTree + documents map[string]*markdownDocument +} + +func newMarkdownCache(tree *CandidateTree) *markdownCache { + return &markdownCache{ + tree: tree, + documents: make(map[string]*markdownDocument), + } +} + +func (cache *markdownCache) document(ctx context.Context, path string) (*markdownDocument, error) { + if document, ok := cache.documents[path]; ok { + return document, nil + } + source, err := cache.tree.ReadFile(ctx, path) + if err != nil { + return nil, err + } + document := parseMarkdown(source) + cache.documents[path] = document + return document, nil +} + +func parseMarkdown(source []byte) *markdownDocument { + root := parser.New().Parse(source) + document := &markdownDocument{ + anchors: make(map[string]struct{}), + } + lineStarts := markdownLineStarts(source) + usedAnchors := make(map[string]struct{}) + nextSuffix := make(map[string]int) + _ = ast.Walk(root, func(node ast.Node, entering bool) (ast.WalkStatus, error) { + if !entering { + return ast.WalkContinue, nil + } + switch node := node.(type) { + case *ast.Link: + document.links = append(document.links, markdownLink{ + Destination: node.Destination.Value(source), + Title: node.Title.Value(source), + Reference: node.Reference != nil, + Location: markdownLocationAt(lineStarts, node.Pos()), + }) + case *ast.Image: + document.links = append(document.links, markdownLink{ + Destination: node.Destination.Value(source), + Title: node.Title.Value(source), + Image: true, + Reference: node.Reference != nil, + Location: markdownLocationAt(lineStarts, node.Pos()), + }) + case *ast.AutoLink: + document.links = append(document.links, markdownLink{ + Destination: node.Destination.Value(source), + Location: markdownLocationAt(lineStarts, node.Pos()), + }) + case *ast.Heading: + base := markdownAnchor(markdownHeadingText(node, source)) + anchor := base + if _, exists := usedAnchors[anchor]; exists { + suffix, ok := nextSuffix[base] + if !ok { + suffix = 1 + } + for { + anchor = base + "-" + strconv.Itoa(suffix) + suffix++ + if _, exists := usedAnchors[anchor]; !exists { + break + } + } + nextSuffix[base] = suffix + } + usedAnchors[anchor] = struct{}{} + document.anchors[anchor] = struct{}{} + } + return ast.WalkContinue, nil + }) + return document +} + +func markdownHeadingText(heading *ast.Heading, source []byte) string { + var result strings.Builder + var appendNode func(ast.Node) + appendNode = func(node ast.Node) { + switch node := node.(type) { + case *ast.RawHTML: + return + case *ast.Text: + result.WriteString(node.Value.Value(source)) + if node.SoftLineBreak() || node.HardLineBreak() { + result.WriteByte(' ') + } + return + case *ast.CodeSpan: + result.WriteString(node.Value.Value(source)) + return + } + for child := node.FirstChild(); child != nil; child = child.NextSibling() { + appendNode(child) + } + } + for child := heading.FirstChild(); child != nil; child = child.NextSibling() { + appendNode(child) + } + return result.String() +} + +func markdownAnchor(heading string) string { + heading = strings.ToLower(heading) + var result strings.Builder + for _, r := range heading { + if unicode.IsLetter(r) || unicode.IsNumber(r) || r == '_' || r == '-' || r == ' ' { + result.WriteRune(r) + } + } + return strings.ReplaceAll(result.String(), " ", "-") +} + +func markdownLineStarts(source []byte) []int { + starts := []int{0} + for offset, value := range source { + if value == '\n' { + starts = append(starts, offset+1) + } + } + return starts +} + +func markdownLocationAt(lineStarts []int, offset int) markdownLocation { + if offset < 0 { + return markdownLocation{} + } + lineIndex := sort.Search(len(lineStarts), func(index int) bool { + return lineStarts[index] > offset + }) - 1 + return markdownLocation{Line: lineIndex + 1, Column: offset - lineStarts[lineIndex] + 1} +} diff --git a/internal/validation/markdown_test.go b/internal/validation/markdown_test.go new file mode 100644 index 0000000..f66abc3 --- /dev/null +++ b/internal/validation/markdown_test.go @@ -0,0 +1,118 @@ +package validation + +import ( + "context" + "strings" + "testing" +) + +func TestParseMarkdownLinksAndSourceLocations(t *testing.T) { + source := []byte("" + + "é [inline](docs/a\\(b\\).md \"inline title\").\n" + + "![chart]( 'image title')\n" + + "[ref][target] then [ref][target].\n" + + "[collapsed][] and [shortcut].\n" + + "![logo][badge]\n" + + "\n" + + "[target]: docs/target.md \"reference title\"\n" + + "[collapsed]: child.md\n" + + "[shortcut]: short.md\n" + + "[badge]: badge.png\n" + + "Inline example: `[fake](fake.md)`.\n" + + "```md\n[also fake](fenced.md)\n```\n" + + "[undefined]\n" + + "Autolink .\n" + + "HTML\n") + document := parseMarkdown(source) + + want := []markdownLink{ + {Destination: "docs/a(b).md", Title: "inline title", Location: markdownLocation{Line: 1, Column: 4}}, + {Destination: "images/chart one.svg", Title: "image title", Image: true, Location: markdownLocation{Line: 2, Column: 1}}, + {Destination: "docs/target.md", Title: "reference title", Reference: true, Location: markdownLocation{Line: 3, Column: 1}}, + {Destination: "docs/target.md", Title: "reference title", Reference: true, Location: markdownLocation{Line: 3, Column: 20}}, + {Destination: "child.md", Reference: true, Location: markdownLocation{Line: 4, Column: 1}}, + {Destination: "short.md", Reference: true, Location: markdownLocation{Line: 4, Column: 19}}, + {Destination: "badge.png", Image: true, Reference: true, Location: markdownLocation{Line: 5, Column: 1}}, + {Destination: "https://example.invalid", Location: markdownLocation{Line: 16, Column: 10}}, + } + if len(document.links) != len(want) { + t.Fatalf("found %d links, want %d: %#v", len(document.links), len(want), document.links) + } + for index := range want { + if got := document.links[index]; got != want[index] { + t.Errorf("link %d = %#v, want %#v", index, got, want[index]) + } + } +} + +func TestParseMarkdownDecodesEntitiesInDestinations(t *testing.T) { + document := parseMarkdown([]byte("[entity]()\n")) + if len(document.links) != 1 || document.links[0].Destination != "a&b.md" { + t.Fatalf("parsed links = %#v, want decoded destination a&b.md", document.links) + } +} + +func TestParseMarkdownHeadingAnchors(t *testing.T) { + source := []byte("" + + "# Café & 東京 `Code Here` Bold!\n" + + "Setext **Heading**\n" + + "---\n" + + "# Foo\n" + + "# Foo\n" + + "# Foo-1\n" + + "# Foo\n" + + "# Foo-1\n") + document := parseMarkdown(source) + for _, anchor := range []string{ + "café--東京-code-here-bold", + "setext-heading", + "foo", + "foo-1", + "foo-1-1", + "foo-2", + "foo-1-2", + } { + if _, ok := document.anchors[anchor]; !ok { + t.Errorf("missing anchor %q in %#v", anchor, document.anchors) + } + } + if len(document.anchors) != 7 { + t.Fatalf("found %d anchors, want 7: %#v", len(document.anchors), document.anchors) + } +} + +func TestMarkdownCacheKeepsParsedDocumentsPerCandidatePath(t *testing.T) { + repository := newTreeRepository(t) + writeTreeFile(t, repository, "guide.md", []byte("# Guide\n[other](other.md)\n")) + writeTreeFile(t, repository, "other.md", []byte("# Other\n")) + gitTree(t, repository, "add", "guide.md", "other.md") + gitTree(t, repository, "commit", "-m", "add markdown files") + + commit := strings.TrimSpace(gitTree(t, repository, "rev-parse", "HEAD")) + tree, err := ReadCandidateTree(context.Background(), repository, commit) + if err != nil { + t.Fatal(err) + } + cache := newMarkdownCache(tree) + first, err := cache.document(context.Background(), "guide.md") + if err != nil { + t.Fatal(err) + } + second, err := cache.document(context.Background(), "guide.md") + if err != nil { + t.Fatal(err) + } + if first != second { + t.Fatal("repeated document lookup returned a different parsed document") + } + other, err := cache.document(context.Background(), "other.md") + if err != nil { + t.Fatal(err) + } + if other == first { + t.Fatal("different candidate paths shared a parsed document") + } + if _, ok := other.anchors["other"]; !ok { + t.Fatalf("other document anchors = %#v", other.anchors) + } +}