Record validation performance baseline

2026-10-09 01:02:05 +00:00
parent ca571aca4f
commit 29e1bab228
6 changed files with 457 additions and 9 deletions
--- a/docs/roadmap/implementation.md
+++ b/docs/roadmap/implementation.md
@@ -1,6 +1,6 @@
 # Computational Efficiency Implementation Plan

-**Status:** Ready for implementation; all stages pending.
+**Status:** Stage 1 complete; Stages 2–11 pending.

 ## Purpose And Baseline

@@ -51,6 +51,50 @@ initialization mechanism.

 **Owners:** Benchmark files in `internal/validation` and root facade fixtures.

+**Completion:** Complete. Measurement-only benchmarks cover tree metadata,
+content reads and formatting for 1,000 unique or repeated-ID Go blobs in loose
+and packed repositories, repeated links to identical 1 MiB Markdown documents,
+retained content after processing, and a real offline facade `Check`. Fixtures
+verify committed bytes, unique/repeated object IDs, link counts, and Git pack
+state. The facade fixture selects Go formatting, local-replacement inspection,
+and Markdown while using test-owned module and build caches.
+
+### Baseline Measurement Record
+
+The baseline is commit `ca571aca4f2fd43798174a6d0d7c5a588b5347c9`, before adding
+measurement support. Results use Go `go1.26.8 linux/amd64`, Git `2.53.0`, and a
+Linux x86_64 Intel Core i5-7260U at 2.20 GHz. The command was
+`go test ./internal/validation . -run '^$' -bench '^BenchmarkPerformance' -benchmem -count=3`.
+Each candidate-content operation used a fresh empty content cache with tree
+metadata prepared outside its timer. The loose fixtures had loose objects and
+no pack; packed fixtures were checked after `git gc --aggressive --prune=now`.
+The unique Go fixture contains 1,000 distinct, formatted files totaling 49,000
+bytes. The repeated-ID fixture contains 1,000 paths sharing one 15-byte blob.
+The Markdown fixture contains two identical 1 MiB documents, a 47,780-byte
+index with 1,000 repeated links, and 2,144,932 bytes processed per check. The
+complete-check module contains 906 bytes of Go/module input. Its `GOMODCACHE`
+and `GOCACHE` are test-owned temporary directories, and module network access
+is disabled; the build cache is warmed before timing.
+
+| Measurement | Baseline observations | Final comparison |
+| --- | --- | --- |
+| Tree metadata, 1,000 entries | Loose: 27.5–28.2 ms; packed: 25.0–28.1 ms. About 1.28 MB allocated per operation. | Pending Stage 11. |
+| Unique-blob reads, 49,000 bytes | Loose: 1.86–1.96 s; packed: 2.57–3.08 s. About 34.57 MB allocated per operation. | Pending Stage 11. |
+| Repeated-ID reads, 15,000 bytes across 1,000 paths | Loose: 2.12–3.00 ms; packed: 3.42–7.75 ms. About 50.7 KB allocated per operation. | Pending Stage 11. |
+| Formatting, 1,000 unique files | Loose: 1.87–2.94 s; packed: 2.12–4.55 s. About 36.9 MB allocated per operation. | Pending Stage 11. |
+| Formatting, 1,000 paths sharing one object | Loose: 4.52–12.66 ms; packed: 6.11–8.62 ms. About 1.49 MB allocated per operation. | Pending Stage 11. |
+| Markdown links and parsing | 19.4–21.5 ms for two identical 1 MiB documents and 1,000 links; about 7.67 MB allocated per operation. | Pending Stage 11. |
+| Facade check, standard Go suite only | 385–690 ms; 906 module bytes; 336–340 KB allocated in the benchmark process. | Pending Stage 11. |
+| Facade check with selected file checks | 528–634 ms; 906 module bytes; 632–646 KB allocated in the benchmark process. | Pending Stage 11. |
+| Retained content after formatting and Markdown parsing | 1,385,040–1,385,616 bytes above the post-GC metadata baseline, with the candidate and Markdown cache kept alive. | Pending Stage 11. |
+
+Raw output is in temporary storage at `/tmp/releasekit-performance-baseline.txt`.
+The shared machine produced substantial timing variation, especially for packed
+content and the full check; allocations and retained-byte observations were
+more stable. `B/op` for facade checks measures allocations in the benchmark
+process and does not include allocations inside its Go subprocess. These are
+workload-specific component measurements, not runtime guarantees.
+
 1. Add benchmarks named with the `BenchmarkPerformance` prefix. Reuse local Git
   fixture patterns; adapt shared helper parameters to `testing.TB` only where
   a benchmark needs them.
--- a/internal/validation/local_replacements_test.go
+++ b/internal/validation/local_replacements_test.go
@@ -167,7 +167,7 @@ func TestRunAttributesCandidateTreeFailureToLocalReplacements(t *testing.T) {
 	}
 }

-func commitCandidateTree(t *testing.T, repository string) string {
+func commitCandidateTree(t testing.TB, repository string) string {
 	t.Helper()
 	gitTree(t, repository, "add", ".")
 	gitTree(t, repository, "commit", "-m", "capture module declarations")
--- a/internal/validation/performance_test.go
+++ b/internal/validation/performance_test.go
@@ -0,0 +1,327 @@
+package validation
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io"
+	"runtime"
+	"strconv"
+	"strings"
+	"testing"
+)
+
+const performanceGoFileCount = 1000
+const performanceMarkdownSize = 1 << 20
+const performanceMarkdownLinkCount = 500
+
+type performanceTree struct {
+	repository string
+	commit     string
+	paths      []string
+	expected   map[string][]byte
+	totalBytes int64
+}
+
+func BenchmarkPerformanceValidation(b *testing.B) {
+	cases := []struct {
+		name     string
+		repeated bool
+		packed   bool
+	}{
+		{name: "unique-loose"},
+		{name: "unique-packed", packed: true},
+		{name: "repeated-loose", repeated: true},
+		{name: "repeated-packed", repeated: true, packed: true},
+	}
+	fixtures := make(map[string]performanceTree, len(cases))
+	for _, testCase := range cases {
+		fixtures[testCase.name] = newPerformanceTree(b, performanceGoFileCount, testCase.repeated, testCase.packed, false)
+	}
+	markdown := newPerformanceTree(b, 0, false, false, true)
+
+	for _, testCase := range cases {
+		fixture := fixtures[testCase.name]
+		b.Run("TreeMetadata/"+testCase.name, func(b *testing.B) {
+			b.ReportAllocs()
+			b.ResetTimer()
+			b.ReportMetric(float64(len(fixture.paths)), "files/op")
+			for i := 0; i < b.N; i++ {
+				tree, err := ReadCandidateTree(context.Background(), fixture.repository, fixture.commit)
+				if err != nil {
+					b.Fatal(err)
+				}
+				if len(tree.Entries()) != len(fixture.paths) {
+					b.Fatalf("tree entries = %d, want %d", len(tree.Entries()), len(fixture.paths))
+				}
+				runtime.KeepAlive(tree)
+			}
+		})
+		b.Run("ContentAccess/"+testCase.name, func(b *testing.B) {
+			trees := freshPerformanceTrees(b, fixture, b.N)
+			b.ReportAllocs()
+			b.SetBytes(fixture.totalBytes)
+			b.ResetTimer()
+			b.ReportMetric(float64(fixture.totalBytes), "bytes/op")
+			b.ReportMetric(float64(len(fixture.paths)), "files/op")
+			b.ReportMetric(float64(performanceGoFileCount), "go-files/op")
+			for _, tree := range trees {
+				var bytesRead int64
+				for _, path := range fixture.paths {
+					contents, err := tree.ReadFile(context.Background(), path)
+					if err != nil {
+						b.Fatal(err)
+					}
+					bytesRead += int64(len(contents))
+				}
+				if bytesRead != fixture.totalBytes {
+					b.Fatalf("read %d bytes, want %d", bytesRead, fixture.totalBytes)
+				}
+				runtime.KeepAlive(tree)
+			}
+			runtime.KeepAlive(trees)
+		})
+		b.Run("GoFormat/"+testCase.name, func(b *testing.B) {
+			trees := freshPerformanceTrees(b, fixture, b.N)
+			b.ReportAllocs()
+			b.SetBytes(fixture.totalBytes)
+			b.ResetTimer()
+			b.ReportMetric(float64(fixture.totalBytes), "bytes/op")
+			b.ReportMetric(float64(performanceGoFileCount), "go-files/op")
+			for _, tree := range trees {
+				if err := checkGoFormat(context.Background(), tree); err != nil {
+					b.Fatal(err)
+				}
+				runtime.KeepAlive(tree)
+			}
+			runtime.KeepAlive(trees)
+		})
+	}
+
+	b.Run("Markdown/one-megabyte-repeated-links-and-identical-content", func(b *testing.B) {
+		trees := freshPerformanceTrees(b, markdown, b.N)
+		b.ReportAllocs()
+		b.SetBytes(markdown.totalBytes)
+		b.ResetTimer()
+		b.ReportMetric(float64(markdown.totalBytes), "bytes/op")
+		b.ReportMetric(float64(performanceMarkdownLinkCount*2), "repeated-links/op")
+		for _, tree := range trees {
+			if err := checkMarkdownLinks(context.Background(), tree, false, io.Discard); err != nil {
+				b.Fatal(err)
+			}
+			runtime.KeepAlive(tree)
+		}
+		runtime.KeepAlive(trees)
+	})
+}
+
+func BenchmarkPerformanceRetainedContent(b *testing.B) {
+	fixture := newPerformanceTree(b, performanceGoFileCount, false, false, true)
+	tree, err := ReadCandidateTree(context.Background(), fixture.repository, fixture.commit)
+	if err != nil {
+		b.Fatal(err)
+	}
+	cache := newMarkdownCache(tree)
+	var markdownPaths []string
+	for _, entry := range tree.Entries() {
+		if strings.HasSuffix(strings.ToLower(entry.Path), ".md") {
+			markdownPaths = append(markdownPaths, entry.Path)
+		}
+	}
+	if len(markdownPaths) != 3 {
+		b.Fatalf("Markdown paths = %v, want README and two identical documents", markdownPaths)
+	}
+
+	runtime.GC()
+	var before runtime.MemStats
+	runtime.ReadMemStats(&before)
+	b.ReportAllocs()
+	b.ReportMetric(float64(len(fixture.paths)), "go-files/op")
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		if err := checkGoFormat(context.Background(), tree); err != nil {
+			b.Fatal(err)
+		}
+		for _, path := range markdownPaths {
+			if _, err := cache.document(context.Background(), path); err != nil {
+				b.Fatal(err)
+			}
+		}
+	}
+	b.StopTimer()
+	runtime.GC()
+	var after runtime.MemStats
+	runtime.ReadMemStats(&after)
+	retained := int64(after.HeapAlloc) - int64(before.HeapAlloc)
+	if retained < 0 {
+		retained = 0
+	}
+	b.ReportMetric(float64(retained), "retained-B")
+	runtime.KeepAlive(tree)
+	runtime.KeepAlive(cache)
+	runtime.KeepAlive(fixture)
+}
+
+func newPerformanceTree(tb testing.TB, goFileCount int, repeated, packed, includeMarkdown bool) performanceTree {
+	tb.Helper()
+	repository := newTreeRepository(tb)
+	fixture := performanceTree{
+		repository: repository,
+		paths:      make([]string, 0, goFileCount),
+		expected:   make(map[string][]byte, goFileCount+3),
+	}
+	for i := 0; i < goFileCount; i++ {
+		name := fmt.Sprintf("source/file-%04d.go", i)
+		contents := performanceGoSource(i)
+		if repeated {
+			contents = []byte("package source\n")
+		}
+		writeTreeFile(tb, repository, name, contents)
+		fixture.paths = append(fixture.paths, name)
+		fixture.expected[name] = contents
+		fixture.totalBytes += int64(len(contents))
+	}
+	if includeMarkdown {
+		markdown := performanceMarkdownDocument()
+		readme := performanceMarkdownIndex()
+		for name, contents := range map[string][]byte{
+			"README.md":          readme,
+			"archive/payload.md": markdown,
+			"docs/payload.md":    markdown,
+		} {
+			writeTreeFile(tb, repository, name, contents)
+			fixture.expected[name] = contents
+			fixture.totalBytes += int64(len(contents))
+		}
+	}
+
+	fixture.commit = commitCandidateTree(tb, repository)
+	if packed {
+		gitTree(tb, repository, "gc", "--aggressive", "--prune=now")
+	}
+	verifyPerformanceTree(tb, fixture, repeated, packed, includeMarkdown)
+	return fixture
+}
+
+func freshPerformanceTrees(tb testing.TB, fixture performanceTree, count int) []*CandidateTree {
+	tb.Helper()
+	base, err := ReadCandidateTree(context.Background(), fixture.repository, fixture.commit)
+	if err != nil {
+		tb.Fatal(err)
+	}
+	trees := make([]*CandidateTree, count)
+	for index := range trees {
+		tree := *base
+		tree.blobs = make(map[string][]byte)
+		trees[index] = &tree
+	}
+	runtime.KeepAlive(base)
+	return trees
+}
+
+func verifyPerformanceTree(tb testing.TB, fixture performanceTree, repeated, packed, includeMarkdown bool) {
+	tb.Helper()
+	tree, err := ReadCandidateTree(context.Background(), fixture.repository, fixture.commit)
+	if err != nil {
+		tb.Fatal(err)
+	}
+	if len(tree.Entries()) != len(fixture.expected) {
+		tb.Fatalf("tree entries = %d, want %d", len(tree.Entries()), len(fixture.expected))
+	}
+	uniqueObjects := make(map[string]struct{}, len(fixture.paths))
+	for _, path := range fixture.paths {
+		contents, err := tree.ReadFile(context.Background(), path)
+		if err != nil {
+			tb.Fatal(err)
+		}
+		if !bytes.Equal(contents, fixture.expected[path]) {
+			tb.Fatalf("committed content at %q differs from fixture bytes", path)
+		}
+		uniqueObjects[tree.entries[tree.byPath[path]].ObjectID] = struct{}{}
+	}
+	wantObjects := len(fixture.paths)
+	if repeated {
+		wantObjects = 1
+	}
+	if len(uniqueObjects) != wantObjects {
+		tb.Fatalf("unique Go object IDs = %d, want %d", len(uniqueObjects), wantObjects)
+	}
+	if includeMarkdown {
+		for _, path := range []string{"README.md", "docs/payload.md", "archive/payload.md"} {
+			contents, err := tree.ReadFile(context.Background(), path)
+			if err != nil {
+				tb.Fatal(err)
+			}
+			if !bytes.Equal(contents, fixture.expected[path]) {
+				tb.Fatalf("committed Markdown content at %q differs from fixture bytes", path)
+			}
+		}
+		if len(fixture.expected["docs/payload.md"]) != performanceMarkdownSize {
+			tb.Fatalf("large Markdown fixture size = %d, want %d", len(fixture.expected["docs/payload.md"]), performanceMarkdownSize)
+		}
+		if !bytes.Equal(fixture.expected["docs/payload.md"], fixture.expected["archive/payload.md"]) {
+			tb.Fatal("Markdown copies differ")
+		}
+		if tree.entries[tree.byPath["docs/payload.md"]].ObjectID != tree.entries[tree.byPath["archive/payload.md"]].ObjectID {
+			tb.Fatal("identical Markdown copies do not share an object ID")
+		}
+		index := string(fixture.expected["README.md"])
+		if strings.Count(index, "docs/payload.md#shared-payload") != performanceMarkdownLinkCount || strings.Count(index, "archive/payload.md#shared-payload") != performanceMarkdownLinkCount {
+			tb.Fatal("Markdown link fixture does not contain the expected repeated targets")
+		}
+	}
+
+	objects := gitTree(tb, fixture.repository, "count-objects", "-v")
+	loose, packedCount := performanceObjectCounts(tb, objects)
+	if packed {
+		if loose != 0 || packedCount == 0 {
+			tb.Fatalf("packed fixture object counts = loose %d, in-pack %d", loose, packedCount)
+		}
+	} else if loose == 0 || packedCount != 0 {
+		tb.Fatalf("loose fixture object counts = loose %d, in-pack %d", loose, packedCount)
+	}
+}
+
+func performanceObjectCounts(tb testing.TB, output string) (loose, packed int) {
+	tb.Helper()
+	for _, line := range strings.Split(output, "\n") {
+		key, value, ok := strings.Cut(line, ": ")
+		if !ok || (key != "count" && key != "in-pack") {
+			continue
+		}
+		count, err := strconv.Atoi(value)
+		if err != nil {
+			tb.Fatalf("parse git count-objects output %q: %v", line, err)
+		}
+		if key == "count" {
+			loose = count
+		} else {
+			packed = count
+		}
+	}
+	return loose, packed
+}
+
+func performanceGoSource(index int) []byte {
+	return []byte(fmt.Sprintf("package source\n\nconst value%04d = %q\n", index, fmt.Sprintf("payload-%04d", index)))
+}
+
+func performanceMarkdownDocument() []byte {
+	contents := make([]byte, performanceMarkdownSize)
+	prefix := []byte("# Shared Payload\n\n")
+	copy(contents, prefix)
+	for index := len(prefix); index < len(contents)-1; index++ {
+		contents[index] = 'x'
+	}
+	contents[len(contents)-1] = '\n'
+	return contents
+}
+
+func performanceMarkdownIndex() []byte {
+	var contents strings.Builder
+	for index := 0; index < performanceMarkdownLinkCount; index++ {
+		fmt.Fprintf(&contents, "- [payload %d](docs/payload.md#shared-payload)\n", index)
+		fmt.Fprintf(&contents, "- [copy %d](archive/payload.md#shared-payload)\n", index)
+	}
+	return []byte(contents.String())
+}
--- a/internal/validation/tree_test.go
+++ b/internal/validation/tree_test.go
@@ -93,7 +93,7 @@ type gitTreeEntry struct {
 	hasSize bool
 }

-func newTreeRepository(t *testing.T) string {
+func newTreeRepository(t testing.TB) string {
 	t.Helper()
 	repository := t.TempDir()
 	gitTree(t, repository, "init", "-b", "main")
@@ -102,7 +102,7 @@ func newTreeRepository(t *testing.T) string {
 	return repository
 }

-func writeTreeFile(t *testing.T, repository, name string, content []byte) {
+func writeTreeFile(t testing.TB, repository, name string, content []byte) {
 	t.Helper()
 	path := filepath.Join(repository, filepath.FromSlash(name))
 	if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
@@ -113,7 +113,7 @@ func writeTreeFile(t *testing.T, repository, name string, content []byte) {
 	}
 }

-func gitTree(t *testing.T, repository string, args ...string) string {
+func gitTree(t testing.TB, repository string, args ...string) string {
 	t.Helper()
 	command := exec.Command("git", args...)
 	command.Dir = repository
--- a/performance_benchmark_test.go
+++ b/performance_benchmark_test.go
@@ -0,0 +1,77 @@
+package releasekit_test
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"gitea.maximumdirect.net/eric/ReleaseKit"
+)
+
+func BenchmarkPerformanceFacadeCheck(b *testing.B) {
+	repository, moduleBytes := performanceCheckFixture(b)
+	warmRequest := releasekit.ReleaseRequest{
+		RepositoryDir: repository.work,
+		Product:       "Sample App",
+		Version:       "v1.2.3",
+	}
+	if _, err := releasekit.Check(context.Background(), warmRequest); err != nil {
+		b.Fatalf("warm benchmark Go build cache: %v", err)
+	}
+	b.Run("GoSuiteOnly", func(b *testing.B) {
+		benchmarkFacadeCheck(b, repository, moduleBytes, releasekit.ValidationSpec{})
+	})
+	b.Run("GoSuiteAndSelectedChecks", func(b *testing.B) {
+		benchmarkFacadeCheck(b, repository, moduleBytes, releasekit.ValidationSpec{
+			GoFormat: true,
+			Markdown: &releasekit.MarkdownValidationSpec{},
+			ModuleDistribution: releasekit.ModuleDistributionValidationSpec{
+				LocalReplacements: true,
+			},
+		})
+	})
+}
+
+func benchmarkFacadeCheck(b *testing.B, repository publishFixture, moduleBytes int64, validation releasekit.ValidationSpec) {
+	b.Helper()
+	request := releasekit.ReleaseRequest{
+		RepositoryDir: repository.work,
+		Product:       "Sample App",
+		Version:       "v1.2.3",
+		Validation:    validation,
+	}
+	b.ReportAllocs()
+	b.ResetTimer()
+	b.ReportMetric(float64(moduleBytes), "module-bytes/op")
+	for i := 0; i < b.N; i++ {
+		if _, err := releasekit.Check(context.Background(), request); err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func performanceCheckFixture(b *testing.B) (publishFixture, int64) {
+	b.Helper()
+	repository := newPublishFixture(b)
+	writeFile(b, repository.work, "source.go", "package fixture\n\nconst name = \"candidate\"\n")
+	gitRun(b, repository.work, "add", "source.go")
+	gitRun(b, repository.work, "commit", "-m", "Add check benchmark source")
+	if status := gitText(b, repository.work, "status", "--porcelain"); status != "" {
+		b.Fatalf("benchmark candidate is not clean:\n%s", status)
+	}
+	b.Setenv("GOMODCACHE", b.TempDir())
+	b.Setenv("GOCACHE", b.TempDir())
+	b.Setenv("GOWORK", "off")
+	b.Setenv("GOPROXY", "off")
+	b.Setenv("GOSUMDB", "off")
+	var moduleBytes int64
+	for _, name := range []string{"go.mod", "source.go", "fixture_test.go", "race_fixture_test.go"} {
+		contents, err := os.ReadFile(filepath.Join(repository.work, name))
+		if err != nil {
+			b.Fatal(err)
+		}
+		moduleBytes += int64(len(contents))
+	}
+	return repository, moduleBytes
+}
--- a/publish_test.go
+++ b/publish_test.go
@@ -797,7 +797,7 @@ func shellQuote(value string) string {
 	return "'" + strings.ReplaceAll(value, "'", "'\\''") + "'"
 }

-func newPublishFixture(t *testing.T) publishFixture {
+func newPublishFixture(t testing.TB) publishFixture {
 	t.Helper()
 	base := t.TempDir()
 	work := filepath.Join(base, "work")
@@ -870,7 +870,7 @@ func TestRaceBuildSelection(t *testing.T) {
 	return publishFixture{work: work, bare: bare, commit: commit, unrelatedCommit: other}
 }

-func gitRun(t *testing.T, dir string, args ...string) {
+func gitRun(t testing.TB, dir string, args ...string) {
 	t.Helper()
 	cmd := exec.Command("git", args...)
 	cmd.Dir = dir
@@ -879,7 +879,7 @@ func gitRun(t *testing.T, dir string, args ...string) {
 	}
 }

-func gitText(t *testing.T, dir string, args ...string) string {
+func gitText(t testing.TB, dir string, args ...string) string {
 	t.Helper()
 	cmd := exec.Command("git", args...)
 	cmd.Dir = dir
@@ -901,7 +901,7 @@ func gitBytes(t *testing.T, dir string, args ...string) string {
 	return string(output)
 }

-func writeFile(t *testing.T, root, name, contents string) {
+func writeFile(t testing.TB, root, name, contents string) {
 	t.Helper()
 	path := filepath.Join(root, filepath.FromSlash(name))
 	if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {