Record validation performance baseline

internal/validation/local_replacements_test.go

@@ -167,7 +167,7 @@ func TestRunAttributesCandidateTreeFailureToLocalReplacements(t *testing.T) {
 	}
 }
 
-func commitCandidateTree(t *testing.T, repository string) string {
+func commitCandidateTree(t testing.TB, repository string) string {
 	t.Helper()
 	gitTree(t, repository, "add", ".")
 	gitTree(t, repository, "commit", "-m", "capture module declarations")

internal/validation/performance_test.go

@@ -0,0 +1,327 @@
+package validation
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io"
+	"runtime"
+	"strconv"
+	"strings"
+	"testing"
+)
+
+const performanceGoFileCount = 1000
+const performanceMarkdownSize = 1 << 20
+const performanceMarkdownLinkCount = 500
+
+type performanceTree struct {
+	repository string
+	commit     string
+	paths      []string
+	expected   map[string][]byte
+	totalBytes int64
+}
+
+func BenchmarkPerformanceValidation(b *testing.B) {
+	cases := []struct {
+		name     string
+		repeated bool
+		packed   bool
+	}{
+		{name: "unique-loose"},
+		{name: "unique-packed", packed: true},
+		{name: "repeated-loose", repeated: true},
+		{name: "repeated-packed", repeated: true, packed: true},
+	}
+	fixtures := make(map[string]performanceTree, len(cases))
+	for _, testCase := range cases {
+		fixtures[testCase.name] = newPerformanceTree(b, performanceGoFileCount, testCase.repeated, testCase.packed, false)
+	}
+	markdown := newPerformanceTree(b, 0, false, false, true)
+
+	for _, testCase := range cases {
+		fixture := fixtures[testCase.name]
+		b.Run("TreeMetadata/"+testCase.name, func(b *testing.B) {
+			b.ReportAllocs()
+			b.ResetTimer()
+			b.ReportMetric(float64(len(fixture.paths)), "files/op")
+			for i := 0; i < b.N; i++ {
+				tree, err := ReadCandidateTree(context.Background(), fixture.repository, fixture.commit)
+				if err != nil {
+					b.Fatal(err)
+				}
+				if len(tree.Entries()) != len(fixture.paths) {
+					b.Fatalf("tree entries = %d, want %d", len(tree.Entries()), len(fixture.paths))
+				}
+				runtime.KeepAlive(tree)
+			}
+		})
+		b.Run("ContentAccess/"+testCase.name, func(b *testing.B) {
+			trees := freshPerformanceTrees(b, fixture, b.N)
+			b.ReportAllocs()
+			b.SetBytes(fixture.totalBytes)
+			b.ResetTimer()
+			b.ReportMetric(float64(fixture.totalBytes), "bytes/op")
+			b.ReportMetric(float64(len(fixture.paths)), "files/op")
+			b.ReportMetric(float64(performanceGoFileCount), "go-files/op")
+			for _, tree := range trees {
+				var bytesRead int64
+				for _, path := range fixture.paths {
+					contents, err := tree.ReadFile(context.Background(), path)
+					if err != nil {
+						b.Fatal(err)
+					}
+					bytesRead += int64(len(contents))
+				}
+				if bytesRead != fixture.totalBytes {
+					b.Fatalf("read %d bytes, want %d", bytesRead, fixture.totalBytes)
+				}
+				runtime.KeepAlive(tree)
+			}
+			runtime.KeepAlive(trees)
+		})
+		b.Run("GoFormat/"+testCase.name, func(b *testing.B) {
+			trees := freshPerformanceTrees(b, fixture, b.N)
+			b.ReportAllocs()
+			b.SetBytes(fixture.totalBytes)
+			b.ResetTimer()
+			b.ReportMetric(float64(fixture.totalBytes), "bytes/op")
+			b.ReportMetric(float64(performanceGoFileCount), "go-files/op")
+			for _, tree := range trees {
+				if err := checkGoFormat(context.Background(), tree); err != nil {
+					b.Fatal(err)
+				}
+				runtime.KeepAlive(tree)
+			}
+			runtime.KeepAlive(trees)
+		})
+	}
+
+	b.Run("Markdown/one-megabyte-repeated-links-and-identical-content", func(b *testing.B) {
+		trees := freshPerformanceTrees(b, markdown, b.N)
+		b.ReportAllocs()
+		b.SetBytes(markdown.totalBytes)
+		b.ResetTimer()
+		b.ReportMetric(float64(markdown.totalBytes), "bytes/op")
+		b.ReportMetric(float64(performanceMarkdownLinkCount*2), "repeated-links/op")
+		for _, tree := range trees {
+			if err := checkMarkdownLinks(context.Background(), tree, false, io.Discard); err != nil {
+				b.Fatal(err)
+			}
+			runtime.KeepAlive(tree)
+		}
+		runtime.KeepAlive(trees)
+	})
+}
+
+func BenchmarkPerformanceRetainedContent(b *testing.B) {
+	fixture := newPerformanceTree(b, performanceGoFileCount, false, false, true)
+	tree, err := ReadCandidateTree(context.Background(), fixture.repository, fixture.commit)
+	if err != nil {
+		b.Fatal(err)
+	}
+	cache := newMarkdownCache(tree)
+	var markdownPaths []string
+	for _, entry := range tree.Entries() {
+		if strings.HasSuffix(strings.ToLower(entry.Path), ".md") {
+			markdownPaths = append(markdownPaths, entry.Path)
+		}
+	}
+	if len(markdownPaths) != 3 {
+		b.Fatalf("Markdown paths = %v, want README and two identical documents", markdownPaths)
+	}
+
+	runtime.GC()
+	var before runtime.MemStats
+	runtime.ReadMemStats(&before)
+	b.ReportAllocs()
+	b.ReportMetric(float64(len(fixture.paths)), "go-files/op")
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		if err := checkGoFormat(context.Background(), tree); err != nil {
+			b.Fatal(err)
+		}
+		for _, path := range markdownPaths {
+			if _, err := cache.document(context.Background(), path); err != nil {
+				b.Fatal(err)
+			}
+		}
+	}
+	b.StopTimer()
+	runtime.GC()
+	var after runtime.MemStats
+	runtime.ReadMemStats(&after)
+	retained := int64(after.HeapAlloc) - int64(before.HeapAlloc)
+	if retained < 0 {
+		retained = 0
+	}
+	b.ReportMetric(float64(retained), "retained-B")
+	runtime.KeepAlive(tree)
+	runtime.KeepAlive(cache)
+	runtime.KeepAlive(fixture)
+}
+
+func newPerformanceTree(tb testing.TB, goFileCount int, repeated, packed, includeMarkdown bool) performanceTree {
+	tb.Helper()
+	repository := newTreeRepository(tb)
+	fixture := performanceTree{
+		repository: repository,
+		paths:      make([]string, 0, goFileCount),
+		expected:   make(map[string][]byte, goFileCount+3),
+	}
+	for i := 0; i < goFileCount; i++ {
+		name := fmt.Sprintf("source/file-%04d.go", i)
+		contents := performanceGoSource(i)
+		if repeated {
+			contents = []byte("package source\n")
+		}
+		writeTreeFile(tb, repository, name, contents)
+		fixture.paths = append(fixture.paths, name)
+		fixture.expected[name] = contents
+		fixture.totalBytes += int64(len(contents))
+	}
+	if includeMarkdown {
+		markdown := performanceMarkdownDocument()
+		readme := performanceMarkdownIndex()
+		for name, contents := range map[string][]byte{
+			"README.md":          readme,
+			"archive/payload.md": markdown,
+			"docs/payload.md":    markdown,
+		} {
+			writeTreeFile(tb, repository, name, contents)
+			fixture.expected[name] = contents
+			fixture.totalBytes += int64(len(contents))
+		}
+	}
+
+	fixture.commit = commitCandidateTree(tb, repository)
+	if packed {
+		gitTree(tb, repository, "gc", "--aggressive", "--prune=now")
+	}
+	verifyPerformanceTree(tb, fixture, repeated, packed, includeMarkdown)
+	return fixture
+}
+
+func freshPerformanceTrees(tb testing.TB, fixture performanceTree, count int) []*CandidateTree {
+	tb.Helper()
+	base, err := ReadCandidateTree(context.Background(), fixture.repository, fixture.commit)
+	if err != nil {
+		tb.Fatal(err)
+	}
+	trees := make([]*CandidateTree, count)
+	for index := range trees {
+		tree := *base
+		tree.blobs = make(map[string][]byte)
+		trees[index] = &tree
+	}
+	runtime.KeepAlive(base)
+	return trees
+}
+
+func verifyPerformanceTree(tb testing.TB, fixture performanceTree, repeated, packed, includeMarkdown bool) {
+	tb.Helper()
+	tree, err := ReadCandidateTree(context.Background(), fixture.repository, fixture.commit)
+	if err != nil {
+		tb.Fatal(err)
+	}
+	if len(tree.Entries()) != len(fixture.expected) {
+		tb.Fatalf("tree entries = %d, want %d", len(tree.Entries()), len(fixture.expected))
+	}
+	uniqueObjects := make(map[string]struct{}, len(fixture.paths))
+	for _, path := range fixture.paths {
+		contents, err := tree.ReadFile(context.Background(), path)
+		if err != nil {
+			tb.Fatal(err)
+		}
+		if !bytes.Equal(contents, fixture.expected[path]) {
+			tb.Fatalf("committed content at %q differs from fixture bytes", path)
+		}
+		uniqueObjects[tree.entries[tree.byPath[path]].ObjectID] = struct{}{}
+	}
+	wantObjects := len(fixture.paths)
+	if repeated {
+		wantObjects = 1
+	}
+	if len(uniqueObjects) != wantObjects {
+		tb.Fatalf("unique Go object IDs = %d, want %d", len(uniqueObjects), wantObjects)
+	}
+	if includeMarkdown {
+		for _, path := range []string{"README.md", "docs/payload.md", "archive/payload.md"} {
+			contents, err := tree.ReadFile(context.Background(), path)
+			if err != nil {
+				tb.Fatal(err)
+			}
+			if !bytes.Equal(contents, fixture.expected[path]) {
+				tb.Fatalf("committed Markdown content at %q differs from fixture bytes", path)
+			}
+		}
+		if len(fixture.expected["docs/payload.md"]) != performanceMarkdownSize {
+			tb.Fatalf("large Markdown fixture size = %d, want %d", len(fixture.expected["docs/payload.md"]), performanceMarkdownSize)
+		}
+		if !bytes.Equal(fixture.expected["docs/payload.md"], fixture.expected["archive/payload.md"]) {
+			tb.Fatal("Markdown copies differ")
+		}
+		if tree.entries[tree.byPath["docs/payload.md"]].ObjectID != tree.entries[tree.byPath["archive/payload.md"]].ObjectID {
+			tb.Fatal("identical Markdown copies do not share an object ID")
+		}
+		index := string(fixture.expected["README.md"])
+		if strings.Count(index, "docs/payload.md#shared-payload") != performanceMarkdownLinkCount || strings.Count(index, "archive/payload.md#shared-payload") != performanceMarkdownLinkCount {
+			tb.Fatal("Markdown link fixture does not contain the expected repeated targets")
+		}
+	}
+
+	objects := gitTree(tb, fixture.repository, "count-objects", "-v")
+	loose, packedCount := performanceObjectCounts(tb, objects)
+	if packed {
+		if loose != 0 || packedCount == 0 {
+			tb.Fatalf("packed fixture object counts = loose %d, in-pack %d", loose, packedCount)
+		}
+	} else if loose == 0 || packedCount != 0 {
+		tb.Fatalf("loose fixture object counts = loose %d, in-pack %d", loose, packedCount)
+	}
+}
+
+func performanceObjectCounts(tb testing.TB, output string) (loose, packed int) {
+	tb.Helper()
+	for _, line := range strings.Split(output, "\n") {
+		key, value, ok := strings.Cut(line, ": ")
+		if !ok || (key != "count" && key != "in-pack") {
+			continue
+		}
+		count, err := strconv.Atoi(value)
+		if err != nil {
+			tb.Fatalf("parse git count-objects output %q: %v", line, err)
+		}
+		if key == "count" {
+			loose = count
+		} else {
+			packed = count
+		}
+	}
+	return loose, packed
+}
+
+func performanceGoSource(index int) []byte {
+	return []byte(fmt.Sprintf("package source\n\nconst value%04d = %q\n", index, fmt.Sprintf("payload-%04d", index)))
+}
+
+func performanceMarkdownDocument() []byte {
+	contents := make([]byte, performanceMarkdownSize)
+	prefix := []byte("# Shared Payload\n\n")
+	copy(contents, prefix)
+	for index := len(prefix); index < len(contents)-1; index++ {
+		contents[index] = 'x'
+	}
+	contents[len(contents)-1] = '\n'
+	return contents
+}
+
+func performanceMarkdownIndex() []byte {
+	var contents strings.Builder
+	for index := 0; index < performanceMarkdownLinkCount; index++ {
+		fmt.Fprintf(&contents, "- [payload %d](docs/payload.md#shared-payload)\n", index)
+		fmt.Fprintf(&contents, "- [copy %d](archive/payload.md#shared-payload)\n", index)
+	}
+	return []byte(contents.String())
+}

internal/validation/tree_test.go

@@ -93,7 +93,7 @@ type gitTreeEntry struct {
 	hasSize bool
 }
 
-func newTreeRepository(t *testing.T) string {
+func newTreeRepository(t testing.TB) string {
 	t.Helper()
 	repository := t.TempDir()
 	gitTree(t, repository, "init", "-b", "main")
@@ -102,7 +102,7 @@ func newTreeRepository(t *testing.T) string {
 	return repository
 }
 
-func writeTreeFile(t *testing.T, repository, name string, content []byte) {
+func writeTreeFile(t testing.TB, repository, name string, content []byte) {
 	t.Helper()
 	path := filepath.Join(repository, filepath.FromSlash(name))
 	if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
@@ -113,7 +113,7 @@ func writeTreeFile(t *testing.T, repository, name string, content []byte) {
 	}
 }
 
-func gitTree(t *testing.T, repository string, args ...string) string {
+func gitTree(t testing.TB, repository string, args ...string) string {
 	t.Helper()
 	command := exec.Command("git", args...)
 	command.Dir = repository