diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index ba6bb59..e508797 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -1,6 +1,6 @@ # Computational Efficiency Implementation Plan -**Status:** Ready for implementation; all stages pending. +**Status:** Stage 1 complete; Stages 2–11 pending. ## Purpose And Baseline @@ -51,6 +51,50 @@ initialization mechanism. **Owners:** Benchmark files in `internal/validation` and root facade fixtures. +**Completion:** Complete. Measurement-only benchmarks cover tree metadata, +content reads and formatting for 1,000 unique or repeated-ID Go blobs in loose +and packed repositories, repeated links to identical 1 MiB Markdown documents, +retained content after processing, and a real offline facade `Check`. Fixtures +verify committed bytes, unique/repeated object IDs, link counts, and Git pack +state. The facade fixture selects Go formatting, local-replacement inspection, +and Markdown while using test-owned module and build caches. + +### Baseline Measurement Record + +The baseline is commit `ca571aca4f2fd43798174a6d0d7c5a588b5347c9`, before adding +measurement support. Results use Go `go1.26.8 linux/amd64`, Git `2.53.0`, and a +Linux x86_64 Intel Core i5-7260U at 2.20 GHz. The command was +`go test ./internal/validation . -run '^$' -bench '^BenchmarkPerformance' -benchmem -count=3`. +Each candidate-content operation used a fresh empty content cache with tree +metadata prepared outside its timer. The loose fixtures had loose objects and +no pack; packed fixtures were checked after `git gc --aggressive --prune=now`. +The unique Go fixture contains 1,000 distinct, formatted files totaling 49,000 +bytes. The repeated-ID fixture contains 1,000 paths sharing one 15-byte blob. +The Markdown fixture contains two identical 1 MiB documents, a 47,780-byte +index with 1,000 repeated links, and 2,144,932 bytes processed per check. The +complete-check module contains 906 bytes of Go/module input. Its `GOMODCACHE` +and `GOCACHE` are test-owned temporary directories, and module network access +is disabled; the build cache is warmed before timing. + +| Measurement | Baseline observations | Final comparison | +| --- | --- | --- | +| Tree metadata, 1,000 entries | Loose: 27.5–28.2 ms; packed: 25.0–28.1 ms. About 1.28 MB allocated per operation. | Pending Stage 11. | +| Unique-blob reads, 49,000 bytes | Loose: 1.86–1.96 s; packed: 2.57–3.08 s. About 34.57 MB allocated per operation. | Pending Stage 11. | +| Repeated-ID reads, 15,000 bytes across 1,000 paths | Loose: 2.12–3.00 ms; packed: 3.42–7.75 ms. About 50.7 KB allocated per operation. | Pending Stage 11. | +| Formatting, 1,000 unique files | Loose: 1.87–2.94 s; packed: 2.12–4.55 s. About 36.9 MB allocated per operation. | Pending Stage 11. | +| Formatting, 1,000 paths sharing one object | Loose: 4.52–12.66 ms; packed: 6.11–8.62 ms. About 1.49 MB allocated per operation. | Pending Stage 11. | +| Markdown links and parsing | 19.4–21.5 ms for two identical 1 MiB documents and 1,000 links; about 7.67 MB allocated per operation. | Pending Stage 11. | +| Facade check, standard Go suite only | 385–690 ms; 906 module bytes; 336–340 KB allocated in the benchmark process. | Pending Stage 11. | +| Facade check with selected file checks | 528–634 ms; 906 module bytes; 632–646 KB allocated in the benchmark process. | Pending Stage 11. | +| Retained content after formatting and Markdown parsing | 1,385,040–1,385,616 bytes above the post-GC metadata baseline, with the candidate and Markdown cache kept alive. | Pending Stage 11. | + +Raw output is in temporary storage at `/tmp/releasekit-performance-baseline.txt`. +The shared machine produced substantial timing variation, especially for packed +content and the full check; allocations and retained-byte observations were +more stable. `B/op` for facade checks measures allocations in the benchmark +process and does not include allocations inside its Go subprocess. These are +workload-specific component measurements, not runtime guarantees. + 1. Add benchmarks named with the `BenchmarkPerformance` prefix. Reuse local Git fixture patterns; adapt shared helper parameters to `testing.TB` only where a benchmark needs them. diff --git a/internal/validation/local_replacements_test.go b/internal/validation/local_replacements_test.go index 6c6d07c..80901ff 100644 --- a/internal/validation/local_replacements_test.go +++ b/internal/validation/local_replacements_test.go @@ -167,7 +167,7 @@ func TestRunAttributesCandidateTreeFailureToLocalReplacements(t *testing.T) { } } -func commitCandidateTree(t *testing.T, repository string) string { +func commitCandidateTree(t testing.TB, repository string) string { t.Helper() gitTree(t, repository, "add", ".") gitTree(t, repository, "commit", "-m", "capture module declarations") diff --git a/internal/validation/performance_test.go b/internal/validation/performance_test.go new file mode 100644 index 0000000..f8f3413 --- /dev/null +++ b/internal/validation/performance_test.go @@ -0,0 +1,327 @@ +package validation + +import ( + "bytes" + "context" + "fmt" + "io" + "runtime" + "strconv" + "strings" + "testing" +) + +const performanceGoFileCount = 1000 +const performanceMarkdownSize = 1 << 20 +const performanceMarkdownLinkCount = 500 + +type performanceTree struct { + repository string + commit string + paths []string + expected map[string][]byte + totalBytes int64 +} + +func BenchmarkPerformanceValidation(b *testing.B) { + cases := []struct { + name string + repeated bool + packed bool + }{ + {name: "unique-loose"}, + {name: "unique-packed", packed: true}, + {name: "repeated-loose", repeated: true}, + {name: "repeated-packed", repeated: true, packed: true}, + } + fixtures := make(map[string]performanceTree, len(cases)) + for _, testCase := range cases { + fixtures[testCase.name] = newPerformanceTree(b, performanceGoFileCount, testCase.repeated, testCase.packed, false) + } + markdown := newPerformanceTree(b, 0, false, false, true) + + for _, testCase := range cases { + fixture := fixtures[testCase.name] + b.Run("TreeMetadata/"+testCase.name, func(b *testing.B) { + b.ReportAllocs() + b.ResetTimer() + b.ReportMetric(float64(len(fixture.paths)), "files/op") + for i := 0; i < b.N; i++ { + tree, err := ReadCandidateTree(context.Background(), fixture.repository, fixture.commit) + if err != nil { + b.Fatal(err) + } + if len(tree.Entries()) != len(fixture.paths) { + b.Fatalf("tree entries = %d, want %d", len(tree.Entries()), len(fixture.paths)) + } + runtime.KeepAlive(tree) + } + }) + b.Run("ContentAccess/"+testCase.name, func(b *testing.B) { + trees := freshPerformanceTrees(b, fixture, b.N) + b.ReportAllocs() + b.SetBytes(fixture.totalBytes) + b.ResetTimer() + b.ReportMetric(float64(fixture.totalBytes), "bytes/op") + b.ReportMetric(float64(len(fixture.paths)), "files/op") + b.ReportMetric(float64(performanceGoFileCount), "go-files/op") + for _, tree := range trees { + var bytesRead int64 + for _, path := range fixture.paths { + contents, err := tree.ReadFile(context.Background(), path) + if err != nil { + b.Fatal(err) + } + bytesRead += int64(len(contents)) + } + if bytesRead != fixture.totalBytes { + b.Fatalf("read %d bytes, want %d", bytesRead, fixture.totalBytes) + } + runtime.KeepAlive(tree) + } + runtime.KeepAlive(trees) + }) + b.Run("GoFormat/"+testCase.name, func(b *testing.B) { + trees := freshPerformanceTrees(b, fixture, b.N) + b.ReportAllocs() + b.SetBytes(fixture.totalBytes) + b.ResetTimer() + b.ReportMetric(float64(fixture.totalBytes), "bytes/op") + b.ReportMetric(float64(performanceGoFileCount), "go-files/op") + for _, tree := range trees { + if err := checkGoFormat(context.Background(), tree); err != nil { + b.Fatal(err) + } + runtime.KeepAlive(tree) + } + runtime.KeepAlive(trees) + }) + } + + b.Run("Markdown/one-megabyte-repeated-links-and-identical-content", func(b *testing.B) { + trees := freshPerformanceTrees(b, markdown, b.N) + b.ReportAllocs() + b.SetBytes(markdown.totalBytes) + b.ResetTimer() + b.ReportMetric(float64(markdown.totalBytes), "bytes/op") + b.ReportMetric(float64(performanceMarkdownLinkCount*2), "repeated-links/op") + for _, tree := range trees { + if err := checkMarkdownLinks(context.Background(), tree, false, io.Discard); err != nil { + b.Fatal(err) + } + runtime.KeepAlive(tree) + } + runtime.KeepAlive(trees) + }) +} + +func BenchmarkPerformanceRetainedContent(b *testing.B) { + fixture := newPerformanceTree(b, performanceGoFileCount, false, false, true) + tree, err := ReadCandidateTree(context.Background(), fixture.repository, fixture.commit) + if err != nil { + b.Fatal(err) + } + cache := newMarkdownCache(tree) + var markdownPaths []string + for _, entry := range tree.Entries() { + if strings.HasSuffix(strings.ToLower(entry.Path), ".md") { + markdownPaths = append(markdownPaths, entry.Path) + } + } + if len(markdownPaths) != 3 { + b.Fatalf("Markdown paths = %v, want README and two identical documents", markdownPaths) + } + + runtime.GC() + var before runtime.MemStats + runtime.ReadMemStats(&before) + b.ReportAllocs() + b.ReportMetric(float64(len(fixture.paths)), "go-files/op") + b.ResetTimer() + for i := 0; i < b.N; i++ { + if err := checkGoFormat(context.Background(), tree); err != nil { + b.Fatal(err) + } + for _, path := range markdownPaths { + if _, err := cache.document(context.Background(), path); err != nil { + b.Fatal(err) + } + } + } + b.StopTimer() + runtime.GC() + var after runtime.MemStats + runtime.ReadMemStats(&after) + retained := int64(after.HeapAlloc) - int64(before.HeapAlloc) + if retained < 0 { + retained = 0 + } + b.ReportMetric(float64(retained), "retained-B") + runtime.KeepAlive(tree) + runtime.KeepAlive(cache) + runtime.KeepAlive(fixture) +} + +func newPerformanceTree(tb testing.TB, goFileCount int, repeated, packed, includeMarkdown bool) performanceTree { + tb.Helper() + repository := newTreeRepository(tb) + fixture := performanceTree{ + repository: repository, + paths: make([]string, 0, goFileCount), + expected: make(map[string][]byte, goFileCount+3), + } + for i := 0; i < goFileCount; i++ { + name := fmt.Sprintf("source/file-%04d.go", i) + contents := performanceGoSource(i) + if repeated { + contents = []byte("package source\n") + } + writeTreeFile(tb, repository, name, contents) + fixture.paths = append(fixture.paths, name) + fixture.expected[name] = contents + fixture.totalBytes += int64(len(contents)) + } + if includeMarkdown { + markdown := performanceMarkdownDocument() + readme := performanceMarkdownIndex() + for name, contents := range map[string][]byte{ + "README.md": readme, + "archive/payload.md": markdown, + "docs/payload.md": markdown, + } { + writeTreeFile(tb, repository, name, contents) + fixture.expected[name] = contents + fixture.totalBytes += int64(len(contents)) + } + } + + fixture.commit = commitCandidateTree(tb, repository) + if packed { + gitTree(tb, repository, "gc", "--aggressive", "--prune=now") + } + verifyPerformanceTree(tb, fixture, repeated, packed, includeMarkdown) + return fixture +} + +func freshPerformanceTrees(tb testing.TB, fixture performanceTree, count int) []*CandidateTree { + tb.Helper() + base, err := ReadCandidateTree(context.Background(), fixture.repository, fixture.commit) + if err != nil { + tb.Fatal(err) + } + trees := make([]*CandidateTree, count) + for index := range trees { + tree := *base + tree.blobs = make(map[string][]byte) + trees[index] = &tree + } + runtime.KeepAlive(base) + return trees +} + +func verifyPerformanceTree(tb testing.TB, fixture performanceTree, repeated, packed, includeMarkdown bool) { + tb.Helper() + tree, err := ReadCandidateTree(context.Background(), fixture.repository, fixture.commit) + if err != nil { + tb.Fatal(err) + } + if len(tree.Entries()) != len(fixture.expected) { + tb.Fatalf("tree entries = %d, want %d", len(tree.Entries()), len(fixture.expected)) + } + uniqueObjects := make(map[string]struct{}, len(fixture.paths)) + for _, path := range fixture.paths { + contents, err := tree.ReadFile(context.Background(), path) + if err != nil { + tb.Fatal(err) + } + if !bytes.Equal(contents, fixture.expected[path]) { + tb.Fatalf("committed content at %q differs from fixture bytes", path) + } + uniqueObjects[tree.entries[tree.byPath[path]].ObjectID] = struct{}{} + } + wantObjects := len(fixture.paths) + if repeated { + wantObjects = 1 + } + if len(uniqueObjects) != wantObjects { + tb.Fatalf("unique Go object IDs = %d, want %d", len(uniqueObjects), wantObjects) + } + if includeMarkdown { + for _, path := range []string{"README.md", "docs/payload.md", "archive/payload.md"} { + contents, err := tree.ReadFile(context.Background(), path) + if err != nil { + tb.Fatal(err) + } + if !bytes.Equal(contents, fixture.expected[path]) { + tb.Fatalf("committed Markdown content at %q differs from fixture bytes", path) + } + } + if len(fixture.expected["docs/payload.md"]) != performanceMarkdownSize { + tb.Fatalf("large Markdown fixture size = %d, want %d", len(fixture.expected["docs/payload.md"]), performanceMarkdownSize) + } + if !bytes.Equal(fixture.expected["docs/payload.md"], fixture.expected["archive/payload.md"]) { + tb.Fatal("Markdown copies differ") + } + if tree.entries[tree.byPath["docs/payload.md"]].ObjectID != tree.entries[tree.byPath["archive/payload.md"]].ObjectID { + tb.Fatal("identical Markdown copies do not share an object ID") + } + index := string(fixture.expected["README.md"]) + if strings.Count(index, "docs/payload.md#shared-payload") != performanceMarkdownLinkCount || strings.Count(index, "archive/payload.md#shared-payload") != performanceMarkdownLinkCount { + tb.Fatal("Markdown link fixture does not contain the expected repeated targets") + } + } + + objects := gitTree(tb, fixture.repository, "count-objects", "-v") + loose, packedCount := performanceObjectCounts(tb, objects) + if packed { + if loose != 0 || packedCount == 0 { + tb.Fatalf("packed fixture object counts = loose %d, in-pack %d", loose, packedCount) + } + } else if loose == 0 || packedCount != 0 { + tb.Fatalf("loose fixture object counts = loose %d, in-pack %d", loose, packedCount) + } +} + +func performanceObjectCounts(tb testing.TB, output string) (loose, packed int) { + tb.Helper() + for _, line := range strings.Split(output, "\n") { + key, value, ok := strings.Cut(line, ": ") + if !ok || (key != "count" && key != "in-pack") { + continue + } + count, err := strconv.Atoi(value) + if err != nil { + tb.Fatalf("parse git count-objects output %q: %v", line, err) + } + if key == "count" { + loose = count + } else { + packed = count + } + } + return loose, packed +} + +func performanceGoSource(index int) []byte { + return []byte(fmt.Sprintf("package source\n\nconst value%04d = %q\n", index, fmt.Sprintf("payload-%04d", index))) +} + +func performanceMarkdownDocument() []byte { + contents := make([]byte, performanceMarkdownSize) + prefix := []byte("# Shared Payload\n\n") + copy(contents, prefix) + for index := len(prefix); index < len(contents)-1; index++ { + contents[index] = 'x' + } + contents[len(contents)-1] = '\n' + return contents +} + +func performanceMarkdownIndex() []byte { + var contents strings.Builder + for index := 0; index < performanceMarkdownLinkCount; index++ { + fmt.Fprintf(&contents, "- [payload %d](docs/payload.md#shared-payload)\n", index) + fmt.Fprintf(&contents, "- [copy %d](archive/payload.md#shared-payload)\n", index) + } + return []byte(contents.String()) +} diff --git a/internal/validation/tree_test.go b/internal/validation/tree_test.go index 8cb9bf4..f01211d 100644 --- a/internal/validation/tree_test.go +++ b/internal/validation/tree_test.go @@ -93,7 +93,7 @@ type gitTreeEntry struct { hasSize bool } -func newTreeRepository(t *testing.T) string { +func newTreeRepository(t testing.TB) string { t.Helper() repository := t.TempDir() gitTree(t, repository, "init", "-b", "main") @@ -102,7 +102,7 @@ func newTreeRepository(t *testing.T) string { return repository } -func writeTreeFile(t *testing.T, repository, name string, content []byte) { +func writeTreeFile(t testing.TB, repository, name string, content []byte) { t.Helper() path := filepath.Join(repository, filepath.FromSlash(name)) if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { @@ -113,7 +113,7 @@ func writeTreeFile(t *testing.T, repository, name string, content []byte) { } } -func gitTree(t *testing.T, repository string, args ...string) string { +func gitTree(t testing.TB, repository string, args ...string) string { t.Helper() command := exec.Command("git", args...) command.Dir = repository diff --git a/performance_benchmark_test.go b/performance_benchmark_test.go new file mode 100644 index 0000000..fba8bf1 --- /dev/null +++ b/performance_benchmark_test.go @@ -0,0 +1,77 @@ +package releasekit_test + +import ( + "context" + "os" + "path/filepath" + "testing" + + "gitea.maximumdirect.net/eric/ReleaseKit" +) + +func BenchmarkPerformanceFacadeCheck(b *testing.B) { + repository, moduleBytes := performanceCheckFixture(b) + warmRequest := releasekit.ReleaseRequest{ + RepositoryDir: repository.work, + Product: "Sample App", + Version: "v1.2.3", + } + if _, err := releasekit.Check(context.Background(), warmRequest); err != nil { + b.Fatalf("warm benchmark Go build cache: %v", err) + } + b.Run("GoSuiteOnly", func(b *testing.B) { + benchmarkFacadeCheck(b, repository, moduleBytes, releasekit.ValidationSpec{}) + }) + b.Run("GoSuiteAndSelectedChecks", func(b *testing.B) { + benchmarkFacadeCheck(b, repository, moduleBytes, releasekit.ValidationSpec{ + GoFormat: true, + Markdown: &releasekit.MarkdownValidationSpec{}, + ModuleDistribution: releasekit.ModuleDistributionValidationSpec{ + LocalReplacements: true, + }, + }) + }) +} + +func benchmarkFacadeCheck(b *testing.B, repository publishFixture, moduleBytes int64, validation releasekit.ValidationSpec) { + b.Helper() + request := releasekit.ReleaseRequest{ + RepositoryDir: repository.work, + Product: "Sample App", + Version: "v1.2.3", + Validation: validation, + } + b.ReportAllocs() + b.ResetTimer() + b.ReportMetric(float64(moduleBytes), "module-bytes/op") + for i := 0; i < b.N; i++ { + if _, err := releasekit.Check(context.Background(), request); err != nil { + b.Fatal(err) + } + } +} + +func performanceCheckFixture(b *testing.B) (publishFixture, int64) { + b.Helper() + repository := newPublishFixture(b) + writeFile(b, repository.work, "source.go", "package fixture\n\nconst name = \"candidate\"\n") + gitRun(b, repository.work, "add", "source.go") + gitRun(b, repository.work, "commit", "-m", "Add check benchmark source") + if status := gitText(b, repository.work, "status", "--porcelain"); status != "" { + b.Fatalf("benchmark candidate is not clean:\n%s", status) + } + b.Setenv("GOMODCACHE", b.TempDir()) + b.Setenv("GOCACHE", b.TempDir()) + b.Setenv("GOWORK", "off") + b.Setenv("GOPROXY", "off") + b.Setenv("GOSUMDB", "off") + var moduleBytes int64 + for _, name := range []string{"go.mod", "source.go", "fixture_test.go", "race_fixture_test.go"} { + contents, err := os.ReadFile(filepath.Join(repository.work, name)) + if err != nil { + b.Fatal(err) + } + moduleBytes += int64(len(contents)) + } + return repository, moduleBytes +} diff --git a/publish_test.go b/publish_test.go index 6ddf8f3..fec8245 100644 --- a/publish_test.go +++ b/publish_test.go @@ -797,7 +797,7 @@ func shellQuote(value string) string { return "'" + strings.ReplaceAll(value, "'", "'\\''") + "'" } -func newPublishFixture(t *testing.T) publishFixture { +func newPublishFixture(t testing.TB) publishFixture { t.Helper() base := t.TempDir() work := filepath.Join(base, "work") @@ -870,7 +870,7 @@ func TestRaceBuildSelection(t *testing.T) { return publishFixture{work: work, bare: bare, commit: commit, unrelatedCommit: other} } -func gitRun(t *testing.T, dir string, args ...string) { +func gitRun(t testing.TB, dir string, args ...string) { t.Helper() cmd := exec.Command("git", args...) cmd.Dir = dir @@ -879,7 +879,7 @@ func gitRun(t *testing.T, dir string, args ...string) { } } -func gitText(t *testing.T, dir string, args ...string) string { +func gitText(t testing.TB, dir string, args ...string) string { t.Helper() cmd := exec.Command("git", args...) cmd.Dir = dir @@ -901,7 +901,7 @@ func gitBytes(t *testing.T, dir string, args ...string) string { return string(output) } -func writeFile(t *testing.T, root, name, contents string) { +func writeFile(t testing.TB, root, name, contents string) { t.Helper() path := filepath.Join(root, filepath.FromSlash(name)) if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {