Select module files with ancestor boundary lookups
This commit is contained in:
@@ -32,7 +32,11 @@ contents; the Go-only path does not enumerate the tree.
|
|||||||
`CandidateTree` keeps the full committed repository lookup separate from the
|
`CandidateTree` keeps the full committed repository lookup separate from the
|
||||||
ordered file input set. The selected module's committed regular-file inputs
|
ordered file input set. The selected module's committed regular-file inputs
|
||||||
exclude independently released nested modules, while vendored contents stay
|
exclude independently released nested modules, while vendored contents stay
|
||||||
in scope. Explicit shared-file selections are normalized from repository-root
|
in scope. Selection indexes boundary directories once and checks each entry's
|
||||||
|
ancestors, so unrelated boundaries are not scanned for each file. The
|
||||||
|
[scope benchmark](../../internal/validation/scope_performance_test.go) varies
|
||||||
|
boundary count while retaining 20,000 input files.
|
||||||
|
Explicit shared-file selections are normalized from repository-root
|
||||||
paths, added as individual files, and deduplicated with module inputs by path.
|
paths, added as individual files, and deduplicated with module inputs by path.
|
||||||
File-size exclusions are normalized from the selected module directory. Link
|
File-size exclusions are normalized from the selected module directory. Link
|
||||||
resolution can still inspect a committed destination outside the selected input
|
resolution can still inspect a committed destination outside the selected input
|
||||||
|
|||||||
@@ -336,7 +336,15 @@ test-owned caches. This adds no live module-service completion gate.
|
|||||||
|
|
||||||
## Issue 10: Remove the repeated boundary scan from file selection (P2)
|
## Issue 10: Remove the repeated boundary scan from file selection (P2)
|
||||||
|
|
||||||
Status: open. Category: code quality and avoidable computational cost.
|
Status: complete. Category: code quality and avoidable computational cost.
|
||||||
|
|
||||||
|
**Completion:** File selection indexes boundary directories and looks up each
|
||||||
|
entry's ancestors. Existing ordering, vendor/shared selection, full reference
|
||||||
|
lookup, and reader/cache behavior pass the complete offline validation suite.
|
||||||
|
A component benchmark retains 20,000 inputs with 0, 1,000, and 4,000 boundaries.
|
||||||
|
On this host, the 4,000-boundary case decreased from about 2.73 seconds to 38
|
||||||
|
milliseconds per selection; the new cases ranged from 29 to 38 milliseconds.
|
||||||
|
These are illustrative measurements without timing gates or input limits.
|
||||||
|
|
||||||
**Evidence:** [`selectInputEntries`](../../internal/validation/tree.go) loops
|
**Evidence:** [`selectInputEntries`](../../internal/validation/tree.go) loops
|
||||||
over all nested-module boundaries for every candidate entry. This costs
|
over all nested-module boundaries for every candidate entry. This costs
|
||||||
|
|||||||
38
internal/validation/scope_performance_test.go
Normal file
38
internal/validation/scope_performance_test.go
Normal file
@@ -0,0 +1,38 @@
|
|||||||
|
package validation
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/ReleaseKit/internal/git"
|
||||||
|
)
|
||||||
|
|
||||||
|
func BenchmarkSelectInputEntriesByModuleCount(b *testing.B) {
|
||||||
|
const retainedFiles = 20000
|
||||||
|
for _, boundaries := range []int{0, 1000, 4000} {
|
||||||
|
b.Run(fmt.Sprintf("boundaries=%d", boundaries), func(b *testing.B) {
|
||||||
|
entries := make([]git.TreeEntry, 0, retainedFiles+2*boundaries)
|
||||||
|
entries = append(entries, git.TreeEntry{Path: "go.mod", Type: "blob", Mode: "100644"})
|
||||||
|
for file := 1; file < retainedFiles; file++ {
|
||||||
|
entries = append(entries, git.TreeEntry{Path: fmt.Sprintf("source/file-%06d.txt", file), Type: "blob", Mode: "100644"})
|
||||||
|
}
|
||||||
|
for module := 0; module < boundaries; module++ {
|
||||||
|
directory := fmt.Sprintf("modules/unit-%06d", module)
|
||||||
|
entries = append(entries,
|
||||||
|
git.TreeEntry{Path: directory + "/go.mod", Type: "blob", Mode: "100644"},
|
||||||
|
git.TreeEntry{Path: directory + "/source.go", Type: "blob", Mode: "100644"})
|
||||||
|
}
|
||||||
|
byPath := make(map[string]int, len(entries))
|
||||||
|
for i, entry := range entries {
|
||||||
|
byPath[entry.Path] = i
|
||||||
|
}
|
||||||
|
b.ReportAllocs()
|
||||||
|
for b.Loop() {
|
||||||
|
selected, err := selectInputEntries(entries, byPath, ".", []string{})
|
||||||
|
if err != nil || len(selected) != retainedFiles {
|
||||||
|
b.Fatalf("selected %d files, error=%v; want %d", len(selected), err, retainedFiles)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -95,7 +95,7 @@ func (tree *CandidateTree) InputEntries() []git.TreeEntry {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func selectInputEntries(entries []git.TreeEntry, byPath map[string]int, modulePath string, sharedFiles []string) ([]git.TreeEntry, error) {
|
func selectInputEntries(entries []git.TreeEntry, byPath map[string]int, modulePath string, sharedFiles []string) ([]git.TreeEntry, error) {
|
||||||
boundaries := make([]string, 0)
|
boundaries := make(map[string]struct{})
|
||||||
for _, entry := range entries {
|
for _, entry := range entries {
|
||||||
moduleRelative, ok := relativeToModule(entry.Path, modulePath)
|
moduleRelative, ok := relativeToModule(entry.Path, modulePath)
|
||||||
if !ok || moduleRelative == "go.mod" || path.Base(moduleRelative) != "go.mod" {
|
if !ok || moduleRelative == "go.mod" || path.Base(moduleRelative) != "go.mod" {
|
||||||
@@ -104,7 +104,7 @@ func selectInputEntries(entries []git.TreeEntry, byPath map[string]int, modulePa
|
|||||||
if insideVendor(moduleRelative) {
|
if insideVendor(moduleRelative) {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
boundaries = append(boundaries, path.Dir(entry.Path))
|
boundaries[path.Dir(entry.Path)] = struct{}{}
|
||||||
}
|
}
|
||||||
selected := make(map[string]git.TreeEntry)
|
selected := make(map[string]git.TreeEntry)
|
||||||
for _, entry := range entries {
|
for _, entry := range entries {
|
||||||
@@ -113,8 +113,8 @@ func selectInputEntries(entries []git.TreeEntry, byPath map[string]int, modulePa
|
|||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
excluded := false
|
excluded := false
|
||||||
for _, boundary := range boundaries {
|
for ancestor := entry.Path; ancestor != "." && ancestor != modulePath; ancestor = path.Dir(ancestor) {
|
||||||
if entry.Path == boundary || strings.HasPrefix(entry.Path, boundary+"/") {
|
if _, isBoundary := boundaries[ancestor]; isBoundary {
|
||||||
excluded = true
|
excluded = true
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user