diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index e508797..63c8021 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -1,6 +1,6 @@ # Computational Efficiency Implementation Plan -**Status:** Stage 1 complete; Stages 2–11 pending. +**Status:** Stages 1–2 complete; Stages 3–11 pending. ## Purpose And Baseline @@ -126,6 +126,53 @@ measurement support only. **Owners:** Benchmark files in `internal/delivery`, `internal/gotool`, `internal/runtimeinfo`, and root delivery fixtures. +**Completion:** Complete. Measurements cover declarations and local +verification of two deterministic 16 MiB files, prebuilt facade delivery for +fresh uploads and matching retries, combined output capture and forwarding at +4 KiB and 8 MiB, and injected and real metadata version access. The local HTTP +fixture hashes and counts incoming multipart bytes incrementally and streams +matching files from immutable fixture data. It verifies uploaded identities and +that retries match all three declared assets without uploading again. + +### Delivery, Output, And Runtime Baseline + +The baseline is commit `29e1bab`, before production refactoring, using the same +Go, Git, operating system, processor, benchmark command, and three repetitions +recorded above. Artifact declaration and preflight fixtures contain two +deterministic 16 MiB files (33,554,432 binary bytes per operation); setup and +initial writes are outside the timers. Declaration currently hashes each file, +and declaration plus local verification therefore measures both hashing passes. +The facade benchmark uses a committed tagged source fixture and prebuilt +artifacts, so its time includes local source checks, hashing, HTTP requests, and +streamed body comparison or upload, but no Go compilation. Fresh hosted state is +reset outside the timed region for every operation. Retry state is prepared +before timing. The fixture server retains only digest and size metadata for +uploads and streams known matching bytes directly from shared fixture content. + +Output measurements call the combined writer directly, with forwarding to +`io.Discard`, then capture its completed buffer. They include the current +completed-output copy. Runtime fallback measurements call `Version()` repeatedly +against real build metadata; injected access measures the selection helper's +injected-version branch. The benchmark process reports its own allocations, +including local HTTP fixture work; streamed artifact bodies are not buffered by +the fixture server. + +| Measurement | Baseline observations | Final comparison | +| --- | --- | --- | +| Artifact declarations, two 16 MiB files | 94.3–95.4 ms; 33,554,432 bytes processed; about 68.9 KB allocated. | Pending Stage 11. | +| Declarations plus local verification | 186.7–188.4 ms; 33,554,432 binary bytes per each of two passes; about 136.2 KB allocated. | Pending Stage 11. | +| Facade delivery, fresh uploads | 429–447 ms; 32 MiB binary payload, two files plus manifest; 1.43–1.46 MB allocated. | Pending Stage 11. | +| Facade delivery, matching retry | 309.1–310.0 ms; 32 MiB of local verification plus streamed remote comparisons; 1.21–1.22 MB allocated. | Pending Stage 11. | +| Combined output, 4 KiB and `io.Discard` | 1.82–1.87 μs; 8,256 B and 3 allocations per operation. | Pending Stage 11. | +| Combined output, 8 MiB and `io.Discard` | 2.02–2.36 ms; 16.78 MB and 3 allocations per operation, including the completed capture copy. | Pending Stage 11. | +| Injected version selection | 0.745–0.754 ns; 0 B and 0 allocations. | Pending Stage 11. | +| Repeated real metadata fallback | 1.438–1.474 μs; 1,184 B and 7 allocations per operation. | Pending Stage 11. | + +The delivery observations include local HTTP and filesystem costs specific to +these fixtures. The output component benchmark does not include child startup; +the real delivery benchmark does include HTTP request handling. These results +describe measured workloads and do not establish runtime guarantees. + 1. Extend the same benchmark prefix to declaration/preflight processing of two deterministic 16 MiB artifact files. Measure prebuilt facade `Deliver` against a local HTTP fixture for fresh uploads and matching retries. diff --git a/internal/delivery/performance_test.go b/internal/delivery/performance_test.go new file mode 100644 index 0000000..3be26b9 --- /dev/null +++ b/internal/delivery/performance_test.go @@ -0,0 +1,85 @@ +package delivery + +import ( + "crypto/sha256" + "encoding/hex" + "fmt" + "os" + "path/filepath" + "testing" + + internalbuild "gitea.maximumdirect.net/eric/ReleaseKit/internal/build" + "gitea.maximumdirect.net/eric/ReleaseKit/internal/gitea" +) + +const performanceArtifactSize = 16 << 20 + +func BenchmarkPerformanceAssetPreflight(b *testing.B) { + result, totalBytes := performanceBuildResult(b) + declarations, err := declaredBuildAssets(result) + if err != nil { + b.Fatal(err) + } + + b.Run("Declarations", func(b *testing.B) { + b.ReportAllocs() + b.SetBytes(totalBytes) + b.ResetTimer() + b.ReportMetric(float64(totalBytes), "bytes/op") + for i := 0; i < b.N; i++ { + assets, err := declaredBuildAssets(result) + if err != nil || len(assets) != len(declarations) { + b.Fatalf("declaredBuildAssets() returned %d assets, err %v", len(assets), err) + } + } + }) + + b.Run("DeclarationsAndLocalVerification", func(b *testing.B) { + b.ReportAllocs() + b.SetBytes(totalBytes) + b.ResetTimer() + b.ReportMetric(float64(totalBytes), "bytes/op") + for i := 0; i < b.N; i++ { + assets, err := declaredBuildAssets(result) + if err != nil { + b.Fatal(err) + } + if err := gitea.VerifyLocalAssets(assets); err != nil { + b.Fatal(err) + } + } + }) +} + +func performanceBuildResult(b *testing.B) (internalbuild.Result, int64) { + b.Helper() + outputDir := b.TempDir() + data := make([]byte, performanceArtifactSize) + for i := range data { + data[i] = byte((i*31 + i/251 + 17) % 251) + } + digest := sha256.Sum256(data) + result := internalbuild.Result{ + Version: "v1.2.3", + BinaryName: "releasekit", + Targets: []internalbuild.Target{ + {OS: "linux", Arch: "amd64"}, + {OS: "windows", Arch: "amd64"}, + }, + } + for _, target := range result.Targets { + name := fmt.Sprintf("releasekit-v1.2.3-%s-%s", target.OS, target.Arch) + if target.OS == "windows" { + name += ".exe" + } + path := filepath.Join(outputDir, name) + if err := os.WriteFile(path, data, 0o600); err != nil { + b.Fatal(err) + } + result.Artifacts = append(result.Artifacts, internalbuild.Artifact{ + Path: path, Filename: name, Target: target, + SHA256: hex.EncodeToString(digest[:]), + }) + } + return result, 2 * performanceArtifactSize +} diff --git a/internal/gotool/performance_test.go b/internal/gotool/performance_test.go new file mode 100644 index 0000000..ac87f37 --- /dev/null +++ b/internal/gotool/performance_test.go @@ -0,0 +1,38 @@ +package gotool + +import ( + "bytes" + "io" + "testing" +) + +const performanceDiagnosticSize = 8 << 20 + +func BenchmarkPerformanceCombinedOutput(b *testing.B) { + for _, testCase := range []struct { + name string + size int + }{ + {name: "modest", size: 4 << 10}, + {name: "diagnostic-8MiB", size: performanceDiagnosticSize}, + } { + input := bytes.Repeat([]byte("go tool diagnostic output\n"), (testCase.size+25)/26) + input = input[:testCase.size] + b.Run(testCase.name+"/discard-forwarding", func(b *testing.B) { + b.ReportAllocs() + b.SetBytes(int64(len(input))) + b.ResetTimer() + b.ReportMetric(float64(len(input)), "bytes/op") + for i := 0; i < b.N; i++ { + writer := &combinedWriter{output: io.Discard} + if written, err := writer.Write(input); err != nil || written != len(input) { + b.Fatalf("write captured output = %d, err %v; want %d bytes", written, err, len(input)) + } + captured := writer.Bytes() + if len(captured) != len(input) { + b.Fatalf("captured output = %d bytes, want %d", len(captured), len(input)) + } + } + }) + } +} diff --git a/internal/runtimeinfo/performance_test.go b/internal/runtimeinfo/performance_test.go new file mode 100644 index 0000000..ac90f24 --- /dev/null +++ b/internal/runtimeinfo/performance_test.go @@ -0,0 +1,31 @@ +package runtimeinfo + +import ( + "runtime/debug" + "testing" +) + +var performanceVersionSink string + +func BenchmarkPerformanceVersion(b *testing.B) { + b.Run("Injected", func(b *testing.B) { + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + performanceVersionSink = version("v1.2.3", debug.ReadBuildInfo) + if performanceVersionSink != "v1.2.3" { + b.Fatalf("version() = %q, want injected version", performanceVersionSink) + } + } + }) + b.Run("RealMetadataFallback", func(b *testing.B) { + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + performanceVersionSink = Version() + if performanceVersionSink == "" { + b.Fatal("Version() returned an empty version") + } + } + }) +} diff --git a/performance_delivery_test.go b/performance_delivery_test.go new file mode 100644 index 0000000..83569bf --- /dev/null +++ b/performance_delivery_test.go @@ -0,0 +1,304 @@ +package releasekit_test + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "hash" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "sort" + "strings" + "sync" + "testing" + + releasekit "gitea.maximumdirect.net/eric/ReleaseKit" + "gitea.maximumdirect.net/eric/ReleaseKit/internal/gitea" +) + +const performanceDeliveryArtifactSize = 16 << 20 + +type deliveryBenchmarkAsset struct { + name string + data []byte + digest string +} + +type deliveryBenchmarkServer struct { + mu sync.Mutex + baseURL string + commit string + assets []deliveryBenchmarkAsset + page bool + attached map[string]deliveryBenchmarkIdentity +} + +type deliveryBenchmarkIdentity struct { + digest string + size int64 +} + +func BenchmarkPerformanceFacadeDeliver(b *testing.B) { + repository, build, expected := newDeliveryBenchmarkFixture(b) + state := &deliveryBenchmarkServer{commit: gitText(b, repository, "rev-parse", "HEAD"), assets: expected} + server := httptest.NewServer(http.HandlerFunc(state.serveHTTP)) + state.baseURL = server.URL + b.Cleanup(server.Close) + request := releasekit.DeliveryRequest{ + RepositoryDir: repository, + Product: "Sample App", + Version: "v1.2.3", + Destination: facadeDeliveryDestination(server.URL), + Prebuilt: &build, + } + totalBytes := int64(2 * performanceDeliveryArtifactSize) + + b.Run("FreshUploads", func(b *testing.B) { + b.ReportAllocs() + b.SetBytes(totalBytes) + b.ReportMetric(float64(totalBytes), "artifact-bytes/op") + b.ReportMetric(2, "binary-files/op") + for i := 0; i < b.N; i++ { + b.StopTimer() + state.reset(false) + b.StartTimer() + result, err := releasekit.Deliver(context.Background(), request) + b.StopTimer() + if err != nil { + b.Fatalf("Deliver() error = %v", err) + } + if len(result.Uploaded) != 3 { + b.Fatalf("uploaded %d assets, want two binaries and a manifest", len(result.Uploaded)) + } + state.verifyUploads(b) + b.StartTimer() + } + }) + + b.Run("MatchingRetry", func(b *testing.B) { + b.StopTimer() + state.reset(true) + b.ReportAllocs() + b.SetBytes(totalBytes) + b.ReportMetric(float64(totalBytes), "artifact-bytes/op") + b.ReportMetric(2, "binary-files/op") + b.ResetTimer() + b.StartTimer() + for i := 0; i < b.N; i++ { + result, err := releasekit.Deliver(context.Background(), request) + if err != nil { + b.Fatalf("Deliver() error = %v", err) + } + if len(result.Uploaded) != 0 || len(result.Matching) != 3 { + b.Fatalf("matching retry uploaded %d and matched %d assets; want 0 and 3", len(result.Uploaded), len(result.Matching)) + } + } + }) +} + +func newDeliveryBenchmarkFixture(b *testing.B) (string, releasekit.BuildResult, []deliveryBenchmarkAsset) { + b.Helper() + repository := filepath.Join(b.TempDir(), "repository") + if err := os.MkdirAll(repository, 0o755); err != nil { + b.Fatal(err) + } + gitRun(b, repository, "init", "-b", "main") + gitRun(b, repository, "config", "user.name", "ReleaseKit Benchmark") + gitRun(b, repository, "config", "user.email", "benchmark@example.invalid") + writeFile(b, repository, "docs/releases/v1.2.3.md", "# Sample App v1.2.3\n\nDelivery fixture.\n") + gitRun(b, repository, "add", ".") + gitRun(b, repository, "commit", "-m", "prepare delivery benchmark") + gitRun(b, repository, "tag", "v1.2.3") + + data := make([]byte, performanceDeliveryArtifactSize) + for i := range data { + data[i] = byte((i*17 + i/127 + 29) % 251) + } + digest := sha256.Sum256(data) + digestText := hex.EncodeToString(digest[:]) + outputDir := filepath.Join(b.TempDir(), "artifacts") + if err := os.MkdirAll(outputDir, 0o700); err != nil { + b.Fatal(err) + } + targets := []releasekit.Target{{OS: "linux", Arch: "amd64"}, {OS: "windows", Arch: "amd64"}} + build := releasekit.BuildResult{ + RepositoryDir: repository, CandidateCommit: gitText(b, repository, "rev-parse", "HEAD"), + SourceVerified: true, Version: "v1.2.3", MainPackage: "./cmd/app", + BinaryName: "releasekit", Targets: targets, OutputDir: outputDir, + } + assets := make([]deliveryBenchmarkAsset, 0, len(targets)+1) + for _, target := range targets { + filename := fmt.Sprintf("releasekit-v1.2.3-%s-%s", target.OS, target.Arch) + if target.OS == "windows" { + filename += ".exe" + } + path := filepath.Join(outputDir, filename) + if err := os.WriteFile(path, data, 0o600); err != nil { + b.Fatal(err) + } + build.Artifacts = append(build.Artifacts, releasekit.BuildArtifact{Path: path, Filename: filename, Target: target, SHA256: digestText}) + assets = append(assets, deliveryBenchmarkAsset{name: filename, data: data, digest: digestText}) + } + manifest := performanceDeliveryManifest(build.Artifacts) + manifestHash := sha256.Sum256(manifest) + assets = append(assets, deliveryBenchmarkAsset{name: gitea.ManifestFilename, data: manifest, digest: hex.EncodeToString(manifestHash[:])}) + return repository, build, assets +} + +func performanceDeliveryManifest(artifacts []releasekit.BuildArtifact) []byte { + ordered := append([]releasekit.BuildArtifact(nil), artifacts...) + sort.Slice(ordered, func(i, j int) bool { return ordered[i].Filename < ordered[j].Filename }) + var manifest strings.Builder + for _, artifact := range ordered { + fmt.Fprintf(&manifest, "%s %s\n", artifact.SHA256, artifact.Filename) + } + return []byte(manifest.String()) +} + +func (state *deliveryBenchmarkServer) reset(populated bool) { + state.mu.Lock() + defer state.mu.Unlock() + state.page = populated + state.attached = make(map[string]deliveryBenchmarkIdentity) + if populated { + for _, asset := range state.assets { + state.attached[asset.name] = deliveryBenchmarkIdentity{digest: asset.digest, size: int64(len(asset.data))} + } + } +} + +func (state *deliveryBenchmarkServer) verifyUploads(b *testing.B) { + b.Helper() + state.mu.Lock() + defer state.mu.Unlock() + if !state.page || len(state.attached) != len(state.assets) { + b.Fatalf("fresh upload left release page %t and %d attached assets, want page and %d assets", state.page, len(state.attached), len(state.assets)) + } + for _, asset := range state.assets { + identity, ok := state.attached[asset.name] + if !ok || identity.digest != asset.digest || identity.size != int64(len(asset.data)) { + b.Fatalf("uploaded %q identity = %#v, present %t; want digest %s and size %d", asset.name, identity, ok, asset.digest, len(asset.data)) + } + } +} + +func (state *deliveryBenchmarkServer) serveHTTP(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/api/v1/repos/owner/example/tags/v1.2.3": + _, _ = fmt.Fprintf(w, `{"name":"v1.2.3","commit":{"sha":%q}}`, state.commit) + case "/api/v1/repos/owner/example/releases/tags/v1.2.3": + state.mu.Lock() + page := state.page + state.mu.Unlock() + if !page { + http.NotFound(w, r) + return + } + state.writeRelease(w) + case "/api/v1/repos/owner/example/releases": + state.mu.Lock() + state.page = true + state.mu.Unlock() + state.writeRelease(w) + case "/api/v1/repos/owner/example/releases/10/assets": + if r.Method == http.MethodGet { + state.writeAssetList(w) + return + } + state.receiveUpload(w, r) + default: + if strings.HasPrefix(r.URL.Path, "/files/") { + state.serveFile(w, r) + return + } + http.NotFound(w, r) + } +} + +func (state *deliveryBenchmarkServer) writeRelease(w http.ResponseWriter) { + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{ + "id": 10, "html_url": state.baseURL + "/releases/10", "tag_name": "v1.2.3", + "target_commitish": state.commit, "name": "Fixture v1.2.3", + "body": "# Sample App v1.2.3\n\nDelivery fixture.\n", "draft": false, "prerelease": false, + }) +} + +func (state *deliveryBenchmarkServer) writeAssetList(w http.ResponseWriter) { + state.mu.Lock() + assets := make([]map[string]any, 0, len(state.attached)) + for name, identity := range state.attached { + assets = append(assets, map[string]any{ + "id": len(assets) + 1, "name": name, "size": identity.size, + "browser_download_url": state.baseURL + "/files/" + name, + }) + } + state.mu.Unlock() + _ = json.NewEncoder(w).Encode(assets) +} + +func (state *deliveryBenchmarkServer) receiveUpload(w http.ResponseWriter, r *http.Request) { + name := r.URL.Query().Get("name") + reader, err := r.MultipartReader() + if err != nil { + http.Error(w, "invalid multipart request", http.StatusBadRequest) + return + } + var digest hash.Hash = sha256.New() + var size int64 + for { + part, err := reader.NextPart() + if err == io.EOF { + break + } + if err != nil { + http.Error(w, "invalid multipart body", http.StatusBadRequest) + return + } + if part.FormName() == "attachment" { + size, err = io.Copy(digest, part) + if err != nil { + _ = part.Close() + http.Error(w, "cannot read attachment", http.StatusBadRequest) + return + } + } + _ = part.Close() + } + identity := deliveryBenchmarkIdentity{digest: hex.EncodeToString(digest.Sum(nil)), size: size} + var expected *deliveryBenchmarkAsset + for i := range state.assets { + if state.assets[i].name == name { + expected = &state.assets[i] + break + } + } + if expected == nil || identity.digest != expected.digest || identity.size != int64(len(expected.data)) { + http.Error(w, "attachment digest or size mismatch", http.StatusBadRequest) + return + } + state.mu.Lock() + state.attached[name] = identity + assetID := len(state.attached) + state.mu.Unlock() + w.WriteHeader(http.StatusCreated) + _ = json.NewEncoder(w).Encode(map[string]any{"id": assetID, "name": name, "size": size}) +} + +func (state *deliveryBenchmarkServer) serveFile(w http.ResponseWriter, r *http.Request) { + name := strings.TrimPrefix(r.URL.Path, "/files/") + for _, asset := range state.assets { + if asset.name == name { + w.Header().Set("Content-Length", fmt.Sprint(len(asset.data))) + _, _ = io.Copy(w, bytes.NewReader(asset.data)) + return + } + } + http.NotFound(w, r) +}