Measure delivery output and version costs

This commit is contained in:
2026-10-09 01:11:07 +00:00
parent 29e1bab228
commit 41e58ef894
5 changed files with 506 additions and 1 deletions

View File

@@ -1,6 +1,6 @@
# Computational Efficiency Implementation Plan
**Status:** Stage 1 complete; Stages 2–11 pending.
**Status:** Stages 1–2 complete; Stages 3–11 pending.
## Purpose And Baseline
@@ -126,6 +126,53 @@ measurement support only.
**Owners:** Benchmark files in `internal/delivery`, `internal/gotool`,
`internal/runtimeinfo`, and root delivery fixtures.
**Completion:** Complete. Measurements cover declarations and local
verification of two deterministic 16 MiB files, prebuilt facade delivery for
fresh uploads and matching retries, combined output capture and forwarding at
4 KiB and 8 MiB, and injected and real metadata version access. The local HTTP
fixture hashes and counts incoming multipart bytes incrementally and streams
matching files from immutable fixture data. It verifies uploaded identities and
that retries match all three declared assets without uploading again.
### Delivery, Output, And Runtime Baseline
The baseline is commit `29e1bab`, before production refactoring, using the same
Go, Git, operating system, processor, benchmark command, and three repetitions
recorded above. Artifact declaration and preflight fixtures contain two
deterministic 16 MiB files (33,554,432 binary bytes per operation); setup and
initial writes are outside the timers. Declaration currently hashes each file,
and declaration plus local verification therefore measures both hashing passes.
The facade benchmark uses a committed tagged source fixture and prebuilt
artifacts, so its time includes local source checks, hashing, HTTP requests, and
streamed body comparison or upload, but no Go compilation. Fresh hosted state is
reset outside the timed region for every operation. Retry state is prepared
before timing. The fixture server retains only digest and size metadata for
uploads and streams known matching bytes directly from shared fixture content.
Output measurements call the combined writer directly, with forwarding to
`io.Discard`, then capture its completed buffer. They include the current
completed-output copy. Runtime fallback measurements call `Version()` repeatedly
against real build metadata; injected access measures the selection helper's
injected-version branch. The benchmark process reports its own allocations,
including local HTTP fixture work; streamed artifact bodies are not buffered by
the fixture server.
| Measurement | Baseline observations | Final comparison |
| --- | --- | --- |
| Artifact declarations, two 16 MiB files | 94.3–95.4 ms; 33,554,432 bytes processed; about 68.9 KB allocated. | Pending Stage 11. |
| Declarations plus local verification | 186.7–188.4 ms; 33,554,432 binary bytes per each of two passes; about 136.2 KB allocated. | Pending Stage 11. |
| Facade delivery, fresh uploads | 429–447 ms; 32 MiB binary payload, two files plus manifest; 1.43–1.46 MB allocated. | Pending Stage 11. |
| Facade delivery, matching retry | 309.1–310.0 ms; 32 MiB of local verification plus streamed remote comparisons; 1.21–1.22 MB allocated. | Pending Stage 11. |
| Combined output, 4 KiB and `io.Discard` | 1.82–1.87 μs; 8,256 B and 3 allocations per operation. | Pending Stage 11. |
| Combined output, 8 MiB and `io.Discard` | 2.02–2.36 ms; 16.78 MB and 3 allocations per operation, including the completed capture copy. | Pending Stage 11. |
| Injected version selection | 0.745–0.754 ns; 0 B and 0 allocations. | Pending Stage 11. |
| Repeated real metadata fallback | 1.438–1.474 μs; 1,184 B and 7 allocations per operation. | Pending Stage 11. |
The delivery observations include local HTTP and filesystem costs specific to
these fixtures. The output component benchmark does not include child startup;
the real delivery benchmark does include HTTP request handling. These results
describe measured workloads and do not establish runtime guarantees.
1. Extend the same benchmark prefix to declaration/preflight processing of two
deterministic 16 MiB artifact files. Measure prebuilt facade `Deliver`
against a local HTTP fixture for fresh uploads and matching retries.

View File

@@ -0,0 +1,85 @@
package delivery
import (
"crypto/sha256"
"encoding/hex"
"fmt"
"os"
"path/filepath"
"testing"
internalbuild "gitea.maximumdirect.net/eric/ReleaseKit/internal/build"
"gitea.maximumdirect.net/eric/ReleaseKit/internal/gitea"
)
const performanceArtifactSize = 16 << 20
func BenchmarkPerformanceAssetPreflight(b *testing.B) {
result, totalBytes := performanceBuildResult(b)
declarations, err := declaredBuildAssets(result)
if err != nil {
b.Fatal(err)
}
b.Run("Declarations", func(b *testing.B) {
b.ReportAllocs()
b.SetBytes(totalBytes)
b.ResetTimer()
b.ReportMetric(float64(totalBytes), "bytes/op")
for i := 0; i < b.N; i++ {
assets, err := declaredBuildAssets(result)
if err != nil || len(assets) != len(declarations) {
b.Fatalf("declaredBuildAssets() returned %d assets, err %v", len(assets), err)
}
}
})
b.Run("DeclarationsAndLocalVerification", func(b *testing.B) {
b.ReportAllocs()
b.SetBytes(totalBytes)
b.ResetTimer()
b.ReportMetric(float64(totalBytes), "bytes/op")
for i := 0; i < b.N; i++ {
assets, err := declaredBuildAssets(result)
if err != nil {
b.Fatal(err)
}
if err := gitea.VerifyLocalAssets(assets); err != nil {
b.Fatal(err)
}
}
})
}
func performanceBuildResult(b *testing.B) (internalbuild.Result, int64) {
b.Helper()
outputDir := b.TempDir()
data := make([]byte, performanceArtifactSize)
for i := range data {
data[i] = byte((i*31 + i/251 + 17) % 251)
}
digest := sha256.Sum256(data)
result := internalbuild.Result{
Version: "v1.2.3",
BinaryName: "releasekit",
Targets: []internalbuild.Target{
{OS: "linux", Arch: "amd64"},
{OS: "windows", Arch: "amd64"},
},
}
for _, target := range result.Targets {
name := fmt.Sprintf("releasekit-v1.2.3-%s-%s", target.OS, target.Arch)
if target.OS == "windows" {
name += ".exe"
}
path := filepath.Join(outputDir, name)
if err := os.WriteFile(path, data, 0o600); err != nil {
b.Fatal(err)
}
result.Artifacts = append(result.Artifacts, internalbuild.Artifact{
Path: path, Filename: name, Target: target,
SHA256: hex.EncodeToString(digest[:]),
})
}
return result, 2 * performanceArtifactSize
}

View File

@@ -0,0 +1,38 @@
package gotool
import (
"bytes"
"io"
"testing"
)
const performanceDiagnosticSize = 8 << 20
func BenchmarkPerformanceCombinedOutput(b *testing.B) {
for _, testCase := range []struct {
name string
size int
}{
{name: "modest", size: 4 << 10},
{name: "diagnostic-8MiB", size: performanceDiagnosticSize},
} {
input := bytes.Repeat([]byte("go tool diagnostic output\n"), (testCase.size+25)/26)
input = input[:testCase.size]
b.Run(testCase.name+"/discard-forwarding", func(b *testing.B) {
b.ReportAllocs()
b.SetBytes(int64(len(input)))
b.ResetTimer()
b.ReportMetric(float64(len(input)), "bytes/op")
for i := 0; i < b.N; i++ {
writer := &combinedWriter{output: io.Discard}
if written, err := writer.Write(input); err != nil || written != len(input) {
b.Fatalf("write captured output = %d, err %v; want %d bytes", written, err, len(input))
}
captured := writer.Bytes()
if len(captured) != len(input) {
b.Fatalf("captured output = %d bytes, want %d", len(captured), len(input))
}
}
})
}
}

View File

@@ -0,0 +1,31 @@
package runtimeinfo
import (
"runtime/debug"
"testing"
)
var performanceVersionSink string
func BenchmarkPerformanceVersion(b *testing.B) {
b.Run("Injected", func(b *testing.B) {
b.ReportAllocs()
b.ResetTimer()
for i := 0; i < b.N; i++ {
performanceVersionSink = version("v1.2.3", debug.ReadBuildInfo)
if performanceVersionSink != "v1.2.3" {
b.Fatalf("version() = %q, want injected version", performanceVersionSink)
}
}
})
b.Run("RealMetadataFallback", func(b *testing.B) {
b.ReportAllocs()
b.ResetTimer()
for i := 0; i < b.N; i++ {
performanceVersionSink = Version()
if performanceVersionSink == "" {
b.Fatal("Version() returned an empty version")
}
}
})
}

View File

@@ -0,0 +1,304 @@
package releasekit_test
import (
"bytes"
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"fmt"
"hash"
"io"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"sort"
"strings"
"sync"
"testing"
releasekit "gitea.maximumdirect.net/eric/ReleaseKit"
"gitea.maximumdirect.net/eric/ReleaseKit/internal/gitea"
)
const performanceDeliveryArtifactSize = 16 << 20
type deliveryBenchmarkAsset struct {
name string
data []byte
digest string
}
type deliveryBenchmarkServer struct {
mu sync.Mutex
baseURL string
commit string
assets []deliveryBenchmarkAsset
page bool
attached map[string]deliveryBenchmarkIdentity
}
type deliveryBenchmarkIdentity struct {
digest string
size int64
}
func BenchmarkPerformanceFacadeDeliver(b *testing.B) {
repository, build, expected := newDeliveryBenchmarkFixture(b)
state := &deliveryBenchmarkServer{commit: gitText(b, repository, "rev-parse", "HEAD"), assets: expected}
server := httptest.NewServer(http.HandlerFunc(state.serveHTTP))
state.baseURL = server.URL
b.Cleanup(server.Close)
request := releasekit.DeliveryRequest{
RepositoryDir: repository,
Product: "Sample App",
Version: "v1.2.3",
Destination: facadeDeliveryDestination(server.URL),
Prebuilt: &build,
}
totalBytes := int64(2 * performanceDeliveryArtifactSize)
b.Run("FreshUploads", func(b *testing.B) {
b.ReportAllocs()
b.SetBytes(totalBytes)
b.ReportMetric(float64(totalBytes), "artifact-bytes/op")
b.ReportMetric(2, "binary-files/op")
for i := 0; i < b.N; i++ {
b.StopTimer()
state.reset(false)
b.StartTimer()
result, err := releasekit.Deliver(context.Background(), request)
b.StopTimer()
if err != nil {
b.Fatalf("Deliver() error = %v", err)
}
if len(result.Uploaded) != 3 {
b.Fatalf("uploaded %d assets, want two binaries and a manifest", len(result.Uploaded))
}
state.verifyUploads(b)
b.StartTimer()
}
})
b.Run("MatchingRetry", func(b *testing.B) {
b.StopTimer()
state.reset(true)
b.ReportAllocs()
b.SetBytes(totalBytes)
b.ReportMetric(float64(totalBytes), "artifact-bytes/op")
b.ReportMetric(2, "binary-files/op")
b.ResetTimer()
b.StartTimer()
for i := 0; i < b.N; i++ {
result, err := releasekit.Deliver(context.Background(), request)
if err != nil {
b.Fatalf("Deliver() error = %v", err)
}
if len(result.Uploaded) != 0 || len(result.Matching) != 3 {
b.Fatalf("matching retry uploaded %d and matched %d assets; want 0 and 3", len(result.Uploaded), len(result.Matching))
}
}
})
}
func newDeliveryBenchmarkFixture(b *testing.B) (string, releasekit.BuildResult, []deliveryBenchmarkAsset) {
b.Helper()
repository := filepath.Join(b.TempDir(), "repository")
if err := os.MkdirAll(repository, 0o755); err != nil {
b.Fatal(err)
}
gitRun(b, repository, "init", "-b", "main")
gitRun(b, repository, "config", "user.name", "ReleaseKit Benchmark")
gitRun(b, repository, "config", "user.email", "benchmark@example.invalid")
writeFile(b, repository, "docs/releases/v1.2.3.md", "# Sample App v1.2.3\n\nDelivery fixture.\n")
gitRun(b, repository, "add", ".")
gitRun(b, repository, "commit", "-m", "prepare delivery benchmark")
gitRun(b, repository, "tag", "v1.2.3")
data := make([]byte, performanceDeliveryArtifactSize)
for i := range data {
data[i] = byte((i*17 + i/127 + 29) % 251)
}
digest := sha256.Sum256(data)
digestText := hex.EncodeToString(digest[:])
outputDir := filepath.Join(b.TempDir(), "artifacts")
if err := os.MkdirAll(outputDir, 0o700); err != nil {
b.Fatal(err)
}
targets := []releasekit.Target{{OS: "linux", Arch: "amd64"}, {OS: "windows", Arch: "amd64"}}
build := releasekit.BuildResult{
RepositoryDir: repository, CandidateCommit: gitText(b, repository, "rev-parse", "HEAD"),
SourceVerified: true, Version: "v1.2.3", MainPackage: "./cmd/app",
BinaryName: "releasekit", Targets: targets, OutputDir: outputDir,
}
assets := make([]deliveryBenchmarkAsset, 0, len(targets)+1)
for _, target := range targets {
filename := fmt.Sprintf("releasekit-v1.2.3-%s-%s", target.OS, target.Arch)
if target.OS == "windows" {
filename += ".exe"
}
path := filepath.Join(outputDir, filename)
if err := os.WriteFile(path, data, 0o600); err != nil {
b.Fatal(err)
}
build.Artifacts = append(build.Artifacts, releasekit.BuildArtifact{Path: path, Filename: filename, Target: target, SHA256: digestText})
assets = append(assets, deliveryBenchmarkAsset{name: filename, data: data, digest: digestText})
}
manifest := performanceDeliveryManifest(build.Artifacts)
manifestHash := sha256.Sum256(manifest)
assets = append(assets, deliveryBenchmarkAsset{name: gitea.ManifestFilename, data: manifest, digest: hex.EncodeToString(manifestHash[:])})
return repository, build, assets
}
func performanceDeliveryManifest(artifacts []releasekit.BuildArtifact) []byte {
ordered := append([]releasekit.BuildArtifact(nil), artifacts...)
sort.Slice(ordered, func(i, j int) bool { return ordered[i].Filename < ordered[j].Filename })
var manifest strings.Builder
for _, artifact := range ordered {
fmt.Fprintf(&manifest, "%s %s\n", artifact.SHA256, artifact.Filename)
}
return []byte(manifest.String())
}
func (state *deliveryBenchmarkServer) reset(populated bool) {
state.mu.Lock()
defer state.mu.Unlock()
state.page = populated
state.attached = make(map[string]deliveryBenchmarkIdentity)
if populated {
for _, asset := range state.assets {
state.attached[asset.name] = deliveryBenchmarkIdentity{digest: asset.digest, size: int64(len(asset.data))}
}
}
}
func (state *deliveryBenchmarkServer) verifyUploads(b *testing.B) {
b.Helper()
state.mu.Lock()
defer state.mu.Unlock()
if !state.page || len(state.attached) != len(state.assets) {
b.Fatalf("fresh upload left release page %t and %d attached assets, want page and %d assets", state.page, len(state.attached), len(state.assets))
}
for _, asset := range state.assets {
identity, ok := state.attached[asset.name]
if !ok || identity.digest != asset.digest || identity.size != int64(len(asset.data)) {
b.Fatalf("uploaded %q identity = %#v, present %t; want digest %s and size %d", asset.name, identity, ok, asset.digest, len(asset.data))
}
}
}
func (state *deliveryBenchmarkServer) serveHTTP(w http.ResponseWriter, r *http.Request) {
switch r.URL.Path {
case "/api/v1/repos/owner/example/tags/v1.2.3":
_, _ = fmt.Fprintf(w, `{"name":"v1.2.3","commit":{"sha":%q}}`, state.commit)
case "/api/v1/repos/owner/example/releases/tags/v1.2.3":
state.mu.Lock()
page := state.page
state.mu.Unlock()
if !page {
http.NotFound(w, r)
return
}
state.writeRelease(w)
case "/api/v1/repos/owner/example/releases":
state.mu.Lock()
state.page = true
state.mu.Unlock()
state.writeRelease(w)
case "/api/v1/repos/owner/example/releases/10/assets":
if r.Method == http.MethodGet {
state.writeAssetList(w)
return
}
state.receiveUpload(w, r)
default:
if strings.HasPrefix(r.URL.Path, "/files/") {
state.serveFile(w, r)
return
}
http.NotFound(w, r)
}
}
func (state *deliveryBenchmarkServer) writeRelease(w http.ResponseWriter) {
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(map[string]any{
"id": 10, "html_url": state.baseURL + "/releases/10", "tag_name": "v1.2.3",
"target_commitish": state.commit, "name": "Fixture v1.2.3",
"body": "# Sample App v1.2.3\n\nDelivery fixture.\n", "draft": false, "prerelease": false,
})
}
func (state *deliveryBenchmarkServer) writeAssetList(w http.ResponseWriter) {
state.mu.Lock()
assets := make([]map[string]any, 0, len(state.attached))
for name, identity := range state.attached {
assets = append(assets, map[string]any{
"id": len(assets) + 1, "name": name, "size": identity.size,
"browser_download_url": state.baseURL + "/files/" + name,
})
}
state.mu.Unlock()
_ = json.NewEncoder(w).Encode(assets)
}
func (state *deliveryBenchmarkServer) receiveUpload(w http.ResponseWriter, r *http.Request) {
name := r.URL.Query().Get("name")
reader, err := r.MultipartReader()
if err != nil {
http.Error(w, "invalid multipart request", http.StatusBadRequest)
return
}
var digest hash.Hash = sha256.New()
var size int64
for {
part, err := reader.NextPart()
if err == io.EOF {
break
}
if err != nil {
http.Error(w, "invalid multipart body", http.StatusBadRequest)
return
}
if part.FormName() == "attachment" {
size, err = io.Copy(digest, part)
if err != nil {
_ = part.Close()
http.Error(w, "cannot read attachment", http.StatusBadRequest)
return
}
}
_ = part.Close()
}
identity := deliveryBenchmarkIdentity{digest: hex.EncodeToString(digest.Sum(nil)), size: size}
var expected *deliveryBenchmarkAsset
for i := range state.assets {
if state.assets[i].name == name {
expected = &state.assets[i]
break
}
}
if expected == nil || identity.digest != expected.digest || identity.size != int64(len(expected.data)) {
http.Error(w, "attachment digest or size mismatch", http.StatusBadRequest)
return
}
state.mu.Lock()
state.attached[name] = identity
assetID := len(state.attached)
state.mu.Unlock()
w.WriteHeader(http.StatusCreated)
_ = json.NewEncoder(w).Encode(map[string]any{"id": assetID, "name": name, "size": size})
}
func (state *deliveryBenchmarkServer) serveFile(w http.ResponseWriter, r *http.Request) {
name := strings.TrimPrefix(r.URL.Path, "/files/")
for _, asset := range state.assets {
if asset.name == name {
w.Header().Set("Content-Length", fmt.Sprint(len(asset.data)))
_, _ = io.Copy(w, bytes.NewReader(asset.data))
return
}
}
http.NotFound(w, r)
}