Add local batched Git blob reader
This commit is contained in:
@@ -40,7 +40,7 @@ reports these implemented packages:
|
|||||||
| `internal/build` | Candidate-build coordination, selected Go target compilation, version injection, native smoke check, and artifact digests. |
|
| `internal/build` | Candidate-build coordination, selected Go target compilation, version injection, native smoke check, and artifact digests. |
|
||||||
| `internal/candidate` | Stable-version, note, repository-state, candidate capture, and post-validation checks. |
|
| `internal/candidate` | Stable-version, note, repository-state, candidate capture, and post-validation checks. |
|
||||||
| `internal/delivery` | Tagged-source and release-note checks, managed builds in disposable storage outside the checkout, prebuilt artifact validation, manifest assembly, and hosted delivery coordination. |
|
| `internal/delivery` | Tagged-source and release-note checks, managed builds in disposable storage outside the checkout, prebuilt artifact validation, manifest assembly, and hosted delivery coordination. |
|
||||||
| `internal/git` | Git command boundary, committed-tree and blob reads, push-destination resolution, exact ref inspection, tagged checkout preparation, and tag operations, including captured-endpoint verification during recovery. |
|
| `internal/git` | Git command boundary, committed-tree reads, lazy local-only batched blob reads, push-destination resolution, exact ref inspection, tagged checkout preparation, and tag operations, including captured-endpoint verification during recovery. |
|
||||||
| `internal/gitea` | Gitea API transport, hosted release pages, asset inspection, manifests, and uploads. |
|
| `internal/gitea` | Gitea API transport, hosted release pages, asset inspection, manifests, and uploads. |
|
||||||
| `internal/gotool` | Effective Go configuration discovery, shared native-validation and static-build environments, and network-capable dependency preparation; see [Go command environments](gotool.md). |
|
| `internal/gotool` | Effective Go configuration discovery, shared native-validation and static-build environments, and network-capable dependency preparation; see [Go command environments](gotool.md). |
|
||||||
| `internal/tempdir` | Disposable system temporary storage outside source worktrees. |
|
| `internal/tempdir` | Disposable system temporary storage outside source worktrees. |
|
||||||
@@ -82,6 +82,7 @@ not a separate release matrix:
|
|||||||
| Committed-file validation: formatting, workspace files, local replacements, and file size | `internal/validation`, `internal/candidate`, `releasekit.Check`, `releasekit.Publish`, CLI profile conversion | `TestCheckGoFormatUsesCapturedRegularGoBlobs`, `TestCheckWorkspaceFilesReportsCommittedBlobNames`, `TestCheckLocalReplacementsParsesCommittedModuleDeclarations`, `TestCheckLocalReplacementsFollowsCommittedModuleSymlinks`, `TestCheckFileSizeUsesStrictLimitsAndDefault`, `TestCheckFileSizeAppliesLiteralPathExclusions`, `TestCheckFileSizeUsesCommittedBlobSizes`, `TestCheckFileSizeUsesOnlyCandidateFilesIndividually` |
|
| Committed-file validation: formatting, workspace files, local replacements, and file size | `internal/validation`, `internal/candidate`, `releasekit.Check`, `releasekit.Publish`, CLI profile conversion | `TestCheckGoFormatUsesCapturedRegularGoBlobs`, `TestCheckWorkspaceFilesReportsCommittedBlobNames`, `TestCheckLocalReplacementsParsesCommittedModuleDeclarations`, `TestCheckLocalReplacementsFollowsCommittedModuleSymlinks`, `TestCheckFileSizeUsesStrictLimitsAndDefault`, `TestCheckFileSizeAppliesLiteralPathExclusions`, `TestCheckFileSizeUsesCommittedBlobSizes`, `TestCheckFileSizeUsesOnlyCandidateFilesIndividually` |
|
||||||
| Internal Markdown parsing, committed destinations, fragments, boundary observations, and public selection | `releasekit.Check`, `releasekit.Publish`, `internal/candidate`, `internal/validation`, `cmd/releasekit/internal/cli` | `TestParseMarkdownLinksAndSourceLocations`, `TestParseMarkdownDecodesEntitiesInDestinations`, `TestParseMarkdownHeadingAnchors`, `TestMarkdownCacheKeepsParsedDocumentsPerCandidatePath`, `TestCheckMarkdownLinksResolvesCommittedDestinationsAndFragments`, `TestCheckMarkdownLinksReportsMissingPathsAnchorsAndCycles`, `TestCheckMarkdownLinksUsesResolvedMarkdownIdentity`, `TestCheckMarkdownLinksReportsBoundaryAndSubmoduleObservations`, `TestCheckMarkdownLinksPreservesResolverCauseAndCancellation`, `TestCheckMarkdownLinksPreservesAnchorReadCause`, `TestRunSelectsMarkdownWithCombinedGoChecksAndReportsOutOfScopeLinks`, `TestMarkdownValidationRunsThroughCheckAndPublish`, `TestLoadSettingsPreservesMarkdownObjectSelection`, `TestValidationSpecPreservesMarkdownObjectSelection` |
|
| Internal Markdown parsing, committed destinations, fragments, boundary observations, and public selection | `releasekit.Check`, `releasekit.Publish`, `internal/candidate`, `internal/validation`, `cmd/releasekit/internal/cli` | `TestParseMarkdownLinksAndSourceLocations`, `TestParseMarkdownDecodesEntitiesInDestinations`, `TestParseMarkdownHeadingAnchors`, `TestMarkdownCacheKeepsParsedDocumentsPerCandidatePath`, `TestCheckMarkdownLinksResolvesCommittedDestinationsAndFragments`, `TestCheckMarkdownLinksReportsMissingPathsAnchorsAndCycles`, `TestCheckMarkdownLinksUsesResolvedMarkdownIdentity`, `TestCheckMarkdownLinksReportsBoundaryAndSubmoduleObservations`, `TestCheckMarkdownLinksPreservesResolverCauseAndCancellation`, `TestCheckMarkdownLinksPreservesAnchorReadCause`, `TestRunSelectsMarkdownWithCombinedGoChecksAndReportsOutOfScopeLinks`, `TestMarkdownValidationRunsThroughCheckAndPublish`, `TestLoadSettingsPreservesMarkdownObjectSelection`, `TestValidationSpecPreservesMarkdownObjectSelection` |
|
||||||
| Shared Go settings for validation and builds | `internal/gotool`, `internal/process`, `internal/candidate`, `internal/build` | `TestDefaultValidationDoesNotInheritGOFLAGSRunFilter`, `TestNativeAndStaticCommandsUseOwnedBaseline`, `TestPersistentSettingsPreserveCachesAndNativeCTools`, `TestBuildIgnoresAmbientAndPersistentArchitectureTuning` |
|
| Shared Go settings for validation and builds | `internal/gotool`, `internal/process`, `internal/candidate`, `internal/build` | `TestDefaultValidationDoesNotInheritGOFLAGSRunFilter`, `TestNativeAndStaticCommandsUseOwnedBaseline`, `TestPersistentSettingsPreserveCachesAndNativeCTools`, `TestBuildIgnoresAmbientAndPersistentArchitectureTuning` |
|
||||||
|
| Batched local Git blob reads | `internal/git`, `internal/process` | `TestBlobReaderReadsLooseAndPackedObjects`, `TestBlobReaderMissingObjectKeepsBatchSessionUsable`, `TestCandidateObjectReadsStayLocalInPartialClones` |
|
||||||
| Piped child-process exchanges and cancellation cleanup | `internal/process` | `TestStartPipedExchangesBeforeInputEOFAndPreservesSettings`, `TestStartPipedCancellationAfterReadinessReapsChild`, `TestStartPipedPreservesNonzeroExit` |
|
| Piped child-process exchanges and cancellation cleanup | `internal/process` | `TestStartPipedExchangesBeforeInputEOFAndPreservesSettings`, `TestStartPipedCancellationAfterReadinessReapsChild`, `TestStartPipedPreservesNonzeroExit` |
|
||||||
| Authoritative endpoint and exact annotated tag publication | `releasekit.Publish`, `internal/publish`, `internal/git` | `TestPublishPushesOnlyTheCapturedAnnotatedTag`, `TestPublishRejectsUnexpectedPushURLBeforeValidation`, `TestPublishComparesExpectedURLWithResolvedInsteadOfDestination`, `TestPublishExpectedPushURLUsesPushURLWhenFetchAndPushDiffer`, `TestCheckIgnoresExpectedPushURL`, `TestResolveUsesPushURLAndQueriesWithoutMutation` |
|
| Authoritative endpoint and exact annotated tag publication | `releasekit.Publish`, `internal/publish`, `internal/git` | `TestPublishPushesOnlyTheCapturedAnnotatedTag`, `TestPublishRejectsUnexpectedPushURLBeforeValidation`, `TestPublishComparesExpectedURLWithResolvedInsteadOfDestination`, `TestPublishExpectedPushURLUsesPushURLWhenFetchAndPushDiffer`, `TestCheckIgnoresExpectedPushURL`, `TestResolveUsesPushURLAndQueriesWithoutMutation` |
|
||||||
| Publication partial and uncertain recovery | `internal/publish`, `internal/git` | `TestPublishRejectedPushKeepsLocalTagAndReportsNotPublished`, `TestPublishLostAcknowledgmentConfirmsExactRemoteObject`, `TestPublishLostAcknowledgmentWithChangedEndpointRemainsUnknown`, `TestPublishUnavailableReconciliationRemainsUnknownAndPreservesPushError` |
|
| Publication partial and uncertain recovery | `internal/publish`, `internal/git` | `TestPublishRejectedPushKeepsLocalTagAndReportsNotPublished`, `TestPublishLostAcknowledgmentConfirmsExactRemoteObject`, `TestPublishLostAcknowledgmentWithChangedEndpointRemainsUnknown`, `TestPublishUnavailableReconciliationRemainsUnknownAndPreservesPushError` |
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
# Computational Efficiency Implementation Plan
|
# Computational Efficiency Implementation Plan
|
||||||
|
|
||||||
**Status:** Stages 1–3 complete; Stages 4–11 pending.
|
**Status:** Stages 1–4 complete; Stages 5–11 pending.
|
||||||
|
|
||||||
## Purpose And Baseline
|
## Purpose And Baseline
|
||||||
|
|
||||||
@@ -236,6 +236,24 @@ by deterministic cleanup. Update the process inventory description when needed.
|
|||||||
**Owners:** A focused reader file and tests under `internal/git`, reusing
|
**Owners:** A focused reader file and tests under `internal/git`, reusing
|
||||||
[source.go](../../internal/git/source.go).
|
[source.go](../../internal/git/source.go).
|
||||||
|
|
||||||
|
**Completion:** Complete. `BlobReader` lazily starts one local-only
|
||||||
|
`git cat-file --batch` process, serializes request/response exchanges, and
|
||||||
|
returns caller-owned body bytes after validating the object ID, blob type,
|
||||||
|
representable size, and framing. Missing objects produce contextual errors
|
||||||
|
without invalidating a healthy session. Broken exchanges and cancellation close
|
||||||
|
the pipes and reap the child, preserving the original failure, Git stderr, and
|
||||||
|
process causes. Existing standalone blob and committed-file readers remain in
|
||||||
|
place. Tests cover repeated loose and packed reads, empty and binary bodies,
|
||||||
|
missing objects, partial clones, malformed/truncated responses, cancellation,
|
||||||
|
and child failures.
|
||||||
|
|
||||||
|
The process-reuse benchmark used one 9-byte blob. A new reader per read took
|
||||||
|
1.98–2.24 ms and allocated 37.9 KB in 190 allocations per operation; a shared
|
||||||
|
session took 57–82 μs and allocated 425 B in 8 allocations. The fresh-session
|
||||||
|
measurement includes process startup and clean close per read. The shared-session
|
||||||
|
timer includes lazy startup on its first read; final cleanup is outside the
|
||||||
|
timer. This is a fixture-specific measurement, not a subprocess-count contract.
|
||||||
|
|
||||||
1. Add an internal `BlobReader` that stores a repository directory and parent
|
1. Add an internal `BlobReader` that stores a repository directory and parent
|
||||||
context. Create its process lazily on the first content request through
|
context. Create its process lazily on the first content request through
|
||||||
`process.StartPiped` with `git cat-file --batch`.
|
`process.StartPiped` with `git cat-file --batch`.
|
||||||
|
|||||||
222
internal/git/blob_reader.go
Normal file
222
internal/git/blob_reader.go
Normal file
@@ -0,0 +1,222 @@
|
|||||||
|
package git
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bufio"
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"os/exec"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/ReleaseKit/internal/process"
|
||||||
|
)
|
||||||
|
|
||||||
|
var errBlobObjectMissing = errors.New("Git object is missing")
|
||||||
|
|
||||||
|
// BlobReader reads requested blobs through one lazy git cat-file --batch
|
||||||
|
// process. Reads are serialized; callers must close the reader when finished.
|
||||||
|
type BlobReader struct {
|
||||||
|
repositoryDir string
|
||||||
|
parent context.Context
|
||||||
|
|
||||||
|
mu sync.Mutex
|
||||||
|
child *process.PipedProcess
|
||||||
|
stdout *bufio.Reader
|
||||||
|
stderr bytes.Buffer
|
||||||
|
closed bool
|
||||||
|
aborted bool
|
||||||
|
terminal error
|
||||||
|
closeErr error
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewBlobReader creates a lazy reader for one repository. It starts no child
|
||||||
|
// process until the first valid blob is requested.
|
||||||
|
func NewBlobReader(ctx context.Context, repositoryDir string) *BlobReader {
|
||||||
|
return &BlobReader{repositoryDir: repositoryDir, parent: ctx}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Read returns one blob body from the candidate repository. The returned bytes
|
||||||
|
// are newly allocated and owned by the caller.
|
||||||
|
func (reader *BlobReader) Read(ctx context.Context, objectID string) ([]byte, error) {
|
||||||
|
if !validObjectID(objectID) {
|
||||||
|
return nil, fmt.Errorf("invalid blob object ID %q", objectID)
|
||||||
|
}
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return nil, fmt.Errorf("read blob %s: %w", objectID, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
reader.mu.Lock()
|
||||||
|
defer reader.mu.Unlock()
|
||||||
|
if reader.terminal != nil {
|
||||||
|
return nil, reader.terminal
|
||||||
|
}
|
||||||
|
if reader.closed {
|
||||||
|
return nil, fmt.Errorf("read blob %s: Git blob reader is closed", objectID)
|
||||||
|
}
|
||||||
|
if err := reader.startLocked(); err != nil {
|
||||||
|
return nil, fmt.Errorf("read blob %s: %w", objectID, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
child := reader.child
|
||||||
|
callbackDone := make(chan struct{})
|
||||||
|
stopCancellation := context.AfterFunc(ctx, func() {
|
||||||
|
defer close(callbackDone)
|
||||||
|
child.Cancel()
|
||||||
|
_ = child.Stdin().Close()
|
||||||
|
_ = child.Stdout().Close()
|
||||||
|
})
|
||||||
|
body, err := reader.exchange(objectID)
|
||||||
|
if !stopCancellation() {
|
||||||
|
<-callbackDone
|
||||||
|
}
|
||||||
|
if contextErr := ctx.Err(); contextErr != nil {
|
||||||
|
err = contextErr
|
||||||
|
} else if contextErr := reader.parent.Err(); contextErr != nil {
|
||||||
|
err = contextErr
|
||||||
|
}
|
||||||
|
if errors.Is(err, errBlobObjectMissing) {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return nil, reader.abortLocked(objectID, err)
|
||||||
|
}
|
||||||
|
return body, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Close closes a healthy session cleanly or reaps an aborted session. It is
|
||||||
|
// safe to call more than once.
|
||||||
|
func (reader *BlobReader) Close() error {
|
||||||
|
reader.mu.Lock()
|
||||||
|
defer reader.mu.Unlock()
|
||||||
|
if reader.closed {
|
||||||
|
return reader.closeErr
|
||||||
|
}
|
||||||
|
reader.closed = true
|
||||||
|
if reader.aborted {
|
||||||
|
reader.closeErr = reader.terminal
|
||||||
|
return reader.closeErr
|
||||||
|
}
|
||||||
|
if reader.child == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if err := reader.child.Stdin().Close(); err != nil {
|
||||||
|
reader.closeErr = reader.abortLocked("", fmt.Errorf("close Git cat-file input: %w", err))
|
||||||
|
return reader.closeErr
|
||||||
|
}
|
||||||
|
if _, err := io.Copy(io.Discard, reader.child.Stdout()); err != nil {
|
||||||
|
reader.closeErr = reader.abortLocked("", fmt.Errorf("finish Git cat-file output: %w", err))
|
||||||
|
return reader.closeErr
|
||||||
|
}
|
||||||
|
if err := reader.child.Wait(); err != nil {
|
||||||
|
reader.closeErr = reader.processError("close Git cat-file batch", err)
|
||||||
|
}
|
||||||
|
return reader.closeErr
|
||||||
|
}
|
||||||
|
|
||||||
|
func (reader *BlobReader) startLocked() error {
|
||||||
|
if reader.child != nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if err := reader.parent.Err(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
child, err := process.StartPiped(reader.parent, process.Invocation{
|
||||||
|
Args: []string{"git", "cat-file", "--batch"},
|
||||||
|
Dir: reader.repositoryDir,
|
||||||
|
Env: map[string]string{
|
||||||
|
"GIT_NO_LAZY_FETCH": "1",
|
||||||
|
"GIT_ALLOW_PROTOCOL": "",
|
||||||
|
},
|
||||||
|
Stderr: &reader.stderr,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
reader.child = child
|
||||||
|
reader.stdout = bufio.NewReader(child.Stdout())
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (reader *BlobReader) exchange(objectID string) ([]byte, error) {
|
||||||
|
if _, err := io.WriteString(reader.child.Stdin(), objectID+"\n"); err != nil {
|
||||||
|
return nil, fmt.Errorf("write object request: %w", err)
|
||||||
|
}
|
||||||
|
header, err := reader.stdout.ReadString('\n')
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("read object header: %w", err)
|
||||||
|
}
|
||||||
|
fields := strings.Fields(strings.TrimSuffix(header, "\n"))
|
||||||
|
if len(fields) == 2 && strings.EqualFold(fields[0], objectID) && fields[1] == "missing" {
|
||||||
|
return nil, fmt.Errorf("read blob %s: %w", objectID, errBlobObjectMissing)
|
||||||
|
}
|
||||||
|
if len(fields) != 3 {
|
||||||
|
return nil, fmt.Errorf("invalid Git blob header %q", strings.TrimSuffix(header, "\n"))
|
||||||
|
}
|
||||||
|
if !strings.EqualFold(fields[0], objectID) {
|
||||||
|
return nil, fmt.Errorf("Git blob response object ID %q does not match request %q", fields[0], objectID)
|
||||||
|
}
|
||||||
|
if fields[1] != "blob" {
|
||||||
|
return nil, fmt.Errorf("Git object %s has type %q, want blob", objectID, fields[1])
|
||||||
|
}
|
||||||
|
size, err := strconv.ParseInt(fields[2], 10, strconv.IntSize)
|
||||||
|
if err != nil || size < 0 {
|
||||||
|
if err == nil {
|
||||||
|
err = errors.New("negative object size")
|
||||||
|
}
|
||||||
|
return nil, fmt.Errorf("invalid Git blob size %q: %w", fields[2], err)
|
||||||
|
}
|
||||||
|
body := make([]byte, int(size))
|
||||||
|
if _, err := io.ReadFull(reader.stdout, body); err != nil {
|
||||||
|
return nil, fmt.Errorf("read %d-byte Git blob body: %w", size, err)
|
||||||
|
}
|
||||||
|
framing, err := reader.stdout.ReadByte()
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("read Git blob framing: %w", err)
|
||||||
|
}
|
||||||
|
if framing != '\n' {
|
||||||
|
return nil, fmt.Errorf("invalid Git blob framing byte %q", framing)
|
||||||
|
}
|
||||||
|
return body, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (reader *BlobReader) abortLocked(objectID string, cause error) error {
|
||||||
|
if reader.aborted {
|
||||||
|
return reader.terminal
|
||||||
|
}
|
||||||
|
reader.aborted = true
|
||||||
|
var waitErr error
|
||||||
|
if reader.child != nil {
|
||||||
|
reader.child.Cancel()
|
||||||
|
_ = reader.child.Stdin().Close()
|
||||||
|
_ = reader.child.Stdout().Close()
|
||||||
|
waitErr = reader.child.Wait()
|
||||||
|
}
|
||||||
|
if contextErr := reader.parent.Err(); contextErr != nil {
|
||||||
|
cause = contextErr
|
||||||
|
}
|
||||||
|
reader.terminal = reader.processError("read blob "+objectID, errors.Join(cause, relevantProcessError(waitErr, cause)))
|
||||||
|
return reader.terminal
|
||||||
|
}
|
||||||
|
|
||||||
|
func (reader *BlobReader) processError(operation string, cause error) error {
|
||||||
|
message := strings.TrimSpace(reader.stderr.String())
|
||||||
|
if message != "" {
|
||||||
|
return fmt.Errorf("%s: %s: %w", operation, message, cause)
|
||||||
|
}
|
||||||
|
return fmt.Errorf("%s: %w", operation, cause)
|
||||||
|
}
|
||||||
|
|
||||||
|
func relevantProcessError(waitErr, cause error) error {
|
||||||
|
if waitErr == nil || errors.Is(cause, context.Canceled) || errors.Is(cause, context.DeadlineExceeded) {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
var exitErr *exec.ExitError
|
||||||
|
if errors.As(waitErr, &exitErr) && exitErr.ExitCode() == -1 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return waitErr
|
||||||
|
}
|
||||||
195
internal/git/blob_reader_test.go
Normal file
195
internal/git/blob_reader_test.go
Normal file
@@ -0,0 +1,195 @@
|
|||||||
|
package git
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"os"
|
||||||
|
"os/exec"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestBlobReaderReadsLooseAndPackedObjects(t *testing.T) {
|
||||||
|
for _, packed := range []bool{false, true} {
|
||||||
|
name := "loose"
|
||||||
|
if packed {
|
||||||
|
name = "packed"
|
||||||
|
}
|
||||||
|
t.Run(name, func(t *testing.T) {
|
||||||
|
fixture := newRemoteFixture(t)
|
||||||
|
binary := []byte{0, '\n', 0xff, 'b', '\n', 0}
|
||||||
|
writeRepoFile(t, fixture.work, "empty.dat", "")
|
||||||
|
writeRepoFile(t, fixture.work, "binary.dat", string(binary))
|
||||||
|
gitRun(t, fixture.work, "add", ".")
|
||||||
|
gitRun(t, fixture.work, "commit", "-m", "add binary fixtures")
|
||||||
|
want := map[string][]byte{
|
||||||
|
gitText(t, fixture.work, "rev-parse", "HEAD:source.txt"): []byte("candidate"),
|
||||||
|
gitText(t, fixture.work, "rev-parse", "HEAD:empty.dat"): {},
|
||||||
|
gitText(t, fixture.work, "rev-parse", "HEAD:binary.dat"): binary,
|
||||||
|
}
|
||||||
|
if packed {
|
||||||
|
gitRun(t, fixture.work, "gc", "--aggressive", "--prune=now")
|
||||||
|
}
|
||||||
|
reader := NewBlobReader(t.Context(), fixture.work)
|
||||||
|
defer reader.Close()
|
||||||
|
for repeat := 0; repeat < 2; repeat++ {
|
||||||
|
for objectID, expected := range want {
|
||||||
|
got, err := reader.Read(t.Context(), objectID)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Read(%s) error = %v", objectID, err)
|
||||||
|
}
|
||||||
|
if !bytes.Equal(got, expected) {
|
||||||
|
t.Errorf("Read(%s) = %v, want %v", objectID, got, expected)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBlobReaderMissingObjectKeepsBatchSessionUsable(t *testing.T) {
|
||||||
|
fixture := newRemoteFixture(t)
|
||||||
|
objectID := gitText(t, fixture.work, "rev-parse", "HEAD:source.txt")
|
||||||
|
reader := NewBlobReader(t.Context(), fixture.work)
|
||||||
|
defer reader.Close()
|
||||||
|
missing := strings.Repeat("0", 40)
|
||||||
|
if _, err := reader.Read(t.Context(), missing); err == nil || !strings.Contains(err.Error(), "missing") {
|
||||||
|
t.Fatalf("Read(missing) error = %v, want contextual missing-object error", err)
|
||||||
|
}
|
||||||
|
contents, err := reader.Read(t.Context(), objectID)
|
||||||
|
if err != nil || string(contents) != "candidate" {
|
||||||
|
t.Fatalf("Read(existing) = %q, %v after missing request", contents, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBlobReaderPreservesHexCaseAndArbitraryBodyBytes(t *testing.T) {
|
||||||
|
installFakeGit(t, `while IFS= read -r object; do
|
||||||
|
upper=$(printf '%s' "$object" | tr 'a-f' 'A-F')
|
||||||
|
printf '%s blob 4\n' "$upper"
|
||||||
|
printf 'a\000b\n\n'
|
||||||
|
done`)
|
||||||
|
objectID := strings.Repeat("abcdef0123", 4)
|
||||||
|
reader := NewBlobReader(t.Context(), t.TempDir())
|
||||||
|
contents, err := reader.Read(t.Context(), objectID)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if want := []byte{'a', 0, 'b', '\n'}; !bytes.Equal(contents, want) {
|
||||||
|
t.Fatalf("Read() = %v, want %v", contents, want)
|
||||||
|
}
|
||||||
|
if err := reader.Close(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBlobReaderRejectsInvalidHeadersAndFraming(t *testing.T) {
|
||||||
|
for _, testCase := range []struct {
|
||||||
|
name string
|
||||||
|
response string
|
||||||
|
want string
|
||||||
|
}{
|
||||||
|
{name: "wrong object", response: `printf '%040d blob 0\n\n' 0`, want: "does not match request"},
|
||||||
|
{name: "wrong type", response: `printf '%s tree 0\n\n' "$object"`, want: `has type "tree"`},
|
||||||
|
{name: "invalid size", response: `printf '%s blob -1\n' "$object"`, want: "invalid Git blob size"},
|
||||||
|
{name: "invalid framing", response: `printf '%s blob 0\nx' "$object"`, want: "invalid Git blob framing"},
|
||||||
|
} {
|
||||||
|
t.Run(testCase.name, func(t *testing.T) {
|
||||||
|
installFakeGit(t, "IFS= read -r object\n"+testCase.response)
|
||||||
|
reader := NewBlobReader(t.Context(), t.TempDir())
|
||||||
|
_, err := reader.Read(t.Context(), strings.Repeat("d", 40))
|
||||||
|
if err == nil || !strings.Contains(err.Error(), testCase.want) {
|
||||||
|
t.Fatalf("Read() error = %v, want %q", err, testCase.want)
|
||||||
|
}
|
||||||
|
if closeErr := reader.Close(); !strings.Contains(closeErr.Error(), testCase.want) {
|
||||||
|
t.Fatalf("Close() error = %v, want original protocol error", closeErr)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBlobReaderTruncatedBodyAbortsAndKeepsOriginalCause(t *testing.T) {
|
||||||
|
installFakeGit(t, `IFS= read -r object
|
||||||
|
printf '%s blob 8\n' "$object"
|
||||||
|
printf 'short'
|
||||||
|
exit 0`)
|
||||||
|
reader := NewBlobReader(t.Context(), t.TempDir())
|
||||||
|
_, err := reader.Read(t.Context(), strings.Repeat("a", 40))
|
||||||
|
if !errors.Is(err, io.ErrUnexpectedEOF) {
|
||||||
|
t.Fatalf("Read() error = %v, want truncated-body cause", err)
|
||||||
|
}
|
||||||
|
if closeErr := reader.Close(); !errors.Is(closeErr, io.ErrUnexpectedEOF) {
|
||||||
|
t.Fatalf("Close() error = %v, want original read cause", closeErr)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBlobReaderCancellationDuringBodyReadReapsChild(t *testing.T) {
|
||||||
|
ready := filepath.Join(t.TempDir(), "ready")
|
||||||
|
installFakeGit(t, `IFS= read -r object
|
||||||
|
printf '%s blob 1048576\npartial' "$object"
|
||||||
|
: > "$RELEASEKIT_GIT_READER_READY"
|
||||||
|
exec sleep 60`)
|
||||||
|
t.Setenv("RELEASEKIT_GIT_READER_READY", ready)
|
||||||
|
ctx, cancel := context.WithCancel(t.Context())
|
||||||
|
reader := NewBlobReader(ctx, t.TempDir())
|
||||||
|
result := make(chan error, 1)
|
||||||
|
go func() {
|
||||||
|
_, err := reader.Read(ctx, strings.Repeat("b", 40))
|
||||||
|
result <- err
|
||||||
|
}()
|
||||||
|
waitForFile(t, ready)
|
||||||
|
cancel()
|
||||||
|
select {
|
||||||
|
case err := <-result:
|
||||||
|
if !errors.Is(err, context.Canceled) {
|
||||||
|
t.Fatalf("Read() error = %v, want context cancellation", err)
|
||||||
|
}
|
||||||
|
case <-time.After(5 * time.Second):
|
||||||
|
t.Fatal("Read() did not return after cancellation")
|
||||||
|
}
|
||||||
|
if err := reader.Close(); !errors.Is(err, context.Canceled) {
|
||||||
|
t.Fatalf("Close() error = %v, want original cancellation cause", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBlobReaderPreservesGitFailureAndStderr(t *testing.T) {
|
||||||
|
installFakeGit(t, `printf 'fixture Git failure\n' >&2
|
||||||
|
exit 23`)
|
||||||
|
reader := NewBlobReader(t.Context(), t.TempDir())
|
||||||
|
_, err := reader.Read(t.Context(), strings.Repeat("c", 40))
|
||||||
|
var exitErr *exec.ExitError
|
||||||
|
if !errors.As(err, &exitErr) || exitErr.ExitCode() != 23 {
|
||||||
|
t.Fatalf("Read() error = %v, want Git exit code 23", err)
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "fixture Git failure") {
|
||||||
|
t.Fatalf("Read() error = %v, want captured Git stderr", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func installFakeGit(t *testing.T, script string) {
|
||||||
|
t.Helper()
|
||||||
|
directory := t.TempDir()
|
||||||
|
path := filepath.Join(directory, "git")
|
||||||
|
if err := os.WriteFile(path, []byte("#!/bin/sh\n"+script+"\n"), 0o700); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Setenv("PATH", directory+string(os.PathListSeparator)+os.Getenv("PATH"))
|
||||||
|
}
|
||||||
|
|
||||||
|
func waitForFile(t *testing.T, path string) {
|
||||||
|
t.Helper()
|
||||||
|
deadline := time.Now().Add(5 * time.Second)
|
||||||
|
for time.Now().Before(deadline) {
|
||||||
|
if _, err := os.Stat(path); err == nil {
|
||||||
|
return
|
||||||
|
} else if !errors.Is(err, os.ErrNotExist) {
|
||||||
|
t.Fatal(fmt.Errorf("inspect readiness signal: %w", err))
|
||||||
|
}
|
||||||
|
time.Sleep(5 * time.Millisecond)
|
||||||
|
}
|
||||||
|
t.Fatal("child did not create readiness signal")
|
||||||
|
}
|
||||||
59
internal/git/performance_test.go
Normal file
59
internal/git/performance_test.go
Normal file
@@ -0,0 +1,59 @@
|
|||||||
|
package git
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"runtime"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func BenchmarkPerformanceBlobReader(b *testing.B) {
|
||||||
|
fixture := newRemoteFixture(b)
|
||||||
|
objectID := gitText(b, fixture.work, "rev-parse", "HEAD:source.txt")
|
||||||
|
const expectedBytes = int64(len("candidate"))
|
||||||
|
|
||||||
|
b.Run("new-session-per-read", func(b *testing.B) {
|
||||||
|
b.ReportAllocs()
|
||||||
|
b.SetBytes(expectedBytes)
|
||||||
|
b.ResetTimer()
|
||||||
|
b.ReportMetric(float64(expectedBytes), "bytes/op")
|
||||||
|
b.ReportMetric(1, "blobs/op")
|
||||||
|
for i := 0; i < b.N; i++ {
|
||||||
|
reader := NewBlobReader(context.Background(), fixture.work)
|
||||||
|
contents, err := reader.Read(context.Background(), objectID)
|
||||||
|
if err != nil {
|
||||||
|
b.Fatal(err)
|
||||||
|
}
|
||||||
|
if int64(len(contents)) != expectedBytes {
|
||||||
|
b.Fatalf("read %d bytes, want %d", len(contents), expectedBytes)
|
||||||
|
}
|
||||||
|
if err := reader.Close(); err != nil {
|
||||||
|
b.Fatal(err)
|
||||||
|
}
|
||||||
|
runtime.KeepAlive(contents)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
b.Run("shared-session", func(b *testing.B) {
|
||||||
|
reader := NewBlobReader(context.Background(), fixture.work)
|
||||||
|
b.Cleanup(func() {
|
||||||
|
if err := reader.Close(); err != nil {
|
||||||
|
b.Error(err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
b.ReportAllocs()
|
||||||
|
b.SetBytes(expectedBytes)
|
||||||
|
b.ResetTimer()
|
||||||
|
b.ReportMetric(float64(expectedBytes), "bytes/op")
|
||||||
|
b.ReportMetric(1, "blobs/op")
|
||||||
|
for i := 0; i < b.N; i++ {
|
||||||
|
contents, err := reader.Read(context.Background(), objectID)
|
||||||
|
if err != nil {
|
||||||
|
b.Fatal(err)
|
||||||
|
}
|
||||||
|
if int64(len(contents)) != expectedBytes {
|
||||||
|
b.Fatalf("read %d bytes, want %d", len(contents), expectedBytes)
|
||||||
|
}
|
||||||
|
runtime.KeepAlive(contents)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
@@ -164,7 +164,7 @@ type remoteFixture struct {
|
|||||||
commit string
|
commit string
|
||||||
}
|
}
|
||||||
|
|
||||||
func newRemoteFixture(t *testing.T) remoteFixture {
|
func newRemoteFixture(t testing.TB) remoteFixture {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
base := t.TempDir()
|
base := t.TempDir()
|
||||||
work := filepath.Join(base, "work")
|
work := filepath.Join(base, "work")
|
||||||
@@ -194,7 +194,7 @@ func resolveFixture(t *testing.T, repo remoteFixture) Endpoint {
|
|||||||
return endpoint
|
return endpoint
|
||||||
}
|
}
|
||||||
|
|
||||||
func gitRun(t *testing.T, dir string, args ...string) {
|
func gitRun(t testing.TB, dir string, args ...string) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
cmd := exec.Command("git", args...)
|
cmd := exec.Command("git", args...)
|
||||||
cmd.Dir = dir
|
cmd.Dir = dir
|
||||||
@@ -203,7 +203,7 @@ func gitRun(t *testing.T, dir string, args ...string) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func gitText(t *testing.T, dir string, args ...string) string {
|
func gitText(t testing.TB, dir string, args ...string) string {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
cmd := exec.Command("git", args...)
|
cmd := exec.Command("git", args...)
|
||||||
cmd.Dir = dir
|
cmd.Dir = dir
|
||||||
@@ -214,7 +214,7 @@ func gitText(t *testing.T, dir string, args ...string) string {
|
|||||||
return strings.TrimSuffix(string(outputBytes), "\n")
|
return strings.TrimSuffix(string(outputBytes), "\n")
|
||||||
}
|
}
|
||||||
|
|
||||||
func writeRepoFile(t *testing.T, repoDir, name, contents string) {
|
func writeRepoFile(t testing.TB, repoDir, name, contents string) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
path := filepath.Join(repoDir, name)
|
path := filepath.Join(repoDir, name)
|
||||||
if err := os.WriteFile(path, []byte(contents), 0o644); err != nil {
|
if err := os.WriteFile(path, []byte(contents), 0o644); err != nil {
|
||||||
|
|||||||
@@ -62,6 +62,13 @@ func TestCandidateObjectReadsStayLocalInPartialClones(t *testing.T) {
|
|||||||
if !errors.As(err, &exitError) {
|
if !errors.As(err, &exitError) {
|
||||||
t.Fatalf("blob error %v lost its Git process cause", err)
|
t.Fatalf("blob error %v lost its Git process cause", err)
|
||||||
}
|
}
|
||||||
|
reader := NewBlobReader(t.Context(), repository)
|
||||||
|
if _, err := reader.Read(t.Context(), blob); err == nil || !strings.Contains(err.Error(), "missing") {
|
||||||
|
t.Fatalf("batch blob read error = %v, want missing-object inspection error", err)
|
||||||
|
}
|
||||||
|
if err := reader.Close(); err != nil {
|
||||||
|
t.Fatalf("close missing-object batch reader: %v", err)
|
||||||
|
}
|
||||||
if data, err := os.ReadFile(attempts); !errors.Is(err, os.ErrNotExist) {
|
if data, err := os.ReadFile(attempts); !errors.Is(err, os.ErrNotExist) {
|
||||||
t.Fatalf("object inspection invoked remote transport: %q, %v", data, err)
|
t.Fatalf("object inspection invoked remote transport: %q, %v", data, err)
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user