eric/distributor
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 7 Wiki Activity

Add HTTP upload idempotency support

Browse Source
This commit is contained in:
Eric Rakestraw
2026-06-04 14:03:26 +00:00
parent 1a402e6cfa
commit a15722571f
9 changed files with 603 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files

129
internal/app/upload_coordinator.go
View File

@@ -13,6 +13,7 @@ import (
"gitea.maximumdirect.net/eric/distributor/internal/config"
"gitea.maximumdirect.net/eric/distributor/internal/ingest"
sourcebundle "gitea.maximumdirect.net/eric/distributor/pkg/bundle"
)
const DefaultUploadMaxFileCount = 4096
@@ -43,12 +44,13 @@ type UploadRunRecord struct {
}
type UploadRequest struct {
PipelineID string
ContentType string
Body io.Reader
DryRun bool
Force bool
MaxFileCount int
PipelineID string
ContentType string
Body io.Reader
IdempotencyKey string
DryRun bool
Force bool
MaxFileCount int
}
type UploadQueueFullError struct {
@@ -64,6 +66,22 @@ func IsUploadQueueFull(err error) bool {
return errors.As(err, &full)
}
type UploadIdempotencyConflictError struct {
Retryable bool
}
func (err UploadIdempotencyConflictError) Error() string {
if err.Retryable {
return "upload idempotency key is already being processed"
}
return "upload idempotency key conflicts with a different source manifest"
}
func IsUploadIdempotencyConflict(err error) bool {
var conflict UploadIdempotencyConflictError
return errors.As(err, &conflict)
}
type UploadCoordinator struct {
ctx context.Context
cfg config.Config
@@ -82,6 +100,7 @@ type UploadCoordinator struct {
activePipeline map[string]bool
pending []*uploadJob
records map[UploadRunID]UploadRunRecord
idempotency map[uploadIdempotencyScope]uploadIdempotencyRecord
}
type uploadStageFunc func(context.Context, ingest.StageOptions) (ingest.StagedBundle, error)
@@ -95,6 +114,17 @@ type uploadJob struct {
stagedRoot string
}
type uploadIdempotencyScope struct {
PipelineID string
Key string
}
type uploadIdempotencyRecord struct {
RunID UploadRunID
Manifest sourcebundle.Manifest
Pending bool
}
type uploadCoordinatorHooks struct {
stage uploadStageFunc
run uploadRunFunc
@@ -140,6 +170,7 @@ func newUploadCoordinator(ctx context.Context, cfg config.Config, hooks uploadCo
maxConcurrency: cfg.Server.HTTP.MaxConcurrency,
activePipeline: map[string]bool{},
records: map[UploadRunID]UploadRunRecord{},
idempotency: map[uploadIdempotencyScope]uploadIdempotencyRecord{},
}
go coordinator.dispatchLoop()
return coordinator
@@ -169,14 +200,26 @@ func (coordinator *UploadCoordinator) Submit(ctx context.Context, request Upload
if err := ingest.ValidateContentType(request.ContentType); err != nil {
return UploadRunRecord{}, err
}
scope, hasKey := uploadRequestIdempotencyScope(pipeline.ID, request.IdempotencyKey)
coordinator.mu.Lock()
coordinator.expireLocked(coordinator.now().UTC())
if coordinator.queueFullLocked() {
existingIdempotency, hasExistingIdempotency := coordinator.idempotency[scope]
if hasKey && hasExistingIdempotency && existingIdempotency.Pending {
coordinator.mu.Unlock()
return UploadRunRecord{}, UploadQueueFullError{QueueSize: coordinator.queueSize}
return UploadRunRecord{}, UploadIdempotencyConflictError{Retryable: true}
}
needsReservation := !hasKey || !hasExistingIdempotency
if needsReservation {
if coordinator.queueFullLocked() {
coordinator.mu.Unlock()
return UploadRunRecord{}, UploadQueueFullError{QueueSize: coordinator.queueSize}
}
coordinator.reservedCount++
if hasKey {
coordinator.idempotency[scope] = uploadIdempotencyRecord{Pending: true}
}
}
coordinator.reservedCount++
coordinator.mu.Unlock()
staged, err := coordinator.stage(ctx, ingest.StageOptions{
@@ -189,13 +232,36 @@ func (coordinator *UploadCoordinator) Submit(ctx context.Context, request Upload
MaxFileCount: uploadMaxFileCount(request.MaxFileCount),
})
if err != nil {
coordinator.releaseReservation()
if needsReservation {
coordinator.releaseReservation(scope, hasKey)
}
return UploadRunRecord{}, err
}
coordinator.mu.Lock()
defer coordinator.mu.Unlock()
coordinator.reservedCount--
if needsReservation {
coordinator.reservedCount--
}
if hasKey {
existingIdempotency, hasExistingIdempotency = coordinator.idempotency[scope]
if hasExistingIdempotency && !existingIdempotency.Pending {
if uploadManifestsEqual(existingIdempotency.Manifest, staged.Manifest) {
_ = os.RemoveAll(staged.Root)
record, ok := coordinator.records[existingIdempotency.RunID]
if !ok {
return UploadRunRecord{}, fmt.Errorf("idempotency record references missing run")
}
return record, nil
}
_ = os.RemoveAll(staged.Root)
return UploadRunRecord{}, UploadIdempotencyConflictError{}
}
if !hasExistingIdempotency && !needsReservation && coordinator.queueFullLocked() {
_ = os.RemoveAll(staged.Root)
return UploadRunRecord{}, UploadQueueFullError{QueueSize: coordinator.queueSize}
}
}
record := UploadRunRecord{
ID: runID,
PipelineID: pipeline.ID,
@@ -204,6 +270,12 @@ func (coordinator *UploadCoordinator) Submit(ctx context.Context, request Upload
StagedRoot: staged.Root,
}
coordinator.records[runID] = record
if hasKey {
coordinator.idempotency[scope] = uploadIdempotencyRecord{
RunID: runID,
Manifest: staged.Manifest,
}
}
coordinator.pending = append(coordinator.pending, &uploadJob{
recordID: runID,
request: request,
@@ -214,6 +286,13 @@ func (coordinator *UploadCoordinator) Submit(ctx context.Context, request Upload
return record, nil
}
func uploadRequestIdempotencyScope(pipelineID, key string) (uploadIdempotencyScope, bool) {
if key == "" {
return uploadIdempotencyScope{}, false
}
return uploadIdempotencyScope{PipelineID: pipelineID, Key: key}, true
}
func (coordinator *UploadCoordinator) Status(runID UploadRunID) (UploadRunRecord, bool) {
coordinator.mu.Lock()
defer coordinator.mu.Unlock()
@@ -326,10 +405,15 @@ func (coordinator *UploadCoordinator) runJob(job *uploadJob) {
coordinator.complete(job, &report, err)
}
func (coordinator *UploadCoordinator) releaseReservation() {
func (coordinator *UploadCoordinator) releaseReservation(scope uploadIdempotencyScope, hasKey bool) {
coordinator.mu.Lock()
defer coordinator.mu.Unlock()
coordinator.reservedCount--
if hasKey {
if record, ok := coordinator.idempotency[scope]; ok && record.Pending {
delete(coordinator.idempotency, scope)
}
}
}
func (coordinator *UploadCoordinator) queueFullLocked() bool {
@@ -379,10 +463,31 @@ func (coordinator *UploadCoordinator) expireLocked(now time.Time) []UploadRunRec
record.Error = ""
expired = append(expired, record)
delete(coordinator.records, runID)
for scope, idempotencyRecord := range coordinator.idempotency {
if idempotencyRecord.RunID == runID {
delete(coordinator.idempotency, scope)
}
}
}
return expired
}
func uploadManifestsEqual(a, b sourcebundle.Manifest) bool {
if a.SchemaVersion != b.SchemaVersion ||
a.ID != b.ID ||
a.Digest != b.Digest ||
!a.Created.Equal(b.Created) ||
len(a.Files) != len(b.Files) {
return false
}
for index := range a.Files {
if a.Files[index] != b.Files[index] {
return false
}
}
return true
}
func (coordinator *UploadCoordinator) notify() {
select {
case coordinator.signal <- struct{}{}: