Add upload archive staging

This commit is contained in:
2026-06-03 15:05:56 +00:00
parent 35c5237dfc
commit 65dd22f974
3 changed files with 771 additions and 0 deletions

43
docs/internal/ingest.md Normal file
View File

@@ -0,0 +1,43 @@
# Ingestion Internals
## Purpose
`internal/ingest` stages uploaded source bundle archives into local per-run directories. It does not authenticate requests, manage upload queues, publish destinations, or start an HTTP server.
## Archive staging
`StageArchive` accepts one upload body, content type, pipeline staging path, run id, and explicit size and file-count limits. It writes the request body to temporary storage while enforcing the configured upload size limit, extracts the archive into temporary local storage, validates the extracted source bundle, and then commits the validated bundle to:
```text
<pipeline staging path>/<run id>
```
The returned `StagedBundle.Root` is a local filesystem path to the validated source bundle root.
## Accepted archive formats
The package accepts only:
- `application/x-tar`
- `application/gzip`
- `application/x-gzip`
Gzip uploads must contain a tar archive.
## Extraction rules
Archive entry paths must be clean relative slash-separated paths. Extraction rejects absolute paths, path traversal, backslash paths, duplicate files, symlinks, hardlinks, devices, sockets, and other special entries.
The archive must contain exactly one root-level `manifest.json`. Nested manifests are rejected.
Regular files and directories are the only accepted tar entries. Regular file extraction enforces the explicit maximum extracted byte count and maximum file count supplied by the caller.
## Bundle validation
After extraction, the package loads and validates the staged bundle through `pkg/bundle`. Manifest parsing, source path validation, file existence checks, regular-file checks, file sizes, file SHA-256 digests, and bundle digest validation use the existing source bundle contract.
Validation happens before the staged bundle is committed to its final per-run path.
## Failure behavior
Failed staging removes temporary archive and extraction data created by the package. A failed call does not publish anything and does not leave a committed per-run bundle directory.

346
internal/ingest/archive.go Normal file
View File

@@ -0,0 +1,346 @@
package ingest
import (
"archive/tar"
"compress/gzip"
"context"
"errors"
"fmt"
"io"
"mime"
"os"
"path"
"path/filepath"
"strings"
sourcebundle "gitea.maximumdirect.net/eric/distributor/pkg/bundle"
)
const (
ContentTypeTar = "application/x-tar"
ContentTypeGzip = "application/gzip"
ContentTypeXGzip = "application/x-gzip"
)
var (
ErrUnsupportedContentType = errors.New("unsupported archive content type")
ErrUploadTooLarge = errors.New("upload exceeds maximum size")
ErrExtractedTooLarge = errors.New("extracted bundle exceeds maximum size")
ErrTooManyFiles = errors.New("extracted bundle has too many files")
ErrUnsafeArchivePath = errors.New("unsafe archive path")
)
type StageOptions struct {
Body io.Reader
ContentType string
PipelineStagingPath string
RunID string
MaxUploadSize int64
MaxExtractedSize int64
MaxFileCount int
}
type StagedBundle struct {
Root string
Manifest sourcebundle.Manifest
}
func StageArchive(ctx context.Context, opts StageOptions) (StagedBundle, error) {
if ctx == nil {
ctx = context.Background()
}
if err := validateStageOptions(opts); err != nil {
return StagedBundle{}, err
}
format, err := archiveFormat(opts.ContentType)
if err != nil {
return StagedBundle{}, err
}
if err := os.MkdirAll(opts.PipelineStagingPath, 0o755); err != nil {
return StagedBundle{}, fmt.Errorf("create pipeline staging path: %w", err)
}
tempDir, err := os.MkdirTemp(opts.PipelineStagingPath, "."+opts.RunID+"-")
if err != nil {
return StagedBundle{}, fmt.Errorf("create staging temp dir: %w", err)
}
cleanupTemp := true
defer func() {
if cleanupTemp {
_ = os.RemoveAll(tempDir)
}
}()
archivePath := filepath.Join(tempDir, "upload.archive")
if err := writeLimited(ctx, archivePath, opts.Body, opts.MaxUploadSize); err != nil {
return StagedBundle{}, err
}
extractRoot := filepath.Join(tempDir, "bundle")
if err := os.Mkdir(extractRoot, 0o755); err != nil {
return StagedBundle{}, fmt.Errorf("create extraction root: %w", err)
}
if err := extractArchive(ctx, archivePath, extractRoot, format, opts.MaxExtractedSize, opts.MaxFileCount); err != nil {
return StagedBundle{}, err
}
manifest, err := sourcebundle.LoadManifest(extractRoot)
if err != nil {
return StagedBundle{}, err
}
if err := sourcebundle.ValidateBundle(extractRoot, manifest); err != nil {
return StagedBundle{}, err
}
finalRoot := filepath.Join(opts.PipelineStagingPath, opts.RunID)
if err := os.Rename(extractRoot, finalRoot); err != nil {
return StagedBundle{}, fmt.Errorf("commit staged bundle: %w", err)
}
cleanupTemp = false
if err := os.RemoveAll(tempDir); err != nil {
return StagedBundle{}, fmt.Errorf("remove staging temp dir: %w", err)
}
return StagedBundle{
Root: finalRoot,
Manifest: manifest,
}, nil
}
func validateStageOptions(opts StageOptions) error {
if opts.Body == nil {
return fmt.Errorf("body is required")
}
if opts.PipelineStagingPath == "" {
return fmt.Errorf("pipeline staging path is required")
}
if err := validateRunID(opts.RunID); err != nil {
return err
}
if opts.MaxUploadSize <= 0 {
return fmt.Errorf("max upload size must be greater than zero")
}
if opts.MaxExtractedSize <= 0 {
return fmt.Errorf("max extracted size must be greater than zero")
}
if opts.MaxFileCount <= 0 {
return fmt.Errorf("max file count must be greater than zero")
}
return nil
}
func validateRunID(value string) error {
if value == "" {
return fmt.Errorf("run id is required")
}
if value == "." || value == ".." || strings.ContainsAny(value, `/\`) {
return fmt.Errorf("run id must be a single filesystem path segment")
}
return nil
}
type archiveKind int
const (
archiveKindTar archiveKind = iota + 1
archiveKindGzip
)
func archiveFormat(contentType string) (archiveKind, error) {
mediaType, _, err := mime.ParseMediaType(contentType)
if err != nil {
mediaType = contentType
}
switch mediaType {
case ContentTypeTar:
return archiveKindTar, nil
case ContentTypeGzip, ContentTypeXGzip:
return archiveKindGzip, nil
default:
return 0, ErrUnsupportedContentType
}
}
func writeLimited(ctx context.Context, destination string, body io.Reader, maxSize int64) error {
file, err := os.OpenFile(destination, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600)
if err != nil {
return fmt.Errorf("create upload archive: %w", err)
}
defer file.Close()
limited := &limitedReader{ctx: ctx, reader: body, limit: maxSize}
if _, err := io.Copy(file, limited); err != nil {
return err
}
if err := file.Close(); err != nil {
return fmt.Errorf("write upload archive: %w", err)
}
return nil
}
type limitedReader struct {
ctx context.Context
reader io.Reader
limit int64
read int64
}
func (r *limitedReader) Read(data []byte) (int, error) {
if err := r.ctx.Err(); err != nil {
return 0, err
}
if r.read == r.limit {
var probe [1]byte
n, err := r.reader.Read(probe[:])
if n > 0 {
return 0, ErrUploadTooLarge
}
return 0, err
}
remaining := r.limit - r.read
if int64(len(data)) > remaining+1 {
data = data[:remaining+1]
}
n, err := r.reader.Read(data)
if r.read+int64(n) > r.limit {
allowed := int(r.limit - r.read)
r.read = r.limit
return allowed, ErrUploadTooLarge
}
r.read += int64(n)
return n, err
}
func extractArchive(ctx context.Context, archivePath, destination string, format archiveKind, maxExtractedSize int64, maxFileCount int) error {
file, err := os.Open(archivePath)
if err != nil {
return fmt.Errorf("open upload archive: %w", err)
}
defer file.Close()
var reader io.Reader = file
var gzipReader *gzip.Reader
if format == archiveKindGzip {
gzipReader, err = gzip.NewReader(file)
if err != nil {
return fmt.Errorf("open gzip archive: %w", err)
}
defer gzipReader.Close()
reader = gzipReader
}
extractor := archiveExtractor{
ctx: ctx,
destination: destination,
maxExtractedSize: maxExtractedSize,
maxFileCount: maxFileCount,
}
if err := extractor.extract(tar.NewReader(reader)); err != nil {
return err
}
if extractor.rootManifestCount != 1 {
return fmt.Errorf("archive must contain exactly one root-level manifest.json")
}
return nil
}
type archiveExtractor struct {
ctx context.Context
destination string
maxExtractedSize int64
maxFileCount int
extractedSize int64
fileCount int
rootManifestCount int
seenFiles map[string]struct{}
}
func (e *archiveExtractor) extract(reader *tar.Reader) error {
e.seenFiles = make(map[string]struct{})
for {
if err := e.ctx.Err(); err != nil {
return err
}
header, err := reader.Next()
if errors.Is(err, io.EOF) {
return nil
}
if err != nil {
return fmt.Errorf("read tar archive: %w", err)
}
name, err := cleanArchivePath(header.Name)
if err != nil {
return err
}
if path.Base(name) == sourcebundle.ManifestName {
if name != sourcebundle.ManifestName {
return fmt.Errorf("nested manifest %q is not allowed", name)
}
e.rootManifestCount++
}
switch header.Typeflag {
case tar.TypeDir:
if err := os.MkdirAll(filepath.Join(e.destination, filepath.FromSlash(name)), 0o755); err != nil {
return fmt.Errorf("create archive directory %q: %w", name, err)
}
case tar.TypeReg, tar.TypeRegA:
if err := e.extractFile(reader, name, header.Size); err != nil {
return err
}
default:
return fmt.Errorf("archive entry %q has unsupported type %c", name, header.Typeflag)
}
}
}
func (e *archiveExtractor) extractFile(reader *tar.Reader, name string, size int64) error {
if size < 0 {
return fmt.Errorf("archive entry %q has invalid size", name)
}
e.fileCount++
if e.fileCount > e.maxFileCount {
return ErrTooManyFiles
}
e.extractedSize += size
if e.extractedSize > e.maxExtractedSize {
return ErrExtractedTooLarge
}
if _, exists := e.seenFiles[name]; exists {
return fmt.Errorf("archive entry %q is duplicated", name)
}
e.seenFiles[name] = struct{}{}
fullPath := filepath.Join(e.destination, filepath.FromSlash(name))
if err := os.MkdirAll(filepath.Dir(fullPath), 0o755); err != nil {
return fmt.Errorf("create archive parent for %q: %w", name, err)
}
file, err := os.OpenFile(fullPath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o644)
if err != nil {
return fmt.Errorf("create archive file %q: %w", name, err)
}
defer file.Close()
if _, err := io.CopyN(file, reader, size); err != nil {
return fmt.Errorf("extract archive file %q: %w", name, err)
}
if err := file.Close(); err != nil {
return fmt.Errorf("extract archive file %q: %w", name, err)
}
return nil
}
func cleanArchivePath(value string) (string, error) {
if value == "" || strings.Contains(value, `\`) || strings.HasPrefix(value, "/") {
return "", ErrUnsafeArchivePath
}
cleaned := path.Clean(value)
if cleaned != value {
return "", ErrUnsafeArchivePath
}
for _, segment := range strings.Split(value, "/") {
if segment == "" || segment == "." || segment == ".." {
return "", ErrUnsafeArchivePath
}
}
return value, nil
}

View File

@@ -0,0 +1,382 @@
package ingest
import (
"archive/tar"
"bytes"
"compress/gzip"
"context"
"errors"
"io/fs"
"os"
"path/filepath"
"strings"
"testing"
"time"
sourcebundle "gitea.maximumdirect.net/eric/distributor/pkg/bundle"
)
func TestStageArchiveAcceptsTar(t *testing.T) {
archive := validArchive(t, false)
staged := stageArchive(t, archive, ContentTypeTar)
if got, want := staged.Manifest.ID, "reports.ingest"; got != want {
t.Fatalf("manifest id = %q, want %q", got, want)
}
if got := readFile(t, staged.Root, "report.md"); got != "# Report\n" {
t.Fatalf("report = %q", got)
}
}
func TestStageArchiveAcceptsGzipTar(t *testing.T) {
archive := validArchive(t, true)
staged := stageArchive(t, archive, ContentTypeGzip+"; charset=binary")
if got, want := staged.Manifest.ID, "reports.ingest"; got != want {
t.Fatalf("manifest id = %q, want %q", got, want)
}
if got := readFile(t, staged.Root, "summary.txt"); got != "Summary\n" {
t.Fatalf("summary = %q", got)
}
}
func TestStageArchiveRejectsUnsupportedContentType(t *testing.T) {
err := stageArchiveError(t, validArchive(t, false), "application/zip", nil)
if !errors.Is(err, ErrUnsupportedContentType) {
t.Fatalf("StageArchive() error = %v, want ErrUnsupportedContentType", err)
}
}
func TestStageArchiveEnforcesMaxUploadSize(t *testing.T) {
archive := validArchive(t, false)
err := stageArchiveError(t, archive, ContentTypeTar, func(opts *StageOptions) {
opts.MaxUploadSize = int64(len(archive) - 1)
})
if !errors.Is(err, ErrUploadTooLarge) {
t.Fatalf("StageArchive() error = %v, want ErrUploadTooLarge", err)
}
}
func TestStageArchiveEnforcesExtractionLimits(t *testing.T) {
archive := validArchive(t, false)
tests := map[string]struct {
mutate func(*StageOptions)
wantErr error
}{
"size": {
mutate: func(opts *StageOptions) {
opts.MaxExtractedSize = 1
},
wantErr: ErrExtractedTooLarge,
},
"files": {
mutate: func(opts *StageOptions) {
opts.MaxFileCount = 1
},
wantErr: ErrTooManyFiles,
},
}
for name, tt := range tests {
t.Run(name, func(t *testing.T) {
err := stageArchiveError(t, archive, ContentTypeTar, tt.mutate)
if !errors.Is(err, tt.wantErr) {
t.Fatalf("StageArchive() error = %v, want %v", err, tt.wantErr)
}
})
}
}
func TestStageArchiveRejectsUnsafeEntries(t *testing.T) {
tests := map[string][]tarEntry{
"absolute path": {
fileEntry("/report.md", "report"),
},
"path traversal": {
fileEntry("../report.md", "report"),
},
"backslash path": {
fileEntry(`nested\report.md`, "report"),
},
"symlink": {
{name: "link.md", typeflag: tar.TypeSymlink, linkname: "report.md"},
},
"hardlink": {
{name: "link.md", typeflag: tar.TypeLink, linkname: "report.md"},
},
"device": {
{name: "device", typeflag: tar.TypeChar},
},
}
for name, entries := range tests {
t.Run(name, func(t *testing.T) {
err := stageArchiveError(t, makeArchive(t, false, entries...), ContentTypeTar, nil)
if err == nil {
t.Fatal("StageArchive() error = nil, want error")
}
})
}
}
func TestStageArchiveRejectsBundleValidationFailures(t *testing.T) {
tests := map[string][]tarEntry{
"missing manifest": {
fileEntry("report.md", "report"),
},
"nested manifest": {
fileEntry("manifest.json", manifestJSON(t, manifestFor("reports.nested", fileSpec{path: "report.md", body: "report"}))),
fileEntry("nested/manifest.json", "{}"),
fileEntry("report.md", "report"),
},
"missing listed file": {
fileEntry("manifest.json", manifestJSON(t, manifestFor("reports.missing", fileSpec{path: "missing.md", body: "missing"}))),
},
"digest mismatch": {
fileEntry("manifest.json", manifestJSON(t, manifestFor("reports.digest", fileSpec{path: "report.md", body: "expected"}))),
fileEntry("report.md", "actual"),
},
"non regular listed file": {
fileEntry("manifest.json", manifestJSON(t, manifestFor("reports.directory", fileSpec{path: "report.md", body: "report"}))),
{name: "report.md", typeflag: tar.TypeDir},
},
}
for name, entries := range tests {
t.Run(name, func(t *testing.T) {
err := stageArchiveError(t, makeArchive(t, false, entries...), ContentTypeTar, nil)
if err == nil {
t.Fatal("StageArchive() error = nil, want error")
}
})
}
}
func TestStageArchiveCleansUpFailedExtraction(t *testing.T) {
stagingPath := filepath.Join(t.TempDir(), "staging")
archive := makeArchive(t, false, fileEntry("../report.md", "report"))
_, err := StageArchive(context.Background(), StageOptions{
Body: bytes.NewReader(archive),
ContentType: ContentTypeTar,
PipelineStagingPath: stagingPath,
RunID: "reports.20260603T120000Z.abcd",
MaxUploadSize: int64(len(archive)),
MaxExtractedSize: 1024 * 1024,
MaxFileCount: 10,
})
if err == nil {
t.Fatal("StageArchive() error = nil, want error")
}
entries, err := os.ReadDir(stagingPath)
if err != nil {
t.Fatalf("ReadDir() error = %v", err)
}
if len(entries) != 0 {
t.Fatalf("staging entries = %d, want cleanup", len(entries))
}
}
func stageArchive(t *testing.T, archive []byte, contentType string) StagedBundle {
t.Helper()
staged, err := StageArchive(context.Background(), defaultStageOptions(t, archive, contentType))
if err != nil {
t.Fatalf("StageArchive() error = %v", err)
}
return staged
}
func stageArchiveError(t *testing.T, archive []byte, contentType string, mutate func(*StageOptions)) error {
t.Helper()
opts := defaultStageOptions(t, archive, contentType)
if mutate != nil {
mutate(&opts)
}
_, err := StageArchive(context.Background(), opts)
if err == nil {
t.Fatal("StageArchive() error = nil, want error")
}
return err
}
func defaultStageOptions(t *testing.T, archive []byte, contentType string) StageOptions {
t.Helper()
return StageOptions{
Body: bytes.NewReader(archive),
ContentType: contentType,
PipelineStagingPath: filepath.Join(t.TempDir(), "staging"),
RunID: "reports.20260603T120000Z.abcd",
MaxUploadSize: int64(len(archive)),
MaxExtractedSize: 1024 * 1024,
MaxFileCount: 10,
}
}
func validArchive(t *testing.T, compressed bool) []byte {
t.Helper()
root := filepath.Join(t.TempDir(), "bundle")
sourceRoot := t.TempDir()
writeFile(t, sourceRoot, "report.md", "# Report\n")
writeFile(t, sourceRoot, "summary.txt", "Summary\n")
_, err := sourcebundle.WriteBundle(sourcebundle.WriteBundleOptions{
Root: root,
ID: "reports.ingest",
Created: time.Date(2026, 6, 3, 12, 0, 0, 0, time.UTC),
Files: []sourcebundle.BundleFile{
{SourcePath: filepath.Join(sourceRoot, "report.md"), Path: "report.md"},
{SourcePath: filepath.Join(sourceRoot, "summary.txt"), Path: "summary.txt"},
},
})
if err != nil {
t.Fatalf("WriteBundle() error = %v", err)
}
var entries []tarEntry
if err := filepath.WalkDir(root, func(filePath string, entry fs.DirEntry, err error) error {
if err != nil {
return err
}
if entry.IsDir() {
return nil
}
relative, err := filepath.Rel(root, filePath)
if err != nil {
return err
}
data, err := os.ReadFile(filePath)
if err != nil {
return err
}
entries = append(entries, fileEntry(filepath.ToSlash(relative), string(data)))
return nil
}); err != nil {
t.Fatalf("walk bundle: %v", err)
}
return makeArchive(t, compressed, entries...)
}
type tarEntry struct {
name string
typeflag byte
body []byte
linkname string
}
func fileEntry(name, body string) tarEntry {
return tarEntry{name: name, typeflag: tar.TypeReg, body: []byte(body)}
}
func makeArchive(t *testing.T, compressed bool, entries ...tarEntry) []byte {
t.Helper()
var output bytes.Buffer
var writer *tar.Writer
var gzipWriter *gzip.Writer
if compressed {
gzipWriter = gzip.NewWriter(&output)
writer = tar.NewWriter(gzipWriter)
} else {
writer = tar.NewWriter(&output)
}
for _, entry := range entries {
header := &tar.Header{
Name: entry.name,
Typeflag: entry.typeflag,
Size: int64(len(entry.body)),
Mode: 0o644,
Linkname: entry.linkname,
}
if entry.typeflag == tar.TypeDir {
header.Size = 0
header.Mode = 0o755
}
if err := writer.WriteHeader(header); err != nil {
t.Fatalf("WriteHeader(%q) error = %v", entry.name, err)
}
if len(entry.body) > 0 {
if _, err := writer.Write(entry.body); err != nil {
t.Fatalf("Write(%q) error = %v", entry.name, err)
}
}
}
if err := writer.Close(); err != nil {
t.Fatalf("close tar writer: %v", err)
}
if gzipWriter != nil {
if err := gzipWriter.Close(); err != nil {
t.Fatalf("close gzip writer: %v", err)
}
}
return output.Bytes()
}
type fileSpec struct {
path string
body string
}
func manifestFor(id string, files ...fileSpec) sourcebundle.Manifest {
manifest := sourcebundle.Manifest{
SchemaVersion: sourcebundle.SchemaVersion,
ID: id,
Created: time.Date(2026, 6, 3, 12, 0, 0, 0, time.UTC),
}
for _, file := range files {
manifest.Files = append(manifest.Files, sourcebundle.ManifestFile{
Path: file.path,
SHA256: sourcebundle.FileDigest([]byte(file.body)),
Size: int64(len(file.body)),
})
}
manifest.Digest = sourcebundle.BundleDigest(manifest.Files)
return manifest
}
func manifestJSON(t *testing.T, manifest sourcebundle.Manifest) string {
t.Helper()
data, err := sourcebundle.MarshalManifest(manifest)
if err != nil {
t.Fatalf("MarshalManifest() error = %v", err)
}
return string(data)
}
func writeFile(t *testing.T, root, relative, body string) {
t.Helper()
fullPath := filepath.Join(root, filepath.FromSlash(relative))
if err := os.MkdirAll(filepath.Dir(fullPath), 0o755); err != nil {
t.Fatalf("MkdirAll() error = %v", err)
}
if err := os.WriteFile(fullPath, []byte(body), 0o644); err != nil {
t.Fatalf("WriteFile() error = %v", err)
}
}
func readFile(t *testing.T, root, relative string) string {
t.Helper()
data, err := os.ReadFile(filepath.Join(root, filepath.FromSlash(relative)))
if err != nil {
t.Fatalf("ReadFile() error = %v", err)
}
return string(data)
}
func TestCleanArchivePath(t *testing.T) {
tests := map[string]bool{
"manifest.json": true,
"nested/report.md": true,
"": false,
"/absolute.md": false,
"../escape.md": false,
"nested/../report.md": false,
`nested\report.md`: false,
"./report.md": false,
"nested//report.md": false,
}
for value, wantOK := range tests {
t.Run(strings.ReplaceAll(value, "/", "_"), func(t *testing.T) {
_, err := cleanArchivePath(value)
if wantOK && err != nil {
t.Fatalf("cleanArchivePath(%q) error = %v", value, err)
}
if !wantOK && err == nil {
t.Fatalf("cleanArchivePath(%q) error = nil, want error", value)
}
})
}
}