Add upload archive staging

This commit is contained in:
2026-06-03 15:05:56 +00:00
parent 35c5237dfc
commit 65dd22f974
3 changed files with 771 additions and 0 deletions

346
internal/ingest/archive.go Normal file
View File

@@ -0,0 +1,346 @@
package ingest
import (
"archive/tar"
"compress/gzip"
"context"
"errors"
"fmt"
"io"
"mime"
"os"
"path"
"path/filepath"
"strings"
sourcebundle "gitea.maximumdirect.net/eric/distributor/pkg/bundle"
)
const (
ContentTypeTar = "application/x-tar"
ContentTypeGzip = "application/gzip"
ContentTypeXGzip = "application/x-gzip"
)
var (
ErrUnsupportedContentType = errors.New("unsupported archive content type")
ErrUploadTooLarge = errors.New("upload exceeds maximum size")
ErrExtractedTooLarge = errors.New("extracted bundle exceeds maximum size")
ErrTooManyFiles = errors.New("extracted bundle has too many files")
ErrUnsafeArchivePath = errors.New("unsafe archive path")
)
type StageOptions struct {
Body io.Reader
ContentType string
PipelineStagingPath string
RunID string
MaxUploadSize int64
MaxExtractedSize int64
MaxFileCount int
}
type StagedBundle struct {
Root string
Manifest sourcebundle.Manifest
}
func StageArchive(ctx context.Context, opts StageOptions) (StagedBundle, error) {
if ctx == nil {
ctx = context.Background()
}
if err := validateStageOptions(opts); err != nil {
return StagedBundle{}, err
}
format, err := archiveFormat(opts.ContentType)
if err != nil {
return StagedBundle{}, err
}
if err := os.MkdirAll(opts.PipelineStagingPath, 0o755); err != nil {
return StagedBundle{}, fmt.Errorf("create pipeline staging path: %w", err)
}
tempDir, err := os.MkdirTemp(opts.PipelineStagingPath, "."+opts.RunID+"-")
if err != nil {
return StagedBundle{}, fmt.Errorf("create staging temp dir: %w", err)
}
cleanupTemp := true
defer func() {
if cleanupTemp {
_ = os.RemoveAll(tempDir)
}
}()
archivePath := filepath.Join(tempDir, "upload.archive")
if err := writeLimited(ctx, archivePath, opts.Body, opts.MaxUploadSize); err != nil {
return StagedBundle{}, err
}
extractRoot := filepath.Join(tempDir, "bundle")
if err := os.Mkdir(extractRoot, 0o755); err != nil {
return StagedBundle{}, fmt.Errorf("create extraction root: %w", err)
}
if err := extractArchive(ctx, archivePath, extractRoot, format, opts.MaxExtractedSize, opts.MaxFileCount); err != nil {
return StagedBundle{}, err
}
manifest, err := sourcebundle.LoadManifest(extractRoot)
if err != nil {
return StagedBundle{}, err
}
if err := sourcebundle.ValidateBundle(extractRoot, manifest); err != nil {
return StagedBundle{}, err
}
finalRoot := filepath.Join(opts.PipelineStagingPath, opts.RunID)
if err := os.Rename(extractRoot, finalRoot); err != nil {
return StagedBundle{}, fmt.Errorf("commit staged bundle: %w", err)
}
cleanupTemp = false
if err := os.RemoveAll(tempDir); err != nil {
return StagedBundle{}, fmt.Errorf("remove staging temp dir: %w", err)
}
return StagedBundle{
Root: finalRoot,
Manifest: manifest,
}, nil
}
func validateStageOptions(opts StageOptions) error {
if opts.Body == nil {
return fmt.Errorf("body is required")
}
if opts.PipelineStagingPath == "" {
return fmt.Errorf("pipeline staging path is required")
}
if err := validateRunID(opts.RunID); err != nil {
return err
}
if opts.MaxUploadSize <= 0 {
return fmt.Errorf("max upload size must be greater than zero")
}
if opts.MaxExtractedSize <= 0 {
return fmt.Errorf("max extracted size must be greater than zero")
}
if opts.MaxFileCount <= 0 {
return fmt.Errorf("max file count must be greater than zero")
}
return nil
}
func validateRunID(value string) error {
if value == "" {
return fmt.Errorf("run id is required")
}
if value == "." || value == ".." || strings.ContainsAny(value, `/\`) {
return fmt.Errorf("run id must be a single filesystem path segment")
}
return nil
}
type archiveKind int
const (
archiveKindTar archiveKind = iota + 1
archiveKindGzip
)
func archiveFormat(contentType string) (archiveKind, error) {
mediaType, _, err := mime.ParseMediaType(contentType)
if err != nil {
mediaType = contentType
}
switch mediaType {
case ContentTypeTar:
return archiveKindTar, nil
case ContentTypeGzip, ContentTypeXGzip:
return archiveKindGzip, nil
default:
return 0, ErrUnsupportedContentType
}
}
func writeLimited(ctx context.Context, destination string, body io.Reader, maxSize int64) error {
file, err := os.OpenFile(destination, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600)
if err != nil {
return fmt.Errorf("create upload archive: %w", err)
}
defer file.Close()
limited := &limitedReader{ctx: ctx, reader: body, limit: maxSize}
if _, err := io.Copy(file, limited); err != nil {
return err
}
if err := file.Close(); err != nil {
return fmt.Errorf("write upload archive: %w", err)
}
return nil
}
type limitedReader struct {
ctx context.Context
reader io.Reader
limit int64
read int64
}
func (r *limitedReader) Read(data []byte) (int, error) {
if err := r.ctx.Err(); err != nil {
return 0, err
}
if r.read == r.limit {
var probe [1]byte
n, err := r.reader.Read(probe[:])
if n > 0 {
return 0, ErrUploadTooLarge
}
return 0, err
}
remaining := r.limit - r.read
if int64(len(data)) > remaining+1 {
data = data[:remaining+1]
}
n, err := r.reader.Read(data)
if r.read+int64(n) > r.limit {
allowed := int(r.limit - r.read)
r.read = r.limit
return allowed, ErrUploadTooLarge
}
r.read += int64(n)
return n, err
}
func extractArchive(ctx context.Context, archivePath, destination string, format archiveKind, maxExtractedSize int64, maxFileCount int) error {
file, err := os.Open(archivePath)
if err != nil {
return fmt.Errorf("open upload archive: %w", err)
}
defer file.Close()
var reader io.Reader = file
var gzipReader *gzip.Reader
if format == archiveKindGzip {
gzipReader, err = gzip.NewReader(file)
if err != nil {
return fmt.Errorf("open gzip archive: %w", err)
}
defer gzipReader.Close()
reader = gzipReader
}
extractor := archiveExtractor{
ctx: ctx,
destination: destination,
maxExtractedSize: maxExtractedSize,
maxFileCount: maxFileCount,
}
if err := extractor.extract(tar.NewReader(reader)); err != nil {
return err
}
if extractor.rootManifestCount != 1 {
return fmt.Errorf("archive must contain exactly one root-level manifest.json")
}
return nil
}
type archiveExtractor struct {
ctx context.Context
destination string
maxExtractedSize int64
maxFileCount int
extractedSize int64
fileCount int
rootManifestCount int
seenFiles map[string]struct{}
}
func (e *archiveExtractor) extract(reader *tar.Reader) error {
e.seenFiles = make(map[string]struct{})
for {
if err := e.ctx.Err(); err != nil {
return err
}
header, err := reader.Next()
if errors.Is(err, io.EOF) {
return nil
}
if err != nil {
return fmt.Errorf("read tar archive: %w", err)
}
name, err := cleanArchivePath(header.Name)
if err != nil {
return err
}
if path.Base(name) == sourcebundle.ManifestName {
if name != sourcebundle.ManifestName {
return fmt.Errorf("nested manifest %q is not allowed", name)
}
e.rootManifestCount++
}
switch header.Typeflag {
case tar.TypeDir:
if err := os.MkdirAll(filepath.Join(e.destination, filepath.FromSlash(name)), 0o755); err != nil {
return fmt.Errorf("create archive directory %q: %w", name, err)
}
case tar.TypeReg, tar.TypeRegA:
if err := e.extractFile(reader, name, header.Size); err != nil {
return err
}
default:
return fmt.Errorf("archive entry %q has unsupported type %c", name, header.Typeflag)
}
}
}
func (e *archiveExtractor) extractFile(reader *tar.Reader, name string, size int64) error {
if size < 0 {
return fmt.Errorf("archive entry %q has invalid size", name)
}
e.fileCount++
if e.fileCount > e.maxFileCount {
return ErrTooManyFiles
}
e.extractedSize += size
if e.extractedSize > e.maxExtractedSize {
return ErrExtractedTooLarge
}
if _, exists := e.seenFiles[name]; exists {
return fmt.Errorf("archive entry %q is duplicated", name)
}
e.seenFiles[name] = struct{}{}
fullPath := filepath.Join(e.destination, filepath.FromSlash(name))
if err := os.MkdirAll(filepath.Dir(fullPath), 0o755); err != nil {
return fmt.Errorf("create archive parent for %q: %w", name, err)
}
file, err := os.OpenFile(fullPath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o644)
if err != nil {
return fmt.Errorf("create archive file %q: %w", name, err)
}
defer file.Close()
if _, err := io.CopyN(file, reader, size); err != nil {
return fmt.Errorf("extract archive file %q: %w", name, err)
}
if err := file.Close(); err != nil {
return fmt.Errorf("extract archive file %q: %w", name, err)
}
return nil
}
func cleanArchivePath(value string) (string, error) {
if value == "" || strings.Contains(value, `\`) || strings.HasPrefix(value, "/") {
return "", ErrUnsafeArchivePath
}
cleaned := path.Clean(value)
if cleaned != value {
return "", ErrUnsafeArchivePath
}
for _, segment := range strings.Split(value, "/") {
if segment == "" || segment == "." || segment == ".." {
return "", ErrUnsafeArchivePath
}
}
return value, nil
}

View File

@@ -0,0 +1,382 @@
package ingest
import (
"archive/tar"
"bytes"
"compress/gzip"
"context"
"errors"
"io/fs"
"os"
"path/filepath"
"strings"
"testing"
"time"
sourcebundle "gitea.maximumdirect.net/eric/distributor/pkg/bundle"
)
func TestStageArchiveAcceptsTar(t *testing.T) {
archive := validArchive(t, false)
staged := stageArchive(t, archive, ContentTypeTar)
if got, want := staged.Manifest.ID, "reports.ingest"; got != want {
t.Fatalf("manifest id = %q, want %q", got, want)
}
if got := readFile(t, staged.Root, "report.md"); got != "# Report\n" {
t.Fatalf("report = %q", got)
}
}
func TestStageArchiveAcceptsGzipTar(t *testing.T) {
archive := validArchive(t, true)
staged := stageArchive(t, archive, ContentTypeGzip+"; charset=binary")
if got, want := staged.Manifest.ID, "reports.ingest"; got != want {
t.Fatalf("manifest id = %q, want %q", got, want)
}
if got := readFile(t, staged.Root, "summary.txt"); got != "Summary\n" {
t.Fatalf("summary = %q", got)
}
}
func TestStageArchiveRejectsUnsupportedContentType(t *testing.T) {
err := stageArchiveError(t, validArchive(t, false), "application/zip", nil)
if !errors.Is(err, ErrUnsupportedContentType) {
t.Fatalf("StageArchive() error = %v, want ErrUnsupportedContentType", err)
}
}
func TestStageArchiveEnforcesMaxUploadSize(t *testing.T) {
archive := validArchive(t, false)
err := stageArchiveError(t, archive, ContentTypeTar, func(opts *StageOptions) {
opts.MaxUploadSize = int64(len(archive) - 1)
})
if !errors.Is(err, ErrUploadTooLarge) {
t.Fatalf("StageArchive() error = %v, want ErrUploadTooLarge", err)
}
}
func TestStageArchiveEnforcesExtractionLimits(t *testing.T) {
archive := validArchive(t, false)
tests := map[string]struct {
mutate func(*StageOptions)
wantErr error
}{
"size": {
mutate: func(opts *StageOptions) {
opts.MaxExtractedSize = 1
},
wantErr: ErrExtractedTooLarge,
},
"files": {
mutate: func(opts *StageOptions) {
opts.MaxFileCount = 1
},
wantErr: ErrTooManyFiles,
},
}
for name, tt := range tests {
t.Run(name, func(t *testing.T) {
err := stageArchiveError(t, archive, ContentTypeTar, tt.mutate)
if !errors.Is(err, tt.wantErr) {
t.Fatalf("StageArchive() error = %v, want %v", err, tt.wantErr)
}
})
}
}
func TestStageArchiveRejectsUnsafeEntries(t *testing.T) {
tests := map[string][]tarEntry{
"absolute path": {
fileEntry("/report.md", "report"),
},
"path traversal": {
fileEntry("../report.md", "report"),
},
"backslash path": {
fileEntry(`nested\report.md`, "report"),
},
"symlink": {
{name: "link.md", typeflag: tar.TypeSymlink, linkname: "report.md"},
},
"hardlink": {
{name: "link.md", typeflag: tar.TypeLink, linkname: "report.md"},
},
"device": {
{name: "device", typeflag: tar.TypeChar},
},
}
for name, entries := range tests {
t.Run(name, func(t *testing.T) {
err := stageArchiveError(t, makeArchive(t, false, entries...), ContentTypeTar, nil)
if err == nil {
t.Fatal("StageArchive() error = nil, want error")
}
})
}
}
func TestStageArchiveRejectsBundleValidationFailures(t *testing.T) {
tests := map[string][]tarEntry{
"missing manifest": {
fileEntry("report.md", "report"),
},
"nested manifest": {
fileEntry("manifest.json", manifestJSON(t, manifestFor("reports.nested", fileSpec{path: "report.md", body: "report"}))),
fileEntry("nested/manifest.json", "{}"),
fileEntry("report.md", "report"),
},
"missing listed file": {
fileEntry("manifest.json", manifestJSON(t, manifestFor("reports.missing", fileSpec{path: "missing.md", body: "missing"}))),
},
"digest mismatch": {
fileEntry("manifest.json", manifestJSON(t, manifestFor("reports.digest", fileSpec{path: "report.md", body: "expected"}))),
fileEntry("report.md", "actual"),
},
"non regular listed file": {
fileEntry("manifest.json", manifestJSON(t, manifestFor("reports.directory", fileSpec{path: "report.md", body: "report"}))),
{name: "report.md", typeflag: tar.TypeDir},
},
}
for name, entries := range tests {
t.Run(name, func(t *testing.T) {
err := stageArchiveError(t, makeArchive(t, false, entries...), ContentTypeTar, nil)
if err == nil {
t.Fatal("StageArchive() error = nil, want error")
}
})
}
}
func TestStageArchiveCleansUpFailedExtraction(t *testing.T) {
stagingPath := filepath.Join(t.TempDir(), "staging")
archive := makeArchive(t, false, fileEntry("../report.md", "report"))
_, err := StageArchive(context.Background(), StageOptions{
Body: bytes.NewReader(archive),
ContentType: ContentTypeTar,
PipelineStagingPath: stagingPath,
RunID: "reports.20260603T120000Z.abcd",
MaxUploadSize: int64(len(archive)),
MaxExtractedSize: 1024 * 1024,
MaxFileCount: 10,
})
if err == nil {
t.Fatal("StageArchive() error = nil, want error")
}
entries, err := os.ReadDir(stagingPath)
if err != nil {
t.Fatalf("ReadDir() error = %v", err)
}
if len(entries) != 0 {
t.Fatalf("staging entries = %d, want cleanup", len(entries))
}
}
func stageArchive(t *testing.T, archive []byte, contentType string) StagedBundle {
t.Helper()
staged, err := StageArchive(context.Background(), defaultStageOptions(t, archive, contentType))
if err != nil {
t.Fatalf("StageArchive() error = %v", err)
}
return staged
}
func stageArchiveError(t *testing.T, archive []byte, contentType string, mutate func(*StageOptions)) error {
t.Helper()
opts := defaultStageOptions(t, archive, contentType)
if mutate != nil {
mutate(&opts)
}
_, err := StageArchive(context.Background(), opts)
if err == nil {
t.Fatal("StageArchive() error = nil, want error")
}
return err
}
func defaultStageOptions(t *testing.T, archive []byte, contentType string) StageOptions {
t.Helper()
return StageOptions{
Body: bytes.NewReader(archive),
ContentType: contentType,
PipelineStagingPath: filepath.Join(t.TempDir(), "staging"),
RunID: "reports.20260603T120000Z.abcd",
MaxUploadSize: int64(len(archive)),
MaxExtractedSize: 1024 * 1024,
MaxFileCount: 10,
}
}
func validArchive(t *testing.T, compressed bool) []byte {
t.Helper()
root := filepath.Join(t.TempDir(), "bundle")
sourceRoot := t.TempDir()
writeFile(t, sourceRoot, "report.md", "# Report\n")
writeFile(t, sourceRoot, "summary.txt", "Summary\n")
_, err := sourcebundle.WriteBundle(sourcebundle.WriteBundleOptions{
Root: root,
ID: "reports.ingest",
Created: time.Date(2026, 6, 3, 12, 0, 0, 0, time.UTC),
Files: []sourcebundle.BundleFile{
{SourcePath: filepath.Join(sourceRoot, "report.md"), Path: "report.md"},
{SourcePath: filepath.Join(sourceRoot, "summary.txt"), Path: "summary.txt"},
},
})
if err != nil {
t.Fatalf("WriteBundle() error = %v", err)
}
var entries []tarEntry
if err := filepath.WalkDir(root, func(filePath string, entry fs.DirEntry, err error) error {
if err != nil {
return err
}
if entry.IsDir() {
return nil
}
relative, err := filepath.Rel(root, filePath)
if err != nil {
return err
}
data, err := os.ReadFile(filePath)
if err != nil {
return err
}
entries = append(entries, fileEntry(filepath.ToSlash(relative), string(data)))
return nil
}); err != nil {
t.Fatalf("walk bundle: %v", err)
}
return makeArchive(t, compressed, entries...)
}
type tarEntry struct {
name string
typeflag byte
body []byte
linkname string
}
func fileEntry(name, body string) tarEntry {
return tarEntry{name: name, typeflag: tar.TypeReg, body: []byte(body)}
}
func makeArchive(t *testing.T, compressed bool, entries ...tarEntry) []byte {
t.Helper()
var output bytes.Buffer
var writer *tar.Writer
var gzipWriter *gzip.Writer
if compressed {
gzipWriter = gzip.NewWriter(&output)
writer = tar.NewWriter(gzipWriter)
} else {
writer = tar.NewWriter(&output)
}
for _, entry := range entries {
header := &tar.Header{
Name: entry.name,
Typeflag: entry.typeflag,
Size: int64(len(entry.body)),
Mode: 0o644,
Linkname: entry.linkname,
}
if entry.typeflag == tar.TypeDir {
header.Size = 0
header.Mode = 0o755
}
if err := writer.WriteHeader(header); err != nil {
t.Fatalf("WriteHeader(%q) error = %v", entry.name, err)
}
if len(entry.body) > 0 {
if _, err := writer.Write(entry.body); err != nil {
t.Fatalf("Write(%q) error = %v", entry.name, err)
}
}
}
if err := writer.Close(); err != nil {
t.Fatalf("close tar writer: %v", err)
}
if gzipWriter != nil {
if err := gzipWriter.Close(); err != nil {
t.Fatalf("close gzip writer: %v", err)
}
}
return output.Bytes()
}
type fileSpec struct {
path string
body string
}
func manifestFor(id string, files ...fileSpec) sourcebundle.Manifest {
manifest := sourcebundle.Manifest{
SchemaVersion: sourcebundle.SchemaVersion,
ID: id,
Created: time.Date(2026, 6, 3, 12, 0, 0, 0, time.UTC),
}
for _, file := range files {
manifest.Files = append(manifest.Files, sourcebundle.ManifestFile{
Path: file.path,
SHA256: sourcebundle.FileDigest([]byte(file.body)),
Size: int64(len(file.body)),
})
}
manifest.Digest = sourcebundle.BundleDigest(manifest.Files)
return manifest
}
func manifestJSON(t *testing.T, manifest sourcebundle.Manifest) string {
t.Helper()
data, err := sourcebundle.MarshalManifest(manifest)
if err != nil {
t.Fatalf("MarshalManifest() error = %v", err)
}
return string(data)
}
func writeFile(t *testing.T, root, relative, body string) {
t.Helper()
fullPath := filepath.Join(root, filepath.FromSlash(relative))
if err := os.MkdirAll(filepath.Dir(fullPath), 0o755); err != nil {
t.Fatalf("MkdirAll() error = %v", err)
}
if err := os.WriteFile(fullPath, []byte(body), 0o644); err != nil {
t.Fatalf("WriteFile() error = %v", err)
}
}
func readFile(t *testing.T, root, relative string) string {
t.Helper()
data, err := os.ReadFile(filepath.Join(root, filepath.FromSlash(relative)))
if err != nil {
t.Fatalf("ReadFile() error = %v", err)
}
return string(data)
}
func TestCleanArchivePath(t *testing.T) {
tests := map[string]bool{
"manifest.json": true,
"nested/report.md": true,
"": false,
"/absolute.md": false,
"../escape.md": false,
"nested/../report.md": false,
`nested\report.md`: false,
"./report.md": false,
"nested//report.md": false,
}
for value, wantOK := range tests {
t.Run(strings.ReplaceAll(value, "/", "_"), func(t *testing.T) {
_, err := cleanArchivePath(value)
if wantOK && err != nil {
t.Fatalf("cleanArchivePath(%q) error = %v", value, err)
}
if !wantOK && err == nil {
t.Fatalf("cleanArchivePath(%q) error = nil, want error", value)
}
})
}
}