Add upload archive staging
This commit is contained in:
43
docs/internal/ingest.md
Normal file
43
docs/internal/ingest.md
Normal file
@@ -0,0 +1,43 @@
|
||||
# Ingestion Internals
|
||||
|
||||
## Purpose
|
||||
|
||||
`internal/ingest` stages uploaded source bundle archives into local per-run directories. It does not authenticate requests, manage upload queues, publish destinations, or start an HTTP server.
|
||||
|
||||
## Archive staging
|
||||
|
||||
`StageArchive` accepts one upload body, content type, pipeline staging path, run id, and explicit size and file-count limits. It writes the request body to temporary storage while enforcing the configured upload size limit, extracts the archive into temporary local storage, validates the extracted source bundle, and then commits the validated bundle to:
|
||||
|
||||
```text
|
||||
<pipeline staging path>/<run id>
|
||||
```
|
||||
|
||||
The returned `StagedBundle.Root` is a local filesystem path to the validated source bundle root.
|
||||
|
||||
## Accepted archive formats
|
||||
|
||||
The package accepts only:
|
||||
|
||||
- `application/x-tar`
|
||||
- `application/gzip`
|
||||
- `application/x-gzip`
|
||||
|
||||
Gzip uploads must contain a tar archive.
|
||||
|
||||
## Extraction rules
|
||||
|
||||
Archive entry paths must be clean relative slash-separated paths. Extraction rejects absolute paths, path traversal, backslash paths, duplicate files, symlinks, hardlinks, devices, sockets, and other special entries.
|
||||
|
||||
The archive must contain exactly one root-level `manifest.json`. Nested manifests are rejected.
|
||||
|
||||
Regular files and directories are the only accepted tar entries. Regular file extraction enforces the explicit maximum extracted byte count and maximum file count supplied by the caller.
|
||||
|
||||
## Bundle validation
|
||||
|
||||
After extraction, the package loads and validates the staged bundle through `pkg/bundle`. Manifest parsing, source path validation, file existence checks, regular-file checks, file sizes, file SHA-256 digests, and bundle digest validation use the existing source bundle contract.
|
||||
|
||||
Validation happens before the staged bundle is committed to its final per-run path.
|
||||
|
||||
## Failure behavior
|
||||
|
||||
Failed staging removes temporary archive and extraction data created by the package. A failed call does not publish anything and does not leave a committed per-run bundle directory.
|
||||
346
internal/ingest/archive.go
Normal file
346
internal/ingest/archive.go
Normal file
@@ -0,0 +1,346 @@
|
||||
package ingest
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"mime"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
sourcebundle "gitea.maximumdirect.net/eric/distributor/pkg/bundle"
|
||||
)
|
||||
|
||||
const (
|
||||
ContentTypeTar = "application/x-tar"
|
||||
ContentTypeGzip = "application/gzip"
|
||||
ContentTypeXGzip = "application/x-gzip"
|
||||
)
|
||||
|
||||
var (
|
||||
ErrUnsupportedContentType = errors.New("unsupported archive content type")
|
||||
ErrUploadTooLarge = errors.New("upload exceeds maximum size")
|
||||
ErrExtractedTooLarge = errors.New("extracted bundle exceeds maximum size")
|
||||
ErrTooManyFiles = errors.New("extracted bundle has too many files")
|
||||
ErrUnsafeArchivePath = errors.New("unsafe archive path")
|
||||
)
|
||||
|
||||
type StageOptions struct {
|
||||
Body io.Reader
|
||||
ContentType string
|
||||
PipelineStagingPath string
|
||||
RunID string
|
||||
MaxUploadSize int64
|
||||
MaxExtractedSize int64
|
||||
MaxFileCount int
|
||||
}
|
||||
|
||||
type StagedBundle struct {
|
||||
Root string
|
||||
Manifest sourcebundle.Manifest
|
||||
}
|
||||
|
||||
func StageArchive(ctx context.Context, opts StageOptions) (StagedBundle, error) {
|
||||
if ctx == nil {
|
||||
ctx = context.Background()
|
||||
}
|
||||
if err := validateStageOptions(opts); err != nil {
|
||||
return StagedBundle{}, err
|
||||
}
|
||||
format, err := archiveFormat(opts.ContentType)
|
||||
if err != nil {
|
||||
return StagedBundle{}, err
|
||||
}
|
||||
|
||||
if err := os.MkdirAll(opts.PipelineStagingPath, 0o755); err != nil {
|
||||
return StagedBundle{}, fmt.Errorf("create pipeline staging path: %w", err)
|
||||
}
|
||||
tempDir, err := os.MkdirTemp(opts.PipelineStagingPath, "."+opts.RunID+"-")
|
||||
if err != nil {
|
||||
return StagedBundle{}, fmt.Errorf("create staging temp dir: %w", err)
|
||||
}
|
||||
cleanupTemp := true
|
||||
defer func() {
|
||||
if cleanupTemp {
|
||||
_ = os.RemoveAll(tempDir)
|
||||
}
|
||||
}()
|
||||
|
||||
archivePath := filepath.Join(tempDir, "upload.archive")
|
||||
if err := writeLimited(ctx, archivePath, opts.Body, opts.MaxUploadSize); err != nil {
|
||||
return StagedBundle{}, err
|
||||
}
|
||||
|
||||
extractRoot := filepath.Join(tempDir, "bundle")
|
||||
if err := os.Mkdir(extractRoot, 0o755); err != nil {
|
||||
return StagedBundle{}, fmt.Errorf("create extraction root: %w", err)
|
||||
}
|
||||
if err := extractArchive(ctx, archivePath, extractRoot, format, opts.MaxExtractedSize, opts.MaxFileCount); err != nil {
|
||||
return StagedBundle{}, err
|
||||
}
|
||||
|
||||
manifest, err := sourcebundle.LoadManifest(extractRoot)
|
||||
if err != nil {
|
||||
return StagedBundle{}, err
|
||||
}
|
||||
if err := sourcebundle.ValidateBundle(extractRoot, manifest); err != nil {
|
||||
return StagedBundle{}, err
|
||||
}
|
||||
|
||||
finalRoot := filepath.Join(opts.PipelineStagingPath, opts.RunID)
|
||||
if err := os.Rename(extractRoot, finalRoot); err != nil {
|
||||
return StagedBundle{}, fmt.Errorf("commit staged bundle: %w", err)
|
||||
}
|
||||
cleanupTemp = false
|
||||
if err := os.RemoveAll(tempDir); err != nil {
|
||||
return StagedBundle{}, fmt.Errorf("remove staging temp dir: %w", err)
|
||||
}
|
||||
return StagedBundle{
|
||||
Root: finalRoot,
|
||||
Manifest: manifest,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func validateStageOptions(opts StageOptions) error {
|
||||
if opts.Body == nil {
|
||||
return fmt.Errorf("body is required")
|
||||
}
|
||||
if opts.PipelineStagingPath == "" {
|
||||
return fmt.Errorf("pipeline staging path is required")
|
||||
}
|
||||
if err := validateRunID(opts.RunID); err != nil {
|
||||
return err
|
||||
}
|
||||
if opts.MaxUploadSize <= 0 {
|
||||
return fmt.Errorf("max upload size must be greater than zero")
|
||||
}
|
||||
if opts.MaxExtractedSize <= 0 {
|
||||
return fmt.Errorf("max extracted size must be greater than zero")
|
||||
}
|
||||
if opts.MaxFileCount <= 0 {
|
||||
return fmt.Errorf("max file count must be greater than zero")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateRunID(value string) error {
|
||||
if value == "" {
|
||||
return fmt.Errorf("run id is required")
|
||||
}
|
||||
if value == "." || value == ".." || strings.ContainsAny(value, `/\`) {
|
||||
return fmt.Errorf("run id must be a single filesystem path segment")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type archiveKind int
|
||||
|
||||
const (
|
||||
archiveKindTar archiveKind = iota + 1
|
||||
archiveKindGzip
|
||||
)
|
||||
|
||||
func archiveFormat(contentType string) (archiveKind, error) {
|
||||
mediaType, _, err := mime.ParseMediaType(contentType)
|
||||
if err != nil {
|
||||
mediaType = contentType
|
||||
}
|
||||
switch mediaType {
|
||||
case ContentTypeTar:
|
||||
return archiveKindTar, nil
|
||||
case ContentTypeGzip, ContentTypeXGzip:
|
||||
return archiveKindGzip, nil
|
||||
default:
|
||||
return 0, ErrUnsupportedContentType
|
||||
}
|
||||
}
|
||||
|
||||
func writeLimited(ctx context.Context, destination string, body io.Reader, maxSize int64) error {
|
||||
file, err := os.OpenFile(destination, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600)
|
||||
if err != nil {
|
||||
return fmt.Errorf("create upload archive: %w", err)
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
limited := &limitedReader{ctx: ctx, reader: body, limit: maxSize}
|
||||
if _, err := io.Copy(file, limited); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := file.Close(); err != nil {
|
||||
return fmt.Errorf("write upload archive: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type limitedReader struct {
|
||||
ctx context.Context
|
||||
reader io.Reader
|
||||
limit int64
|
||||
read int64
|
||||
}
|
||||
|
||||
func (r *limitedReader) Read(data []byte) (int, error) {
|
||||
if err := r.ctx.Err(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if r.read == r.limit {
|
||||
var probe [1]byte
|
||||
n, err := r.reader.Read(probe[:])
|
||||
if n > 0 {
|
||||
return 0, ErrUploadTooLarge
|
||||
}
|
||||
return 0, err
|
||||
}
|
||||
remaining := r.limit - r.read
|
||||
if int64(len(data)) > remaining+1 {
|
||||
data = data[:remaining+1]
|
||||
}
|
||||
n, err := r.reader.Read(data)
|
||||
if r.read+int64(n) > r.limit {
|
||||
allowed := int(r.limit - r.read)
|
||||
r.read = r.limit
|
||||
return allowed, ErrUploadTooLarge
|
||||
}
|
||||
r.read += int64(n)
|
||||
return n, err
|
||||
}
|
||||
|
||||
func extractArchive(ctx context.Context, archivePath, destination string, format archiveKind, maxExtractedSize int64, maxFileCount int) error {
|
||||
file, err := os.Open(archivePath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("open upload archive: %w", err)
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
var reader io.Reader = file
|
||||
var gzipReader *gzip.Reader
|
||||
if format == archiveKindGzip {
|
||||
gzipReader, err = gzip.NewReader(file)
|
||||
if err != nil {
|
||||
return fmt.Errorf("open gzip archive: %w", err)
|
||||
}
|
||||
defer gzipReader.Close()
|
||||
reader = gzipReader
|
||||
}
|
||||
|
||||
extractor := archiveExtractor{
|
||||
ctx: ctx,
|
||||
destination: destination,
|
||||
maxExtractedSize: maxExtractedSize,
|
||||
maxFileCount: maxFileCount,
|
||||
}
|
||||
if err := extractor.extract(tar.NewReader(reader)); err != nil {
|
||||
return err
|
||||
}
|
||||
if extractor.rootManifestCount != 1 {
|
||||
return fmt.Errorf("archive must contain exactly one root-level manifest.json")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type archiveExtractor struct {
|
||||
ctx context.Context
|
||||
destination string
|
||||
maxExtractedSize int64
|
||||
maxFileCount int
|
||||
extractedSize int64
|
||||
fileCount int
|
||||
rootManifestCount int
|
||||
seenFiles map[string]struct{}
|
||||
}
|
||||
|
||||
func (e *archiveExtractor) extract(reader *tar.Reader) error {
|
||||
e.seenFiles = make(map[string]struct{})
|
||||
for {
|
||||
if err := e.ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
header, err := reader.Next()
|
||||
if errors.Is(err, io.EOF) {
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("read tar archive: %w", err)
|
||||
}
|
||||
name, err := cleanArchivePath(header.Name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if path.Base(name) == sourcebundle.ManifestName {
|
||||
if name != sourcebundle.ManifestName {
|
||||
return fmt.Errorf("nested manifest %q is not allowed", name)
|
||||
}
|
||||
e.rootManifestCount++
|
||||
}
|
||||
|
||||
switch header.Typeflag {
|
||||
case tar.TypeDir:
|
||||
if err := os.MkdirAll(filepath.Join(e.destination, filepath.FromSlash(name)), 0o755); err != nil {
|
||||
return fmt.Errorf("create archive directory %q: %w", name, err)
|
||||
}
|
||||
case tar.TypeReg, tar.TypeRegA:
|
||||
if err := e.extractFile(reader, name, header.Size); err != nil {
|
||||
return err
|
||||
}
|
||||
default:
|
||||
return fmt.Errorf("archive entry %q has unsupported type %c", name, header.Typeflag)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (e *archiveExtractor) extractFile(reader *tar.Reader, name string, size int64) error {
|
||||
if size < 0 {
|
||||
return fmt.Errorf("archive entry %q has invalid size", name)
|
||||
}
|
||||
e.fileCount++
|
||||
if e.fileCount > e.maxFileCount {
|
||||
return ErrTooManyFiles
|
||||
}
|
||||
e.extractedSize += size
|
||||
if e.extractedSize > e.maxExtractedSize {
|
||||
return ErrExtractedTooLarge
|
||||
}
|
||||
if _, exists := e.seenFiles[name]; exists {
|
||||
return fmt.Errorf("archive entry %q is duplicated", name)
|
||||
}
|
||||
e.seenFiles[name] = struct{}{}
|
||||
|
||||
fullPath := filepath.Join(e.destination, filepath.FromSlash(name))
|
||||
if err := os.MkdirAll(filepath.Dir(fullPath), 0o755); err != nil {
|
||||
return fmt.Errorf("create archive parent for %q: %w", name, err)
|
||||
}
|
||||
file, err := os.OpenFile(fullPath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o644)
|
||||
if err != nil {
|
||||
return fmt.Errorf("create archive file %q: %w", name, err)
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
if _, err := io.CopyN(file, reader, size); err != nil {
|
||||
return fmt.Errorf("extract archive file %q: %w", name, err)
|
||||
}
|
||||
if err := file.Close(); err != nil {
|
||||
return fmt.Errorf("extract archive file %q: %w", name, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func cleanArchivePath(value string) (string, error) {
|
||||
if value == "" || strings.Contains(value, `\`) || strings.HasPrefix(value, "/") {
|
||||
return "", ErrUnsafeArchivePath
|
||||
}
|
||||
cleaned := path.Clean(value)
|
||||
if cleaned != value {
|
||||
return "", ErrUnsafeArchivePath
|
||||
}
|
||||
for _, segment := range strings.Split(value, "/") {
|
||||
if segment == "" || segment == "." || segment == ".." {
|
||||
return "", ErrUnsafeArchivePath
|
||||
}
|
||||
}
|
||||
return value, nil
|
||||
}
|
||||
382
internal/ingest/archive_test.go
Normal file
382
internal/ingest/archive_test.go
Normal file
@@ -0,0 +1,382 @@
|
||||
package ingest
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"errors"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
sourcebundle "gitea.maximumdirect.net/eric/distributor/pkg/bundle"
|
||||
)
|
||||
|
||||
func TestStageArchiveAcceptsTar(t *testing.T) {
|
||||
archive := validArchive(t, false)
|
||||
staged := stageArchive(t, archive, ContentTypeTar)
|
||||
|
||||
if got, want := staged.Manifest.ID, "reports.ingest"; got != want {
|
||||
t.Fatalf("manifest id = %q, want %q", got, want)
|
||||
}
|
||||
if got := readFile(t, staged.Root, "report.md"); got != "# Report\n" {
|
||||
t.Fatalf("report = %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStageArchiveAcceptsGzipTar(t *testing.T) {
|
||||
archive := validArchive(t, true)
|
||||
staged := stageArchive(t, archive, ContentTypeGzip+"; charset=binary")
|
||||
|
||||
if got, want := staged.Manifest.ID, "reports.ingest"; got != want {
|
||||
t.Fatalf("manifest id = %q, want %q", got, want)
|
||||
}
|
||||
if got := readFile(t, staged.Root, "summary.txt"); got != "Summary\n" {
|
||||
t.Fatalf("summary = %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStageArchiveRejectsUnsupportedContentType(t *testing.T) {
|
||||
err := stageArchiveError(t, validArchive(t, false), "application/zip", nil)
|
||||
if !errors.Is(err, ErrUnsupportedContentType) {
|
||||
t.Fatalf("StageArchive() error = %v, want ErrUnsupportedContentType", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStageArchiveEnforcesMaxUploadSize(t *testing.T) {
|
||||
archive := validArchive(t, false)
|
||||
err := stageArchiveError(t, archive, ContentTypeTar, func(opts *StageOptions) {
|
||||
opts.MaxUploadSize = int64(len(archive) - 1)
|
||||
})
|
||||
if !errors.Is(err, ErrUploadTooLarge) {
|
||||
t.Fatalf("StageArchive() error = %v, want ErrUploadTooLarge", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStageArchiveEnforcesExtractionLimits(t *testing.T) {
|
||||
archive := validArchive(t, false)
|
||||
tests := map[string]struct {
|
||||
mutate func(*StageOptions)
|
||||
wantErr error
|
||||
}{
|
||||
"size": {
|
||||
mutate: func(opts *StageOptions) {
|
||||
opts.MaxExtractedSize = 1
|
||||
},
|
||||
wantErr: ErrExtractedTooLarge,
|
||||
},
|
||||
"files": {
|
||||
mutate: func(opts *StageOptions) {
|
||||
opts.MaxFileCount = 1
|
||||
},
|
||||
wantErr: ErrTooManyFiles,
|
||||
},
|
||||
}
|
||||
for name, tt := range tests {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
err := stageArchiveError(t, archive, ContentTypeTar, tt.mutate)
|
||||
if !errors.Is(err, tt.wantErr) {
|
||||
t.Fatalf("StageArchive() error = %v, want %v", err, tt.wantErr)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestStageArchiveRejectsUnsafeEntries(t *testing.T) {
|
||||
tests := map[string][]tarEntry{
|
||||
"absolute path": {
|
||||
fileEntry("/report.md", "report"),
|
||||
},
|
||||
"path traversal": {
|
||||
fileEntry("../report.md", "report"),
|
||||
},
|
||||
"backslash path": {
|
||||
fileEntry(`nested\report.md`, "report"),
|
||||
},
|
||||
"symlink": {
|
||||
{name: "link.md", typeflag: tar.TypeSymlink, linkname: "report.md"},
|
||||
},
|
||||
"hardlink": {
|
||||
{name: "link.md", typeflag: tar.TypeLink, linkname: "report.md"},
|
||||
},
|
||||
"device": {
|
||||
{name: "device", typeflag: tar.TypeChar},
|
||||
},
|
||||
}
|
||||
for name, entries := range tests {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
err := stageArchiveError(t, makeArchive(t, false, entries...), ContentTypeTar, nil)
|
||||
if err == nil {
|
||||
t.Fatal("StageArchive() error = nil, want error")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestStageArchiveRejectsBundleValidationFailures(t *testing.T) {
|
||||
tests := map[string][]tarEntry{
|
||||
"missing manifest": {
|
||||
fileEntry("report.md", "report"),
|
||||
},
|
||||
"nested manifest": {
|
||||
fileEntry("manifest.json", manifestJSON(t, manifestFor("reports.nested", fileSpec{path: "report.md", body: "report"}))),
|
||||
fileEntry("nested/manifest.json", "{}"),
|
||||
fileEntry("report.md", "report"),
|
||||
},
|
||||
"missing listed file": {
|
||||
fileEntry("manifest.json", manifestJSON(t, manifestFor("reports.missing", fileSpec{path: "missing.md", body: "missing"}))),
|
||||
},
|
||||
"digest mismatch": {
|
||||
fileEntry("manifest.json", manifestJSON(t, manifestFor("reports.digest", fileSpec{path: "report.md", body: "expected"}))),
|
||||
fileEntry("report.md", "actual"),
|
||||
},
|
||||
"non regular listed file": {
|
||||
fileEntry("manifest.json", manifestJSON(t, manifestFor("reports.directory", fileSpec{path: "report.md", body: "report"}))),
|
||||
{name: "report.md", typeflag: tar.TypeDir},
|
||||
},
|
||||
}
|
||||
for name, entries := range tests {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
err := stageArchiveError(t, makeArchive(t, false, entries...), ContentTypeTar, nil)
|
||||
if err == nil {
|
||||
t.Fatal("StageArchive() error = nil, want error")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestStageArchiveCleansUpFailedExtraction(t *testing.T) {
|
||||
stagingPath := filepath.Join(t.TempDir(), "staging")
|
||||
archive := makeArchive(t, false, fileEntry("../report.md", "report"))
|
||||
_, err := StageArchive(context.Background(), StageOptions{
|
||||
Body: bytes.NewReader(archive),
|
||||
ContentType: ContentTypeTar,
|
||||
PipelineStagingPath: stagingPath,
|
||||
RunID: "reports.20260603T120000Z.abcd",
|
||||
MaxUploadSize: int64(len(archive)),
|
||||
MaxExtractedSize: 1024 * 1024,
|
||||
MaxFileCount: 10,
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("StageArchive() error = nil, want error")
|
||||
}
|
||||
entries, err := os.ReadDir(stagingPath)
|
||||
if err != nil {
|
||||
t.Fatalf("ReadDir() error = %v", err)
|
||||
}
|
||||
if len(entries) != 0 {
|
||||
t.Fatalf("staging entries = %d, want cleanup", len(entries))
|
||||
}
|
||||
}
|
||||
|
||||
func stageArchive(t *testing.T, archive []byte, contentType string) StagedBundle {
|
||||
t.Helper()
|
||||
staged, err := StageArchive(context.Background(), defaultStageOptions(t, archive, contentType))
|
||||
if err != nil {
|
||||
t.Fatalf("StageArchive() error = %v", err)
|
||||
}
|
||||
return staged
|
||||
}
|
||||
|
||||
func stageArchiveError(t *testing.T, archive []byte, contentType string, mutate func(*StageOptions)) error {
|
||||
t.Helper()
|
||||
opts := defaultStageOptions(t, archive, contentType)
|
||||
if mutate != nil {
|
||||
mutate(&opts)
|
||||
}
|
||||
_, err := StageArchive(context.Background(), opts)
|
||||
if err == nil {
|
||||
t.Fatal("StageArchive() error = nil, want error")
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func defaultStageOptions(t *testing.T, archive []byte, contentType string) StageOptions {
|
||||
t.Helper()
|
||||
return StageOptions{
|
||||
Body: bytes.NewReader(archive),
|
||||
ContentType: contentType,
|
||||
PipelineStagingPath: filepath.Join(t.TempDir(), "staging"),
|
||||
RunID: "reports.20260603T120000Z.abcd",
|
||||
MaxUploadSize: int64(len(archive)),
|
||||
MaxExtractedSize: 1024 * 1024,
|
||||
MaxFileCount: 10,
|
||||
}
|
||||
}
|
||||
|
||||
func validArchive(t *testing.T, compressed bool) []byte {
|
||||
t.Helper()
|
||||
root := filepath.Join(t.TempDir(), "bundle")
|
||||
sourceRoot := t.TempDir()
|
||||
writeFile(t, sourceRoot, "report.md", "# Report\n")
|
||||
writeFile(t, sourceRoot, "summary.txt", "Summary\n")
|
||||
_, err := sourcebundle.WriteBundle(sourcebundle.WriteBundleOptions{
|
||||
Root: root,
|
||||
ID: "reports.ingest",
|
||||
Created: time.Date(2026, 6, 3, 12, 0, 0, 0, time.UTC),
|
||||
Files: []sourcebundle.BundleFile{
|
||||
{SourcePath: filepath.Join(sourceRoot, "report.md"), Path: "report.md"},
|
||||
{SourcePath: filepath.Join(sourceRoot, "summary.txt"), Path: "summary.txt"},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("WriteBundle() error = %v", err)
|
||||
}
|
||||
|
||||
var entries []tarEntry
|
||||
if err := filepath.WalkDir(root, func(filePath string, entry fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if entry.IsDir() {
|
||||
return nil
|
||||
}
|
||||
relative, err := filepath.Rel(root, filePath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
data, err := os.ReadFile(filePath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
entries = append(entries, fileEntry(filepath.ToSlash(relative), string(data)))
|
||||
return nil
|
||||
}); err != nil {
|
||||
t.Fatalf("walk bundle: %v", err)
|
||||
}
|
||||
return makeArchive(t, compressed, entries...)
|
||||
}
|
||||
|
||||
type tarEntry struct {
|
||||
name string
|
||||
typeflag byte
|
||||
body []byte
|
||||
linkname string
|
||||
}
|
||||
|
||||
func fileEntry(name, body string) tarEntry {
|
||||
return tarEntry{name: name, typeflag: tar.TypeReg, body: []byte(body)}
|
||||
}
|
||||
|
||||
func makeArchive(t *testing.T, compressed bool, entries ...tarEntry) []byte {
|
||||
t.Helper()
|
||||
var output bytes.Buffer
|
||||
var writer *tar.Writer
|
||||
var gzipWriter *gzip.Writer
|
||||
if compressed {
|
||||
gzipWriter = gzip.NewWriter(&output)
|
||||
writer = tar.NewWriter(gzipWriter)
|
||||
} else {
|
||||
writer = tar.NewWriter(&output)
|
||||
}
|
||||
for _, entry := range entries {
|
||||
header := &tar.Header{
|
||||
Name: entry.name,
|
||||
Typeflag: entry.typeflag,
|
||||
Size: int64(len(entry.body)),
|
||||
Mode: 0o644,
|
||||
Linkname: entry.linkname,
|
||||
}
|
||||
if entry.typeflag == tar.TypeDir {
|
||||
header.Size = 0
|
||||
header.Mode = 0o755
|
||||
}
|
||||
if err := writer.WriteHeader(header); err != nil {
|
||||
t.Fatalf("WriteHeader(%q) error = %v", entry.name, err)
|
||||
}
|
||||
if len(entry.body) > 0 {
|
||||
if _, err := writer.Write(entry.body); err != nil {
|
||||
t.Fatalf("Write(%q) error = %v", entry.name, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
if err := writer.Close(); err != nil {
|
||||
t.Fatalf("close tar writer: %v", err)
|
||||
}
|
||||
if gzipWriter != nil {
|
||||
if err := gzipWriter.Close(); err != nil {
|
||||
t.Fatalf("close gzip writer: %v", err)
|
||||
}
|
||||
}
|
||||
return output.Bytes()
|
||||
}
|
||||
|
||||
type fileSpec struct {
|
||||
path string
|
||||
body string
|
||||
}
|
||||
|
||||
func manifestFor(id string, files ...fileSpec) sourcebundle.Manifest {
|
||||
manifest := sourcebundle.Manifest{
|
||||
SchemaVersion: sourcebundle.SchemaVersion,
|
||||
ID: id,
|
||||
Created: time.Date(2026, 6, 3, 12, 0, 0, 0, time.UTC),
|
||||
}
|
||||
for _, file := range files {
|
||||
manifest.Files = append(manifest.Files, sourcebundle.ManifestFile{
|
||||
Path: file.path,
|
||||
SHA256: sourcebundle.FileDigest([]byte(file.body)),
|
||||
Size: int64(len(file.body)),
|
||||
})
|
||||
}
|
||||
manifest.Digest = sourcebundle.BundleDigest(manifest.Files)
|
||||
return manifest
|
||||
}
|
||||
|
||||
func manifestJSON(t *testing.T, manifest sourcebundle.Manifest) string {
|
||||
t.Helper()
|
||||
data, err := sourcebundle.MarshalManifest(manifest)
|
||||
if err != nil {
|
||||
t.Fatalf("MarshalManifest() error = %v", err)
|
||||
}
|
||||
return string(data)
|
||||
}
|
||||
|
||||
func writeFile(t *testing.T, root, relative, body string) {
|
||||
t.Helper()
|
||||
fullPath := filepath.Join(root, filepath.FromSlash(relative))
|
||||
if err := os.MkdirAll(filepath.Dir(fullPath), 0o755); err != nil {
|
||||
t.Fatalf("MkdirAll() error = %v", err)
|
||||
}
|
||||
if err := os.WriteFile(fullPath, []byte(body), 0o644); err != nil {
|
||||
t.Fatalf("WriteFile() error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func readFile(t *testing.T, root, relative string) string {
|
||||
t.Helper()
|
||||
data, err := os.ReadFile(filepath.Join(root, filepath.FromSlash(relative)))
|
||||
if err != nil {
|
||||
t.Fatalf("ReadFile() error = %v", err)
|
||||
}
|
||||
return string(data)
|
||||
}
|
||||
|
||||
func TestCleanArchivePath(t *testing.T) {
|
||||
tests := map[string]bool{
|
||||
"manifest.json": true,
|
||||
"nested/report.md": true,
|
||||
"": false,
|
||||
"/absolute.md": false,
|
||||
"../escape.md": false,
|
||||
"nested/../report.md": false,
|
||||
`nested\report.md`: false,
|
||||
"./report.md": false,
|
||||
"nested//report.md": false,
|
||||
}
|
||||
for value, wantOK := range tests {
|
||||
t.Run(strings.ReplaceAll(value, "/", "_"), func(t *testing.T) {
|
||||
_, err := cleanArchivePath(value)
|
||||
if wantOK && err != nil {
|
||||
t.Fatalf("cleanArchivePath(%q) error = %v", value, err)
|
||||
}
|
||||
if !wantOK && err == nil {
|
||||
t.Fatalf("cleanArchivePath(%q) error = nil, want error", value)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user