Add upload archive staging
This commit is contained in:
346
internal/ingest/archive.go
Normal file
346
internal/ingest/archive.go
Normal file
@@ -0,0 +1,346 @@
|
||||
package ingest
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"mime"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
sourcebundle "gitea.maximumdirect.net/eric/distributor/pkg/bundle"
|
||||
)
|
||||
|
||||
const (
|
||||
ContentTypeTar = "application/x-tar"
|
||||
ContentTypeGzip = "application/gzip"
|
||||
ContentTypeXGzip = "application/x-gzip"
|
||||
)
|
||||
|
||||
var (
|
||||
ErrUnsupportedContentType = errors.New("unsupported archive content type")
|
||||
ErrUploadTooLarge = errors.New("upload exceeds maximum size")
|
||||
ErrExtractedTooLarge = errors.New("extracted bundle exceeds maximum size")
|
||||
ErrTooManyFiles = errors.New("extracted bundle has too many files")
|
||||
ErrUnsafeArchivePath = errors.New("unsafe archive path")
|
||||
)
|
||||
|
||||
type StageOptions struct {
|
||||
Body io.Reader
|
||||
ContentType string
|
||||
PipelineStagingPath string
|
||||
RunID string
|
||||
MaxUploadSize int64
|
||||
MaxExtractedSize int64
|
||||
MaxFileCount int
|
||||
}
|
||||
|
||||
type StagedBundle struct {
|
||||
Root string
|
||||
Manifest sourcebundle.Manifest
|
||||
}
|
||||
|
||||
func StageArchive(ctx context.Context, opts StageOptions) (StagedBundle, error) {
|
||||
if ctx == nil {
|
||||
ctx = context.Background()
|
||||
}
|
||||
if err := validateStageOptions(opts); err != nil {
|
||||
return StagedBundle{}, err
|
||||
}
|
||||
format, err := archiveFormat(opts.ContentType)
|
||||
if err != nil {
|
||||
return StagedBundle{}, err
|
||||
}
|
||||
|
||||
if err := os.MkdirAll(opts.PipelineStagingPath, 0o755); err != nil {
|
||||
return StagedBundle{}, fmt.Errorf("create pipeline staging path: %w", err)
|
||||
}
|
||||
tempDir, err := os.MkdirTemp(opts.PipelineStagingPath, "."+opts.RunID+"-")
|
||||
if err != nil {
|
||||
return StagedBundle{}, fmt.Errorf("create staging temp dir: %w", err)
|
||||
}
|
||||
cleanupTemp := true
|
||||
defer func() {
|
||||
if cleanupTemp {
|
||||
_ = os.RemoveAll(tempDir)
|
||||
}
|
||||
}()
|
||||
|
||||
archivePath := filepath.Join(tempDir, "upload.archive")
|
||||
if err := writeLimited(ctx, archivePath, opts.Body, opts.MaxUploadSize); err != nil {
|
||||
return StagedBundle{}, err
|
||||
}
|
||||
|
||||
extractRoot := filepath.Join(tempDir, "bundle")
|
||||
if err := os.Mkdir(extractRoot, 0o755); err != nil {
|
||||
return StagedBundle{}, fmt.Errorf("create extraction root: %w", err)
|
||||
}
|
||||
if err := extractArchive(ctx, archivePath, extractRoot, format, opts.MaxExtractedSize, opts.MaxFileCount); err != nil {
|
||||
return StagedBundle{}, err
|
||||
}
|
||||
|
||||
manifest, err := sourcebundle.LoadManifest(extractRoot)
|
||||
if err != nil {
|
||||
return StagedBundle{}, err
|
||||
}
|
||||
if err := sourcebundle.ValidateBundle(extractRoot, manifest); err != nil {
|
||||
return StagedBundle{}, err
|
||||
}
|
||||
|
||||
finalRoot := filepath.Join(opts.PipelineStagingPath, opts.RunID)
|
||||
if err := os.Rename(extractRoot, finalRoot); err != nil {
|
||||
return StagedBundle{}, fmt.Errorf("commit staged bundle: %w", err)
|
||||
}
|
||||
cleanupTemp = false
|
||||
if err := os.RemoveAll(tempDir); err != nil {
|
||||
return StagedBundle{}, fmt.Errorf("remove staging temp dir: %w", err)
|
||||
}
|
||||
return StagedBundle{
|
||||
Root: finalRoot,
|
||||
Manifest: manifest,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func validateStageOptions(opts StageOptions) error {
|
||||
if opts.Body == nil {
|
||||
return fmt.Errorf("body is required")
|
||||
}
|
||||
if opts.PipelineStagingPath == "" {
|
||||
return fmt.Errorf("pipeline staging path is required")
|
||||
}
|
||||
if err := validateRunID(opts.RunID); err != nil {
|
||||
return err
|
||||
}
|
||||
if opts.MaxUploadSize <= 0 {
|
||||
return fmt.Errorf("max upload size must be greater than zero")
|
||||
}
|
||||
if opts.MaxExtractedSize <= 0 {
|
||||
return fmt.Errorf("max extracted size must be greater than zero")
|
||||
}
|
||||
if opts.MaxFileCount <= 0 {
|
||||
return fmt.Errorf("max file count must be greater than zero")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateRunID(value string) error {
|
||||
if value == "" {
|
||||
return fmt.Errorf("run id is required")
|
||||
}
|
||||
if value == "." || value == ".." || strings.ContainsAny(value, `/\`) {
|
||||
return fmt.Errorf("run id must be a single filesystem path segment")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type archiveKind int
|
||||
|
||||
const (
|
||||
archiveKindTar archiveKind = iota + 1
|
||||
archiveKindGzip
|
||||
)
|
||||
|
||||
func archiveFormat(contentType string) (archiveKind, error) {
|
||||
mediaType, _, err := mime.ParseMediaType(contentType)
|
||||
if err != nil {
|
||||
mediaType = contentType
|
||||
}
|
||||
switch mediaType {
|
||||
case ContentTypeTar:
|
||||
return archiveKindTar, nil
|
||||
case ContentTypeGzip, ContentTypeXGzip:
|
||||
return archiveKindGzip, nil
|
||||
default:
|
||||
return 0, ErrUnsupportedContentType
|
||||
}
|
||||
}
|
||||
|
||||
func writeLimited(ctx context.Context, destination string, body io.Reader, maxSize int64) error {
|
||||
file, err := os.OpenFile(destination, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600)
|
||||
if err != nil {
|
||||
return fmt.Errorf("create upload archive: %w", err)
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
limited := &limitedReader{ctx: ctx, reader: body, limit: maxSize}
|
||||
if _, err := io.Copy(file, limited); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := file.Close(); err != nil {
|
||||
return fmt.Errorf("write upload archive: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type limitedReader struct {
|
||||
ctx context.Context
|
||||
reader io.Reader
|
||||
limit int64
|
||||
read int64
|
||||
}
|
||||
|
||||
func (r *limitedReader) Read(data []byte) (int, error) {
|
||||
if err := r.ctx.Err(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if r.read == r.limit {
|
||||
var probe [1]byte
|
||||
n, err := r.reader.Read(probe[:])
|
||||
if n > 0 {
|
||||
return 0, ErrUploadTooLarge
|
||||
}
|
||||
return 0, err
|
||||
}
|
||||
remaining := r.limit - r.read
|
||||
if int64(len(data)) > remaining+1 {
|
||||
data = data[:remaining+1]
|
||||
}
|
||||
n, err := r.reader.Read(data)
|
||||
if r.read+int64(n) > r.limit {
|
||||
allowed := int(r.limit - r.read)
|
||||
r.read = r.limit
|
||||
return allowed, ErrUploadTooLarge
|
||||
}
|
||||
r.read += int64(n)
|
||||
return n, err
|
||||
}
|
||||
|
||||
func extractArchive(ctx context.Context, archivePath, destination string, format archiveKind, maxExtractedSize int64, maxFileCount int) error {
|
||||
file, err := os.Open(archivePath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("open upload archive: %w", err)
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
var reader io.Reader = file
|
||||
var gzipReader *gzip.Reader
|
||||
if format == archiveKindGzip {
|
||||
gzipReader, err = gzip.NewReader(file)
|
||||
if err != nil {
|
||||
return fmt.Errorf("open gzip archive: %w", err)
|
||||
}
|
||||
defer gzipReader.Close()
|
||||
reader = gzipReader
|
||||
}
|
||||
|
||||
extractor := archiveExtractor{
|
||||
ctx: ctx,
|
||||
destination: destination,
|
||||
maxExtractedSize: maxExtractedSize,
|
||||
maxFileCount: maxFileCount,
|
||||
}
|
||||
if err := extractor.extract(tar.NewReader(reader)); err != nil {
|
||||
return err
|
||||
}
|
||||
if extractor.rootManifestCount != 1 {
|
||||
return fmt.Errorf("archive must contain exactly one root-level manifest.json")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type archiveExtractor struct {
|
||||
ctx context.Context
|
||||
destination string
|
||||
maxExtractedSize int64
|
||||
maxFileCount int
|
||||
extractedSize int64
|
||||
fileCount int
|
||||
rootManifestCount int
|
||||
seenFiles map[string]struct{}
|
||||
}
|
||||
|
||||
func (e *archiveExtractor) extract(reader *tar.Reader) error {
|
||||
e.seenFiles = make(map[string]struct{})
|
||||
for {
|
||||
if err := e.ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
header, err := reader.Next()
|
||||
if errors.Is(err, io.EOF) {
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("read tar archive: %w", err)
|
||||
}
|
||||
name, err := cleanArchivePath(header.Name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if path.Base(name) == sourcebundle.ManifestName {
|
||||
if name != sourcebundle.ManifestName {
|
||||
return fmt.Errorf("nested manifest %q is not allowed", name)
|
||||
}
|
||||
e.rootManifestCount++
|
||||
}
|
||||
|
||||
switch header.Typeflag {
|
||||
case tar.TypeDir:
|
||||
if err := os.MkdirAll(filepath.Join(e.destination, filepath.FromSlash(name)), 0o755); err != nil {
|
||||
return fmt.Errorf("create archive directory %q: %w", name, err)
|
||||
}
|
||||
case tar.TypeReg, tar.TypeRegA:
|
||||
if err := e.extractFile(reader, name, header.Size); err != nil {
|
||||
return err
|
||||
}
|
||||
default:
|
||||
return fmt.Errorf("archive entry %q has unsupported type %c", name, header.Typeflag)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (e *archiveExtractor) extractFile(reader *tar.Reader, name string, size int64) error {
|
||||
if size < 0 {
|
||||
return fmt.Errorf("archive entry %q has invalid size", name)
|
||||
}
|
||||
e.fileCount++
|
||||
if e.fileCount > e.maxFileCount {
|
||||
return ErrTooManyFiles
|
||||
}
|
||||
e.extractedSize += size
|
||||
if e.extractedSize > e.maxExtractedSize {
|
||||
return ErrExtractedTooLarge
|
||||
}
|
||||
if _, exists := e.seenFiles[name]; exists {
|
||||
return fmt.Errorf("archive entry %q is duplicated", name)
|
||||
}
|
||||
e.seenFiles[name] = struct{}{}
|
||||
|
||||
fullPath := filepath.Join(e.destination, filepath.FromSlash(name))
|
||||
if err := os.MkdirAll(filepath.Dir(fullPath), 0o755); err != nil {
|
||||
return fmt.Errorf("create archive parent for %q: %w", name, err)
|
||||
}
|
||||
file, err := os.OpenFile(fullPath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o644)
|
||||
if err != nil {
|
||||
return fmt.Errorf("create archive file %q: %w", name, err)
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
if _, err := io.CopyN(file, reader, size); err != nil {
|
||||
return fmt.Errorf("extract archive file %q: %w", name, err)
|
||||
}
|
||||
if err := file.Close(); err != nil {
|
||||
return fmt.Errorf("extract archive file %q: %w", name, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func cleanArchivePath(value string) (string, error) {
|
||||
if value == "" || strings.Contains(value, `\`) || strings.HasPrefix(value, "/") {
|
||||
return "", ErrUnsafeArchivePath
|
||||
}
|
||||
cleaned := path.Clean(value)
|
||||
if cleaned != value {
|
||||
return "", ErrUnsafeArchivePath
|
||||
}
|
||||
for _, segment := range strings.Split(value, "/") {
|
||||
if segment == "" || segment == "." || segment == ".." {
|
||||
return "", ErrUnsafeArchivePath
|
||||
}
|
||||
}
|
||||
return value, nil
|
||||
}
|
||||
382
internal/ingest/archive_test.go
Normal file
382
internal/ingest/archive_test.go
Normal file
@@ -0,0 +1,382 @@
|
||||
package ingest
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"errors"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
sourcebundle "gitea.maximumdirect.net/eric/distributor/pkg/bundle"
|
||||
)
|
||||
|
||||
func TestStageArchiveAcceptsTar(t *testing.T) {
|
||||
archive := validArchive(t, false)
|
||||
staged := stageArchive(t, archive, ContentTypeTar)
|
||||
|
||||
if got, want := staged.Manifest.ID, "reports.ingest"; got != want {
|
||||
t.Fatalf("manifest id = %q, want %q", got, want)
|
||||
}
|
||||
if got := readFile(t, staged.Root, "report.md"); got != "# Report\n" {
|
||||
t.Fatalf("report = %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStageArchiveAcceptsGzipTar(t *testing.T) {
|
||||
archive := validArchive(t, true)
|
||||
staged := stageArchive(t, archive, ContentTypeGzip+"; charset=binary")
|
||||
|
||||
if got, want := staged.Manifest.ID, "reports.ingest"; got != want {
|
||||
t.Fatalf("manifest id = %q, want %q", got, want)
|
||||
}
|
||||
if got := readFile(t, staged.Root, "summary.txt"); got != "Summary\n" {
|
||||
t.Fatalf("summary = %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStageArchiveRejectsUnsupportedContentType(t *testing.T) {
|
||||
err := stageArchiveError(t, validArchive(t, false), "application/zip", nil)
|
||||
if !errors.Is(err, ErrUnsupportedContentType) {
|
||||
t.Fatalf("StageArchive() error = %v, want ErrUnsupportedContentType", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStageArchiveEnforcesMaxUploadSize(t *testing.T) {
|
||||
archive := validArchive(t, false)
|
||||
err := stageArchiveError(t, archive, ContentTypeTar, func(opts *StageOptions) {
|
||||
opts.MaxUploadSize = int64(len(archive) - 1)
|
||||
})
|
||||
if !errors.Is(err, ErrUploadTooLarge) {
|
||||
t.Fatalf("StageArchive() error = %v, want ErrUploadTooLarge", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStageArchiveEnforcesExtractionLimits(t *testing.T) {
|
||||
archive := validArchive(t, false)
|
||||
tests := map[string]struct {
|
||||
mutate func(*StageOptions)
|
||||
wantErr error
|
||||
}{
|
||||
"size": {
|
||||
mutate: func(opts *StageOptions) {
|
||||
opts.MaxExtractedSize = 1
|
||||
},
|
||||
wantErr: ErrExtractedTooLarge,
|
||||
},
|
||||
"files": {
|
||||
mutate: func(opts *StageOptions) {
|
||||
opts.MaxFileCount = 1
|
||||
},
|
||||
wantErr: ErrTooManyFiles,
|
||||
},
|
||||
}
|
||||
for name, tt := range tests {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
err := stageArchiveError(t, archive, ContentTypeTar, tt.mutate)
|
||||
if !errors.Is(err, tt.wantErr) {
|
||||
t.Fatalf("StageArchive() error = %v, want %v", err, tt.wantErr)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestStageArchiveRejectsUnsafeEntries(t *testing.T) {
|
||||
tests := map[string][]tarEntry{
|
||||
"absolute path": {
|
||||
fileEntry("/report.md", "report"),
|
||||
},
|
||||
"path traversal": {
|
||||
fileEntry("../report.md", "report"),
|
||||
},
|
||||
"backslash path": {
|
||||
fileEntry(`nested\report.md`, "report"),
|
||||
},
|
||||
"symlink": {
|
||||
{name: "link.md", typeflag: tar.TypeSymlink, linkname: "report.md"},
|
||||
},
|
||||
"hardlink": {
|
||||
{name: "link.md", typeflag: tar.TypeLink, linkname: "report.md"},
|
||||
},
|
||||
"device": {
|
||||
{name: "device", typeflag: tar.TypeChar},
|
||||
},
|
||||
}
|
||||
for name, entries := range tests {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
err := stageArchiveError(t, makeArchive(t, false, entries...), ContentTypeTar, nil)
|
||||
if err == nil {
|
||||
t.Fatal("StageArchive() error = nil, want error")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestStageArchiveRejectsBundleValidationFailures(t *testing.T) {
|
||||
tests := map[string][]tarEntry{
|
||||
"missing manifest": {
|
||||
fileEntry("report.md", "report"),
|
||||
},
|
||||
"nested manifest": {
|
||||
fileEntry("manifest.json", manifestJSON(t, manifestFor("reports.nested", fileSpec{path: "report.md", body: "report"}))),
|
||||
fileEntry("nested/manifest.json", "{}"),
|
||||
fileEntry("report.md", "report"),
|
||||
},
|
||||
"missing listed file": {
|
||||
fileEntry("manifest.json", manifestJSON(t, manifestFor("reports.missing", fileSpec{path: "missing.md", body: "missing"}))),
|
||||
},
|
||||
"digest mismatch": {
|
||||
fileEntry("manifest.json", manifestJSON(t, manifestFor("reports.digest", fileSpec{path: "report.md", body: "expected"}))),
|
||||
fileEntry("report.md", "actual"),
|
||||
},
|
||||
"non regular listed file": {
|
||||
fileEntry("manifest.json", manifestJSON(t, manifestFor("reports.directory", fileSpec{path: "report.md", body: "report"}))),
|
||||
{name: "report.md", typeflag: tar.TypeDir},
|
||||
},
|
||||
}
|
||||
for name, entries := range tests {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
err := stageArchiveError(t, makeArchive(t, false, entries...), ContentTypeTar, nil)
|
||||
if err == nil {
|
||||
t.Fatal("StageArchive() error = nil, want error")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestStageArchiveCleansUpFailedExtraction(t *testing.T) {
|
||||
stagingPath := filepath.Join(t.TempDir(), "staging")
|
||||
archive := makeArchive(t, false, fileEntry("../report.md", "report"))
|
||||
_, err := StageArchive(context.Background(), StageOptions{
|
||||
Body: bytes.NewReader(archive),
|
||||
ContentType: ContentTypeTar,
|
||||
PipelineStagingPath: stagingPath,
|
||||
RunID: "reports.20260603T120000Z.abcd",
|
||||
MaxUploadSize: int64(len(archive)),
|
||||
MaxExtractedSize: 1024 * 1024,
|
||||
MaxFileCount: 10,
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("StageArchive() error = nil, want error")
|
||||
}
|
||||
entries, err := os.ReadDir(stagingPath)
|
||||
if err != nil {
|
||||
t.Fatalf("ReadDir() error = %v", err)
|
||||
}
|
||||
if len(entries) != 0 {
|
||||
t.Fatalf("staging entries = %d, want cleanup", len(entries))
|
||||
}
|
||||
}
|
||||
|
||||
func stageArchive(t *testing.T, archive []byte, contentType string) StagedBundle {
|
||||
t.Helper()
|
||||
staged, err := StageArchive(context.Background(), defaultStageOptions(t, archive, contentType))
|
||||
if err != nil {
|
||||
t.Fatalf("StageArchive() error = %v", err)
|
||||
}
|
||||
return staged
|
||||
}
|
||||
|
||||
func stageArchiveError(t *testing.T, archive []byte, contentType string, mutate func(*StageOptions)) error {
|
||||
t.Helper()
|
||||
opts := defaultStageOptions(t, archive, contentType)
|
||||
if mutate != nil {
|
||||
mutate(&opts)
|
||||
}
|
||||
_, err := StageArchive(context.Background(), opts)
|
||||
if err == nil {
|
||||
t.Fatal("StageArchive() error = nil, want error")
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func defaultStageOptions(t *testing.T, archive []byte, contentType string) StageOptions {
|
||||
t.Helper()
|
||||
return StageOptions{
|
||||
Body: bytes.NewReader(archive),
|
||||
ContentType: contentType,
|
||||
PipelineStagingPath: filepath.Join(t.TempDir(), "staging"),
|
||||
RunID: "reports.20260603T120000Z.abcd",
|
||||
MaxUploadSize: int64(len(archive)),
|
||||
MaxExtractedSize: 1024 * 1024,
|
||||
MaxFileCount: 10,
|
||||
}
|
||||
}
|
||||
|
||||
func validArchive(t *testing.T, compressed bool) []byte {
|
||||
t.Helper()
|
||||
root := filepath.Join(t.TempDir(), "bundle")
|
||||
sourceRoot := t.TempDir()
|
||||
writeFile(t, sourceRoot, "report.md", "# Report\n")
|
||||
writeFile(t, sourceRoot, "summary.txt", "Summary\n")
|
||||
_, err := sourcebundle.WriteBundle(sourcebundle.WriteBundleOptions{
|
||||
Root: root,
|
||||
ID: "reports.ingest",
|
||||
Created: time.Date(2026, 6, 3, 12, 0, 0, 0, time.UTC),
|
||||
Files: []sourcebundle.BundleFile{
|
||||
{SourcePath: filepath.Join(sourceRoot, "report.md"), Path: "report.md"},
|
||||
{SourcePath: filepath.Join(sourceRoot, "summary.txt"), Path: "summary.txt"},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("WriteBundle() error = %v", err)
|
||||
}
|
||||
|
||||
var entries []tarEntry
|
||||
if err := filepath.WalkDir(root, func(filePath string, entry fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if entry.IsDir() {
|
||||
return nil
|
||||
}
|
||||
relative, err := filepath.Rel(root, filePath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
data, err := os.ReadFile(filePath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
entries = append(entries, fileEntry(filepath.ToSlash(relative), string(data)))
|
||||
return nil
|
||||
}); err != nil {
|
||||
t.Fatalf("walk bundle: %v", err)
|
||||
}
|
||||
return makeArchive(t, compressed, entries...)
|
||||
}
|
||||
|
||||
type tarEntry struct {
|
||||
name string
|
||||
typeflag byte
|
||||
body []byte
|
||||
linkname string
|
||||
}
|
||||
|
||||
func fileEntry(name, body string) tarEntry {
|
||||
return tarEntry{name: name, typeflag: tar.TypeReg, body: []byte(body)}
|
||||
}
|
||||
|
||||
func makeArchive(t *testing.T, compressed bool, entries ...tarEntry) []byte {
|
||||
t.Helper()
|
||||
var output bytes.Buffer
|
||||
var writer *tar.Writer
|
||||
var gzipWriter *gzip.Writer
|
||||
if compressed {
|
||||
gzipWriter = gzip.NewWriter(&output)
|
||||
writer = tar.NewWriter(gzipWriter)
|
||||
} else {
|
||||
writer = tar.NewWriter(&output)
|
||||
}
|
||||
for _, entry := range entries {
|
||||
header := &tar.Header{
|
||||
Name: entry.name,
|
||||
Typeflag: entry.typeflag,
|
||||
Size: int64(len(entry.body)),
|
||||
Mode: 0o644,
|
||||
Linkname: entry.linkname,
|
||||
}
|
||||
if entry.typeflag == tar.TypeDir {
|
||||
header.Size = 0
|
||||
header.Mode = 0o755
|
||||
}
|
||||
if err := writer.WriteHeader(header); err != nil {
|
||||
t.Fatalf("WriteHeader(%q) error = %v", entry.name, err)
|
||||
}
|
||||
if len(entry.body) > 0 {
|
||||
if _, err := writer.Write(entry.body); err != nil {
|
||||
t.Fatalf("Write(%q) error = %v", entry.name, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
if err := writer.Close(); err != nil {
|
||||
t.Fatalf("close tar writer: %v", err)
|
||||
}
|
||||
if gzipWriter != nil {
|
||||
if err := gzipWriter.Close(); err != nil {
|
||||
t.Fatalf("close gzip writer: %v", err)
|
||||
}
|
||||
}
|
||||
return output.Bytes()
|
||||
}
|
||||
|
||||
type fileSpec struct {
|
||||
path string
|
||||
body string
|
||||
}
|
||||
|
||||
func manifestFor(id string, files ...fileSpec) sourcebundle.Manifest {
|
||||
manifest := sourcebundle.Manifest{
|
||||
SchemaVersion: sourcebundle.SchemaVersion,
|
||||
ID: id,
|
||||
Created: time.Date(2026, 6, 3, 12, 0, 0, 0, time.UTC),
|
||||
}
|
||||
for _, file := range files {
|
||||
manifest.Files = append(manifest.Files, sourcebundle.ManifestFile{
|
||||
Path: file.path,
|
||||
SHA256: sourcebundle.FileDigest([]byte(file.body)),
|
||||
Size: int64(len(file.body)),
|
||||
})
|
||||
}
|
||||
manifest.Digest = sourcebundle.BundleDigest(manifest.Files)
|
||||
return manifest
|
||||
}
|
||||
|
||||
func manifestJSON(t *testing.T, manifest sourcebundle.Manifest) string {
|
||||
t.Helper()
|
||||
data, err := sourcebundle.MarshalManifest(manifest)
|
||||
if err != nil {
|
||||
t.Fatalf("MarshalManifest() error = %v", err)
|
||||
}
|
||||
return string(data)
|
||||
}
|
||||
|
||||
func writeFile(t *testing.T, root, relative, body string) {
|
||||
t.Helper()
|
||||
fullPath := filepath.Join(root, filepath.FromSlash(relative))
|
||||
if err := os.MkdirAll(filepath.Dir(fullPath), 0o755); err != nil {
|
||||
t.Fatalf("MkdirAll() error = %v", err)
|
||||
}
|
||||
if err := os.WriteFile(fullPath, []byte(body), 0o644); err != nil {
|
||||
t.Fatalf("WriteFile() error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func readFile(t *testing.T, root, relative string) string {
|
||||
t.Helper()
|
||||
data, err := os.ReadFile(filepath.Join(root, filepath.FromSlash(relative)))
|
||||
if err != nil {
|
||||
t.Fatalf("ReadFile() error = %v", err)
|
||||
}
|
||||
return string(data)
|
||||
}
|
||||
|
||||
func TestCleanArchivePath(t *testing.T) {
|
||||
tests := map[string]bool{
|
||||
"manifest.json": true,
|
||||
"nested/report.md": true,
|
||||
"": false,
|
||||
"/absolute.md": false,
|
||||
"../escape.md": false,
|
||||
"nested/../report.md": false,
|
||||
`nested\report.md`: false,
|
||||
"./report.md": false,
|
||||
"nested//report.md": false,
|
||||
}
|
||||
for value, wantOK := range tests {
|
||||
t.Run(strings.ReplaceAll(value, "/", "_"), func(t *testing.T) {
|
||||
_, err := cleanArchivePath(value)
|
||||
if wantOK && err != nil {
|
||||
t.Fatalf("cleanArchivePath(%q) error = %v", value, err)
|
||||
}
|
||||
if !wantOK && err == nil {
|
||||
t.Fatalf("cleanArchivePath(%q) error = nil, want error", value)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user