402 lines
11 KiB
Go
402 lines
11 KiB
Go
package ingest
|
|
|
|
import (
|
|
"archive/tar"
|
|
"bytes"
|
|
"compress/gzip"
|
|
"context"
|
|
"errors"
|
|
"io/fs"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
sourcebundle "gitea.maximumdirect.net/eric/distributor/pkg/bundle"
|
|
)
|
|
|
|
func TestStageArchiveAcceptsTar(t *testing.T) {
|
|
archive := validArchive(t, false)
|
|
staged := stageArchive(t, archive, ContentTypeTar)
|
|
|
|
if got, want := staged.Manifest.ID, "reports.ingest"; got != want {
|
|
t.Fatalf("manifest id = %q, want %q", got, want)
|
|
}
|
|
if got := readFile(t, staged.Root, "report.md"); got != "# Report\n" {
|
|
t.Fatalf("report = %q", got)
|
|
}
|
|
}
|
|
|
|
func TestStageArchiveAcceptsGzipTar(t *testing.T) {
|
|
archive := validArchive(t, true)
|
|
staged := stageArchive(t, archive, ContentTypeGzip+"; charset=binary")
|
|
|
|
if got, want := staged.Manifest.ID, "reports.ingest"; got != want {
|
|
t.Fatalf("manifest id = %q, want %q", got, want)
|
|
}
|
|
if got := readFile(t, staged.Root, "summary.txt"); got != "Summary\n" {
|
|
t.Fatalf("summary = %q", got)
|
|
}
|
|
}
|
|
|
|
func TestStageArchiveRejectsUnsupportedContentType(t *testing.T) {
|
|
err := stageArchiveError(t, validArchive(t, false), "application/zip", nil)
|
|
if !errors.Is(err, ErrUnsupportedContentType) {
|
|
t.Fatalf("StageArchive() error = %v, want ErrUnsupportedContentType", err)
|
|
}
|
|
}
|
|
|
|
func TestValidateContentType(t *testing.T) {
|
|
for _, contentType := range []string{
|
|
ContentTypeTar,
|
|
ContentTypeGzip,
|
|
ContentTypeXGzip,
|
|
ContentTypeGzip + "; charset=binary",
|
|
} {
|
|
t.Run(contentType, func(t *testing.T) {
|
|
if err := ValidateContentType(contentType); err != nil {
|
|
t.Fatalf("ValidateContentType() error = %v", err)
|
|
}
|
|
})
|
|
}
|
|
|
|
if err := ValidateContentType("application/zip"); !errors.Is(err, ErrUnsupportedContentType) {
|
|
t.Fatalf("ValidateContentType() error = %v, want ErrUnsupportedContentType", err)
|
|
}
|
|
}
|
|
|
|
func TestStageArchiveEnforcesMaxUploadSize(t *testing.T) {
|
|
archive := validArchive(t, false)
|
|
err := stageArchiveError(t, archive, ContentTypeTar, func(opts *StageOptions) {
|
|
opts.MaxUploadSize = int64(len(archive) - 1)
|
|
})
|
|
if !errors.Is(err, ErrUploadTooLarge) {
|
|
t.Fatalf("StageArchive() error = %v, want ErrUploadTooLarge", err)
|
|
}
|
|
}
|
|
|
|
func TestStageArchiveEnforcesExtractionLimits(t *testing.T) {
|
|
archive := validArchive(t, false)
|
|
tests := map[string]struct {
|
|
mutate func(*StageOptions)
|
|
wantErr error
|
|
}{
|
|
"size": {
|
|
mutate: func(opts *StageOptions) {
|
|
opts.MaxExtractedSize = 1
|
|
},
|
|
wantErr: ErrExtractedTooLarge,
|
|
},
|
|
"files": {
|
|
mutate: func(opts *StageOptions) {
|
|
opts.MaxFileCount = 1
|
|
},
|
|
wantErr: ErrTooManyFiles,
|
|
},
|
|
}
|
|
for name, tt := range tests {
|
|
t.Run(name, func(t *testing.T) {
|
|
err := stageArchiveError(t, archive, ContentTypeTar, tt.mutate)
|
|
if !errors.Is(err, tt.wantErr) {
|
|
t.Fatalf("StageArchive() error = %v, want %v", err, tt.wantErr)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestStageArchiveRejectsUnsafeEntries(t *testing.T) {
|
|
tests := map[string][]tarEntry{
|
|
"absolute path": {
|
|
fileEntry("/report.md", "report"),
|
|
},
|
|
"path traversal": {
|
|
fileEntry("../report.md", "report"),
|
|
},
|
|
"backslash path": {
|
|
fileEntry(`nested\report.md`, "report"),
|
|
},
|
|
"symlink": {
|
|
{name: "link.md", typeflag: tar.TypeSymlink, linkname: "report.md"},
|
|
},
|
|
"hardlink": {
|
|
{name: "link.md", typeflag: tar.TypeLink, linkname: "report.md"},
|
|
},
|
|
"device": {
|
|
{name: "device", typeflag: tar.TypeChar},
|
|
},
|
|
}
|
|
for name, entries := range tests {
|
|
t.Run(name, func(t *testing.T) {
|
|
err := stageArchiveError(t, makeArchive(t, false, entries...), ContentTypeTar, nil)
|
|
if err == nil {
|
|
t.Fatal("StageArchive() error = nil, want error")
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestStageArchiveRejectsBundleValidationFailures(t *testing.T) {
|
|
tests := map[string][]tarEntry{
|
|
"missing manifest": {
|
|
fileEntry("report.md", "report"),
|
|
},
|
|
"nested manifest": {
|
|
fileEntry("manifest.json", manifestJSON(t, manifestFor("reports.nested", fileSpec{path: "report.md", body: "report"}))),
|
|
fileEntry("nested/manifest.json", "{}"),
|
|
fileEntry("report.md", "report"),
|
|
},
|
|
"missing listed file": {
|
|
fileEntry("manifest.json", manifestJSON(t, manifestFor("reports.missing", fileSpec{path: "missing.md", body: "missing"}))),
|
|
},
|
|
"digest mismatch": {
|
|
fileEntry("manifest.json", manifestJSON(t, manifestFor("reports.digest", fileSpec{path: "report.md", body: "expected"}))),
|
|
fileEntry("report.md", "actual"),
|
|
},
|
|
"non regular listed file": {
|
|
fileEntry("manifest.json", manifestJSON(t, manifestFor("reports.directory", fileSpec{path: "report.md", body: "report"}))),
|
|
{name: "report.md", typeflag: tar.TypeDir},
|
|
},
|
|
}
|
|
for name, entries := range tests {
|
|
t.Run(name, func(t *testing.T) {
|
|
err := stageArchiveError(t, makeArchive(t, false, entries...), ContentTypeTar, nil)
|
|
if err == nil {
|
|
t.Fatal("StageArchive() error = nil, want error")
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestStageArchiveCleansUpFailedExtraction(t *testing.T) {
|
|
stagingPath := filepath.Join(t.TempDir(), "staging")
|
|
archive := makeArchive(t, false, fileEntry("../report.md", "report"))
|
|
_, err := StageArchive(context.Background(), StageOptions{
|
|
Body: bytes.NewReader(archive),
|
|
ContentType: ContentTypeTar,
|
|
PipelineStagingPath: stagingPath,
|
|
RunID: "reports.20260603T120000Z.abcd",
|
|
MaxUploadSize: int64(len(archive)),
|
|
MaxExtractedSize: 1024 * 1024,
|
|
MaxFileCount: 10,
|
|
})
|
|
if err == nil {
|
|
t.Fatal("StageArchive() error = nil, want error")
|
|
}
|
|
entries, err := os.ReadDir(stagingPath)
|
|
if err != nil {
|
|
t.Fatalf("ReadDir() error = %v", err)
|
|
}
|
|
if len(entries) != 0 {
|
|
t.Fatalf("staging entries = %d, want cleanup", len(entries))
|
|
}
|
|
}
|
|
|
|
func stageArchive(t *testing.T, archive []byte, contentType string) StagedBundle {
|
|
t.Helper()
|
|
staged, err := StageArchive(context.Background(), defaultStageOptions(t, archive, contentType))
|
|
if err != nil {
|
|
t.Fatalf("StageArchive() error = %v", err)
|
|
}
|
|
return staged
|
|
}
|
|
|
|
func stageArchiveError(t *testing.T, archive []byte, contentType string, mutate func(*StageOptions)) error {
|
|
t.Helper()
|
|
opts := defaultStageOptions(t, archive, contentType)
|
|
if mutate != nil {
|
|
mutate(&opts)
|
|
}
|
|
_, err := StageArchive(context.Background(), opts)
|
|
if err == nil {
|
|
t.Fatal("StageArchive() error = nil, want error")
|
|
}
|
|
return err
|
|
}
|
|
|
|
func defaultStageOptions(t *testing.T, archive []byte, contentType string) StageOptions {
|
|
t.Helper()
|
|
return StageOptions{
|
|
Body: bytes.NewReader(archive),
|
|
ContentType: contentType,
|
|
PipelineStagingPath: filepath.Join(t.TempDir(), "staging"),
|
|
RunID: "reports.20260603T120000Z.abcd",
|
|
MaxUploadSize: int64(len(archive)),
|
|
MaxExtractedSize: 1024 * 1024,
|
|
MaxFileCount: 10,
|
|
}
|
|
}
|
|
|
|
func validArchive(t *testing.T, compressed bool) []byte {
|
|
t.Helper()
|
|
root := filepath.Join(t.TempDir(), "bundle")
|
|
sourceRoot := t.TempDir()
|
|
writeFile(t, sourceRoot, "report.md", "# Report\n")
|
|
writeFile(t, sourceRoot, "summary.txt", "Summary\n")
|
|
_, err := sourcebundle.WriteBundle(sourcebundle.WriteBundleOptions{
|
|
Root: root,
|
|
ID: "reports.ingest",
|
|
Created: time.Date(2026, 6, 3, 12, 0, 0, 0, time.UTC),
|
|
Files: []sourcebundle.BundleFile{
|
|
{SourcePath: filepath.Join(sourceRoot, "report.md"), Path: "report.md"},
|
|
{SourcePath: filepath.Join(sourceRoot, "summary.txt"), Path: "summary.txt"},
|
|
},
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("WriteBundle() error = %v", err)
|
|
}
|
|
|
|
var entries []tarEntry
|
|
if err := filepath.WalkDir(root, func(filePath string, entry fs.DirEntry, err error) error {
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if entry.IsDir() {
|
|
return nil
|
|
}
|
|
relative, err := filepath.Rel(root, filePath)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
data, err := os.ReadFile(filePath)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
entries = append(entries, fileEntry(filepath.ToSlash(relative), string(data)))
|
|
return nil
|
|
}); err != nil {
|
|
t.Fatalf("walk bundle: %v", err)
|
|
}
|
|
return makeArchive(t, compressed, entries...)
|
|
}
|
|
|
|
type tarEntry struct {
|
|
name string
|
|
typeflag byte
|
|
body []byte
|
|
linkname string
|
|
}
|
|
|
|
func fileEntry(name, body string) tarEntry {
|
|
return tarEntry{name: name, typeflag: tar.TypeReg, body: []byte(body)}
|
|
}
|
|
|
|
func makeArchive(t *testing.T, compressed bool, entries ...tarEntry) []byte {
|
|
t.Helper()
|
|
var output bytes.Buffer
|
|
var writer *tar.Writer
|
|
var gzipWriter *gzip.Writer
|
|
if compressed {
|
|
gzipWriter = gzip.NewWriter(&output)
|
|
writer = tar.NewWriter(gzipWriter)
|
|
} else {
|
|
writer = tar.NewWriter(&output)
|
|
}
|
|
for _, entry := range entries {
|
|
header := &tar.Header{
|
|
Name: entry.name,
|
|
Typeflag: entry.typeflag,
|
|
Size: int64(len(entry.body)),
|
|
Mode: 0o644,
|
|
Linkname: entry.linkname,
|
|
}
|
|
if entry.typeflag == tar.TypeDir {
|
|
header.Size = 0
|
|
header.Mode = 0o755
|
|
}
|
|
if err := writer.WriteHeader(header); err != nil {
|
|
t.Fatalf("WriteHeader(%q) error = %v", entry.name, err)
|
|
}
|
|
if len(entry.body) > 0 {
|
|
if _, err := writer.Write(entry.body); err != nil {
|
|
t.Fatalf("Write(%q) error = %v", entry.name, err)
|
|
}
|
|
}
|
|
}
|
|
if err := writer.Close(); err != nil {
|
|
t.Fatalf("close tar writer: %v", err)
|
|
}
|
|
if gzipWriter != nil {
|
|
if err := gzipWriter.Close(); err != nil {
|
|
t.Fatalf("close gzip writer: %v", err)
|
|
}
|
|
}
|
|
return output.Bytes()
|
|
}
|
|
|
|
type fileSpec struct {
|
|
path string
|
|
body string
|
|
}
|
|
|
|
func manifestFor(id string, files ...fileSpec) sourcebundle.Manifest {
|
|
manifest := sourcebundle.Manifest{
|
|
SchemaVersion: sourcebundle.SchemaVersion,
|
|
ID: id,
|
|
Created: time.Date(2026, 6, 3, 12, 0, 0, 0, time.UTC),
|
|
}
|
|
for _, file := range files {
|
|
manifest.Files = append(manifest.Files, sourcebundle.ManifestFile{
|
|
Path: file.path,
|
|
SHA256: sourcebundle.FileDigest([]byte(file.body)),
|
|
Size: int64(len(file.body)),
|
|
})
|
|
}
|
|
manifest.Digest = sourcebundle.BundleDigest(manifest.Files)
|
|
return manifest
|
|
}
|
|
|
|
func manifestJSON(t *testing.T, manifest sourcebundle.Manifest) string {
|
|
t.Helper()
|
|
data, err := sourcebundle.MarshalManifest(manifest)
|
|
if err != nil {
|
|
t.Fatalf("MarshalManifest() error = %v", err)
|
|
}
|
|
return string(data)
|
|
}
|
|
|
|
func writeFile(t *testing.T, root, relative, body string) {
|
|
t.Helper()
|
|
fullPath := filepath.Join(root, filepath.FromSlash(relative))
|
|
if err := os.MkdirAll(filepath.Dir(fullPath), 0o755); err != nil {
|
|
t.Fatalf("MkdirAll() error = %v", err)
|
|
}
|
|
if err := os.WriteFile(fullPath, []byte(body), 0o644); err != nil {
|
|
t.Fatalf("WriteFile() error = %v", err)
|
|
}
|
|
}
|
|
|
|
func readFile(t *testing.T, root, relative string) string {
|
|
t.Helper()
|
|
data, err := os.ReadFile(filepath.Join(root, filepath.FromSlash(relative)))
|
|
if err != nil {
|
|
t.Fatalf("ReadFile() error = %v", err)
|
|
}
|
|
return string(data)
|
|
}
|
|
|
|
func TestCleanArchivePath(t *testing.T) {
|
|
tests := map[string]bool{
|
|
"manifest.json": true,
|
|
"nested/report.md": true,
|
|
"": false,
|
|
"/absolute.md": false,
|
|
"../escape.md": false,
|
|
"nested/../report.md": false,
|
|
`nested\report.md`: false,
|
|
"./report.md": false,
|
|
"nested//report.md": false,
|
|
}
|
|
for value, wantOK := range tests {
|
|
t.Run(strings.ReplaceAll(value, "/", "_"), func(t *testing.T) {
|
|
_, err := cleanArchivePath(value)
|
|
if wantOK && err != nil {
|
|
t.Fatalf("cleanArchivePath(%q) error = %v", value, err)
|
|
}
|
|
if !wantOK && err == nil {
|
|
t.Fatalf("cleanArchivePath(%q) error = nil, want error", value)
|
|
}
|
|
})
|
|
}
|
|
}
|