package ingest import ( "archive/tar" "bytes" "compress/gzip" "context" "errors" "io/fs" "os" "path/filepath" "strings" "testing" "time" sourcebundle "gitea.maximumdirect.net/eric/distributor/pkg/bundle" ) func TestStageArchiveAcceptsTar(t *testing.T) { archive := validArchive(t, false) staged := stageArchive(t, archive, ContentTypeTar) if got, want := staged.Manifest.ID, "reports.ingest"; got != want { t.Fatalf("manifest id = %q, want %q", got, want) } if got := readFile(t, staged.Root, "report.md"); got != "# Report\n" { t.Fatalf("report = %q", got) } } func TestStageArchiveAcceptsGzipTar(t *testing.T) { archive := validArchive(t, true) staged := stageArchive(t, archive, ContentTypeGzip+"; charset=binary") if got, want := staged.Manifest.ID, "reports.ingest"; got != want { t.Fatalf("manifest id = %q, want %q", got, want) } if got := readFile(t, staged.Root, "summary.txt"); got != "Summary\n" { t.Fatalf("summary = %q", got) } } func TestStageArchiveRejectsUnsupportedContentType(t *testing.T) { err := stageArchiveError(t, validArchive(t, false), "application/zip", nil) if !errors.Is(err, ErrUnsupportedContentType) { t.Fatalf("StageArchive() error = %v, want ErrUnsupportedContentType", err) } } func TestValidateContentType(t *testing.T) { for _, contentType := range []string{ ContentTypeTar, ContentTypeGzip, ContentTypeXGzip, ContentTypeGzip + "; charset=binary", } { t.Run(contentType, func(t *testing.T) { if err := ValidateContentType(contentType); err != nil { t.Fatalf("ValidateContentType() error = %v", err) } }) } if err := ValidateContentType("application/zip"); !errors.Is(err, ErrUnsupportedContentType) { t.Fatalf("ValidateContentType() error = %v, want ErrUnsupportedContentType", err) } } func TestStageArchiveEnforcesMaxUploadSize(t *testing.T) { archive := validArchive(t, false) err := stageArchiveError(t, archive, ContentTypeTar, func(opts *StageOptions) { opts.MaxUploadSize = int64(len(archive) - 1) }) if !errors.Is(err, ErrUploadTooLarge) { t.Fatalf("StageArchive() error = %v, want ErrUploadTooLarge", err) } } func TestStageArchiveEnforcesExtractionLimits(t *testing.T) { archive := validArchive(t, false) tests := map[string]struct { mutate func(*StageOptions) wantErr error }{ "size": { mutate: func(opts *StageOptions) { opts.MaxExtractedSize = 1 }, wantErr: ErrExtractedTooLarge, }, "files": { mutate: func(opts *StageOptions) { opts.MaxFileCount = 1 }, wantErr: ErrTooManyFiles, }, } for name, tt := range tests { t.Run(name, func(t *testing.T) { err := stageArchiveError(t, archive, ContentTypeTar, tt.mutate) if !errors.Is(err, tt.wantErr) { t.Fatalf("StageArchive() error = %v, want %v", err, tt.wantErr) } }) } } func TestStageArchiveRejectsUnsafeEntries(t *testing.T) { tests := map[string][]tarEntry{ "absolute path": { fileEntry("/report.md", "report"), }, "path traversal": { fileEntry("../report.md", "report"), }, "backslash path": { fileEntry(`nested\report.md`, "report"), }, "symlink": { {name: "link.md", typeflag: tar.TypeSymlink, linkname: "report.md"}, }, "hardlink": { {name: "link.md", typeflag: tar.TypeLink, linkname: "report.md"}, }, "device": { {name: "device", typeflag: tar.TypeChar}, }, } for name, entries := range tests { t.Run(name, func(t *testing.T) { err := stageArchiveError(t, makeArchive(t, false, entries...), ContentTypeTar, nil) if err == nil { t.Fatal("StageArchive() error = nil, want error") } }) } } func TestStageArchiveRejectsBundleValidationFailures(t *testing.T) { tests := map[string][]tarEntry{ "missing manifest": { fileEntry("report.md", "report"), }, "nested manifest": { fileEntry("manifest.json", manifestJSON(t, manifestFor("reports.nested", fileSpec{path: "report.md", body: "report"}))), fileEntry("nested/manifest.json", "{}"), fileEntry("report.md", "report"), }, "missing listed file": { fileEntry("manifest.json", manifestJSON(t, manifestFor("reports.missing", fileSpec{path: "missing.md", body: "missing"}))), }, "digest mismatch": { fileEntry("manifest.json", manifestJSON(t, manifestFor("reports.digest", fileSpec{path: "report.md", body: "expected"}))), fileEntry("report.md", "actual"), }, "non regular listed file": { fileEntry("manifest.json", manifestJSON(t, manifestFor("reports.directory", fileSpec{path: "report.md", body: "report"}))), {name: "report.md", typeflag: tar.TypeDir}, }, } for name, entries := range tests { t.Run(name, func(t *testing.T) { err := stageArchiveError(t, makeArchive(t, false, entries...), ContentTypeTar, nil) if err == nil { t.Fatal("StageArchive() error = nil, want error") } }) } } func TestStageArchiveCleansUpFailedExtraction(t *testing.T) { stagingPath := filepath.Join(t.TempDir(), "staging") archive := makeArchive(t, false, fileEntry("../report.md", "report")) _, err := StageArchive(context.Background(), StageOptions{ Body: bytes.NewReader(archive), ContentType: ContentTypeTar, PipelineStagingPath: stagingPath, RunID: "reports.20260603T120000Z.abcd", MaxUploadSize: int64(len(archive)), MaxExtractedSize: 1024 * 1024, MaxFileCount: 10, }) if err == nil { t.Fatal("StageArchive() error = nil, want error") } entries, err := os.ReadDir(stagingPath) if err != nil { t.Fatalf("ReadDir() error = %v", err) } if len(entries) != 0 { t.Fatalf("staging entries = %d, want cleanup", len(entries)) } } func stageArchive(t *testing.T, archive []byte, contentType string) StagedBundle { t.Helper() staged, err := StageArchive(context.Background(), defaultStageOptions(t, archive, contentType)) if err != nil { t.Fatalf("StageArchive() error = %v", err) } return staged } func stageArchiveError(t *testing.T, archive []byte, contentType string, mutate func(*StageOptions)) error { t.Helper() opts := defaultStageOptions(t, archive, contentType) if mutate != nil { mutate(&opts) } _, err := StageArchive(context.Background(), opts) if err == nil { t.Fatal("StageArchive() error = nil, want error") } return err } func defaultStageOptions(t *testing.T, archive []byte, contentType string) StageOptions { t.Helper() return StageOptions{ Body: bytes.NewReader(archive), ContentType: contentType, PipelineStagingPath: filepath.Join(t.TempDir(), "staging"), RunID: "reports.20260603T120000Z.abcd", MaxUploadSize: int64(len(archive)), MaxExtractedSize: 1024 * 1024, MaxFileCount: 10, } } func validArchive(t *testing.T, compressed bool) []byte { t.Helper() root := filepath.Join(t.TempDir(), "bundle") sourceRoot := t.TempDir() writeFile(t, sourceRoot, "report.md", "# Report\n") writeFile(t, sourceRoot, "summary.txt", "Summary\n") _, err := sourcebundle.WriteBundle(sourcebundle.WriteBundleOptions{ Root: root, ID: "reports.ingest", Created: time.Date(2026, 6, 3, 12, 0, 0, 0, time.UTC), Files: []sourcebundle.BundleFile{ {SourcePath: filepath.Join(sourceRoot, "report.md"), Path: "report.md"}, {SourcePath: filepath.Join(sourceRoot, "summary.txt"), Path: "summary.txt"}, }, }) if err != nil { t.Fatalf("WriteBundle() error = %v", err) } var entries []tarEntry if err := filepath.WalkDir(root, func(filePath string, entry fs.DirEntry, err error) error { if err != nil { return err } if entry.IsDir() { return nil } relative, err := filepath.Rel(root, filePath) if err != nil { return err } data, err := os.ReadFile(filePath) if err != nil { return err } entries = append(entries, fileEntry(filepath.ToSlash(relative), string(data))) return nil }); err != nil { t.Fatalf("walk bundle: %v", err) } return makeArchive(t, compressed, entries...) } type tarEntry struct { name string typeflag byte body []byte linkname string } func fileEntry(name, body string) tarEntry { return tarEntry{name: name, typeflag: tar.TypeReg, body: []byte(body)} } func makeArchive(t *testing.T, compressed bool, entries ...tarEntry) []byte { t.Helper() var output bytes.Buffer var writer *tar.Writer var gzipWriter *gzip.Writer if compressed { gzipWriter = gzip.NewWriter(&output) writer = tar.NewWriter(gzipWriter) } else { writer = tar.NewWriter(&output) } for _, entry := range entries { header := &tar.Header{ Name: entry.name, Typeflag: entry.typeflag, Size: int64(len(entry.body)), Mode: 0o644, Linkname: entry.linkname, } if entry.typeflag == tar.TypeDir { header.Size = 0 header.Mode = 0o755 } if err := writer.WriteHeader(header); err != nil { t.Fatalf("WriteHeader(%q) error = %v", entry.name, err) } if len(entry.body) > 0 { if _, err := writer.Write(entry.body); err != nil { t.Fatalf("Write(%q) error = %v", entry.name, err) } } } if err := writer.Close(); err != nil { t.Fatalf("close tar writer: %v", err) } if gzipWriter != nil { if err := gzipWriter.Close(); err != nil { t.Fatalf("close gzip writer: %v", err) } } return output.Bytes() } type fileSpec struct { path string body string } func manifestFor(id string, files ...fileSpec) sourcebundle.Manifest { manifest := sourcebundle.Manifest{ SchemaVersion: sourcebundle.SchemaVersion, ID: id, Created: time.Date(2026, 6, 3, 12, 0, 0, 0, time.UTC), } for _, file := range files { manifest.Files = append(manifest.Files, sourcebundle.ManifestFile{ Path: file.path, SHA256: sourcebundle.FileDigest([]byte(file.body)), Size: int64(len(file.body)), }) } manifest.Digest = sourcebundle.BundleDigest(manifest.Files) return manifest } func manifestJSON(t *testing.T, manifest sourcebundle.Manifest) string { t.Helper() data, err := sourcebundle.MarshalManifest(manifest) if err != nil { t.Fatalf("MarshalManifest() error = %v", err) } return string(data) } func writeFile(t *testing.T, root, relative, body string) { t.Helper() fullPath := filepath.Join(root, filepath.FromSlash(relative)) if err := os.MkdirAll(filepath.Dir(fullPath), 0o755); err != nil { t.Fatalf("MkdirAll() error = %v", err) } if err := os.WriteFile(fullPath, []byte(body), 0o644); err != nil { t.Fatalf("WriteFile() error = %v", err) } } func readFile(t *testing.T, root, relative string) string { t.Helper() data, err := os.ReadFile(filepath.Join(root, filepath.FromSlash(relative))) if err != nil { t.Fatalf("ReadFile() error = %v", err) } return string(data) } func TestCleanArchivePath(t *testing.T) { tests := map[string]bool{ "manifest.json": true, "nested/report.md": true, "": false, "/absolute.md": false, "../escape.md": false, "nested/../report.md": false, `nested\report.md`: false, "./report.md": false, "nested//report.md": false, } for value, wantOK := range tests { t.Run(strings.ReplaceAll(value, "/", "_"), func(t *testing.T) { _, err := cleanArchivePath(value) if wantOK && err != nil { t.Fatalf("cleanArchivePath(%q) error = %v", value, err) } if !wantOK && err == nil { t.Fatalf("cleanArchivePath(%q) error = nil, want error", value) } }) } }