Add source bundle validation and inspection

This commit is contained in:
2026-05-31 02:07:30 +00:00
parent 3c2f36a6e5
commit 518944e601
25 changed files with 931 additions and 17 deletions

49
internal/bundle/digest.go Normal file
View File

@@ -0,0 +1,49 @@
package bundle
import (
"crypto/sha256"
"encoding/hex"
"fmt"
"regexp"
"strconv"
"strings"
)
var digestPattern = regexp.MustCompile(`^sha256:[0-9a-f]{64}$`)
func ValidateDigest(value string) error {
if !digestPattern.MatchString(value) {
return fmt.Errorf("must be lowercase sha256:<64 hex>")
}
return nil
}
func FileDigest(data []byte) string {
sum := sha256.Sum256(data)
return "sha256:" + hex.EncodeToString(sum[:])
}
func BundleDigest(files []ManifestFile) string {
canonical := CanonicalFilePayload(files)
sum := sha256.Sum256([]byte(canonical))
return "sha256:" + hex.EncodeToString(sum[:])
}
func CanonicalFilePayload(files []ManifestFile) string {
var builder strings.Builder
builder.WriteByte('[')
for index, file := range files {
if index > 0 {
builder.WriteByte(',')
}
builder.WriteString(`{"path":`)
builder.WriteString(strconv.Quote(file.Path))
builder.WriteString(`,"sha256":`)
builder.WriteString(strconv.Quote(file.SHA256))
builder.WriteString(`,"size":`)
builder.WriteString(strconv.FormatInt(file.Size, 10))
builder.WriteByte('}')
}
builder.WriteByte(']')
return builder.String()
}

View File

@@ -0,0 +1,21 @@
package bundle
import (
"strings"
"testing"
)
func TestCanonicalBundleDigestReferenceFixture(t *testing.T) {
manifest, err := ParseManifest(readFixture(t, "testdata/valid_bundle/manifest.json"))
if err != nil {
t.Fatalf("ParseManifest() error = %v", err)
}
gotPayload := CanonicalFilePayload(manifest.Files)
wantPayload := strings.TrimSpace(string(readFixture(t, "testdata/canonical_payload.json")))
if gotPayload != wantPayload {
t.Fatalf("canonical payload = %q, want %q", gotPayload, wantPayload)
}
if got, want := BundleDigest(manifest.Files), manifest.Digest; got != want {
t.Fatalf("BundleDigest() = %q, want %q", got, want)
}
}

View File

@@ -0,0 +1,82 @@
package bundle
import (
"context"
"fmt"
"path"
"sort"
"strings"
"gitea.maximumdirect.net/eric/distributor/internal/storage"
)
func Discover(ctx context.Context, backend storage.Backend, sourceRoot string) ([]Bundle, error) {
if err := storage.ValidatePrefix(sourceRoot); err != nil {
return nil, err
}
entries, err := storage.List(ctx, backend, sourceRoot, storage.WalkOptions{Recursive: true})
if err != nil {
return nil, err
}
var roots []string
for _, entry := range entries {
if entry.Type == storage.EntryTypeDirectory {
continue
}
if path.Base(entry.Path) == ManifestName {
roots = append(roots, path.Dir(entry.Path))
}
}
for index, root := range roots {
if root == "." {
roots[index] = ""
}
}
sort.Strings(roots)
if len(roots) == 0 {
return nil, fmt.Errorf("no bundles found under %q", displayRoot(sourceRoot))
}
if err := rejectNestedRoots(roots); err != nil {
return nil, err
}
bundles := make([]Bundle, 0, len(roots))
for _, root := range roots {
relativeRoot := relativeToSource(sourceRoot, root)
sourceBundle, err := validateAt(ctx, backend, root, relativeRoot)
if err != nil {
return nil, err
}
bundles = append(bundles, sourceBundle)
}
return bundles, nil
}
func rejectNestedRoots(roots []string) error {
for index, root := range roots {
for _, candidate := range roots[index+1:] {
if isAncestor(root, candidate) {
return fmt.Errorf("nested manifest %q under bundle %q", displayRoot(candidate), displayRoot(root))
}
}
}
return nil
}
func isAncestor(root, candidate string) bool {
if root == "" {
return candidate != ""
}
return strings.HasPrefix(candidate, root+"/")
}
func relativeToSource(sourceRoot, bundleRoot string) string {
if sourceRoot == "" {
return bundleRoot
}
if bundleRoot == sourceRoot {
return ""
}
return strings.TrimPrefix(bundleRoot, sourceRoot+"/")
}

View File

@@ -0,0 +1,70 @@
package bundle
import (
"context"
"reflect"
"testing"
"gitea.maximumdirect.net/eric/distributor/internal/storage"
"gitea.maximumdirect.net/eric/distributor/internal/storage/fake"
)
func TestDiscoverFindsBundlesInDeterministicOrder(t *testing.T) {
backend := fake.New()
addBundle(t, backend, "z/daily")
addBundle(t, backend, "a/daily")
bundles, err := Discover(context.Background(), backend, "")
if err != nil {
t.Fatalf("Discover() error = %v", err)
}
var paths []string
for _, sourceBundle := range bundles {
paths = append(paths, sourceBundle.RootRelativePath)
}
want := []string{"a/daily", "z/daily"}
if !reflect.DeepEqual(paths, want) {
t.Fatalf("paths = %v, want %v", paths, want)
}
}
func TestDiscoverFindsRootBundle(t *testing.T) {
backend := validFakeBundle(t)
bundles, err := Discover(context.Background(), backend, "")
if err != nil {
t.Fatalf("Discover() error = %v", err)
}
if got, want := len(bundles), 1; got != want {
t.Fatalf("bundle count = %d, want %d", got, want)
}
if bundles[0].RootRelativePath != "" {
t.Fatalf("root = %q, want empty", bundles[0].RootRelativePath)
}
}
func TestDiscoverRejectsNestedManifests(t *testing.T) {
backend := fake.New()
addBundle(t, backend, "daily")
addBundle(t, backend, "daily/nested")
_, err := Discover(context.Background(), backend, "")
assertErrorContains(t, err, "nested manifest")
}
func addBundle(t *testing.T, backend *fake.Backend, root string) {
t.Helper()
writeFakeFile(t, backend, joinTestPath(root, "manifest.json"), string(readFixture(t, "testdata/valid_bundle/manifest.json")))
writeFakeFile(t, backend, joinTestPath(root, "report.md"), string(readFixture(t, "testdata/valid_bundle/report.md")))
writeFakeFile(t, backend, joinTestPath(root, "summary.txt"), string(readFixture(t, "testdata/valid_bundle/summary.txt")))
}
func joinTestPath(root, file string) string {
if root == "" {
return file
}
joined, err := storage.Join(root, file)
if err != nil {
panic(err)
}
return joined
}

127
internal/bundle/manifest.go Normal file
View File

@@ -0,0 +1,127 @@
package bundle
import (
"bytes"
"encoding/json"
"fmt"
"io"
"time"
)
const ManifestName = "manifest.json"
type Manifest struct {
SchemaVersion int `json:"schema_version"`
ID string `json:"id"`
Digest string `json:"digest"`
Created time.Time `json:"created"`
Files []ManifestFile `json:"files"`
}
type ManifestFile struct {
Path string `json:"path"`
SHA256 string `json:"sha256"`
Size int64 `json:"size"`
}
type Bundle struct {
RootRelativePath string
Manifest Manifest
}
type rawManifest struct {
SchemaVersion *int `json:"schema_version"`
ID *string `json:"id"`
Digest *string `json:"digest"`
Created *string `json:"created"`
Files []rawManifestFile `json:"files"`
}
type rawManifestFile struct {
Path *string `json:"path"`
SHA256 *string `json:"sha256"`
Size *int64 `json:"size"`
}
func ParseManifest(data []byte) (Manifest, error) {
decoder := json.NewDecoder(bytes.NewReader(data))
var raw rawManifest
if err := decoder.Decode(&raw); err != nil {
return Manifest{}, fmt.Errorf("parse manifest: %w", err)
}
var extra any
if err := decoder.Decode(&extra); err != io.EOF {
return Manifest{}, fmt.Errorf("parse manifest: trailing data")
}
var manifest Manifest
if raw.SchemaVersion == nil {
return Manifest{}, fmt.Errorf("manifest schema_version is required")
}
manifest.SchemaVersion = *raw.SchemaVersion
if manifest.SchemaVersion != 1 {
return Manifest{}, fmt.Errorf("manifest schema_version must be 1")
}
if raw.ID == nil || *raw.ID == "" {
return Manifest{}, fmt.Errorf("manifest id is required")
}
manifest.ID = *raw.ID
if raw.Digest == nil || *raw.Digest == "" {
return Manifest{}, fmt.Errorf("manifest digest is required")
}
if err := ValidateDigest(*raw.Digest); err != nil {
return Manifest{}, fmt.Errorf("manifest digest: %w", err)
}
manifest.Digest = *raw.Digest
if raw.Created == nil || *raw.Created == "" {
return Manifest{}, fmt.Errorf("manifest created is required")
}
created, err := time.Parse(time.RFC3339, *raw.Created)
if err != nil {
return Manifest{}, fmt.Errorf("manifest created must be RFC3339: %w", err)
}
manifest.Created = created
if len(raw.Files) == 0 {
return Manifest{}, fmt.Errorf("manifest files is required")
}
seen := make(map[string]struct{}, len(raw.Files))
for index, rawFile := range raw.Files {
file, err := parseManifestFile(index, rawFile)
if err != nil {
return Manifest{}, err
}
if _, exists := seen[file.Path]; exists {
return Manifest{}, fmt.Errorf("manifest files[%d].path duplicates %q", index, file.Path)
}
seen[file.Path] = struct{}{}
manifest.Files = append(manifest.Files, file)
}
return manifest, nil
}
func parseManifestFile(index int, raw rawManifestFile) (ManifestFile, error) {
if raw.Path == nil || *raw.Path == "" {
return ManifestFile{}, fmt.Errorf("manifest files[%d].path is required", index)
}
if err := ValidateSourcePath(*raw.Path); err != nil {
return ManifestFile{}, fmt.Errorf("manifest files[%d].path: %w", index, err)
}
if raw.SHA256 == nil || *raw.SHA256 == "" {
return ManifestFile{}, fmt.Errorf("manifest files[%d].sha256 is required", index)
}
if err := ValidateDigest(*raw.SHA256); err != nil {
return ManifestFile{}, fmt.Errorf("manifest files[%d].sha256: %w", index, err)
}
if raw.Size == nil {
return ManifestFile{}, fmt.Errorf("manifest files[%d].size is required", index)
}
if *raw.Size < 0 {
return ManifestFile{}, fmt.Errorf("manifest files[%d].size must be non-negative", index)
}
return ManifestFile{
Path: *raw.Path,
SHA256: *raw.SHA256,
Size: *raw.Size,
}, nil
}

View File

@@ -0,0 +1,119 @@
package bundle
import (
"os"
"strings"
"testing"
)
func TestParseManifestValid(t *testing.T) {
data := readFixture(t, "testdata/valid_bundle/manifest.json")
manifest, err := ParseManifest(data)
if err != nil {
t.Fatalf("ParseManifest() error = %v", err)
}
if manifest.SchemaVersion != 1 {
t.Fatalf("schema version = %d, want 1", manifest.SchemaVersion)
}
if manifest.ID != "weather.daily.brentwood.2026-05-30" {
t.Fatalf("id = %q", manifest.ID)
}
if got, want := len(manifest.Files), 2; got != want {
t.Fatalf("file count = %d, want %d", got, want)
}
}
func TestParseManifestRejectsInvalidJSON(t *testing.T) {
_, err := ParseManifest([]byte(`{"schema_version":`))
assertErrorContains(t, err, "parse manifest")
}
func TestParseManifestRejectsMissingRequiredFields(t *testing.T) {
tests := map[string]string{
"schema_version": `{"id":"id","digest":"sha256:0000000000000000000000000000000000000000000000000000000000000000","created":"2026-05-30T11:10:00Z","files":[{"path":"report.md","sha256":"sha256:0000000000000000000000000000000000000000000000000000000000000000","size":0}]}`,
"id": `{"schema_version":1,"digest":"sha256:0000000000000000000000000000000000000000000000000000000000000000","created":"2026-05-30T11:10:00Z","files":[{"path":"report.md","sha256":"sha256:0000000000000000000000000000000000000000000000000000000000000000","size":0}]}`,
"digest": `{"schema_version":1,"id":"id","created":"2026-05-30T11:10:00Z","files":[{"path":"report.md","sha256":"sha256:0000000000000000000000000000000000000000000000000000000000000000","size":0}]}`,
"created": `{"schema_version":1,"id":"id","digest":"sha256:0000000000000000000000000000000000000000000000000000000000000000","files":[{"path":"report.md","sha256":"sha256:0000000000000000000000000000000000000000000000000000000000000000","size":0}]}`,
"files": `{"schema_version":1,"id":"id","digest":"sha256:0000000000000000000000000000000000000000000000000000000000000000","created":"2026-05-30T11:10:00Z"}`,
"file path": `{"schema_version":1,"id":"id","digest":"sha256:0000000000000000000000000000000000000000000000000000000000000000","created":"2026-05-30T11:10:00Z","files":[{"sha256":"sha256:0000000000000000000000000000000000000000000000000000000000000000","size":0}]}`,
"file digest": `{"schema_version":1,"id":"id","digest":"sha256:0000000000000000000000000000000000000000000000000000000000000000","created":"2026-05-30T11:10:00Z","files":[{"path":"report.md","size":0}]}`,
"file size": `{"schema_version":1,"id":"id","digest":"sha256:0000000000000000000000000000000000000000000000000000000000000000","created":"2026-05-30T11:10:00Z","files":[{"path":"report.md","sha256":"sha256:0000000000000000000000000000000000000000000000000000000000000000"}]}`,
}
for name, body := range tests {
t.Run(name, func(t *testing.T) {
_, err := ParseManifest([]byte(body))
assertErrorContains(t, err, "required")
})
}
}
func TestParseManifestRejectsInvalidSchemaVersion(t *testing.T) {
data := replaceFixture(t, `"schema_version": 1`, `"schema_version": 2`)
_, err := ParseManifest(data)
assertErrorContains(t, err, "schema_version must be 1")
}
func TestParseManifestRejectsInvalidTimestamp(t *testing.T) {
data := replaceFixture(t, `"created": "2026-05-30T11:10:00Z"`, `"created": "May 30"`)
_, err := ParseManifest(data)
assertErrorContains(t, err, "RFC3339")
}
func TestParseManifestRejectsInvalidDigestFormat(t *testing.T) {
data := replaceFixture(t, `"digest": "sha256:099b205780d2b050024868399961b05731729a548d5d6329c7b06a6740dd75fe"`, `"digest": "SHA256:099b205780d2b050024868399961b05731729a548d5d6329c7b06a6740dd75fe"`)
_, err := ParseManifest(data)
assertErrorContains(t, err, "lowercase")
}
func TestParseManifestRejectsUnsafeFilePaths(t *testing.T) {
tests := []string{
`"path": "../report.md"`,
`"path": "/report.md"`,
`"path": "nested/../report.md"`,
`"path": "manifest.json"`,
`"path": ".distributor.json"`,
}
for _, replacement := range tests {
t.Run(replacement, func(t *testing.T) {
data := replaceFixture(t, `"path": "report.md"`, replacement)
_, err := ParseManifest(data)
if err == nil {
t.Fatal("ParseManifest() error = nil, want error")
}
})
}
}
func TestParseManifestRejectsDuplicatePaths(t *testing.T) {
data := replaceFixture(t, `"path": "summary.txt"`, `"path": "report.md"`)
_, err := ParseManifest(data)
assertErrorContains(t, err, "duplicates")
}
func readFixture(t *testing.T, path string) []byte {
t.Helper()
data, err := os.ReadFile(path)
if err != nil {
t.Fatalf("read fixture %s: %v", path, err)
}
return data
}
func replaceFixture(t *testing.T, old, replacement string) []byte {
t.Helper()
body := string(readFixture(t, "testdata/valid_bundle/manifest.json"))
if !strings.Contains(body, old) {
t.Fatalf("fixture does not contain %q", old)
}
return []byte(strings.Replace(body, old, replacement, 1))
}
func assertErrorContains(t *testing.T, err error, want string) {
t.Helper()
if err == nil {
t.Fatalf("error = nil, want substring %q", want)
}
if !strings.Contains(err.Error(), want) {
t.Fatalf("error = %q, want substring %q", err.Error(), want)
}
}

View File

@@ -0,0 +1 @@
[{"path":"report.md","sha256":"sha256:3640fd37140ee4d2e0e93e78834f232ea67a50e7bc6279203690cc7de1975fa6","size":16},{"path":"summary.txt","sha256":"sha256:3cbb36aca330b3bd113955dfbada0adb7a5f95ad9f678bd61f175406c6a37e95","size":8}]

View File

@@ -0,0 +1,18 @@
{
"schema_version": 1,
"id": "weather.daily.brentwood.2026-05-30",
"digest": "sha256:099b205780d2b050024868399961b05731729a548d5d6329c7b06a6740dd75fe",
"created": "2026-05-30T11:10:00Z",
"files": [
{
"path": "report.md",
"sha256": "sha256:3640fd37140ee4d2e0e93e78834f232ea67a50e7bc6279203690cc7de1975fa6",
"size": 16
},
{
"path": "summary.txt",
"sha256": "sha256:3cbb36aca330b3bd113955dfbada0adb7a5f95ad9f678bd61f175406c6a37e95",
"size": 8
}
]
}

View File

@@ -0,0 +1,2 @@
# Report
Sunny.

View File

@@ -0,0 +1 @@
Summary

View File

@@ -0,0 +1,85 @@
package bundle
import (
"context"
"fmt"
"gitea.maximumdirect.net/eric/distributor/internal/storage"
)
func ValidateSourcePath(path string) error {
if err := storage.ValidatePath(path); err != nil {
return err
}
switch path {
case ManifestName, ".distributor.json":
return fmt.Errorf("%q is reserved", path)
}
return nil
}
func Validate(ctx context.Context, backend storage.Backend, bundleRoot string) (Bundle, error) {
return validateAt(ctx, backend, bundleRoot, bundleRoot)
}
func validateAt(ctx context.Context, backend storage.Backend, bundleRoot, relativeRoot string) (Bundle, error) {
if err := storage.ValidatePrefix(bundleRoot); err != nil {
return Bundle{}, err
}
manifestPath, err := storage.Join(bundleRoot, ManifestName)
if err != nil {
return Bundle{}, err
}
manifestData, err := backend.ReadFile(ctx, manifestPath)
if err != nil {
return Bundle{}, fmt.Errorf("read manifest %q: %w", manifestPath, err)
}
manifest, err := ParseManifest(manifestData)
if err != nil {
return Bundle{}, fmt.Errorf("bundle %q: %w", displayRoot(relativeRoot), err)
}
for index, manifestFile := range manifest.Files {
filePath, err := storage.Join(bundleRoot, manifestFile.Path)
if err != nil {
return Bundle{}, fmt.Errorf("bundle %q file %q: %w", displayRoot(relativeRoot), manifestFile.Path, err)
}
entry, err := backend.Stat(ctx, filePath)
if err != nil {
return Bundle{}, fmt.Errorf("bundle %q file %q stat: %w", displayRoot(relativeRoot), manifestFile.Path, err)
}
if entry.Type != storage.EntryTypeFile {
return Bundle{}, fmt.Errorf("bundle %q file %q must be a regular file", displayRoot(relativeRoot), manifestFile.Path)
}
if entry.Size != manifestFile.Size {
return Bundle{}, fmt.Errorf("bundle %q file %q size mismatch: got %d want %d", displayRoot(relativeRoot), manifestFile.Path, entry.Size, manifestFile.Size)
}
data, err := backend.ReadFile(ctx, filePath)
if err != nil {
return Bundle{}, fmt.Errorf("bundle %q file %q read: %w", displayRoot(relativeRoot), manifestFile.Path, err)
}
actualDigest := FileDigest(data)
if actualDigest != manifestFile.SHA256 {
return Bundle{}, fmt.Errorf("bundle %q file %q sha256 mismatch: got %s want %s", displayRoot(relativeRoot), manifestFile.Path, actualDigest, manifestFile.SHA256)
}
manifest.Files[index].SHA256 = actualDigest
manifest.Files[index].Size = int64(len(data))
}
actualBundleDigest := BundleDigest(manifest.Files)
if actualBundleDigest != manifest.Digest {
return Bundle{}, fmt.Errorf("bundle %q digest mismatch: got %s want %s", displayRoot(relativeRoot), actualBundleDigest, manifest.Digest)
}
return Bundle{
RootRelativePath: relativeRoot,
Manifest: manifest,
}, nil
}
func displayRoot(root string) string {
if root == "" {
return "."
}
return root
}

View File

@@ -0,0 +1,88 @@
package bundle
import (
"context"
"strings"
"testing"
"gitea.maximumdirect.net/eric/distributor/internal/storage"
"gitea.maximumdirect.net/eric/distributor/internal/storage/fake"
)
func TestValidateValidBundle(t *testing.T) {
backend := validFakeBundle(t)
sourceBundle, err := Validate(context.Background(), backend, "")
if err != nil {
t.Fatalf("Validate() error = %v", err)
}
if sourceBundle.RootRelativePath != "" {
t.Fatalf("root = %q, want empty", sourceBundle.RootRelativePath)
}
if sourceBundle.Manifest.ID != "weather.daily.brentwood.2026-05-30" {
t.Fatalf("id = %q", sourceBundle.Manifest.ID)
}
}
func TestValidateRejectsMissingFile(t *testing.T) {
backend := validFakeBundle(t)
deleteFakeFile(t, backend, "summary.txt")
_, err := Validate(context.Background(), backend, "")
assertErrorContains(t, err, "stat")
}
func TestValidateRejectsSizeMismatch(t *testing.T) {
backend := validFakeBundle(t)
manifest := strings.Replace(string(readFixture(t, "testdata/valid_bundle/manifest.json")), `"size": 8`, `"size": 9`, 1)
writeFakeFile(t, backend, "manifest.json", manifest)
_, err := Validate(context.Background(), backend, "")
assertErrorContains(t, err, "size mismatch")
}
func TestValidateRejectsPerFileDigestMismatch(t *testing.T) {
backend := validFakeBundle(t)
writeFakeFile(t, backend, "report.md", "# Report\nCloud.\n")
_, err := Validate(context.Background(), backend, "")
assertErrorContains(t, err, "sha256 mismatch")
}
func TestValidateRejectsBundleDigestMismatch(t *testing.T) {
backend := validFakeBundle(t)
manifest := strings.Replace(string(readFixture(t, "testdata/valid_bundle/manifest.json")), `"digest": "sha256:099b205780d2b050024868399961b05731729a548d5d6329c7b06a6740dd75fe"`, `"digest": "sha256:0000000000000000000000000000000000000000000000000000000000000000"`, 1)
writeFakeFile(t, backend, "manifest.json", manifest)
_, err := Validate(context.Background(), backend, "")
assertErrorContains(t, err, "digest mismatch")
}
func TestValidateRejectsSymlinkFile(t *testing.T) {
backend := validFakeBundle(t)
if err := backend.AddSymlink("summary.txt"); err != nil {
t.Fatalf("AddSymlink() error = %v", err)
}
_, err := Validate(context.Background(), backend, "")
assertErrorContains(t, err, "regular file")
}
func validFakeBundle(t *testing.T) *fake.Backend {
t.Helper()
backend := fake.New()
writeFakeFile(t, backend, "manifest.json", string(readFixture(t, "testdata/valid_bundle/manifest.json")))
writeFakeFile(t, backend, "report.md", string(readFixture(t, "testdata/valid_bundle/report.md")))
writeFakeFile(t, backend, "summary.txt", string(readFixture(t, "testdata/valid_bundle/summary.txt")))
return backend
}
func writeFakeFile(t *testing.T, backend *fake.Backend, path, data string) {
t.Helper()
_, err := backend.WriteFile(context.Background(), path, []byte(data), storage.WriteOptions{Overwrite: true})
if err != nil {
t.Fatalf("WriteFile(%q) error = %v", path, err)
}
}
func deleteFakeFile(t *testing.T, backend *fake.Backend, path string) {
t.Helper()
err := backend.DeleteManagedBundle(context.Background(), "", []string{path}, storage.DeleteOptions{IgnoreMissing: true})
if err != nil {
t.Fatalf("DeleteManagedBundle(%q) error = %v", path, err)
}
}