From 518944e6017cfaded36b63c3e0c4b1ba3a1a9c09 Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Sun, 31 May 2026 02:07:30 +0000 Subject: [PATCH] Add source bundle validation and inspection --- README.md | 4 +- docs/cli.md | 22 ++- docs/internal/bundle.md | 39 ++++++ examples/source-bundle/manifest.json | 18 +++ examples/source-bundle/report.md | 2 + examples/source-bundle/summary.txt | 1 + internal/app/inspect.go | 57 +++++++- internal/app/inspect_test.go | 40 ++++++ internal/app/validate.go | 25 +++- internal/app/validate_test.go | 39 ++++++ internal/bundle/digest.go | 49 +++++++ internal/bundle/digest_test.go | 21 +++ internal/bundle/discover.go | 82 +++++++++++ internal/bundle/discover_test.go | 70 ++++++++++ internal/bundle/manifest.go | 127 ++++++++++++++++++ internal/bundle/manifest_test.go | 119 ++++++++++++++++ .../bundle/testdata/canonical_payload.json | 1 + .../testdata/valid_bundle/manifest.json | 18 +++ .../bundle/testdata/valid_bundle/report.md | 2 + .../bundle/testdata/valid_bundle/summary.txt | 1 + internal/bundle/validate.go | 85 ++++++++++++ internal/bundle/validate_test.go | 88 ++++++++++++ internal/cli/inspect.go | 4 +- internal/cli/root_test.go | 30 ++++- internal/cli/validate.go | 4 +- 25 files changed, 931 insertions(+), 17 deletions(-) create mode 100644 docs/internal/bundle.md create mode 100644 examples/source-bundle/manifest.json create mode 100644 examples/source-bundle/report.md create mode 100644 examples/source-bundle/summary.txt create mode 100644 internal/app/inspect_test.go create mode 100644 internal/app/validate_test.go create mode 100644 internal/bundle/digest.go create mode 100644 internal/bundle/digest_test.go create mode 100644 internal/bundle/discover.go create mode 100644 internal/bundle/discover_test.go create mode 100644 internal/bundle/manifest.go create mode 100644 internal/bundle/manifest_test.go create mode 100644 internal/bundle/testdata/canonical_payload.json create mode 100644 internal/bundle/testdata/valid_bundle/manifest.json create mode 100644 internal/bundle/testdata/valid_bundle/report.md create mode 100644 internal/bundle/testdata/valid_bundle/summary.txt create mode 100644 internal/bundle/validate.go create mode 100644 internal/bundle/validate_test.go diff --git a/README.md b/README.md index 12b219c..376f102 100644 --- a/README.md +++ b/README.md @@ -2,10 +2,10 @@ `distributor` is a Go application shell for validating and publishing manifested report bundles. -Current implemented behavior is limited to CLI help, version output, config validation, and placeholder operational commands: +Current implemented behavior is limited to CLI help, version output, config validation, source bundle validation, source bundle inspection, and placeholder run execution: ```sh -go run ./cmd/distributor run --config examples/local-to-local.yml --dry-run +go run ./cmd/distributor validate examples/source-bundle ``` See [docs/cli.md](docs/cli.md) and [docs/config.md](docs/config.md) for the implemented CLI and configuration surface. Current design and implementation planning lives under `docs/roadmap/`. diff --git a/docs/cli.md b/docs/cli.md index 26bebf5..6c75a33 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -3,10 +3,10 @@ ## Shortest useful command ```sh -go run ./cmd/distributor run --config examples/local-to-local.yml --dry-run +go run ./cmd/distributor validate examples/source-bundle ``` -This loads and validates the example config, then prints the resolved pipeline summary without publishing files. +This validates a local source bundle fixture. ## Command overview @@ -22,7 +22,11 @@ distributor inspect `run --config --dry-run` loads and validates configuration, then prints a concise summary of configured pipelines and destinations. It does not discover bundles or publish files yet. -`run` without `--dry-run`, `validate`, and `inspect` intentionally fail with a clear `not implemented` error until the corresponding application behavior exists. +`validate ` validates a local source bundle directory or a local tree containing source bundles. + +`inspect ` validates discovered local source bundles and prints a concise normalized summary. + +`run` without `--dry-run` intentionally fails with a clear `not implemented` error until execution behavior exists. ## Flag reference @@ -41,6 +45,18 @@ Each subcommand supports: ## Common workflows +Validate a source bundle: + +```sh +go run ./cmd/distributor validate examples/source-bundle +``` + +Inspect a source bundle: + +```sh +go run ./cmd/distributor inspect examples/source-bundle +``` + Validate a config file without publishing: ```sh diff --git a/docs/internal/bundle.md b/docs/internal/bundle.md new file mode 100644 index 0000000..dc4d5b5 --- /dev/null +++ b/docs/internal/bundle.md @@ -0,0 +1,39 @@ +# Bundles + +## Purpose + +`internal/bundle` parses, discovers, and validates source bundles through the storage interface. + +## Inputs and outputs + +Input is a backend-rooted directory tree containing one or more `manifest.json` files. Output is a deterministic list of validated bundles with relative bundle paths and normalized manifest data. + +## Manifest behavior + +The source manifest requires: + +- `schema_version: 1` +- `id` +- `digest` +- `created` +- non-empty `files` + +Each file requires `path`, `sha256`, and `size`. Digests must use lowercase `sha256:<64 hex>` format. `created` must parse as RFC3339. + +## Validation + +Bundle validation checks source path safety, duplicate file paths, reserved paths, file existence, regular-file type, file size, per-file SHA-256, and the top-level bundle digest. + +The bundle digest is SHA-256 of a deterministic JSON array of file records in manifest order with fields `path`, `sha256`, and `size`. + +## Discovery + +Discovery walks a storage backend beneath a source root, finds `manifest.json` files, sorts bundle paths lexically, and rejects nested manifests. + +## Boundaries + +Bundle code uses `internal/storage` and does not import local, SSH, or S3 adapters. CLI local path support is wired in `internal/app`. + +## Tests + +Before changing bundle behavior, inspect tests under `internal/bundle`. diff --git a/examples/source-bundle/manifest.json b/examples/source-bundle/manifest.json new file mode 100644 index 0000000..2f9b75f --- /dev/null +++ b/examples/source-bundle/manifest.json @@ -0,0 +1,18 @@ +{ + "schema_version": 1, + "id": "weather.daily.brentwood.2026-05-30", + "digest": "sha256:099b205780d2b050024868399961b05731729a548d5d6329c7b06a6740dd75fe", + "created": "2026-05-30T11:10:00Z", + "files": [ + { + "path": "report.md", + "sha256": "sha256:3640fd37140ee4d2e0e93e78834f232ea67a50e7bc6279203690cc7de1975fa6", + "size": 16 + }, + { + "path": "summary.txt", + "sha256": "sha256:3cbb36aca330b3bd113955dfbada0adb7a5f95ad9f678bd61f175406c6a37e95", + "size": 8 + } + ] +} diff --git a/examples/source-bundle/report.md b/examples/source-bundle/report.md new file mode 100644 index 0000000..6fc3466 --- /dev/null +++ b/examples/source-bundle/report.md @@ -0,0 +1,2 @@ +# Report +Sunny. diff --git a/examples/source-bundle/summary.txt b/examples/source-bundle/summary.txt new file mode 100644 index 0000000..d7235f3 --- /dev/null +++ b/examples/source-bundle/summary.txt @@ -0,0 +1 @@ +Summary diff --git a/internal/app/inspect.go b/internal/app/inspect.go index d9210bf..c347110 100644 --- a/internal/app/inspect.go +++ b/internal/app/inspect.go @@ -3,12 +3,63 @@ package app import ( "context" "fmt" + "io" + + "gitea.maximumdirect.net/eric/distributor/internal/adapters/local" + "gitea.maximumdirect.net/eric/distributor/internal/bundle" ) type InspectOptions struct { - Path string + Path string + Stdout io.Writer } -func Inspect(context.Context, InspectOptions) error { - return fmt.Errorf("inspect command: %w", ErrNotImplemented) +func Inspect(ctx context.Context, options InspectOptions) error { + if options.Path == "" { + return fmt.Errorf("inspect command requires a path") + } + backend, err := local.New(options.Path) + if err != nil { + return err + } + bundles, err := bundle.Discover(ctx, backend, "") + if err != nil { + return err + } + return writeInspection(options.Stdout, bundles) +} + +func writeInspection(w io.Writer, bundles []bundle.Bundle) error { + if w == nil { + return nil + } + if _, err := fmt.Fprintf(w, "Bundles: %d\n", len(bundles)); err != nil { + return err + } + for _, sourceBundle := range bundles { + if _, err := fmt.Fprintf( + w, + "- path=%s id=%s created=%s digest=%s files=%d\n", + displayBundlePath(sourceBundle.RootRelativePath), + sourceBundle.Manifest.ID, + sourceBundle.Manifest.Created.Format("2006-01-02T15:04:05Z07:00"), + sourceBundle.Manifest.Digest, + len(sourceBundle.Manifest.Files), + ); err != nil { + return err + } + for _, file := range sourceBundle.Manifest.Files { + if _, err := fmt.Fprintf(w, " - %s size=%d sha256=%s\n", file.Path, file.Size, file.SHA256); err != nil { + return err + } + } + } + return nil +} + +func displayBundlePath(path string) string { + if path == "" { + return "." + } + return path } diff --git a/internal/app/inspect_test.go b/internal/app/inspect_test.go new file mode 100644 index 0000000..09a3204 --- /dev/null +++ b/internal/app/inspect_test.go @@ -0,0 +1,40 @@ +package app + +import ( + "bytes" + "context" + "path/filepath" + "strings" + "testing" +) + +func TestInspectPrintsBundleSummary(t *testing.T) { + var stdout bytes.Buffer + err := Inspect(context.Background(), InspectOptions{ + Path: filepath.Join("..", "bundle", "testdata", "valid_bundle"), + Stdout: &stdout, + }) + if err != nil { + t.Fatalf("Inspect() error = %v", err) + } + output := stdout.String() + for _, want := range []string{ + "Bundles: 1", + "path=.", + "id=weather.daily.brentwood.2026-05-30", + "created=2026-05-30T11:10:00Z", + "report.md size=16", + "summary.txt size=8", + } { + if !strings.Contains(output, want) { + t.Fatalf("Inspect() output = %q, want substring %q", output, want) + } + } +} + +func TestInspectRequiresPath(t *testing.T) { + err := Inspect(context.Background(), InspectOptions{}) + if err == nil || !strings.Contains(err.Error(), "requires a path") { + t.Fatalf("Inspect() error = %v, want required path", err) + } +} diff --git a/internal/app/validate.go b/internal/app/validate.go index 68e514d..4983f69 100644 --- a/internal/app/validate.go +++ b/internal/app/validate.go @@ -3,12 +3,31 @@ package app import ( "context" "fmt" + "io" + + "gitea.maximumdirect.net/eric/distributor/internal/adapters/local" + "gitea.maximumdirect.net/eric/distributor/internal/bundle" ) type ValidateOptions struct { - Path string + Path string + Stdout io.Writer } -func Validate(context.Context, ValidateOptions) error { - return fmt.Errorf("validate command: %w", ErrNotImplemented) +func Validate(ctx context.Context, options ValidateOptions) error { + if options.Path == "" { + return fmt.Errorf("validate command requires a path") + } + backend, err := local.New(options.Path) + if err != nil { + return err + } + bundles, err := bundle.Discover(ctx, backend, "") + if err != nil { + return err + } + if options.Stdout != nil { + _, err = fmt.Fprintf(options.Stdout, "Validated %d bundle(s)\n", len(bundles)) + } + return err } diff --git a/internal/app/validate_test.go b/internal/app/validate_test.go new file mode 100644 index 0000000..a2e2679 --- /dev/null +++ b/internal/app/validate_test.go @@ -0,0 +1,39 @@ +package app + +import ( + "bytes" + "context" + "path/filepath" + "strings" + "testing" +) + +func TestValidateLocalBundle(t *testing.T) { + var stdout bytes.Buffer + err := Validate(context.Background(), ValidateOptions{ + Path: filepath.Join("..", "bundle", "testdata", "valid_bundle"), + Stdout: &stdout, + }) + if err != nil { + t.Fatalf("Validate() error = %v", err) + } + if got, want := stdout.String(), "Validated 1 bundle(s)\n"; got != want { + t.Fatalf("stdout = %q, want %q", got, want) + } +} + +func TestValidateExampleSourceBundle(t *testing.T) { + err := Validate(context.Background(), ValidateOptions{ + Path: filepath.Join("..", "..", "examples", "source-bundle"), + }) + if err != nil { + t.Fatalf("Validate() example error = %v", err) + } +} + +func TestValidateRequiresPath(t *testing.T) { + err := Validate(context.Background(), ValidateOptions{}) + if err == nil || !strings.Contains(err.Error(), "requires a path") { + t.Fatalf("Validate() error = %v, want required path", err) + } +} diff --git a/internal/bundle/digest.go b/internal/bundle/digest.go new file mode 100644 index 0000000..65f3d0e --- /dev/null +++ b/internal/bundle/digest.go @@ -0,0 +1,49 @@ +package bundle + +import ( + "crypto/sha256" + "encoding/hex" + "fmt" + "regexp" + "strconv" + "strings" +) + +var digestPattern = regexp.MustCompile(`^sha256:[0-9a-f]{64}$`) + +func ValidateDigest(value string) error { + if !digestPattern.MatchString(value) { + return fmt.Errorf("must be lowercase sha256:<64 hex>") + } + return nil +} + +func FileDigest(data []byte) string { + sum := sha256.Sum256(data) + return "sha256:" + hex.EncodeToString(sum[:]) +} + +func BundleDigest(files []ManifestFile) string { + canonical := CanonicalFilePayload(files) + sum := sha256.Sum256([]byte(canonical)) + return "sha256:" + hex.EncodeToString(sum[:]) +} + +func CanonicalFilePayload(files []ManifestFile) string { + var builder strings.Builder + builder.WriteByte('[') + for index, file := range files { + if index > 0 { + builder.WriteByte(',') + } + builder.WriteString(`{"path":`) + builder.WriteString(strconv.Quote(file.Path)) + builder.WriteString(`,"sha256":`) + builder.WriteString(strconv.Quote(file.SHA256)) + builder.WriteString(`,"size":`) + builder.WriteString(strconv.FormatInt(file.Size, 10)) + builder.WriteByte('}') + } + builder.WriteByte(']') + return builder.String() +} diff --git a/internal/bundle/digest_test.go b/internal/bundle/digest_test.go new file mode 100644 index 0000000..766eb5e --- /dev/null +++ b/internal/bundle/digest_test.go @@ -0,0 +1,21 @@ +package bundle + +import ( + "strings" + "testing" +) + +func TestCanonicalBundleDigestReferenceFixture(t *testing.T) { + manifest, err := ParseManifest(readFixture(t, "testdata/valid_bundle/manifest.json")) + if err != nil { + t.Fatalf("ParseManifest() error = %v", err) + } + gotPayload := CanonicalFilePayload(manifest.Files) + wantPayload := strings.TrimSpace(string(readFixture(t, "testdata/canonical_payload.json"))) + if gotPayload != wantPayload { + t.Fatalf("canonical payload = %q, want %q", gotPayload, wantPayload) + } + if got, want := BundleDigest(manifest.Files), manifest.Digest; got != want { + t.Fatalf("BundleDigest() = %q, want %q", got, want) + } +} diff --git a/internal/bundle/discover.go b/internal/bundle/discover.go new file mode 100644 index 0000000..677beac --- /dev/null +++ b/internal/bundle/discover.go @@ -0,0 +1,82 @@ +package bundle + +import ( + "context" + "fmt" + "path" + "sort" + "strings" + + "gitea.maximumdirect.net/eric/distributor/internal/storage" +) + +func Discover(ctx context.Context, backend storage.Backend, sourceRoot string) ([]Bundle, error) { + if err := storage.ValidatePrefix(sourceRoot); err != nil { + return nil, err + } + entries, err := storage.List(ctx, backend, sourceRoot, storage.WalkOptions{Recursive: true}) + if err != nil { + return nil, err + } + + var roots []string + for _, entry := range entries { + if entry.Type == storage.EntryTypeDirectory { + continue + } + if path.Base(entry.Path) == ManifestName { + roots = append(roots, path.Dir(entry.Path)) + } + } + for index, root := range roots { + if root == "." { + roots[index] = "" + } + } + sort.Strings(roots) + if len(roots) == 0 { + return nil, fmt.Errorf("no bundles found under %q", displayRoot(sourceRoot)) + } + if err := rejectNestedRoots(roots); err != nil { + return nil, err + } + + bundles := make([]Bundle, 0, len(roots)) + for _, root := range roots { + relativeRoot := relativeToSource(sourceRoot, root) + sourceBundle, err := validateAt(ctx, backend, root, relativeRoot) + if err != nil { + return nil, err + } + bundles = append(bundles, sourceBundle) + } + return bundles, nil +} + +func rejectNestedRoots(roots []string) error { + for index, root := range roots { + for _, candidate := range roots[index+1:] { + if isAncestor(root, candidate) { + return fmt.Errorf("nested manifest %q under bundle %q", displayRoot(candidate), displayRoot(root)) + } + } + } + return nil +} + +func isAncestor(root, candidate string) bool { + if root == "" { + return candidate != "" + } + return strings.HasPrefix(candidate, root+"/") +} + +func relativeToSource(sourceRoot, bundleRoot string) string { + if sourceRoot == "" { + return bundleRoot + } + if bundleRoot == sourceRoot { + return "" + } + return strings.TrimPrefix(bundleRoot, sourceRoot+"/") +} diff --git a/internal/bundle/discover_test.go b/internal/bundle/discover_test.go new file mode 100644 index 0000000..bfefd49 --- /dev/null +++ b/internal/bundle/discover_test.go @@ -0,0 +1,70 @@ +package bundle + +import ( + "context" + "reflect" + "testing" + + "gitea.maximumdirect.net/eric/distributor/internal/storage" + "gitea.maximumdirect.net/eric/distributor/internal/storage/fake" +) + +func TestDiscoverFindsBundlesInDeterministicOrder(t *testing.T) { + backend := fake.New() + addBundle(t, backend, "z/daily") + addBundle(t, backend, "a/daily") + + bundles, err := Discover(context.Background(), backend, "") + if err != nil { + t.Fatalf("Discover() error = %v", err) + } + var paths []string + for _, sourceBundle := range bundles { + paths = append(paths, sourceBundle.RootRelativePath) + } + want := []string{"a/daily", "z/daily"} + if !reflect.DeepEqual(paths, want) { + t.Fatalf("paths = %v, want %v", paths, want) + } +} + +func TestDiscoverFindsRootBundle(t *testing.T) { + backend := validFakeBundle(t) + bundles, err := Discover(context.Background(), backend, "") + if err != nil { + t.Fatalf("Discover() error = %v", err) + } + if got, want := len(bundles), 1; got != want { + t.Fatalf("bundle count = %d, want %d", got, want) + } + if bundles[0].RootRelativePath != "" { + t.Fatalf("root = %q, want empty", bundles[0].RootRelativePath) + } +} + +func TestDiscoverRejectsNestedManifests(t *testing.T) { + backend := fake.New() + addBundle(t, backend, "daily") + addBundle(t, backend, "daily/nested") + + _, err := Discover(context.Background(), backend, "") + assertErrorContains(t, err, "nested manifest") +} + +func addBundle(t *testing.T, backend *fake.Backend, root string) { + t.Helper() + writeFakeFile(t, backend, joinTestPath(root, "manifest.json"), string(readFixture(t, "testdata/valid_bundle/manifest.json"))) + writeFakeFile(t, backend, joinTestPath(root, "report.md"), string(readFixture(t, "testdata/valid_bundle/report.md"))) + writeFakeFile(t, backend, joinTestPath(root, "summary.txt"), string(readFixture(t, "testdata/valid_bundle/summary.txt"))) +} + +func joinTestPath(root, file string) string { + if root == "" { + return file + } + joined, err := storage.Join(root, file) + if err != nil { + panic(err) + } + return joined +} diff --git a/internal/bundle/manifest.go b/internal/bundle/manifest.go new file mode 100644 index 0000000..f3522b8 --- /dev/null +++ b/internal/bundle/manifest.go @@ -0,0 +1,127 @@ +package bundle + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "time" +) + +const ManifestName = "manifest.json" + +type Manifest struct { + SchemaVersion int `json:"schema_version"` + ID string `json:"id"` + Digest string `json:"digest"` + Created time.Time `json:"created"` + Files []ManifestFile `json:"files"` +} + +type ManifestFile struct { + Path string `json:"path"` + SHA256 string `json:"sha256"` + Size int64 `json:"size"` +} + +type Bundle struct { + RootRelativePath string + Manifest Manifest +} + +type rawManifest struct { + SchemaVersion *int `json:"schema_version"` + ID *string `json:"id"` + Digest *string `json:"digest"` + Created *string `json:"created"` + Files []rawManifestFile `json:"files"` +} + +type rawManifestFile struct { + Path *string `json:"path"` + SHA256 *string `json:"sha256"` + Size *int64 `json:"size"` +} + +func ParseManifest(data []byte) (Manifest, error) { + decoder := json.NewDecoder(bytes.NewReader(data)) + var raw rawManifest + if err := decoder.Decode(&raw); err != nil { + return Manifest{}, fmt.Errorf("parse manifest: %w", err) + } + var extra any + if err := decoder.Decode(&extra); err != io.EOF { + return Manifest{}, fmt.Errorf("parse manifest: trailing data") + } + + var manifest Manifest + if raw.SchemaVersion == nil { + return Manifest{}, fmt.Errorf("manifest schema_version is required") + } + manifest.SchemaVersion = *raw.SchemaVersion + if manifest.SchemaVersion != 1 { + return Manifest{}, fmt.Errorf("manifest schema_version must be 1") + } + if raw.ID == nil || *raw.ID == "" { + return Manifest{}, fmt.Errorf("manifest id is required") + } + manifest.ID = *raw.ID + if raw.Digest == nil || *raw.Digest == "" { + return Manifest{}, fmt.Errorf("manifest digest is required") + } + if err := ValidateDigest(*raw.Digest); err != nil { + return Manifest{}, fmt.Errorf("manifest digest: %w", err) + } + manifest.Digest = *raw.Digest + if raw.Created == nil || *raw.Created == "" { + return Manifest{}, fmt.Errorf("manifest created is required") + } + created, err := time.Parse(time.RFC3339, *raw.Created) + if err != nil { + return Manifest{}, fmt.Errorf("manifest created must be RFC3339: %w", err) + } + manifest.Created = created + if len(raw.Files) == 0 { + return Manifest{}, fmt.Errorf("manifest files is required") + } + + seen := make(map[string]struct{}, len(raw.Files)) + for index, rawFile := range raw.Files { + file, err := parseManifestFile(index, rawFile) + if err != nil { + return Manifest{}, err + } + if _, exists := seen[file.Path]; exists { + return Manifest{}, fmt.Errorf("manifest files[%d].path duplicates %q", index, file.Path) + } + seen[file.Path] = struct{}{} + manifest.Files = append(manifest.Files, file) + } + return manifest, nil +} + +func parseManifestFile(index int, raw rawManifestFile) (ManifestFile, error) { + if raw.Path == nil || *raw.Path == "" { + return ManifestFile{}, fmt.Errorf("manifest files[%d].path is required", index) + } + if err := ValidateSourcePath(*raw.Path); err != nil { + return ManifestFile{}, fmt.Errorf("manifest files[%d].path: %w", index, err) + } + if raw.SHA256 == nil || *raw.SHA256 == "" { + return ManifestFile{}, fmt.Errorf("manifest files[%d].sha256 is required", index) + } + if err := ValidateDigest(*raw.SHA256); err != nil { + return ManifestFile{}, fmt.Errorf("manifest files[%d].sha256: %w", index, err) + } + if raw.Size == nil { + return ManifestFile{}, fmt.Errorf("manifest files[%d].size is required", index) + } + if *raw.Size < 0 { + return ManifestFile{}, fmt.Errorf("manifest files[%d].size must be non-negative", index) + } + return ManifestFile{ + Path: *raw.Path, + SHA256: *raw.SHA256, + Size: *raw.Size, + }, nil +} diff --git a/internal/bundle/manifest_test.go b/internal/bundle/manifest_test.go new file mode 100644 index 0000000..5644b68 --- /dev/null +++ b/internal/bundle/manifest_test.go @@ -0,0 +1,119 @@ +package bundle + +import ( + "os" + "strings" + "testing" +) + +func TestParseManifestValid(t *testing.T) { + data := readFixture(t, "testdata/valid_bundle/manifest.json") + manifest, err := ParseManifest(data) + if err != nil { + t.Fatalf("ParseManifest() error = %v", err) + } + if manifest.SchemaVersion != 1 { + t.Fatalf("schema version = %d, want 1", manifest.SchemaVersion) + } + if manifest.ID != "weather.daily.brentwood.2026-05-30" { + t.Fatalf("id = %q", manifest.ID) + } + if got, want := len(manifest.Files), 2; got != want { + t.Fatalf("file count = %d, want %d", got, want) + } +} + +func TestParseManifestRejectsInvalidJSON(t *testing.T) { + _, err := ParseManifest([]byte(`{"schema_version":`)) + assertErrorContains(t, err, "parse manifest") +} + +func TestParseManifestRejectsMissingRequiredFields(t *testing.T) { + tests := map[string]string{ + "schema_version": `{"id":"id","digest":"sha256:0000000000000000000000000000000000000000000000000000000000000000","created":"2026-05-30T11:10:00Z","files":[{"path":"report.md","sha256":"sha256:0000000000000000000000000000000000000000000000000000000000000000","size":0}]}`, + "id": `{"schema_version":1,"digest":"sha256:0000000000000000000000000000000000000000000000000000000000000000","created":"2026-05-30T11:10:00Z","files":[{"path":"report.md","sha256":"sha256:0000000000000000000000000000000000000000000000000000000000000000","size":0}]}`, + "digest": `{"schema_version":1,"id":"id","created":"2026-05-30T11:10:00Z","files":[{"path":"report.md","sha256":"sha256:0000000000000000000000000000000000000000000000000000000000000000","size":0}]}`, + "created": `{"schema_version":1,"id":"id","digest":"sha256:0000000000000000000000000000000000000000000000000000000000000000","files":[{"path":"report.md","sha256":"sha256:0000000000000000000000000000000000000000000000000000000000000000","size":0}]}`, + "files": `{"schema_version":1,"id":"id","digest":"sha256:0000000000000000000000000000000000000000000000000000000000000000","created":"2026-05-30T11:10:00Z"}`, + "file path": `{"schema_version":1,"id":"id","digest":"sha256:0000000000000000000000000000000000000000000000000000000000000000","created":"2026-05-30T11:10:00Z","files":[{"sha256":"sha256:0000000000000000000000000000000000000000000000000000000000000000","size":0}]}`, + "file digest": `{"schema_version":1,"id":"id","digest":"sha256:0000000000000000000000000000000000000000000000000000000000000000","created":"2026-05-30T11:10:00Z","files":[{"path":"report.md","size":0}]}`, + "file size": `{"schema_version":1,"id":"id","digest":"sha256:0000000000000000000000000000000000000000000000000000000000000000","created":"2026-05-30T11:10:00Z","files":[{"path":"report.md","sha256":"sha256:0000000000000000000000000000000000000000000000000000000000000000"}]}`, + } + for name, body := range tests { + t.Run(name, func(t *testing.T) { + _, err := ParseManifest([]byte(body)) + assertErrorContains(t, err, "required") + }) + } +} + +func TestParseManifestRejectsInvalidSchemaVersion(t *testing.T) { + data := replaceFixture(t, `"schema_version": 1`, `"schema_version": 2`) + _, err := ParseManifest(data) + assertErrorContains(t, err, "schema_version must be 1") +} + +func TestParseManifestRejectsInvalidTimestamp(t *testing.T) { + data := replaceFixture(t, `"created": "2026-05-30T11:10:00Z"`, `"created": "May 30"`) + _, err := ParseManifest(data) + assertErrorContains(t, err, "RFC3339") +} + +func TestParseManifestRejectsInvalidDigestFormat(t *testing.T) { + data := replaceFixture(t, `"digest": "sha256:099b205780d2b050024868399961b05731729a548d5d6329c7b06a6740dd75fe"`, `"digest": "SHA256:099b205780d2b050024868399961b05731729a548d5d6329c7b06a6740dd75fe"`) + _, err := ParseManifest(data) + assertErrorContains(t, err, "lowercase") +} + +func TestParseManifestRejectsUnsafeFilePaths(t *testing.T) { + tests := []string{ + `"path": "../report.md"`, + `"path": "/report.md"`, + `"path": "nested/../report.md"`, + `"path": "manifest.json"`, + `"path": ".distributor.json"`, + } + for _, replacement := range tests { + t.Run(replacement, func(t *testing.T) { + data := replaceFixture(t, `"path": "report.md"`, replacement) + _, err := ParseManifest(data) + if err == nil { + t.Fatal("ParseManifest() error = nil, want error") + } + }) + } +} + +func TestParseManifestRejectsDuplicatePaths(t *testing.T) { + data := replaceFixture(t, `"path": "summary.txt"`, `"path": "report.md"`) + _, err := ParseManifest(data) + assertErrorContains(t, err, "duplicates") +} + +func readFixture(t *testing.T, path string) []byte { + t.Helper() + data, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read fixture %s: %v", path, err) + } + return data +} + +func replaceFixture(t *testing.T, old, replacement string) []byte { + t.Helper() + body := string(readFixture(t, "testdata/valid_bundle/manifest.json")) + if !strings.Contains(body, old) { + t.Fatalf("fixture does not contain %q", old) + } + return []byte(strings.Replace(body, old, replacement, 1)) +} + +func assertErrorContains(t *testing.T, err error, want string) { + t.Helper() + if err == nil { + t.Fatalf("error = nil, want substring %q", want) + } + if !strings.Contains(err.Error(), want) { + t.Fatalf("error = %q, want substring %q", err.Error(), want) + } +} diff --git a/internal/bundle/testdata/canonical_payload.json b/internal/bundle/testdata/canonical_payload.json new file mode 100644 index 0000000..c7cfd43 --- /dev/null +++ b/internal/bundle/testdata/canonical_payload.json @@ -0,0 +1 @@ +[{"path":"report.md","sha256":"sha256:3640fd37140ee4d2e0e93e78834f232ea67a50e7bc6279203690cc7de1975fa6","size":16},{"path":"summary.txt","sha256":"sha256:3cbb36aca330b3bd113955dfbada0adb7a5f95ad9f678bd61f175406c6a37e95","size":8}] diff --git a/internal/bundle/testdata/valid_bundle/manifest.json b/internal/bundle/testdata/valid_bundle/manifest.json new file mode 100644 index 0000000..2f9b75f --- /dev/null +++ b/internal/bundle/testdata/valid_bundle/manifest.json @@ -0,0 +1,18 @@ +{ + "schema_version": 1, + "id": "weather.daily.brentwood.2026-05-30", + "digest": "sha256:099b205780d2b050024868399961b05731729a548d5d6329c7b06a6740dd75fe", + "created": "2026-05-30T11:10:00Z", + "files": [ + { + "path": "report.md", + "sha256": "sha256:3640fd37140ee4d2e0e93e78834f232ea67a50e7bc6279203690cc7de1975fa6", + "size": 16 + }, + { + "path": "summary.txt", + "sha256": "sha256:3cbb36aca330b3bd113955dfbada0adb7a5f95ad9f678bd61f175406c6a37e95", + "size": 8 + } + ] +} diff --git a/internal/bundle/testdata/valid_bundle/report.md b/internal/bundle/testdata/valid_bundle/report.md new file mode 100644 index 0000000..6fc3466 --- /dev/null +++ b/internal/bundle/testdata/valid_bundle/report.md @@ -0,0 +1,2 @@ +# Report +Sunny. diff --git a/internal/bundle/testdata/valid_bundle/summary.txt b/internal/bundle/testdata/valid_bundle/summary.txt new file mode 100644 index 0000000..d7235f3 --- /dev/null +++ b/internal/bundle/testdata/valid_bundle/summary.txt @@ -0,0 +1 @@ +Summary diff --git a/internal/bundle/validate.go b/internal/bundle/validate.go new file mode 100644 index 0000000..c6e56ca --- /dev/null +++ b/internal/bundle/validate.go @@ -0,0 +1,85 @@ +package bundle + +import ( + "context" + "fmt" + + "gitea.maximumdirect.net/eric/distributor/internal/storage" +) + +func ValidateSourcePath(path string) error { + if err := storage.ValidatePath(path); err != nil { + return err + } + switch path { + case ManifestName, ".distributor.json": + return fmt.Errorf("%q is reserved", path) + } + return nil +} + +func Validate(ctx context.Context, backend storage.Backend, bundleRoot string) (Bundle, error) { + return validateAt(ctx, backend, bundleRoot, bundleRoot) +} + +func validateAt(ctx context.Context, backend storage.Backend, bundleRoot, relativeRoot string) (Bundle, error) { + if err := storage.ValidatePrefix(bundleRoot); err != nil { + return Bundle{}, err + } + manifestPath, err := storage.Join(bundleRoot, ManifestName) + if err != nil { + return Bundle{}, err + } + manifestData, err := backend.ReadFile(ctx, manifestPath) + if err != nil { + return Bundle{}, fmt.Errorf("read manifest %q: %w", manifestPath, err) + } + manifest, err := ParseManifest(manifestData) + if err != nil { + return Bundle{}, fmt.Errorf("bundle %q: %w", displayRoot(relativeRoot), err) + } + + for index, manifestFile := range manifest.Files { + filePath, err := storage.Join(bundleRoot, manifestFile.Path) + if err != nil { + return Bundle{}, fmt.Errorf("bundle %q file %q: %w", displayRoot(relativeRoot), manifestFile.Path, err) + } + entry, err := backend.Stat(ctx, filePath) + if err != nil { + return Bundle{}, fmt.Errorf("bundle %q file %q stat: %w", displayRoot(relativeRoot), manifestFile.Path, err) + } + if entry.Type != storage.EntryTypeFile { + return Bundle{}, fmt.Errorf("bundle %q file %q must be a regular file", displayRoot(relativeRoot), manifestFile.Path) + } + if entry.Size != manifestFile.Size { + return Bundle{}, fmt.Errorf("bundle %q file %q size mismatch: got %d want %d", displayRoot(relativeRoot), manifestFile.Path, entry.Size, manifestFile.Size) + } + data, err := backend.ReadFile(ctx, filePath) + if err != nil { + return Bundle{}, fmt.Errorf("bundle %q file %q read: %w", displayRoot(relativeRoot), manifestFile.Path, err) + } + actualDigest := FileDigest(data) + if actualDigest != manifestFile.SHA256 { + return Bundle{}, fmt.Errorf("bundle %q file %q sha256 mismatch: got %s want %s", displayRoot(relativeRoot), manifestFile.Path, actualDigest, manifestFile.SHA256) + } + manifest.Files[index].SHA256 = actualDigest + manifest.Files[index].Size = int64(len(data)) + } + + actualBundleDigest := BundleDigest(manifest.Files) + if actualBundleDigest != manifest.Digest { + return Bundle{}, fmt.Errorf("bundle %q digest mismatch: got %s want %s", displayRoot(relativeRoot), actualBundleDigest, manifest.Digest) + } + + return Bundle{ + RootRelativePath: relativeRoot, + Manifest: manifest, + }, nil +} + +func displayRoot(root string) string { + if root == "" { + return "." + } + return root +} diff --git a/internal/bundle/validate_test.go b/internal/bundle/validate_test.go new file mode 100644 index 0000000..db4ffdb --- /dev/null +++ b/internal/bundle/validate_test.go @@ -0,0 +1,88 @@ +package bundle + +import ( + "context" + "strings" + "testing" + + "gitea.maximumdirect.net/eric/distributor/internal/storage" + "gitea.maximumdirect.net/eric/distributor/internal/storage/fake" +) + +func TestValidateValidBundle(t *testing.T) { + backend := validFakeBundle(t) + sourceBundle, err := Validate(context.Background(), backend, "") + if err != nil { + t.Fatalf("Validate() error = %v", err) + } + if sourceBundle.RootRelativePath != "" { + t.Fatalf("root = %q, want empty", sourceBundle.RootRelativePath) + } + if sourceBundle.Manifest.ID != "weather.daily.brentwood.2026-05-30" { + t.Fatalf("id = %q", sourceBundle.Manifest.ID) + } +} + +func TestValidateRejectsMissingFile(t *testing.T) { + backend := validFakeBundle(t) + deleteFakeFile(t, backend, "summary.txt") + _, err := Validate(context.Background(), backend, "") + assertErrorContains(t, err, "stat") +} + +func TestValidateRejectsSizeMismatch(t *testing.T) { + backend := validFakeBundle(t) + manifest := strings.Replace(string(readFixture(t, "testdata/valid_bundle/manifest.json")), `"size": 8`, `"size": 9`, 1) + writeFakeFile(t, backend, "manifest.json", manifest) + _, err := Validate(context.Background(), backend, "") + assertErrorContains(t, err, "size mismatch") +} + +func TestValidateRejectsPerFileDigestMismatch(t *testing.T) { + backend := validFakeBundle(t) + writeFakeFile(t, backend, "report.md", "# Report\nCloud.\n") + _, err := Validate(context.Background(), backend, "") + assertErrorContains(t, err, "sha256 mismatch") +} + +func TestValidateRejectsBundleDigestMismatch(t *testing.T) { + backend := validFakeBundle(t) + manifest := strings.Replace(string(readFixture(t, "testdata/valid_bundle/manifest.json")), `"digest": "sha256:099b205780d2b050024868399961b05731729a548d5d6329c7b06a6740dd75fe"`, `"digest": "sha256:0000000000000000000000000000000000000000000000000000000000000000"`, 1) + writeFakeFile(t, backend, "manifest.json", manifest) + _, err := Validate(context.Background(), backend, "") + assertErrorContains(t, err, "digest mismatch") +} + +func TestValidateRejectsSymlinkFile(t *testing.T) { + backend := validFakeBundle(t) + if err := backend.AddSymlink("summary.txt"); err != nil { + t.Fatalf("AddSymlink() error = %v", err) + } + _, err := Validate(context.Background(), backend, "") + assertErrorContains(t, err, "regular file") +} + +func validFakeBundle(t *testing.T) *fake.Backend { + t.Helper() + backend := fake.New() + writeFakeFile(t, backend, "manifest.json", string(readFixture(t, "testdata/valid_bundle/manifest.json"))) + writeFakeFile(t, backend, "report.md", string(readFixture(t, "testdata/valid_bundle/report.md"))) + writeFakeFile(t, backend, "summary.txt", string(readFixture(t, "testdata/valid_bundle/summary.txt"))) + return backend +} + +func writeFakeFile(t *testing.T, backend *fake.Backend, path, data string) { + t.Helper() + _, err := backend.WriteFile(context.Background(), path, []byte(data), storage.WriteOptions{Overwrite: true}) + if err != nil { + t.Fatalf("WriteFile(%q) error = %v", path, err) + } +} + +func deleteFakeFile(t *testing.T, backend *fake.Backend, path string) { + t.Helper() + err := backend.DeleteManagedBundle(context.Background(), "", []string{path}, storage.DeleteOptions{IgnoreMissing: true}) + if err != nil { + t.Fatalf("DeleteManagedBundle(%q) error = %v", path, err) + } +} diff --git a/internal/cli/inspect.go b/internal/cli/inspect.go index 9a103ac..a39b568 100644 --- a/internal/cli/inspect.go +++ b/internal/cli/inspect.go @@ -21,7 +21,7 @@ func inspectCommand(ctx context.Context, args []string, stdout, stderr io.Writer if len(args) == 1 { path = args[0] } - if err := app.Inspect(ctx, app.InspectOptions{Path: path}); err != nil { + if err := app.Inspect(ctx, app.InspectOptions{Path: path, Stdout: stdout}); err != nil { return fail(stderr, err) } return exitOK @@ -31,6 +31,6 @@ func printInspectHelp(w io.Writer) { fmt.Fprint(w, `Usage: distributor inspect -The inspect command is present but bundle inspection is not implemented yet. +Print a normalized summary of local source bundles. `) } diff --git a/internal/cli/root_test.go b/internal/cli/root_test.go index fea080b..bcad2ad 100644 --- a/internal/cli/root_test.go +++ b/internal/cli/root_test.go @@ -41,8 +41,8 @@ func TestExecuteVersion(t *testing.T) { } } -func TestPlaceholderCommandsFailClearly(t *testing.T) { - tests := []string{"run", "validate", "inspect"} +func TestRunWithoutDryRunFailsClearly(t *testing.T) { + tests := []string{"run"} for _, command := range tests { t.Run(command, func(t *testing.T) { @@ -63,6 +63,32 @@ func TestPlaceholderCommandsFailClearly(t *testing.T) { } } +func TestExecuteValidate(t *testing.T) { + var stdout, stderr bytes.Buffer + + code := Execute(context.Background(), []string{"validate", filepath.Join("..", "bundle", "testdata", "valid_bundle")}, &stdout, &stderr) + + if code != exitOK { + t.Fatalf("exit code = %d, want %d; stderr = %q", code, exitOK, stderr.String()) + } + if got, want := stdout.String(), "Validated 1 bundle(s)\n"; got != want { + t.Fatalf("stdout = %q, want %q", got, want) + } +} + +func TestExecuteInspect(t *testing.T) { + var stdout, stderr bytes.Buffer + + code := Execute(context.Background(), []string{"inspect", filepath.Join("..", "bundle", "testdata", "valid_bundle")}, &stdout, &stderr) + + if code != exitOK { + t.Fatalf("exit code = %d, want %d; stderr = %q", code, exitOK, stderr.String()) + } + if !strings.Contains(stdout.String(), "id=weather.daily.brentwood.2026-05-30") { + t.Fatalf("stdout = %q, want bundle summary", stdout.String()) + } +} + func TestExecuteRunDryRun(t *testing.T) { configPath := filepath.Join(t.TempDir(), "config.yml") err := os.WriteFile(configPath, []byte(` diff --git a/internal/cli/validate.go b/internal/cli/validate.go index 8eab57b..abfc112 100644 --- a/internal/cli/validate.go +++ b/internal/cli/validate.go @@ -21,7 +21,7 @@ func validateCommand(ctx context.Context, args []string, stdout, stderr io.Write if len(args) == 1 { path = args[0] } - if err := app.Validate(ctx, app.ValidateOptions{Path: path}); err != nil { + if err := app.Validate(ctx, app.ValidateOptions{Path: path, Stdout: stdout}); err != nil { return fail(stderr, err) } return exitOK @@ -31,6 +31,6 @@ func printValidateHelp(w io.Writer) { fmt.Fprint(w, `Usage: distributor validate -The validate command is present but bundle validation is not implemented yet. +Validate a local source bundle directory or a tree containing source bundles. `) }