diff --git a/README.md b/README.md index 18ba95e..3291250 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ It is a local-first CLI with SSH/SFTP and S3-compatible storage support: source bundles can be read from local or remote storage, destinations can be local directories or remote paths, and Markdown files can be rendered to HTML sidecars. -Go producers can use `gitea.maximumdirect.net/eric/distributor/pkg/bundle` to build, parse, write, and validate source bundle manifests with the same contract used by `distributor`. +Go producers can use `gitea.maximumdirect.net/eric/distributor/pkg/bundle` to build, write, parse, and validate complete local source bundles with the same manifest contract used by `distributor`. Run the local example pipeline: diff --git a/docs/internal/bundle.md b/docs/internal/bundle.md index 026abca..2d6181d 100644 --- a/docs/internal/bundle.md +++ b/docs/internal/bundle.md @@ -2,7 +2,7 @@ ## Purpose -`internal/bundle` discovers and validates source bundles through the storage interface. The source manifest model, manifest parsing, manifest validation, path rules, and digest calculation come from `pkg/bundle` so producer-facing APIs and distributor validation share one manifest contract. +`internal/bundle` discovers and validates source bundles through the storage interface. The source manifest model, manifest parsing, manifest validation, path rules, digest calculation, and producer-side local writer come from `pkg/bundle` so producer-facing APIs and distributor validation share one manifest contract. ## Inputs and outputs @@ -38,7 +38,7 @@ Manifest parsing and validation fail before destination planning. Storage-backed ## Boundaries -Internal bundle discovery uses `internal/storage` and does not import concrete adapters. Producer-side local filesystem manifest building and validation belong to `pkg/bundle`. CLI local path support is wired in `internal/app`. +Internal bundle discovery uses `internal/storage` and does not import concrete adapters. Producer-side local filesystem manifest building, complete bundle writing, and validation belong to `pkg/bundle`. CLI local path support is wired in `internal/app`. ## Tests diff --git a/docs/operations.md b/docs/operations.md index 886b347..7bed15b 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -76,27 +76,29 @@ Each published destination bundle contains `.distributor.json`. This file is the Do not edit `.distributor.json` by hand during normal operation. If it is missing or invalid while destination files remain, `distributor` treats the destination as unmanaged or conflicted. -## Go Producer Manifests +## Go Producer Bundles -Go producer applications can import `gitea.maximumdirect.net/eric/distributor/pkg/bundle` to create source manifests with the same path, digest, timestamp, and validation rules used by `distributor`. +Go producer applications can import `gitea.maximumdirect.net/eric/distributor/pkg/bundle` to create complete local source bundles with the same path, digest, timestamp, and validation rules used by `distributor`. -Minimal producer-side manifest creation: +Minimal producer-side bundle creation: ```go -manifest, err := bundle.BuildManifest(bundle.BuildOptions{ +manifest, err := bundle.WriteBundle(bundle.WriteBundleOptions{ Root: outputDir, ID: "reports.example.2026-05-30", - Files: []string{"report.md", "summary.txt"}, + Files: []bundle.BundleFile{ + {SourcePath: reportPath, Path: "report.md"}, + {SourcePath: summaryPath, Path: "summary.txt"}, + }, }) if err != nil { return err } -if err := bundle.WriteManifest(outputDir, manifest, bundle.WriteManifestOptions{}); err != nil { - return err -} ``` -Use explicit `Files` to preserve caller order, or `Scan: true` to recursively include regular files under `Root` in deterministic slash-path order. Scan mode includes dotfiles, excludes files named `manifest.json` or `.distributor.json`, and rejects symlinks. +`WriteBundle` copies local producer files into a sibling temporary directory, writes `manifest.json`, validates the result, and promotes the completed bundle into place. It fails if `Root` already exists unless `Overwrite` is true. With overwrite enabled, it builds and validates the replacement before moving the existing root aside. + +Use `BuildManifest` and `WriteManifest` when a producer already wrote all bundle files into the final root. `BuildManifest` can preserve an explicit file order, or `Scan: true` can recursively include regular files under `Root` in deterministic slash-path order. Scan mode includes dotfiles, excludes files named `manifest.json` or `.distributor.json`, and rejects symlinks. ## Dry Runs diff --git a/docs/policy/architecture.md b/docs/policy/architecture.md index a631b73..8d69d3d 100644 --- a/docs/policy/architecture.md +++ b/docs/policy/architecture.md @@ -187,7 +187,7 @@ Avoid dependencies for small conveniences. Do not let external dependency types Use this current layout unless the project has a documented reason to differ: - `cmd/distributor`: application entrypoint only. -- `pkg/bundle`: public producer-facing source manifest model, digest logic, parsing, building, manifest writing, and local validation helpers. +- `pkg/bundle`: public producer-facing source manifest model, digest logic, parsing, manifest building, complete local bundle writing, and local validation helpers. - `internal/app`: application orchestration and top-level use cases. - `internal/cli`: CLI command definitions, flags, argument parsing, and command wiring. - `internal/config`: configuration structs, defaults, loading, precedence, and validation. diff --git a/docs/policy/development.md b/docs/policy/development.md index b07a9cf..e4de34a 100644 --- a/docs/policy/development.md +++ b/docs/policy/development.md @@ -6,7 +6,7 @@ Use it with `docs/policy/architecture.md` and `docs/policy/documentation.md`. ## Repository Layout - `cmd/distributor`: executable entrypoint only. -- `pkg/bundle`: public producer-facing source manifest helpers. +- `pkg/bundle`: public producer-facing source manifest and local bundle writer helpers. - `internal/app`: top-level use cases for `run`, `validate`, and `inspect`. - `internal/cli`: standard-library command parsing, flags, help text, and command wiring. - `internal/config`: YAML configuration structs, loading, defaults, and validation. diff --git a/pkg/bundle/doc.go b/pkg/bundle/doc.go index 85a6854..12fb6de 100644 --- a/pkg/bundle/doc.go +++ b/pkg/bundle/doc.go @@ -3,6 +3,7 @@ // // A source bundle is a local directory containing a manifest.json file and the // files listed by that manifest. This package owns the public manifest model, -// digest calculation, path validation, manifest parsing, manifest building, and -// local bundle validation used by Go producer applications. +// digest calculation, path validation, manifest parsing, manifest building, +// local bundle writing, and local bundle validation used by Go producer +// applications. package bundle diff --git a/pkg/bundle/example_test.go b/pkg/bundle/example_test.go index 213c3e6..8b14870 100644 --- a/pkg/bundle/example_test.go +++ b/pkg/bundle/example_test.go @@ -38,3 +38,36 @@ func ExampleBuildManifest() { // report.txt // 7 } + +func ExampleWriteBundle() { + sourceRoot, err := os.MkdirTemp("", "distributor-source-*") + if err != nil { + panic(err) + } + defer os.RemoveAll(sourceRoot) + outputRoot := filepath.Join(os.TempDir(), "distributor-bundle-example") + defer os.RemoveAll(outputRoot) + + if err := os.WriteFile(filepath.Join(sourceRoot, "report.txt"), []byte("report\n"), 0o600); err != nil { + panic(err) + } + + manifest, err := bundle.WriteBundle(bundle.WriteBundleOptions{ + Root: outputRoot, + ID: "reports.example", + Created: time.Date(2026, 5, 30, 11, 10, 0, 0, time.UTC), + Files: []bundle.BundleFile{ + {SourcePath: filepath.Join(sourceRoot, "report.txt"), Path: "report.txt"}, + }, + Overwrite: true, + }) + if err != nil { + panic(err) + } + + fmt.Println(manifest.ID) + fmt.Println(manifest.Files[0].Path) + // Output: + // reports.example + // report.txt +} diff --git a/pkg/bundle/types.go b/pkg/bundle/types.go index faf2253..bbcde40 100644 --- a/pkg/bundle/types.go +++ b/pkg/bundle/types.go @@ -31,3 +31,16 @@ type BuildOptions struct { type WriteManifestOptions struct { Overwrite bool } + +type BundleFile struct { + SourcePath string + Path string +} + +type WriteBundleOptions struct { + Root string + ID string + Created time.Time + Files []BundleFile + Overwrite bool +} diff --git a/pkg/bundle/writer.go b/pkg/bundle/writer.go new file mode 100644 index 0000000..33b6a7d --- /dev/null +++ b/pkg/bundle/writer.go @@ -0,0 +1,187 @@ +package bundle + +import ( + "errors" + "fmt" + "io" + "os" + "path/filepath" +) + +func WriteBundle(opts WriteBundleOptions) (Manifest, error) { + root, err := cleanBundleRoot(opts.Root) + if err != nil { + return Manifest{}, err + } + if opts.ID == "" { + return Manifest{}, fmt.Errorf("id is required") + } + if len(opts.Files) == 0 { + return Manifest{}, fmt.Errorf("files is required") + } + + parent := filepath.Dir(root) + if err := os.MkdirAll(parent, 0o755); err != nil { + return Manifest{}, fmt.Errorf("create bundle parent: %w", err) + } + if !opts.Overwrite { + if _, err := os.Lstat(root); err == nil { + return Manifest{}, fmt.Errorf("bundle root %q already exists", root) + } else if !errors.Is(err, os.ErrNotExist) { + return Manifest{}, fmt.Errorf("stat bundle root: %w", err) + } + } + + tempRoot, err := os.MkdirTemp(parent, "."+filepath.Base(root)+"-*.tmp") + if err != nil { + return Manifest{}, fmt.Errorf("create bundle temp root: %w", err) + } + removeTemp := true + defer func() { + if removeTemp { + _ = os.RemoveAll(tempRoot) + } + }() + + paths, err := copyBundleFiles(tempRoot, opts.Files) + if err != nil { + return Manifest{}, err + } + manifest, err := BuildManifest(BuildOptions{ + Root: tempRoot, + ID: opts.ID, + Created: opts.Created, + Files: paths, + }) + if err != nil { + return Manifest{}, err + } + if err := WriteManifest(tempRoot, manifest, WriteManifestOptions{}); err != nil { + return Manifest{}, err + } + if err := ValidateBundle(tempRoot, manifest); err != nil { + return Manifest{}, err + } + + if err := promoteBundleRoot(tempRoot, root, opts.Overwrite); err != nil { + return Manifest{}, err + } + removeTemp = false + return manifest, nil +} + +func cleanBundleRoot(root string) (string, error) { + if root == "" { + return "", fmt.Errorf("root is required") + } + return filepath.Clean(root), nil +} + +func copyBundleFiles(root string, files []BundleFile) ([]string, error) { + paths := make([]string, 0, len(files)) + seen := make(map[string]struct{}, len(files)) + for index, file := range files { + if file.SourcePath == "" { + return nil, fmt.Errorf("files[%d].source_path is required", index) + } + if err := ValidateSourcePath(file.Path); err != nil { + return nil, fmt.Errorf("files[%d].path: %w", index, err) + } + if _, exists := seen[file.Path]; exists { + return nil, fmt.Errorf("files[%d].path duplicates %q", index, file.Path) + } + seen[file.Path] = struct{}{} + if err := copyBundleFile(root, file); err != nil { + return nil, fmt.Errorf("files[%d]: %w", index, err) + } + paths = append(paths, file.Path) + } + return paths, nil +} + +func copyBundleFile(root string, file BundleFile) error { + info, err := os.Lstat(file.SourcePath) + if err != nil { + return fmt.Errorf("stat source %q: %w", file.SourcePath, err) + } + if !info.Mode().IsRegular() { + return fmt.Errorf("source %q must be a regular file", file.SourcePath) + } + destination := filepath.Join(root, filepath.FromSlash(file.Path)) + if err := os.MkdirAll(filepath.Dir(destination), 0o755); err != nil { + return fmt.Errorf("create destination directory: %w", err) + } + + source, err := os.Open(file.SourcePath) + if err != nil { + return fmt.Errorf("open source %q: %w", file.SourcePath, err) + } + defer source.Close() + + mode := info.Mode().Perm() + if mode == 0 { + mode = 0o600 + } + target, err := os.OpenFile(destination, os.O_WRONLY|os.O_CREATE|os.O_EXCL, mode) + if err != nil { + return fmt.Errorf("create destination %q: %w", file.Path, err) + } + if _, err := io.Copy(target, source); err != nil { + _ = target.Close() + return fmt.Errorf("copy to destination %q: %w", file.Path, err) + } + if err := target.Close(); err != nil { + return fmt.Errorf("close destination %q: %w", file.Path, err) + } + return nil +} + +func promoteBundleRoot(tempRoot, root string, overwrite bool) error { + if !overwrite { + if _, err := os.Lstat(root); err == nil { + return fmt.Errorf("bundle root %q already exists", root) + } else if !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("stat bundle root: %w", err) + } + if err := os.Rename(tempRoot, root); err != nil { + return fmt.Errorf("promote bundle root: %w", err) + } + return nil + } + if _, err := os.Lstat(root); errors.Is(err, os.ErrNotExist) { + if err := os.Rename(tempRoot, root); err != nil { + return fmt.Errorf("promote bundle root: %w", err) + } + return nil + } else if err != nil { + return fmt.Errorf("stat bundle root: %w", err) + } + + backupRoot, err := reserveSiblingPath(filepath.Dir(root), "."+filepath.Base(root)+"-backup-*.tmp") + if err != nil { + return err + } + if err := os.Rename(root, backupRoot); err != nil { + return fmt.Errorf("move existing bundle root: %w", err) + } + if err := os.Rename(tempRoot, root); err != nil { + restoreErr := os.Rename(backupRoot, root) + if restoreErr != nil { + return fmt.Errorf("promote bundle root: %w; restore existing bundle root: %v", err, restoreErr) + } + return fmt.Errorf("promote bundle root: %w", err) + } + _ = os.RemoveAll(backupRoot) + return nil +} + +func reserveSiblingPath(parent, pattern string) (string, error) { + path, err := os.MkdirTemp(parent, pattern) + if err != nil { + return "", fmt.Errorf("reserve backup path: %w", err) + } + if err := os.Remove(path); err != nil { + return "", fmt.Errorf("reserve backup path: %w", err) + } + return path, nil +} diff --git a/pkg/bundle/writer_integration_test.go b/pkg/bundle/writer_integration_test.go new file mode 100644 index 0000000..a70625f --- /dev/null +++ b/pkg/bundle/writer_integration_test.go @@ -0,0 +1,38 @@ +package bundle_test + +import ( + "bytes" + "context" + "os" + "path/filepath" + "testing" + + "gitea.maximumdirect.net/eric/distributor/internal/app" + "gitea.maximumdirect.net/eric/distributor/pkg/bundle" +) + +func TestWriteBundleOutputValidatesThroughDistributor(t *testing.T) { + sourceRoot := t.TempDir() + outputRoot := filepath.Join(t.TempDir(), "bundle") + if err := os.WriteFile(filepath.Join(sourceRoot, "report.md"), []byte("# Report\n"), 0o600); err != nil { + t.Fatalf("write source: %v", err) + } + + if _, err := bundle.WriteBundle(bundle.WriteBundleOptions{ + Root: outputRoot, + ID: "reports.distributor", + Files: []bundle.BundleFile{ + {SourcePath: filepath.Join(sourceRoot, "report.md"), Path: "report.md"}, + }, + }); err != nil { + t.Fatalf("WriteBundle() error = %v", err) + } + + var stdout bytes.Buffer + if err := app.Validate(context.Background(), app.ValidateOptions{Path: outputRoot, Stdout: &stdout}); err != nil { + t.Fatalf("app.Validate() error = %v", err) + } + if got, want := stdout.String(), "Validated 1 bundle(s)\n"; got != want { + t.Fatalf("stdout = %q, want %q", got, want) + } +} diff --git a/pkg/bundle/writer_test.go b/pkg/bundle/writer_test.go new file mode 100644 index 0000000..45e9cd9 --- /dev/null +++ b/pkg/bundle/writer_test.go @@ -0,0 +1,232 @@ +package bundle + +import ( + "os" + "path/filepath" + "reflect" + "strings" + "testing" + "time" +) + +func TestWriteBundleCreatesCompleteBundle(t *testing.T) { + sourceRoot := t.TempDir() + outputRoot := filepath.Join(t.TempDir(), "bundle") + writeFile(t, sourceRoot, "report.md", "# Report\n") + writeFile(t, sourceRoot, "summary.txt", "Summary\n") + created := time.Date(2026, 5, 30, 11, 10, 0, 0, time.UTC) + + manifest, err := WriteBundle(WriteBundleOptions{ + Root: outputRoot, + ID: "reports.writer", + Created: created, + Files: []BundleFile{ + {SourcePath: filepath.Join(sourceRoot, "summary.txt"), Path: "nested/summary.txt"}, + {SourcePath: filepath.Join(sourceRoot, "report.md"), Path: "report.md"}, + }, + }) + if err != nil { + t.Fatalf("WriteBundle() error = %v", err) + } + + if !manifest.Created.Equal(created) { + t.Fatalf("created = %s, want %s", manifest.Created, created) + } + if paths := manifestPaths(manifest); !reflect.DeepEqual(paths, []string{"nested/summary.txt", "report.md"}) { + t.Fatalf("paths = %v, want caller order", paths) + } + if got := readFile(t, outputRoot, "nested/summary.txt"); got != "Summary\n" { + t.Fatalf("nested summary = %q", got) + } + if got := readFile(t, outputRoot, "report.md"); got != "# Report\n" { + t.Fatalf("report = %q", got) + } + loaded, err := LoadManifest(outputRoot) + if err != nil { + t.Fatalf("LoadManifest() error = %v", err) + } + if !reflect.DeepEqual(loaded, manifest) { + t.Fatalf("loaded manifest = %#v, want %#v", loaded, manifest) + } + if err := ValidateBundle(outputRoot, loaded); err != nil { + t.Fatalf("ValidateBundle() error = %v", err) + } +} + +func TestWriteBundleDefaultsCreated(t *testing.T) { + sourceRoot := t.TempDir() + outputRoot := filepath.Join(t.TempDir(), "bundle") + writeFile(t, sourceRoot, "report.md", "# Report\n") + before := time.Now().UTC() + + manifest, err := WriteBundle(WriteBundleOptions{ + Root: outputRoot, + ID: "reports.created", + Files: []BundleFile{ + {SourcePath: filepath.Join(sourceRoot, "report.md"), Path: "report.md"}, + }, + }) + if err != nil { + t.Fatalf("WriteBundle() error = %v", err) + } + after := time.Now().UTC() + if manifest.Created.Before(before) || manifest.Created.After(after) { + t.Fatalf("created = %s, want between %s and %s", manifest.Created, before, after) + } +} + +func TestWriteBundleRejectsDuplicatePaths(t *testing.T) { + sourceRoot := t.TempDir() + writeFile(t, sourceRoot, "report.md", "# Report\n") + + _, err := WriteBundle(WriteBundleOptions{ + Root: filepath.Join(t.TempDir(), "bundle"), + ID: "reports.duplicate", + Files: []BundleFile{ + {SourcePath: filepath.Join(sourceRoot, "report.md"), Path: "report.md"}, + {SourcePath: filepath.Join(sourceRoot, "report.md"), Path: "report.md"}, + }, + }) + if err == nil || !strings.Contains(err.Error(), "duplicates") { + t.Fatalf("WriteBundle() error = %v, want duplicate path error", err) + } +} + +func TestWriteBundleRejectsSymlinkSource(t *testing.T) { + sourceRoot := t.TempDir() + writeFile(t, sourceRoot, "target.md", "# Report\n") + linkPath := filepath.Join(sourceRoot, "link.md") + if err := os.Symlink("target.md", linkPath); err != nil { + t.Skipf("symlink unavailable: %v", err) + } + + _, err := WriteBundle(WriteBundleOptions{ + Root: filepath.Join(t.TempDir(), "bundle"), + ID: "reports.symlink", + Files: []BundleFile{ + {SourcePath: linkPath, Path: "report.md"}, + }, + }) + if err == nil || !strings.Contains(err.Error(), "regular file") { + t.Fatalf("WriteBundle() error = %v, want regular file error", err) + } +} + +func TestWriteBundleDoesNotOverwriteByDefault(t *testing.T) { + sourceRoot := t.TempDir() + outputRoot := filepath.Join(t.TempDir(), "bundle") + writeFile(t, sourceRoot, "report.md", "old\n") + if _, err := WriteBundle(WriteBundleOptions{ + Root: outputRoot, + ID: "reports.old", + Files: []BundleFile{ + {SourcePath: filepath.Join(sourceRoot, "report.md"), Path: "report.md"}, + }, + }); err != nil { + t.Fatalf("initial WriteBundle() error = %v", err) + } + writeFile(t, sourceRoot, "report.md", "new\n") + + _, err := WriteBundle(WriteBundleOptions{ + Root: outputRoot, + ID: "reports.new", + Files: []BundleFile{ + {SourcePath: filepath.Join(sourceRoot, "report.md"), Path: "report.md"}, + }, + }) + if err == nil || !strings.Contains(err.Error(), "already exists") { + t.Fatalf("WriteBundle() error = %v, want exists error", err) + } + if got := readFile(t, outputRoot, "report.md"); got != "old\n" { + t.Fatalf("report = %q, want old content", got) + } +} + +func TestWriteBundleOverwritesExistingRoot(t *testing.T) { + sourceRoot := t.TempDir() + outputRoot := filepath.Join(t.TempDir(), "bundle") + writeFile(t, sourceRoot, "old.md", "old\n") + if _, err := WriteBundle(WriteBundleOptions{ + Root: outputRoot, + ID: "reports.old", + Files: []BundleFile{ + {SourcePath: filepath.Join(sourceRoot, "old.md"), Path: "old.md"}, + }, + }); err != nil { + t.Fatalf("initial WriteBundle() error = %v", err) + } + writeFile(t, sourceRoot, "new.md", "new\n") + + manifest, err := WriteBundle(WriteBundleOptions{ + Root: outputRoot, + ID: "reports.new", + Overwrite: true, + Files: []BundleFile{ + {SourcePath: filepath.Join(sourceRoot, "new.md"), Path: "new.md"}, + }, + }) + if err != nil { + t.Fatalf("WriteBundle(overwrite) error = %v", err) + } + + if manifest.ID != "reports.new" { + t.Fatalf("manifest id = %q, want reports.new", manifest.ID) + } + if _, err := os.Stat(filepath.Join(outputRoot, "old.md")); !os.IsNotExist(err) { + t.Fatalf("old file stat error = %v, want not exist", err) + } + if got := readFile(t, outputRoot, "new.md"); got != "new\n" { + t.Fatalf("new file = %q", got) + } + if err := ValidateBundle(outputRoot, manifest); err != nil { + t.Fatalf("ValidateBundle() error = %v", err) + } +} + +func TestWriteBundleKeepsExistingRootWhenReplacementBuildFails(t *testing.T) { + sourceRoot := t.TempDir() + outputRoot := filepath.Join(t.TempDir(), "bundle") + writeFile(t, sourceRoot, "report.md", "old\n") + oldManifest, err := WriteBundle(WriteBundleOptions{ + Root: outputRoot, + ID: "reports.old", + Files: []BundleFile{ + {SourcePath: filepath.Join(sourceRoot, "report.md"), Path: "report.md"}, + }, + }) + if err != nil { + t.Fatalf("initial WriteBundle() error = %v", err) + } + + _, err = WriteBundle(WriteBundleOptions{ + Root: outputRoot, + ID: "reports.new", + Overwrite: true, + Files: []BundleFile{ + {SourcePath: filepath.Join(sourceRoot, "missing.md"), Path: "report.md"}, + }, + }) + if err == nil { + t.Fatal("WriteBundle(overwrite) error = nil, want missing source error") + } + + loaded, err := LoadManifest(outputRoot) + if err != nil { + t.Fatalf("LoadManifest() error = %v", err) + } + if !reflect.DeepEqual(loaded, oldManifest) { + t.Fatalf("loaded manifest = %#v, want old manifest %#v", loaded, oldManifest) + } + if got := readFile(t, outputRoot, "report.md"); got != "old\n" { + t.Fatalf("report = %q, want old content", got) + } +} + +func readFile(t *testing.T, root, relative string) string { + t.Helper() + data, err := os.ReadFile(filepath.Join(root, filepath.FromSlash(relative))) + if err != nil { + t.Fatalf("read %s: %v", relative, err) + } + return string(data) +}