Add manifest creation command

2026-06-01 21:02:34 +00:00
parent 04557f610d
commit 8b1e5abf68
6 changed files with 538 additions and 1 deletions
--- a/docs/cli.md
+++ b/docs/cli.md
@@ -16,12 +16,14 @@ distributor version [--format text|json]
 distributor run [--config <path>] [--dry-run] [--force] [--format text|json]
 distributor validate [--format text|json] <path>
 distributor inspect [--format text|json] <path>
+distributor manifest create <bundle-path> --id <bundle-id> [options]
 ```

 - `version`: prints the application name and version. Development builds print `distributor dev`.
 - `run`: loads a YAML config, discovers source bundles, plans each configured destination, writes selected outputs unless `--dry-run` is set, and prints a final status summary.
 - `validate`: validates a local source bundle directory or a local tree containing source bundles.
 - `inspect`: validates local source bundles and prints normalized bundle metadata.
+- `manifest create`: creates `manifest.json` for a local source bundle directory.

 `validate` and `inspect` accept local paths only. `run` executes `local`, `ssh`, and `s3` backends.

@@ -45,6 +47,13 @@ Output-producing subcommands:
 - `--dry-run`: load config, discover bundles, inspect destination state, print planned actions and final status, and do not write output files, destination state, or SSH `known_hosts` entries.
 - `--force`: allow explicit destructive replacement for supported conflict cases in this run only.

+`manifest create` flags:
+
+- `--id <bundle-id>`: source bundle id. Required.
+- `--file <path>`: bundle-relative file to include. Repeatable. If omitted, files are scanned recursively.
+- `--created <time>`: RFC3339 source created timestamp. If omitted, the current UTC time is used.
+- `--overwrite`: replace an existing `manifest.json`.
+
 `run` does not accept positional arguments. `validate` and `inspect` accept at most one path; omitting the path returns a required-path error.

 ## Common workflows
@@ -61,6 +70,22 @@ Inspect a source bundle:
 go run ./cmd/distributor inspect examples/source-bundle
 ```

+Create a manifest for a local producer bundle:
+
+```sh
+go run ./cmd/distributor manifest create <bundle-path> --id <bundle-id>
+```
+
+Create a manifest with explicit file order:
+
+```sh
+go run ./cmd/distributor manifest create <bundle-path> \
+  --id <bundle-id> \
+  --created 2026-06-01T11:00:00Z \
+  --file report.md \
+  --file summary.txt
+```
+
 Preview local publication without writing:

 ```sh
@@ -129,10 +154,11 @@ Command-specific JSON results:
 - `version`: application name and version.
 - `validate`: bundle count and discovered bundle identifiers.
 - `inspect`: bundle path, id, created timestamp, digest, file count, total size, and manifest file records.
+- `manifest create`: manifest path, bundle root, id, created timestamp, digest, file count, and file records.
 - `run`: dry-run status, pipeline summaries, destination action records, output records, final counters, warnings, and partial failure records.

 ## Diagnostics

-Use `validate` before publication when a producer has written a new bundle. Use `inspect` to confirm normalized ids, timestamps, digests, file paths, and file sizes.
+Use `manifest create` when a local producer has written bundle files but not `manifest.json`. Use `validate` before publication when a producer has written a new bundle. Use `inspect` to confirm normalized ids, timestamps, digests, file paths, and file sizes.

 For symptom-oriented recovery steps, see [troubleshooting](troubleshooting.md). For destination state and retry behavior, see [operations](operations.md). For config fields and defaults, see [configuration](config.md).
--- a/docs/operations.md
+++ b/docs/operations.md
@@ -100,6 +100,15 @@ if err != nil {

 Use `BuildManifest` and `WriteManifest` when a producer already wrote all bundle files into the final root. `BuildManifest` can preserve an explicit file order, or `Scan: true` can recursively include regular files under `Root` in deterministic slash-path order. Scan mode includes dotfiles, excludes files named `manifest.json` or `.distributor.json`, and rejects symlinks.

+Shell producers can create the same manifest through the CLI after writing bundle files:
+
+```sh
+go run ./cmd/distributor manifest create <bundle-path> --id reports.example.2026-05-30
+go run ./cmd/distributor validate <bundle-path>
+```
+
+Use repeated `--file` flags to preserve a specific file order. If no `--file` flags are provided, the command scans the bundle directory recursively using the same filtering rules as `pkg/bundle.BuildManifest`.
+
 ## Dry Runs

 `--dry-run` loads and validates config, discovers source bundles, inspects destination state, plans outputs, and prints summary lines. It does not write output files, destination state, or SSH `known_hosts` entries.
--- a/internal/app/manifest.go
+++ b/internal/app/manifest.go
@@ -0,0 +1,128 @@
+package app
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"path/filepath"
+	"strings"
+	"time"
+
+	producerbundle "gitea.maximumdirect.net/eric/distributor/pkg/bundle"
+)
+
+type ManifestCreateOptions struct {
+	Root         string
+	ID           string
+	Created      string
+	Files        []string
+	Overwrite    bool
+	Stdout       io.Writer
+	OutputFormat OutputFormat
+}
+
+func ManifestCreate(ctx context.Context, options ManifestCreateOptions) error {
+	if err := ValidateOutputFormat(options.OutputFormat); err != nil {
+		return err
+	}
+	if err := ctx.Err(); err != nil {
+		return err
+	}
+	if options.Root == "" {
+		return fmt.Errorf("manifest create command requires a bundle path")
+	}
+	if options.ID == "" {
+		return fmt.Errorf("manifest create command requires --id")
+	}
+
+	created, err := parseOptionalCreated(options.Created)
+	if err != nil {
+		return err
+	}
+	files := normalizeManifestFiles(options.Files)
+	buildOptions := producerbundle.BuildOptions{
+		Root:    options.Root,
+		ID:      options.ID,
+		Created: created,
+		Files:   files,
+		Scan:    len(files) == 0,
+	}
+	manifest, err := producerbundle.BuildManifest(buildOptions)
+	if err != nil {
+		return err
+	}
+	if err := producerbundle.WriteManifest(options.Root, manifest, producerbundle.WriteManifestOptions{Overwrite: options.Overwrite}); err != nil {
+		return err
+	}
+	loaded, err := producerbundle.LoadManifest(options.Root)
+	if err != nil {
+		return err
+	}
+	if err := producerbundle.ValidateBundle(options.Root, loaded); err != nil {
+		return err
+	}
+
+	result := manifestCreateResultFromManifest(options.Root, loaded)
+	if IsJSONOutput(options.OutputFormat) {
+		return WriteJSONEnvelope(options.Stdout, "manifest create", true, nil, result, nil)
+	}
+	if options.Stdout != nil {
+		_, err = fmt.Fprintf(options.Stdout, "created %s\nbundle: %s\nfiles: %d\ndigest: %s\n", producerbundle.ManifestName, result.ID, result.FileCount, result.Digest)
+	}
+	return err
+}
+
+func parseOptionalCreated(value string) (time.Time, error) {
+	if value == "" {
+		return time.Time{}, nil
+	}
+	created, err := time.Parse(time.RFC3339, value)
+	if err != nil {
+		return time.Time{}, fmt.Errorf("created must be RFC3339: %w", err)
+	}
+	return created, nil
+}
+
+func normalizeManifestFiles(files []string) []string {
+	normalized := make([]string, 0, len(files))
+	for _, file := range files {
+		normalized = append(normalized, filepath.ToSlash(filepath.Clean(strings.ReplaceAll(file, "\\", string(filepath.Separator)))))
+	}
+	return normalized
+}
+
+type manifestCreateResult struct {
+	ManifestPath string                     `json:"manifest_path"`
+	Root         string                     `json:"root"`
+	ID           string                     `json:"id"`
+	Created      string                     `json:"created"`
+	Digest       string                     `json:"digest"`
+	FileCount    int                        `json:"file_count"`
+	Files        []manifestCreateFileResult `json:"files"`
+}
+
+type manifestCreateFileResult struct {
+	Path   string `json:"path"`
+	SHA256 string `json:"sha256"`
+	Size   int64  `json:"size"`
+}
+
+func manifestCreateResultFromManifest(root string, manifest producerbundle.Manifest) manifestCreateResult {
+	result := manifestCreateResult{
+		ManifestPath: filepath.ToSlash(filepath.Join(root, producerbundle.ManifestName)),
+		Root:         filepath.ToSlash(root),
+		ID:           manifest.ID,
+		Created:      manifest.Created.Format(time.RFC3339),
+		Digest:       manifest.Digest,
+		FileCount:    len(manifest.Files),
+		Files:        make([]manifestCreateFileResult, 0, len(manifest.Files)),
+	}
+	for _, file := range manifest.Files {
+		result.Files = append(result.Files, manifestCreateFileResult{
+			Path:   file.Path,
+			SHA256: file.SHA256,
+			Size:   file.Size,
+		})
+	}
+	return result
+}
--- a/internal/cli/manifest.go
+++ b/internal/cli/manifest.go
@@ -0,0 +1,139 @@
+package cli
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"io"
+	"strings"
+
+	"gitea.maximumdirect.net/eric/distributor/internal/app"
+)
+
+func manifestCommand(ctx context.Context, args []string, stdout, stderr io.Writer) int {
+	if len(args) == 0 || args[0] == "-h" || args[0] == "--help" || args[0] == "help" {
+		printManifestHelp(stdout)
+		return exitOK
+	}
+	switch args[0] {
+	case "create":
+		return manifestCreateCommand(ctx, args[1:], stdout, stderr)
+	default:
+		fmt.Fprintf(stderr, "%s: manifest unknown command %q\n\n", app.Name, args[0])
+		printManifestHelp(stderr)
+		return exitUsage
+	}
+}
+
+func manifestCreateCommand(ctx context.Context, args []string, stdout, stderr io.Writer) int {
+	if hasHelp(args) {
+		printManifestCreateHelp(stdout)
+		return exitOK
+	}
+	flags := flag.NewFlagSet("manifest create", flag.ContinueOnError)
+	flags.SetOutput(stderr)
+	id := flags.String("id", "", "source bundle id")
+	created := flags.String("created", "", "source created timestamp")
+	overwrite := flags.Bool("overwrite", false, "replace an existing manifest.json")
+	formatFlag := addFormatFlag(flags)
+	var files repeatedFlag
+	flags.Var(&files, "file", "bundle-relative file to include")
+	flagArgs, positionalArgs, ok := splitManifestCreateArgs(stderr, args)
+	if !ok {
+		return exitUsage
+	}
+	if err := flags.Parse(flagArgs); err != nil {
+		return exitUsage
+	}
+	if len(positionalArgs) != 1 {
+		fmt.Fprintf(stderr, "%s: manifest create requires exactly one bundle path\n", app.Name)
+		return exitUsage
+	}
+	format, ok := parseOutputFormat(stderr, "manifest create", *formatFlag)
+	if !ok {
+		return exitUsage
+	}
+	err := app.ManifestCreate(ctx, app.ManifestCreateOptions{
+		Root:         positionalArgs[0],
+		ID:           *id,
+		Created:      *created,
+		Files:        []string(files),
+		Overwrite:    *overwrite,
+		Stdout:       stdout,
+		OutputFormat: format,
+	})
+	if err != nil {
+		return fail(stderr, err)
+	}
+	return exitOK
+}
+
+func splitManifestCreateArgs(stderr io.Writer, args []string) ([]string, []string, bool) {
+	var flagArgs []string
+	var positionalArgs []string
+	for index := 0; index < len(args); index++ {
+		arg := args[index]
+		switch arg {
+		case "--overwrite":
+			flagArgs = append(flagArgs, arg)
+		case "--id", "--created", "--file", "--format":
+			if index+1 >= len(args) {
+				fmt.Fprintf(stderr, "%s: manifest create %s requires a value\n", app.Name, arg)
+				return nil, nil, false
+			}
+			flagArgs = append(flagArgs, arg, args[index+1])
+			index++
+		default:
+			if strings.HasPrefix(arg, "--id=") ||
+				strings.HasPrefix(arg, "--created=") ||
+				strings.HasPrefix(arg, "--file=") ||
+				strings.HasPrefix(arg, "--format=") {
+				flagArgs = append(flagArgs, arg)
+				continue
+			}
+			if strings.HasPrefix(arg, "-") {
+				flagArgs = append(flagArgs, arg)
+				continue
+			}
+			positionalArgs = append(positionalArgs, arg)
+		}
+	}
+	return flagArgs, positionalArgs, true
+}
+
+type repeatedFlag []string
+
+func (f *repeatedFlag) String() string {
+	return fmt.Sprint([]string(*f))
+}
+
+func (f *repeatedFlag) Set(value string) error {
+	*f = append(*f, value)
+	return nil
+}
+
+func printManifestHelp(w io.Writer) {
+	fmt.Fprint(w, `Usage:
+  distributor manifest <command> [options]
+
+Commands:
+  create    Create a source bundle manifest
+
+Use "distributor manifest <command> --help" for command-specific help.
+`)
+}
+
+func printManifestCreateHelp(w io.Writer) {
+	fmt.Fprint(w, `Usage:
+  distributor manifest create <bundle-path> --id <bundle-id> [options]
+
+Options:
+  --id <bundle-id>     Source bundle id
+  --file <path>        Bundle-relative file to include; repeatable
+  --created <time>     RFC3339 source created timestamp
+  --overwrite          Replace an existing manifest.json
+  --format text|json   Output format
+
+Create manifest.json for a local source bundle directory.
+`)
+}
--- a/internal/cli/root.go
+++ b/internal/cli/root.go
@@ -33,6 +33,8 @@ func Execute(ctx context.Context, args []string, stdout, stderr io.Writer) int {
 		return validateCommand(ctx, args[1:], stdout, stderr)
 	case "inspect":
 		return inspectCommand(ctx, args[1:], stdout, stderr)
+	case "manifest":
+		return manifestCommand(ctx, args[1:], stdout, stderr)
 	default:
 		fmt.Fprintf(stderr, "%s: unknown command %q\n\n", app.Name, args[0])
 		printRootHelp(stderr)
@@ -51,6 +53,7 @@ Commands:
  run        Run configured distribution pipelines
  validate   Validate a source bundle or bundle tree
  inspect    Inspect bundles or distributor state
+  manifest   Create source bundle manifests

 Use "%s <command> --help" for command-specific help.
 `, app.Name, app.Name, app.Name)
--- a/internal/cli/root_test.go
+++ b/internal/cli/root_test.go
@@ -13,6 +13,7 @@ import (

 	"gitea.maximumdirect.net/eric/distributor/internal/storage"
 	"gitea.maximumdirect.net/eric/distributor/internal/testutil"
+	producerbundle "gitea.maximumdirect.net/eric/distributor/pkg/bundle"
 )

 func decodeEnvelope(t *testing.T, stdout *bytes.Buffer) map[string]any {
@@ -267,6 +268,206 @@ func TestExecuteInspectArgs(t *testing.T) {
 	}
 }

+func TestExecuteManifestCreateExplicitFiles(t *testing.T) {
+	root := t.TempDir()
+	writeCLIFile(t, root, "b.txt", "bravo")
+	writeCLIFile(t, root, "nested/a.txt", "alpha")
+	var stdout, stderr bytes.Buffer
+
+	code := Execute(context.Background(), []string{
+		"manifest", "create", root,
+		"--id", "reports.explicit",
+		"--created", "2026-06-01T11:00:00Z",
+		"--file", "b.txt",
+		"--file", "nested/a.txt",
+	}, &stdout, &stderr)
+
+	if code != exitOK {
+		t.Fatalf("exit code = %d, want %d; stderr = %q", code, exitOK, stderr.String())
+	}
+	for _, want := range []string{
+		"created manifest.json",
+		"bundle: reports.explicit",
+		"files: 2",
+		"digest: sha256:",
+	} {
+		if !strings.Contains(stdout.String(), want) {
+			t.Fatalf("stdout = %q, want substring %q", stdout.String(), want)
+		}
+	}
+	manifest, err := producerbundle.LoadManifest(root)
+	if err != nil {
+		t.Fatalf("LoadManifest() error = %v", err)
+	}
+	if got, want := manifestPaths(manifest), []string{"b.txt", "nested/a.txt"}; !equalStrings(got, want) {
+		t.Fatalf("manifest paths = %v, want %v", got, want)
+	}
+	if err := producerbundle.ValidateBundle(root, manifest); err != nil {
+		t.Fatalf("ValidateBundle() error = %v", err)
+	}
+	var validateStdout, validateStderr bytes.Buffer
+	validateCode := Execute(context.Background(), []string{"validate", root}, &validateStdout, &validateStderr)
+	if validateCode != exitOK {
+		t.Fatalf("validate exit code = %d, want %d; stderr = %q", validateCode, exitOK, validateStderr.String())
+	}
+}
+
+func TestExecuteManifestCreateScansBundle(t *testing.T) {
+	root := t.TempDir()
+	writeCLIFile(t, root, "z.txt", "zulu")
+	writeCLIFile(t, root, ".env", "dotfile")
+	writeCLIFile(t, root, "nested/report.md", "# Report\n")
+	writeCLIFile(t, root, storage.StateFileName, "destination state")
+	var stdout, stderr bytes.Buffer
+
+	code := Execute(context.Background(), []string{"manifest", "create", root, "--id", "reports.scan"}, &stdout, &stderr)
+
+	if code != exitOK {
+		t.Fatalf("exit code = %d, want %d; stderr = %q", code, exitOK, stderr.String())
+	}
+	manifest, err := producerbundle.LoadManifest(root)
+	if err != nil {
+		t.Fatalf("LoadManifest() error = %v", err)
+	}
+	if got, want := manifestPaths(manifest), []string{".env", "nested/report.md", "z.txt"}; !equalStrings(got, want) {
+		t.Fatalf("manifest paths = %v, want %v", got, want)
+	}
+}
+
+func TestExecuteManifestCreateJSON(t *testing.T) {
+	root := t.TempDir()
+	writeCLIFile(t, root, "report.md", "# Report\n")
+	var stdout, stderr bytes.Buffer
+
+	code := Execute(context.Background(), []string{"manifest", "create", root, "--id", "reports.json", "--file", "report.md", "--format", "json"}, &stdout, &stderr)
+
+	if code != exitOK {
+		t.Fatalf("exit code = %d, want %d; stderr = %q", code, exitOK, stderr.String())
+	}
+	envelope := decodeEnvelope(t, &stdout)
+	if envelope["command"] != "manifest create" || envelope["ok"] != true {
+		t.Fatalf("envelope = %#v, want manifest create ok", envelope)
+	}
+	result := envelopeResult(t, envelope)
+	if result["id"] != "reports.json" || result["file_count"] != float64(1) {
+		t.Fatalf("result = %#v, want manifest summary", result)
+	}
+	if stderr.Len() != 0 {
+		t.Fatalf("stderr = %q, want empty", stderr.String())
+	}
+}
+
+func TestExecuteManifestCreateOverwrite(t *testing.T) {
+	root := t.TempDir()
+	writeCLIFile(t, root, "report.md", "old\n")
+	var stdout, stderr bytes.Buffer
+	code := Execute(context.Background(), []string{"manifest", "create", root, "--id", "reports.old", "--file", "report.md"}, &stdout, &stderr)
+	if code != exitOK {
+		t.Fatalf("initial exit code = %d, want %d; stderr = %q", code, exitOK, stderr.String())
+	}
+	writeCLIFile(t, root, "report.md", "new\n")
+
+	stdout.Reset()
+	stderr.Reset()
+	code = Execute(context.Background(), []string{"manifest", "create", root, "--id", "reports.new", "--file", "report.md"}, &stdout, &stderr)
+	if code != exitError {
+		t.Fatalf("overwrite exit code = %d, want %d", code, exitError)
+	}
+	if !strings.Contains(stderr.String(), "write manifest") {
+		t.Fatalf("stderr = %q, want write manifest error", stderr.String())
+	}
+	manifest, err := producerbundle.LoadManifest(root)
+	if err != nil {
+		t.Fatalf("LoadManifest() error = %v", err)
+	}
+	if manifest.ID != "reports.old" {
+		t.Fatalf("manifest id = %q, want reports.old", manifest.ID)
+	}
+
+	stdout.Reset()
+	stderr.Reset()
+	code = Execute(context.Background(), []string{"manifest", "create", root, "--id", "reports.new", "--file", "report.md", "--overwrite"}, &stdout, &stderr)
+	if code != exitOK {
+		t.Fatalf("overwrite exit code = %d, want %d; stderr = %q", code, exitOK, stderr.String())
+	}
+	manifest, err = producerbundle.LoadManifest(root)
+	if err != nil {
+		t.Fatalf("LoadManifest() error = %v", err)
+	}
+	if manifest.ID != "reports.new" {
+		t.Fatalf("manifest id = %q, want reports.new", manifest.ID)
+	}
+}
+
+func TestExecuteManifestCreateRejectsSymlink(t *testing.T) {
+	root := t.TempDir()
+	writeCLIFile(t, root, "target.md", "# Report\n")
+	if err := os.Symlink("target.md", filepath.Join(root, "link.md")); err != nil {
+		t.Skipf("symlink unavailable: %v", err)
+	}
+	var stdout, stderr bytes.Buffer
+
+	code := Execute(context.Background(), []string{"manifest", "create", root, "--id", "reports.link", "--file", "link.md"}, &stdout, &stderr)
+
+	if code != exitError {
+		t.Fatalf("exit code = %d, want %d", code, exitError)
+	}
+	if !strings.Contains(stderr.String(), "regular file") {
+		t.Fatalf("stderr = %q, want regular file error", stderr.String())
+	}
+}
+
+func TestExecuteManifestCreateArgs(t *testing.T) {
+	root := t.TempDir()
+	writeCLIFile(t, root, "report.md", "# Report\n")
+	tests := []struct {
+		name       string
+		args       []string
+		wantCode   int
+		wantStderr string
+	}{
+		{
+			name:       "missing path",
+			args:       []string{"manifest", "create", "--id", "reports.missing"},
+			wantCode:   exitUsage,
+			wantStderr: "requires exactly one bundle path",
+		},
+		{
+			name:       "missing id",
+			args:       []string{"manifest", "create", root},
+			wantCode:   exitError,
+			wantStderr: "requires --id",
+		},
+		{
+			name:       "bad created",
+			args:       []string{"manifest", "create", root, "--id", "reports.bad", "--created", "June 1"},
+			wantCode:   exitError,
+			wantStderr: "created must be RFC3339",
+		},
+		{
+			name:       "bad format",
+			args:       []string{"manifest", "create", root, "--id", "reports.bad", "--format", "xml"},
+			wantCode:   exitUsage,
+			wantStderr: "format must be text or json",
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			var stdout, stderr bytes.Buffer
+			code := Execute(context.Background(), tt.args, &stdout, &stderr)
+			if code != tt.wantCode {
+				t.Fatalf("exit code = %d, want %d; stderr = %q", code, tt.wantCode, stderr.String())
+			}
+			if stdout.Len() != 0 {
+				t.Fatalf("stdout = %q, want empty", stdout.String())
+			}
+			if !strings.Contains(stderr.String(), tt.wantStderr) {
+				t.Fatalf("stderr = %q, want substring %q", stderr.String(), tt.wantStderr)
+			}
+		})
+	}
+}
+
 func TestExecuteRunDryRun(t *testing.T) {
 	sourceRoot := t.TempDir()
 	testutil.WriteSourceBundle(t, sourceRoot, "", testutil.BundleOptions{})
@@ -512,3 +713,34 @@ func TestUnknownCommandIsUsageError(t *testing.T) {
 		t.Fatalf("stderr = %q, want unknown command error", stderr.String())
 	}
 }
+
+func manifestPaths(manifest producerbundle.Manifest) []string {
+	paths := make([]string, 0, len(manifest.Files))
+	for _, file := range manifest.Files {
+		paths = append(paths, file.Path)
+	}
+	return paths
+}
+
+func equalStrings(a, b []string) bool {
+	if len(a) != len(b) {
+		return false
+	}
+	for index := range a {
+		if a[index] != b[index] {
+			return false
+		}
+	}
+	return true
+}
+
+func writeCLIFile(t *testing.T, root, relative, body string) {
+	t.Helper()
+	path := filepath.Join(root, filepath.FromSlash(relative))
+	if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
+		t.Fatalf("mkdir %s: %v", filepath.Dir(path), err)
+	}
+	if err := os.WriteFile(path, []byte(body), 0o600); err != nil {
+		t.Fatalf("write %s: %v", path, err)
+	}
+}