From 29dbad29673d66708714196a6b0a2bd9465e9f8b Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Sun, 31 May 2026 01:53:13 +0000 Subject: [PATCH] Add config loading and dry-run validation --- README.md | 6 +- docs/cli.md | 23 ++- docs/config.md | 96 +++++++++++ examples/fan-out.yml | 32 ++++ examples/local-to-local.yml | 9 ++ go.mod | 2 + go.sum | 4 + internal/app/run.go | 50 +++++- internal/app/run_test.go | 56 +++++++ internal/cli/root_test.go | 34 ++++ internal/cli/run.go | 29 +++- internal/config/config.go | 69 +++++++- internal/config/defaults.go | 47 ++++++ internal/config/load.go | 29 ++++ internal/config/load_test.go | 306 +++++++++++++++++++++++++++++++++++ internal/config/validate.go | 153 ++++++++++++++++++ 16 files changed, 928 insertions(+), 17 deletions(-) create mode 100644 docs/config.md create mode 100644 examples/fan-out.yml create mode 100644 examples/local-to-local.yml create mode 100644 go.sum create mode 100644 internal/app/run_test.go create mode 100644 internal/config/load.go create mode 100644 internal/config/load_test.go create mode 100644 internal/config/validate.go diff --git a/README.md b/README.md index 8968ed2..12b219c 100644 --- a/README.md +++ b/README.md @@ -2,10 +2,10 @@ `distributor` is a Go application shell for validating and publishing manifested report bundles. -Current implemented behavior is limited to CLI help, version output, and placeholder operational commands: +Current implemented behavior is limited to CLI help, version output, config validation, and placeholder operational commands: ```sh -go run ./cmd/distributor --help +go run ./cmd/distributor run --config examples/local-to-local.yml --dry-run ``` -See [docs/cli.md](docs/cli.md) for the implemented CLI surface. Current design and implementation planning lives under `docs/roadmap/`. +See [docs/cli.md](docs/cli.md) and [docs/config.md](docs/config.md) for the implemented CLI and configuration surface. Current design and implementation planning lives under `docs/roadmap/`. diff --git a/docs/cli.md b/docs/cli.md index f20ec1b..26bebf5 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -3,10 +3,10 @@ ## Shortest useful command ```sh -go run ./cmd/distributor --help +go run ./cmd/distributor run --config examples/local-to-local.yml --dry-run ``` -This prints the currently implemented command shell. +This loads and validates the example config, then prints the resolved pipeline summary without publishing files. ## Command overview @@ -20,7 +20,9 @@ distributor inspect `version` prints the application name and version. The default development version is `dev`; release builds may replace it at build time. -`run`, `validate`, and `inspect` are command placeholders. They intentionally fail with a clear `not implemented` error until the corresponding application behavior exists. +`run --config --dry-run` loads and validates configuration, then prints a concise summary of configured pipelines and destinations. It does not discover bundles or publish files yet. + +`run` without `--dry-run`, `validate`, and `inspect` intentionally fail with a clear `not implemented` error until the corresponding application behavior exists. ## Flag reference @@ -28,6 +30,19 @@ The root command supports: - `--help`, `-h`: print root help. -Each implemented subcommand supports: +Each subcommand supports: - `--help`, `-h`: print command-specific help. + +`run` supports: + +- `--config `: config file to load. +- `--dry-run`: validate config and print the resolved summary without publishing. + +## Common workflows + +Validate a config file without publishing: + +```sh +go run ./cmd/distributor run --config examples/local-to-local.yml --dry-run +``` diff --git a/docs/config.md b/docs/config.md new file mode 100644 index 0000000..c24217d --- /dev/null +++ b/docs/config.md @@ -0,0 +1,96 @@ +# Distributor Configuration + +## Config file location + +`distributor run --config --dry-run` loads the YAML config at the path provided by `--config`. + +If `--config` is omitted during dry-run, the built-in default path is: + +```text +/usr/local/etc/distributor/config.yml +``` + +The current implementation loads and validates configuration only. Bundle discovery and publication are not implemented yet. + +## Minimal config + +```yaml +pipelines: + - id: reports + source: + backend: local + path: /var/spool/distributor/reports + destinations: + - id: archive + backend: local + path: /srv/reports/archive +``` + +This uses the default publish policy of source files only and the default transfer policy. + +## Production-oriented config + +```yaml +pipelines: + - id: reports + source: + backend: local + path: /var/spool/distributor/reports + validation: + on_digest_mismatch: fail + destinations: + - id: archive + backend: s3 + endpoint: https://s3.example.com + bucket: reports + prefix: archive + region: us-east-1 + force_path_style: true + credentials: + access_key_id_env: DISTRIBUTOR_S3_ACCESS_KEY_ID + secret_access_key_env: DISTRIBUTOR_S3_SECRET_ACCESS_KEY + publish: + source: true + html: false +``` + +## Reference + +Top level: + +- `pipelines`: required non-empty list. + +Pipeline: + +- `id`: required unique identifier. +- `source`: required backend config. +- `destinations`: required non-empty destination list. +- `validation.on_digest_mismatch`: optional, defaults to `fail`; only `fail` is supported. + +Backends: + +- `local`: requires `path`. +- `ssh`: requires `uri` and `path`. +- `s3`: requires `endpoint` and `bucket`; supports optional `prefix`, `region`, `force_path_style`, and `credentials`. + +Destination policy: + +- `publish.source`: publish source artifacts. +- `publish.html`: publish generated HTML artifacts. +- `transfer.on_destination_same`: `skip` or `fail`, defaults to `skip`. +- `transfer.on_destination_older`: `replace` or `fail`, defaults to `replace`. +- `transfer.on_destination_newer`: `skip` or `fail`, defaults to `skip`. +- `transfer.on_conflict`: only `fail`, defaults to `fail`. + +When `publish.html` is true, `transform.markdown_to_html.enabled: true` and `transform.markdown_to_html.mode: sidecar` are required. + +## Secrets + +Do not put literal secrets in config files. S3 credentials may refer to environment variable names with: + +- `credentials.access_key_id_env` +- `credentials.secret_access_key_env` + +## Examples + +Maintained examples live under [examples/](../examples/). diff --git a/examples/fan-out.yml b/examples/fan-out.yml new file mode 100644 index 0000000..d60fdb7 --- /dev/null +++ b/examples/fan-out.yml @@ -0,0 +1,32 @@ +pipelines: + - id: reports + source: + backend: local + path: /var/spool/distributor/reports + validation: + on_digest_mismatch: fail + destinations: + - id: markdown-archive + backend: s3 + endpoint: https://s3.example.com + bucket: reports + prefix: archive + region: us-east-1 + force_path_style: true + credentials: + access_key_id_env: DISTRIBUTOR_S3_ACCESS_KEY_ID + secret_access_key_env: DISTRIBUTOR_S3_SECRET_ACCESS_KEY + publish: + source: true + html: false + - id: static-site + backend: ssh + uri: ssh://deploy@example.com:22 + path: /srv/www/reports + publish: + source: false + html: true + transform: + markdown_to_html: + enabled: true + mode: sidecar diff --git a/examples/local-to-local.yml b/examples/local-to-local.yml new file mode 100644 index 0000000..79d0245 --- /dev/null +++ b/examples/local-to-local.yml @@ -0,0 +1,9 @@ +pipelines: + - id: reports + source: + backend: local + path: /var/spool/distributor/reports + destinations: + - id: archive + backend: local + path: /srv/reports/archive diff --git a/go.mod b/go.mod index 47963af..ea77c63 100644 --- a/go.mod +++ b/go.mod @@ -1,3 +1,5 @@ module gitea.maximumdirect.net/eric/distributor go 1.26 + +require gopkg.in/yaml.v3 v3.0.1 diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..a62c313 --- /dev/null +++ b/go.sum @@ -0,0 +1,4 @@ +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/app/run.go b/internal/app/run.go index a409407..56867fc 100644 --- a/internal/app/run.go +++ b/internal/app/run.go @@ -3,10 +3,52 @@ package app import ( "context" "fmt" + "io" + + "gitea.maximumdirect.net/eric/distributor/internal/config" ) -type RunOptions struct{} - -func Run(context.Context, RunOptions) error { - return fmt.Errorf("run command: %w", ErrNotImplemented) +type RunOptions struct { + ConfigPath string + DryRun bool + Stdout io.Writer +} + +func Run(ctx context.Context, options RunOptions) error { + if err := ctx.Err(); err != nil { + return err + } + if !options.DryRun { + return fmt.Errorf("run command: %w", ErrNotImplemented) + } + + configPath := options.ConfigPath + if configPath == "" { + configPath = config.DefaultConfigPath + } + cfg, err := config.LoadFile(configPath) + if err != nil { + return err + } + return writeRunSummary(options.Stdout, cfg) +} + +func writeRunSummary(w io.Writer, cfg config.Config) error { + if w == nil { + return nil + } + if _, err := fmt.Fprintf(w, "Configured pipelines: %d\n", len(cfg.Pipelines)); err != nil { + return err + } + for _, pipeline := range cfg.Pipelines { + if _, err := fmt.Fprintf(w, "- %s: source=%s destinations=%d\n", pipeline.ID, pipeline.Source.Backend, len(pipeline.Destinations)); err != nil { + return err + } + for _, destination := range pipeline.Destinations { + if _, err := fmt.Fprintf(w, " - %s: backend=%s publish_source=%t publish_html=%t\n", destination.ID, destination.Backend, destination.Publish.Source, destination.Publish.HTML); err != nil { + return err + } + } + } + return nil } diff --git a/internal/app/run_test.go b/internal/app/run_test.go new file mode 100644 index 0000000..88dffbe --- /dev/null +++ b/internal/app/run_test.go @@ -0,0 +1,56 @@ +package app + +import ( + "bytes" + "context" + "os" + "path/filepath" + "strings" + "testing" +) + +func TestRunDryRunPrintsConfigSummary(t *testing.T) { + configPath := filepath.Join(t.TempDir(), "config.yml") + err := os.WriteFile(configPath, []byte(` +pipelines: + - id: reports + source: + backend: local + path: /source + destinations: + - id: archive + backend: local + path: /archive +`), 0o600) + if err != nil { + t.Fatalf("write config: %v", err) + } + + var stdout bytes.Buffer + err = Run(context.Background(), RunOptions{ + ConfigPath: configPath, + DryRun: true, + Stdout: &stdout, + }) + if err != nil { + t.Fatalf("Run() error = %v", err) + } + + output := stdout.String() + for _, want := range []string{ + "Configured pipelines: 1", + "- reports: source=local destinations=1", + "archive: backend=local publish_source=true publish_html=false", + } { + if !strings.Contains(output, want) { + t.Fatalf("Run() output = %q, want substring %q", output, want) + } + } +} + +func TestRunWithoutDryRunIsNotImplemented(t *testing.T) { + err := Run(context.Background(), RunOptions{}) + if err == nil || !strings.Contains(err.Error(), "not implemented") { + t.Fatalf("Run() error = %v, want not implemented", err) + } +} diff --git a/internal/cli/root_test.go b/internal/cli/root_test.go index bee5103..fea080b 100644 --- a/internal/cli/root_test.go +++ b/internal/cli/root_test.go @@ -3,6 +3,8 @@ package cli import ( "bytes" "context" + "os" + "path/filepath" "strings" "testing" ) @@ -61,6 +63,38 @@ func TestPlaceholderCommandsFailClearly(t *testing.T) { } } +func TestExecuteRunDryRun(t *testing.T) { + configPath := filepath.Join(t.TempDir(), "config.yml") + err := os.WriteFile(configPath, []byte(` +pipelines: + - id: reports + source: + backend: local + path: /source + destinations: + - id: archive + backend: local + path: /archive +`), 0o600) + if err != nil { + t.Fatalf("write config: %v", err) + } + + var stdout, stderr bytes.Buffer + + code := Execute(context.Background(), []string{"run", "--config", configPath, "--dry-run"}, &stdout, &stderr) + + if code != exitOK { + t.Fatalf("exit code = %d, want %d; stderr = %q", code, exitOK, stderr.String()) + } + if !strings.Contains(stdout.String(), "Configured pipelines: 1") { + t.Fatalf("stdout = %q, want config summary", stdout.String()) + } + if stderr.Len() != 0 { + t.Fatalf("stderr = %q, want empty", stderr.String()) + } +} + func TestUnknownCommandIsUsageError(t *testing.T) { var stdout, stderr bytes.Buffer diff --git a/internal/cli/run.go b/internal/cli/run.go index 45a2643..940d9c6 100644 --- a/internal/cli/run.go +++ b/internal/cli/run.go @@ -2,6 +2,7 @@ package cli import ( "context" + "flag" "fmt" "io" @@ -13,11 +14,24 @@ func runCommand(ctx context.Context, args []string, stdout, stderr io.Writer) in printRunHelp(stdout) return exitOK } - if len(args) > 0 { - fmt.Fprintf(stderr, "%s: run does not accept options yet: %v\n", app.Name, args) + + flags := flag.NewFlagSet("run", flag.ContinueOnError) + flags.SetOutput(stderr) + configPath := flags.String("config", "", "path to config file") + dryRun := flags.Bool("dry-run", false, "load and validate config without publishing") + if err := flags.Parse(args); err != nil { return exitUsage } - if err := app.Run(ctx, app.RunOptions{}); err != nil { + if flags.NArg() > 0 { + fmt.Fprintf(stderr, "%s: run does not accept positional arguments: %v\n", app.Name, flags.Args()) + return exitUsage + } + + if err := app.Run(ctx, app.RunOptions{ + ConfigPath: *configPath, + DryRun: *dryRun, + Stdout: stdout, + }); err != nil { return fail(stderr, err) } return exitOK @@ -25,8 +39,13 @@ func runCommand(ctx context.Context, args []string, stdout, stderr io.Writer) in func printRunHelp(w io.Writer) { fmt.Fprint(w, `Usage: - distributor run + distributor run --config --dry-run -The run command is present but distribution behavior is not implemented yet. +Options: + --config Path to config file + --dry-run Load and validate config without publishing + +Execution behavior is not implemented yet. Dry-run currently prints a resolved +configuration summary only. `) } diff --git a/internal/config/config.go b/internal/config/config.go index 90b9d4a..865f49a 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -1,3 +1,70 @@ package config -type Config struct{} +type Config struct { + Pipelines []Pipeline `yaml:"pipelines"` +} + +type Pipeline struct { + ID string `yaml:"id"` + Source Backend `yaml:"source"` + Validation ValidationPolicy `yaml:"validation"` + Destinations []Destination `yaml:"destinations"` +} + +type Destination struct { + ID string `yaml:"id"` + Backend string `yaml:"backend"` + Path string `yaml:"path"` + URI string `yaml:"uri"` + Endpoint string `yaml:"endpoint"` + Bucket string `yaml:"bucket"` + Prefix string `yaml:"prefix"` + Region string `yaml:"region"` + ForcePath bool `yaml:"force_path_style"` + Creds Credentials `yaml:"credentials"` + Publish *PublishPolicy `yaml:"publish"` + Transform Transform `yaml:"transform"` + Transfer TransferPolicy `yaml:"transfer"` +} + +type Backend struct { + Backend string `yaml:"backend"` + Path string `yaml:"path"` + URI string `yaml:"uri"` + Endpoint string `yaml:"endpoint"` + Bucket string `yaml:"bucket"` + Prefix string `yaml:"prefix"` + Region string `yaml:"region"` + ForcePath bool `yaml:"force_path_style"` + Creds Credentials `yaml:"credentials"` +} + +type Credentials struct { + AccessKeyIDEnv string `yaml:"access_key_id_env"` + SecretAccessKeyEnv string `yaml:"secret_access_key_env"` +} + +type ValidationPolicy struct { + OnDigestMismatch string `yaml:"on_digest_mismatch"` +} + +type PublishPolicy struct { + Source bool `yaml:"source"` + HTML bool `yaml:"html"` +} + +type Transform struct { + MarkdownToHTML *MarkdownToHTML `yaml:"markdown_to_html"` +} + +type MarkdownToHTML struct { + Enabled bool `yaml:"enabled"` + Mode string `yaml:"mode"` +} + +type TransferPolicy struct { + OnDestinationSame string `yaml:"on_destination_same"` + OnDestinationOlder string `yaml:"on_destination_older"` + OnDestinationNewer string `yaml:"on_destination_newer"` + OnConflict string `yaml:"on_conflict"` +} diff --git a/internal/config/defaults.go b/internal/config/defaults.go index 9f3e1d6..5a4c11d 100644 --- a/internal/config/defaults.go +++ b/internal/config/defaults.go @@ -1,3 +1,50 @@ package config const DefaultConfigPath = "/usr/local/etc/distributor/config.yml" + +const ( + BackendLocal = "local" + BackendSSH = "ssh" + BackendS3 = "s3" +) + +const ( + ValidationActionFail = "fail" +) + +const ( + TransferActionSkip = "skip" + TransferActionReplace = "replace" + TransferActionFail = "fail" +) + +const ( + TransformModeSidecar = "sidecar" +) + +func ApplyDefaults(cfg *Config) { + for pipelineIndex := range cfg.Pipelines { + pipeline := &cfg.Pipelines[pipelineIndex] + if pipeline.Validation.OnDigestMismatch == "" { + pipeline.Validation.OnDigestMismatch = ValidationActionFail + } + for destinationIndex := range pipeline.Destinations { + destination := &pipeline.Destinations[destinationIndex] + if destination.Publish == nil { + destination.Publish = &PublishPolicy{Source: true} + } + if destination.Transfer.OnDestinationSame == "" { + destination.Transfer.OnDestinationSame = TransferActionSkip + } + if destination.Transfer.OnDestinationOlder == "" { + destination.Transfer.OnDestinationOlder = TransferActionReplace + } + if destination.Transfer.OnDestinationNewer == "" { + destination.Transfer.OnDestinationNewer = TransferActionSkip + } + if destination.Transfer.OnConflict == "" { + destination.Transfer.OnConflict = TransferActionFail + } + } + } +} diff --git a/internal/config/load.go b/internal/config/load.go new file mode 100644 index 0000000..04d1245 --- /dev/null +++ b/internal/config/load.go @@ -0,0 +1,29 @@ +package config + +import ( + "fmt" + "os" + + "gopkg.in/yaml.v3" +) + +func LoadFile(path string) (Config, error) { + file, err := os.Open(path) + if err != nil { + return Config{}, fmt.Errorf("load config %q: %w", path, err) + } + defer file.Close() + + var cfg Config + decoder := yaml.NewDecoder(file) + decoder.KnownFields(true) + if err := decoder.Decode(&cfg); err != nil { + return Config{}, fmt.Errorf("parse config %q: %w", path, err) + } + + ApplyDefaults(&cfg) + if err := Validate(cfg); err != nil { + return Config{}, fmt.Errorf("validate config %q: %w", path, err) + } + return cfg, nil +} diff --git a/internal/config/load_test.go b/internal/config/load_test.go new file mode 100644 index 0000000..8769b5f --- /dev/null +++ b/internal/config/load_test.go @@ -0,0 +1,306 @@ +package config + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +func TestLoadFileValidMinimalLocalToLocalConfig(t *testing.T) { + cfg := loadConfig(t, ` +pipelines: + - id: local-copy + source: + backend: local + path: /var/spool/reports + destinations: + - id: archive + backend: local + path: /srv/archive +`) + + if got, want := len(cfg.Pipelines), 1; got != want { + t.Fatalf("pipeline count = %d, want %d", got, want) + } + destination := cfg.Pipelines[0].Destinations[0] + if !destination.Publish.Source || destination.Publish.HTML { + t.Fatalf("publish defaults = source:%t html:%t, want source:true html:false", destination.Publish.Source, destination.Publish.HTML) + } + if got, want := cfg.Pipelines[0].Validation.OnDigestMismatch, ValidationActionFail; got != want { + t.Fatalf("validation default = %q, want %q", got, want) + } + if got, want := destination.Transfer.OnDestinationOlder, TransferActionReplace; got != want { + t.Fatalf("transfer default = %q, want %q", got, want) + } +} + +func TestLoadFileValidFanOutConfig(t *testing.T) { + cfg := loadConfig(t, ` +pipelines: + - id: fan-out + source: + backend: local + path: /var/spool/reports + destinations: + - id: markdown-archive + backend: s3 + endpoint: https://s3.example.com + bucket: reports + prefix: archive + publish: + source: true + html: false + - id: static-site + backend: ssh + uri: ssh://deploy@example.com:22 + path: /srv/www/reports + publish: + source: false + html: true + transform: + markdown_to_html: + enabled: true + mode: sidecar +`) + + if got, want := len(cfg.Pipelines[0].Destinations), 2; got != want { + t.Fatalf("destination count = %d, want %d", got, want) + } +} + +func TestLoadFileValidBackendConfigs(t *testing.T) { + tests := map[string]string{ + "local": ` +pipelines: + - id: local-backend + source: + backend: local + path: /source + destinations: + - id: local-destination + backend: local + path: /destination +`, + "ssh": ` +pipelines: + - id: ssh-backend + source: + backend: ssh + uri: ssh://reports@example.com:22 + path: /source + destinations: + - id: ssh-destination + backend: ssh + uri: ssh://deploy@example.com:22 + path: /destination +`, + "s3": ` +pipelines: + - id: s3-backend + source: + backend: s3 + endpoint: https://s3.example.com + bucket: source + prefix: incoming + region: us-east-1 + force_path_style: true + credentials: + access_key_id_env: DISTRIBUTOR_S3_ACCESS_KEY_ID + secret_access_key_env: DISTRIBUTOR_S3_SECRET_ACCESS_KEY + destinations: + - id: s3-destination + backend: s3 + endpoint: https://s3.example.com + bucket: destination + prefix: archive +`, + } + + for name, body := range tests { + t.Run(name, func(t *testing.T) { + loadConfig(t, body) + }) + } +} + +func TestLoadFileRejectsDuplicatePipelineIDs(t *testing.T) { + assertLoadError(t, ` +pipelines: + - id: duplicate + source: + backend: local + path: /one + destinations: + - id: archive + backend: local + path: /archive + - id: duplicate + source: + backend: local + path: /two + destinations: + - id: archive + backend: local + path: /archive +`, "pipeline id duplicate is duplicated") +} + +func TestLoadFileRejectsDuplicateDestinationIDs(t *testing.T) { + assertLoadError(t, ` +pipelines: + - id: reports + source: + backend: local + path: /source + destinations: + - id: archive + backend: local + path: /archive-one + - id: archive + backend: local + path: /archive-two +`, "destination id archive is duplicated") +} + +func TestLoadFileRejectsMissingRequiredFields(t *testing.T) { + tests := map[string]string{ + "pipelines": ``, + "pipeline id": `pipelines: [{source: {backend: local, path: /source}, destinations: [{id: archive, backend: local, path: /archive}]}]`, + "source backend": `pipelines: [{id: reports, source: {path: /source}, destinations: [{id: archive, backend: local, path: /archive}]}]`, + "destinations": `pipelines: [{id: reports, source: {backend: local, path: /source}}]`, + "destination id": `pipelines: [{id: reports, source: {backend: local, path: /source}, destinations: [{backend: local, path: /archive}]}]`, + "local path": `pipelines: [{id: reports, source: {backend: local}, destinations: [{id: archive, backend: local, path: /archive}]}]`, + "ssh uri": `pipelines: [{id: reports, source: {backend: ssh, path: /source}, destinations: [{id: archive, backend: local, path: /archive}]}]`, + "s3 bucket": `pipelines: [{id: reports, source: {backend: s3, endpoint: "https://s3.example.com"}, destinations: [{id: archive, backend: local, path: /archive}]}]`, + "publish outputs": `pipelines: [{id: reports, source: {backend: local, path: /source}, destinations: [{id: archive, backend: local, path: /archive, publish: {source: false, html: false}}]}]`, + } + + for name, body := range tests { + t.Run(name, func(t *testing.T) { + assertLoadError(t, body, "") + }) + } +} + +func TestLoadFileRejectsUnsupportedBackend(t *testing.T) { + assertLoadError(t, ` +pipelines: + - id: reports + source: + backend: ftp + path: /source + destinations: + - id: archive + backend: local + path: /archive +`, "backend ftp is unsupported") +} + +func TestLoadFileRejectsInvalidTransferAction(t *testing.T) { + assertLoadError(t, ` +pipelines: + - id: reports + source: + backend: local + path: /source + destinations: + - id: archive + backend: local + path: /archive + transfer: + on_destination_older: overwrite +`, "on_destination_older must be replace or fail") +} + +func TestLoadFileRejectsInvalidValidationAction(t *testing.T) { + assertLoadError(t, ` +pipelines: + - id: reports + source: + backend: local + path: /source + validation: + on_digest_mismatch: warn + destinations: + - id: archive + backend: local + path: /archive +`, "on_digest_mismatch must be fail") +} + +func TestLoadFileRejectsHTMLPublishWithoutTransform(t *testing.T) { + assertLoadError(t, ` +pipelines: + - id: reports + source: + backend: local + path: /source + destinations: + - id: static + backend: local + path: /srv/www + publish: + source: false + html: true +`, "markdown_to_html is required") +} + +func TestLoadFileRejectsUnknownFields(t *testing.T) { + assertLoadError(t, ` +pipelines: + - id: reports + surprise: true + source: + backend: local + path: /source + destinations: + - id: archive + backend: local + path: /archive +`, "field surprise not found") +} + +func TestExampleConfigsLoad(t *testing.T) { + for _, path := range []string{ + "../../examples/local-to-local.yml", + "../../examples/fan-out.yml", + } { + t.Run(path, func(t *testing.T) { + if _, err := LoadFile(path); err != nil { + t.Fatalf("LoadFile(%q) error = %v", path, err) + } + }) + } +} + +func loadConfig(t *testing.T, body string) Config { + t.Helper() + path := writeConfig(t, body) + cfg, err := LoadFile(path) + if err != nil { + t.Fatalf("LoadFile() error = %v", err) + } + return cfg +} + +func assertLoadError(t *testing.T, body, want string) { + t.Helper() + path := writeConfig(t, body) + _, err := LoadFile(path) + if err == nil { + t.Fatal("LoadFile() error = nil, want error") + } + if want != "" && !strings.Contains(err.Error(), want) { + t.Fatalf("LoadFile() error = %q, want substring %q", err.Error(), want) + } +} + +func writeConfig(t *testing.T, body string) string { + t.Helper() + path := filepath.Join(t.TempDir(), "config.yml") + if err := os.WriteFile(path, []byte(strings.TrimSpace(body)+"\n"), 0o600); err != nil { + t.Fatalf("write config: %v", err) + } + return path +} diff --git a/internal/config/validate.go b/internal/config/validate.go new file mode 100644 index 0000000..5f60a5c --- /dev/null +++ b/internal/config/validate.go @@ -0,0 +1,153 @@ +package config + +import ( + "fmt" + "regexp" + "strings" +) + +var idPattern = regexp.MustCompile(`^[a-zA-Z0-9][a-zA-Z0-9._-]*$`) + +type ValidationErrors []string + +func (e ValidationErrors) Error() string { + if len(e) == 1 { + return e[0] + } + return strings.Join(e, "; ") +} + +func Validate(cfg Config) error { + var errs ValidationErrors + + if len(cfg.Pipelines) == 0 { + errs = append(errs, "pipelines is required") + } + + pipelineIDs := make(map[string]struct{}, len(cfg.Pipelines)) + for pipelineIndex, pipeline := range cfg.Pipelines { + pipelineContext := fmt.Sprintf("pipelines[%d]", pipelineIndex) + if pipeline.ID == "" { + errs = append(errs, pipelineContext+".id is required") + } else if !idPattern.MatchString(pipeline.ID) { + errs = append(errs, pipelineContext+".id must be a slug-like identifier") + } else if _, exists := pipelineIDs[pipeline.ID]; exists { + errs = append(errs, "pipeline id "+pipeline.ID+" is duplicated") + } else { + pipelineIDs[pipeline.ID] = struct{}{} + } + + errs = validateBackend(errs, pipelineContext+".source", pipeline.Source.Backend, pipeline.Source.Path, pipeline.Source.URI, pipeline.Source.Endpoint, pipeline.Source.Bucket) + errs = validateValidationPolicy(errs, pipelineContext+".validation", pipeline.Validation) + if len(pipeline.Destinations) == 0 { + errs = append(errs, pipelineContext+".destinations is required") + } + + destinationIDs := make(map[string]struct{}, len(pipeline.Destinations)) + for destinationIndex, destination := range pipeline.Destinations { + destinationContext := fmt.Sprintf("%s.destinations[%d]", pipelineContext, destinationIndex) + if destination.ID == "" { + errs = append(errs, destinationContext+".id is required") + } else if !idPattern.MatchString(destination.ID) { + errs = append(errs, destinationContext+".id must be a slug-like identifier") + } else if _, exists := destinationIDs[destination.ID]; exists { + errs = append(errs, "destination id "+destination.ID+" is duplicated in pipeline "+pipeline.ID) + } else { + destinationIDs[destination.ID] = struct{}{} + } + + errs = validateBackend(errs, destinationContext, destination.Backend, destination.Path, destination.URI, destination.Endpoint, destination.Bucket) + errs = validatePublishPolicy(errs, destinationContext+".publish", destination.Publish) + errs = validateTransform(errs, destinationContext+".transform", destination.Publish, destination.Transform) + errs = validateTransferPolicy(errs, destinationContext+".transfer", destination.Transfer) + } + } + + if len(errs) > 0 { + return errs + } + return nil +} + +func validateBackend(errs ValidationErrors, context, backend, path, uri, endpoint, bucket string) ValidationErrors { + switch backend { + case "": + errs = append(errs, context+".backend is required") + case BackendLocal: + if path == "" { + errs = append(errs, context+".path is required for local backend") + } + case BackendSSH: + if uri == "" { + errs = append(errs, context+".uri is required for ssh backend") + } + if path == "" { + errs = append(errs, context+".path is required for ssh backend") + } + case BackendS3: + if endpoint == "" { + errs = append(errs, context+".endpoint is required for s3 backend") + } + if bucket == "" { + errs = append(errs, context+".bucket is required for s3 backend") + } + default: + errs = append(errs, context+".backend "+backend+" is unsupported") + } + return errs +} + +func validateValidationPolicy(errs ValidationErrors, context string, policy ValidationPolicy) ValidationErrors { + if policy.OnDigestMismatch != ValidationActionFail { + errs = append(errs, context+".on_digest_mismatch must be "+ValidationActionFail) + } + return errs +} + +func validatePublishPolicy(errs ValidationErrors, context string, policy *PublishPolicy) ValidationErrors { + if policy == nil { + errs = append(errs, context+" is required") + return errs + } + if !policy.Source && !policy.HTML { + errs = append(errs, context+" must enable source or html") + } + return errs +} + +func validateTransform(errs ValidationErrors, context string, publish *PublishPolicy, transform Transform) ValidationErrors { + publishesHTML := publish != nil && publish.HTML + if transform.MarkdownToHTML == nil { + if publishesHTML { + errs = append(errs, context+".markdown_to_html is required when publish.html is true") + } + return errs + } + + if publishesHTML && !transform.MarkdownToHTML.Enabled { + errs = append(errs, context+".markdown_to_html.enabled must be true when publish.html is true") + } + if transform.MarkdownToHTML.Enabled && transform.MarkdownToHTML.Mode != TransformModeSidecar { + errs = append(errs, context+".markdown_to_html.mode must be "+TransformModeSidecar) + } + if !transform.MarkdownToHTML.Enabled && transform.MarkdownToHTML.Mode != "" && transform.MarkdownToHTML.Mode != TransformModeSidecar { + errs = append(errs, context+".markdown_to_html.mode must be "+TransformModeSidecar) + } + return errs +} + +func validateTransferPolicy(errs ValidationErrors, context string, policy TransferPolicy) ValidationErrors { + if policy.OnDestinationSame != TransferActionSkip && policy.OnDestinationSame != TransferActionFail { + errs = append(errs, context+".on_destination_same must be skip or fail") + } + if policy.OnDestinationOlder != TransferActionReplace && policy.OnDestinationOlder != TransferActionFail { + errs = append(errs, context+".on_destination_older must be replace or fail") + } + if policy.OnDestinationNewer != TransferActionSkip && policy.OnDestinationNewer != TransferActionFail { + errs = append(errs, context+".on_destination_newer must be skip or fail") + } + if policy.OnConflict != TransferActionFail { + errs = append(errs, context+".on_conflict must be fail") + } + return errs +}