Add config loading and dry-run validation

This commit is contained in:
2026-05-31 01:53:13 +00:00
parent 22d0424232
commit 29dbad2967
16 changed files with 928 additions and 17 deletions

View File

@@ -3,10 +3,52 @@ package app
import (
"context"
"fmt"
"io"
"gitea.maximumdirect.net/eric/distributor/internal/config"
)
type RunOptions struct{}
func Run(context.Context, RunOptions) error {
return fmt.Errorf("run command: %w", ErrNotImplemented)
type RunOptions struct {
ConfigPath string
DryRun bool
Stdout io.Writer
}
func Run(ctx context.Context, options RunOptions) error {
if err := ctx.Err(); err != nil {
return err
}
if !options.DryRun {
return fmt.Errorf("run command: %w", ErrNotImplemented)
}
configPath := options.ConfigPath
if configPath == "" {
configPath = config.DefaultConfigPath
}
cfg, err := config.LoadFile(configPath)
if err != nil {
return err
}
return writeRunSummary(options.Stdout, cfg)
}
func writeRunSummary(w io.Writer, cfg config.Config) error {
if w == nil {
return nil
}
if _, err := fmt.Fprintf(w, "Configured pipelines: %d\n", len(cfg.Pipelines)); err != nil {
return err
}
for _, pipeline := range cfg.Pipelines {
if _, err := fmt.Fprintf(w, "- %s: source=%s destinations=%d\n", pipeline.ID, pipeline.Source.Backend, len(pipeline.Destinations)); err != nil {
return err
}
for _, destination := range pipeline.Destinations {
if _, err := fmt.Fprintf(w, " - %s: backend=%s publish_source=%t publish_html=%t\n", destination.ID, destination.Backend, destination.Publish.Source, destination.Publish.HTML); err != nil {
return err
}
}
}
return nil
}

56
internal/app/run_test.go Normal file
View File

@@ -0,0 +1,56 @@
package app
import (
"bytes"
"context"
"os"
"path/filepath"
"strings"
"testing"
)
func TestRunDryRunPrintsConfigSummary(t *testing.T) {
configPath := filepath.Join(t.TempDir(), "config.yml")
err := os.WriteFile(configPath, []byte(`
pipelines:
- id: reports
source:
backend: local
path: /source
destinations:
- id: archive
backend: local
path: /archive
`), 0o600)
if err != nil {
t.Fatalf("write config: %v", err)
}
var stdout bytes.Buffer
err = Run(context.Background(), RunOptions{
ConfigPath: configPath,
DryRun: true,
Stdout: &stdout,
})
if err != nil {
t.Fatalf("Run() error = %v", err)
}
output := stdout.String()
for _, want := range []string{
"Configured pipelines: 1",
"- reports: source=local destinations=1",
"archive: backend=local publish_source=true publish_html=false",
} {
if !strings.Contains(output, want) {
t.Fatalf("Run() output = %q, want substring %q", output, want)
}
}
}
func TestRunWithoutDryRunIsNotImplemented(t *testing.T) {
err := Run(context.Background(), RunOptions{})
if err == nil || !strings.Contains(err.Error(), "not implemented") {
t.Fatalf("Run() error = %v, want not implemented", err)
}
}

View File

@@ -3,6 +3,8 @@ package cli
import (
"bytes"
"context"
"os"
"path/filepath"
"strings"
"testing"
)
@@ -61,6 +63,38 @@ func TestPlaceholderCommandsFailClearly(t *testing.T) {
}
}
func TestExecuteRunDryRun(t *testing.T) {
configPath := filepath.Join(t.TempDir(), "config.yml")
err := os.WriteFile(configPath, []byte(`
pipelines:
- id: reports
source:
backend: local
path: /source
destinations:
- id: archive
backend: local
path: /archive
`), 0o600)
if err != nil {
t.Fatalf("write config: %v", err)
}
var stdout, stderr bytes.Buffer
code := Execute(context.Background(), []string{"run", "--config", configPath, "--dry-run"}, &stdout, &stderr)
if code != exitOK {
t.Fatalf("exit code = %d, want %d; stderr = %q", code, exitOK, stderr.String())
}
if !strings.Contains(stdout.String(), "Configured pipelines: 1") {
t.Fatalf("stdout = %q, want config summary", stdout.String())
}
if stderr.Len() != 0 {
t.Fatalf("stderr = %q, want empty", stderr.String())
}
}
func TestUnknownCommandIsUsageError(t *testing.T) {
var stdout, stderr bytes.Buffer

View File

@@ -2,6 +2,7 @@ package cli
import (
"context"
"flag"
"fmt"
"io"
@@ -13,11 +14,24 @@ func runCommand(ctx context.Context, args []string, stdout, stderr io.Writer) in
printRunHelp(stdout)
return exitOK
}
if len(args) > 0 {
fmt.Fprintf(stderr, "%s: run does not accept options yet: %v\n", app.Name, args)
flags := flag.NewFlagSet("run", flag.ContinueOnError)
flags.SetOutput(stderr)
configPath := flags.String("config", "", "path to config file")
dryRun := flags.Bool("dry-run", false, "load and validate config without publishing")
if err := flags.Parse(args); err != nil {
return exitUsage
}
if err := app.Run(ctx, app.RunOptions{}); err != nil {
if flags.NArg() > 0 {
fmt.Fprintf(stderr, "%s: run does not accept positional arguments: %v\n", app.Name, flags.Args())
return exitUsage
}
if err := app.Run(ctx, app.RunOptions{
ConfigPath: *configPath,
DryRun: *dryRun,
Stdout: stdout,
}); err != nil {
return fail(stderr, err)
}
return exitOK
@@ -25,8 +39,13 @@ func runCommand(ctx context.Context, args []string, stdout, stderr io.Writer) in
func printRunHelp(w io.Writer) {
fmt.Fprint(w, `Usage:
distributor run
distributor run --config <path> --dry-run
The run command is present but distribution behavior is not implemented yet.
Options:
--config <path> Path to config file
--dry-run Load and validate config without publishing
Execution behavior is not implemented yet. Dry-run currently prints a resolved
configuration summary only.
`)
}

View File

@@ -1,3 +1,70 @@
package config
type Config struct{}
type Config struct {
Pipelines []Pipeline `yaml:"pipelines"`
}
type Pipeline struct {
ID string `yaml:"id"`
Source Backend `yaml:"source"`
Validation ValidationPolicy `yaml:"validation"`
Destinations []Destination `yaml:"destinations"`
}
type Destination struct {
ID string `yaml:"id"`
Backend string `yaml:"backend"`
Path string `yaml:"path"`
URI string `yaml:"uri"`
Endpoint string `yaml:"endpoint"`
Bucket string `yaml:"bucket"`
Prefix string `yaml:"prefix"`
Region string `yaml:"region"`
ForcePath bool `yaml:"force_path_style"`
Creds Credentials `yaml:"credentials"`
Publish *PublishPolicy `yaml:"publish"`
Transform Transform `yaml:"transform"`
Transfer TransferPolicy `yaml:"transfer"`
}
type Backend struct {
Backend string `yaml:"backend"`
Path string `yaml:"path"`
URI string `yaml:"uri"`
Endpoint string `yaml:"endpoint"`
Bucket string `yaml:"bucket"`
Prefix string `yaml:"prefix"`
Region string `yaml:"region"`
ForcePath bool `yaml:"force_path_style"`
Creds Credentials `yaml:"credentials"`
}
type Credentials struct {
AccessKeyIDEnv string `yaml:"access_key_id_env"`
SecretAccessKeyEnv string `yaml:"secret_access_key_env"`
}
type ValidationPolicy struct {
OnDigestMismatch string `yaml:"on_digest_mismatch"`
}
type PublishPolicy struct {
Source bool `yaml:"source"`
HTML bool `yaml:"html"`
}
type Transform struct {
MarkdownToHTML *MarkdownToHTML `yaml:"markdown_to_html"`
}
type MarkdownToHTML struct {
Enabled bool `yaml:"enabled"`
Mode string `yaml:"mode"`
}
type TransferPolicy struct {
OnDestinationSame string `yaml:"on_destination_same"`
OnDestinationOlder string `yaml:"on_destination_older"`
OnDestinationNewer string `yaml:"on_destination_newer"`
OnConflict string `yaml:"on_conflict"`
}

View File

@@ -1,3 +1,50 @@
package config
const DefaultConfigPath = "/usr/local/etc/distributor/config.yml"
const (
BackendLocal = "local"
BackendSSH = "ssh"
BackendS3 = "s3"
)
const (
ValidationActionFail = "fail"
)
const (
TransferActionSkip = "skip"
TransferActionReplace = "replace"
TransferActionFail = "fail"
)
const (
TransformModeSidecar = "sidecar"
)
func ApplyDefaults(cfg *Config) {
for pipelineIndex := range cfg.Pipelines {
pipeline := &cfg.Pipelines[pipelineIndex]
if pipeline.Validation.OnDigestMismatch == "" {
pipeline.Validation.OnDigestMismatch = ValidationActionFail
}
for destinationIndex := range pipeline.Destinations {
destination := &pipeline.Destinations[destinationIndex]
if destination.Publish == nil {
destination.Publish = &PublishPolicy{Source: true}
}
if destination.Transfer.OnDestinationSame == "" {
destination.Transfer.OnDestinationSame = TransferActionSkip
}
if destination.Transfer.OnDestinationOlder == "" {
destination.Transfer.OnDestinationOlder = TransferActionReplace
}
if destination.Transfer.OnDestinationNewer == "" {
destination.Transfer.OnDestinationNewer = TransferActionSkip
}
if destination.Transfer.OnConflict == "" {
destination.Transfer.OnConflict = TransferActionFail
}
}
}
}

29
internal/config/load.go Normal file
View File

@@ -0,0 +1,29 @@
package config
import (
"fmt"
"os"
"gopkg.in/yaml.v3"
)
func LoadFile(path string) (Config, error) {
file, err := os.Open(path)
if err != nil {
return Config{}, fmt.Errorf("load config %q: %w", path, err)
}
defer file.Close()
var cfg Config
decoder := yaml.NewDecoder(file)
decoder.KnownFields(true)
if err := decoder.Decode(&cfg); err != nil {
return Config{}, fmt.Errorf("parse config %q: %w", path, err)
}
ApplyDefaults(&cfg)
if err := Validate(cfg); err != nil {
return Config{}, fmt.Errorf("validate config %q: %w", path, err)
}
return cfg, nil
}

View File

@@ -0,0 +1,306 @@
package config
import (
"os"
"path/filepath"
"strings"
"testing"
)
func TestLoadFileValidMinimalLocalToLocalConfig(t *testing.T) {
cfg := loadConfig(t, `
pipelines:
- id: local-copy
source:
backend: local
path: /var/spool/reports
destinations:
- id: archive
backend: local
path: /srv/archive
`)
if got, want := len(cfg.Pipelines), 1; got != want {
t.Fatalf("pipeline count = %d, want %d", got, want)
}
destination := cfg.Pipelines[0].Destinations[0]
if !destination.Publish.Source || destination.Publish.HTML {
t.Fatalf("publish defaults = source:%t html:%t, want source:true html:false", destination.Publish.Source, destination.Publish.HTML)
}
if got, want := cfg.Pipelines[0].Validation.OnDigestMismatch, ValidationActionFail; got != want {
t.Fatalf("validation default = %q, want %q", got, want)
}
if got, want := destination.Transfer.OnDestinationOlder, TransferActionReplace; got != want {
t.Fatalf("transfer default = %q, want %q", got, want)
}
}
func TestLoadFileValidFanOutConfig(t *testing.T) {
cfg := loadConfig(t, `
pipelines:
- id: fan-out
source:
backend: local
path: /var/spool/reports
destinations:
- id: markdown-archive
backend: s3
endpoint: https://s3.example.com
bucket: reports
prefix: archive
publish:
source: true
html: false
- id: static-site
backend: ssh
uri: ssh://deploy@example.com:22
path: /srv/www/reports
publish:
source: false
html: true
transform:
markdown_to_html:
enabled: true
mode: sidecar
`)
if got, want := len(cfg.Pipelines[0].Destinations), 2; got != want {
t.Fatalf("destination count = %d, want %d", got, want)
}
}
func TestLoadFileValidBackendConfigs(t *testing.T) {
tests := map[string]string{
"local": `
pipelines:
- id: local-backend
source:
backend: local
path: /source
destinations:
- id: local-destination
backend: local
path: /destination
`,
"ssh": `
pipelines:
- id: ssh-backend
source:
backend: ssh
uri: ssh://reports@example.com:22
path: /source
destinations:
- id: ssh-destination
backend: ssh
uri: ssh://deploy@example.com:22
path: /destination
`,
"s3": `
pipelines:
- id: s3-backend
source:
backend: s3
endpoint: https://s3.example.com
bucket: source
prefix: incoming
region: us-east-1
force_path_style: true
credentials:
access_key_id_env: DISTRIBUTOR_S3_ACCESS_KEY_ID
secret_access_key_env: DISTRIBUTOR_S3_SECRET_ACCESS_KEY
destinations:
- id: s3-destination
backend: s3
endpoint: https://s3.example.com
bucket: destination
prefix: archive
`,
}
for name, body := range tests {
t.Run(name, func(t *testing.T) {
loadConfig(t, body)
})
}
}
func TestLoadFileRejectsDuplicatePipelineIDs(t *testing.T) {
assertLoadError(t, `
pipelines:
- id: duplicate
source:
backend: local
path: /one
destinations:
- id: archive
backend: local
path: /archive
- id: duplicate
source:
backend: local
path: /two
destinations:
- id: archive
backend: local
path: /archive
`, "pipeline id duplicate is duplicated")
}
func TestLoadFileRejectsDuplicateDestinationIDs(t *testing.T) {
assertLoadError(t, `
pipelines:
- id: reports
source:
backend: local
path: /source
destinations:
- id: archive
backend: local
path: /archive-one
- id: archive
backend: local
path: /archive-two
`, "destination id archive is duplicated")
}
func TestLoadFileRejectsMissingRequiredFields(t *testing.T) {
tests := map[string]string{
"pipelines": ``,
"pipeline id": `pipelines: [{source: {backend: local, path: /source}, destinations: [{id: archive, backend: local, path: /archive}]}]`,
"source backend": `pipelines: [{id: reports, source: {path: /source}, destinations: [{id: archive, backend: local, path: /archive}]}]`,
"destinations": `pipelines: [{id: reports, source: {backend: local, path: /source}}]`,
"destination id": `pipelines: [{id: reports, source: {backend: local, path: /source}, destinations: [{backend: local, path: /archive}]}]`,
"local path": `pipelines: [{id: reports, source: {backend: local}, destinations: [{id: archive, backend: local, path: /archive}]}]`,
"ssh uri": `pipelines: [{id: reports, source: {backend: ssh, path: /source}, destinations: [{id: archive, backend: local, path: /archive}]}]`,
"s3 bucket": `pipelines: [{id: reports, source: {backend: s3, endpoint: "https://s3.example.com"}, destinations: [{id: archive, backend: local, path: /archive}]}]`,
"publish outputs": `pipelines: [{id: reports, source: {backend: local, path: /source}, destinations: [{id: archive, backend: local, path: /archive, publish: {source: false, html: false}}]}]`,
}
for name, body := range tests {
t.Run(name, func(t *testing.T) {
assertLoadError(t, body, "")
})
}
}
func TestLoadFileRejectsUnsupportedBackend(t *testing.T) {
assertLoadError(t, `
pipelines:
- id: reports
source:
backend: ftp
path: /source
destinations:
- id: archive
backend: local
path: /archive
`, "backend ftp is unsupported")
}
func TestLoadFileRejectsInvalidTransferAction(t *testing.T) {
assertLoadError(t, `
pipelines:
- id: reports
source:
backend: local
path: /source
destinations:
- id: archive
backend: local
path: /archive
transfer:
on_destination_older: overwrite
`, "on_destination_older must be replace or fail")
}
func TestLoadFileRejectsInvalidValidationAction(t *testing.T) {
assertLoadError(t, `
pipelines:
- id: reports
source:
backend: local
path: /source
validation:
on_digest_mismatch: warn
destinations:
- id: archive
backend: local
path: /archive
`, "on_digest_mismatch must be fail")
}
func TestLoadFileRejectsHTMLPublishWithoutTransform(t *testing.T) {
assertLoadError(t, `
pipelines:
- id: reports
source:
backend: local
path: /source
destinations:
- id: static
backend: local
path: /srv/www
publish:
source: false
html: true
`, "markdown_to_html is required")
}
func TestLoadFileRejectsUnknownFields(t *testing.T) {
assertLoadError(t, `
pipelines:
- id: reports
surprise: true
source:
backend: local
path: /source
destinations:
- id: archive
backend: local
path: /archive
`, "field surprise not found")
}
func TestExampleConfigsLoad(t *testing.T) {
for _, path := range []string{
"../../examples/local-to-local.yml",
"../../examples/fan-out.yml",
} {
t.Run(path, func(t *testing.T) {
if _, err := LoadFile(path); err != nil {
t.Fatalf("LoadFile(%q) error = %v", path, err)
}
})
}
}
func loadConfig(t *testing.T, body string) Config {
t.Helper()
path := writeConfig(t, body)
cfg, err := LoadFile(path)
if err != nil {
t.Fatalf("LoadFile() error = %v", err)
}
return cfg
}
func assertLoadError(t *testing.T, body, want string) {
t.Helper()
path := writeConfig(t, body)
_, err := LoadFile(path)
if err == nil {
t.Fatal("LoadFile() error = nil, want error")
}
if want != "" && !strings.Contains(err.Error(), want) {
t.Fatalf("LoadFile() error = %q, want substring %q", err.Error(), want)
}
}
func writeConfig(t *testing.T, body string) string {
t.Helper()
path := filepath.Join(t.TempDir(), "config.yml")
if err := os.WriteFile(path, []byte(strings.TrimSpace(body)+"\n"), 0o600); err != nil {
t.Fatalf("write config: %v", err)
}
return path
}

153
internal/config/validate.go Normal file
View File

@@ -0,0 +1,153 @@
package config
import (
"fmt"
"regexp"
"strings"
)
var idPattern = regexp.MustCompile(`^[a-zA-Z0-9][a-zA-Z0-9._-]*$`)
type ValidationErrors []string
func (e ValidationErrors) Error() string {
if len(e) == 1 {
return e[0]
}
return strings.Join(e, "; ")
}
func Validate(cfg Config) error {
var errs ValidationErrors
if len(cfg.Pipelines) == 0 {
errs = append(errs, "pipelines is required")
}
pipelineIDs := make(map[string]struct{}, len(cfg.Pipelines))
for pipelineIndex, pipeline := range cfg.Pipelines {
pipelineContext := fmt.Sprintf("pipelines[%d]", pipelineIndex)
if pipeline.ID == "" {
errs = append(errs, pipelineContext+".id is required")
} else if !idPattern.MatchString(pipeline.ID) {
errs = append(errs, pipelineContext+".id must be a slug-like identifier")
} else if _, exists := pipelineIDs[pipeline.ID]; exists {
errs = append(errs, "pipeline id "+pipeline.ID+" is duplicated")
} else {
pipelineIDs[pipeline.ID] = struct{}{}
}
errs = validateBackend(errs, pipelineContext+".source", pipeline.Source.Backend, pipeline.Source.Path, pipeline.Source.URI, pipeline.Source.Endpoint, pipeline.Source.Bucket)
errs = validateValidationPolicy(errs, pipelineContext+".validation", pipeline.Validation)
if len(pipeline.Destinations) == 0 {
errs = append(errs, pipelineContext+".destinations is required")
}
destinationIDs := make(map[string]struct{}, len(pipeline.Destinations))
for destinationIndex, destination := range pipeline.Destinations {
destinationContext := fmt.Sprintf("%s.destinations[%d]", pipelineContext, destinationIndex)
if destination.ID == "" {
errs = append(errs, destinationContext+".id is required")
} else if !idPattern.MatchString(destination.ID) {
errs = append(errs, destinationContext+".id must be a slug-like identifier")
} else if _, exists := destinationIDs[destination.ID]; exists {
errs = append(errs, "destination id "+destination.ID+" is duplicated in pipeline "+pipeline.ID)
} else {
destinationIDs[destination.ID] = struct{}{}
}
errs = validateBackend(errs, destinationContext, destination.Backend, destination.Path, destination.URI, destination.Endpoint, destination.Bucket)
errs = validatePublishPolicy(errs, destinationContext+".publish", destination.Publish)
errs = validateTransform(errs, destinationContext+".transform", destination.Publish, destination.Transform)
errs = validateTransferPolicy(errs, destinationContext+".transfer", destination.Transfer)
}
}
if len(errs) > 0 {
return errs
}
return nil
}
func validateBackend(errs ValidationErrors, context, backend, path, uri, endpoint, bucket string) ValidationErrors {
switch backend {
case "":
errs = append(errs, context+".backend is required")
case BackendLocal:
if path == "" {
errs = append(errs, context+".path is required for local backend")
}
case BackendSSH:
if uri == "" {
errs = append(errs, context+".uri is required for ssh backend")
}
if path == "" {
errs = append(errs, context+".path is required for ssh backend")
}
case BackendS3:
if endpoint == "" {
errs = append(errs, context+".endpoint is required for s3 backend")
}
if bucket == "" {
errs = append(errs, context+".bucket is required for s3 backend")
}
default:
errs = append(errs, context+".backend "+backend+" is unsupported")
}
return errs
}
func validateValidationPolicy(errs ValidationErrors, context string, policy ValidationPolicy) ValidationErrors {
if policy.OnDigestMismatch != ValidationActionFail {
errs = append(errs, context+".on_digest_mismatch must be "+ValidationActionFail)
}
return errs
}
func validatePublishPolicy(errs ValidationErrors, context string, policy *PublishPolicy) ValidationErrors {
if policy == nil {
errs = append(errs, context+" is required")
return errs
}
if !policy.Source && !policy.HTML {
errs = append(errs, context+" must enable source or html")
}
return errs
}
func validateTransform(errs ValidationErrors, context string, publish *PublishPolicy, transform Transform) ValidationErrors {
publishesHTML := publish != nil && publish.HTML
if transform.MarkdownToHTML == nil {
if publishesHTML {
errs = append(errs, context+".markdown_to_html is required when publish.html is true")
}
return errs
}
if publishesHTML && !transform.MarkdownToHTML.Enabled {
errs = append(errs, context+".markdown_to_html.enabled must be true when publish.html is true")
}
if transform.MarkdownToHTML.Enabled && transform.MarkdownToHTML.Mode != TransformModeSidecar {
errs = append(errs, context+".markdown_to_html.mode must be "+TransformModeSidecar)
}
if !transform.MarkdownToHTML.Enabled && transform.MarkdownToHTML.Mode != "" && transform.MarkdownToHTML.Mode != TransformModeSidecar {
errs = append(errs, context+".markdown_to_html.mode must be "+TransformModeSidecar)
}
return errs
}
func validateTransferPolicy(errs ValidationErrors, context string, policy TransferPolicy) ValidationErrors {
if policy.OnDestinationSame != TransferActionSkip && policy.OnDestinationSame != TransferActionFail {
errs = append(errs, context+".on_destination_same must be skip or fail")
}
if policy.OnDestinationOlder != TransferActionReplace && policy.OnDestinationOlder != TransferActionFail {
errs = append(errs, context+".on_destination_older must be replace or fail")
}
if policy.OnDestinationNewer != TransferActionSkip && policy.OnDestinationNewer != TransferActionFail {
errs = append(errs, context+".on_destination_newer must be skip or fail")
}
if policy.OnConflict != TransferActionFail {
errs = append(errs, context+".on_conflict must be fail")
}
return errs
}