Add destination link metadata
This commit is contained in:
@@ -33,6 +33,7 @@ type Destination struct {
|
||||
Publish *PublishPolicy `yaml:"publish"`
|
||||
Transform Transform `yaml:"transform"`
|
||||
PathMap PathMapping `yaml:"path_mapping"`
|
||||
Links *Links `yaml:"links"`
|
||||
Transfer TransferPolicy `yaml:"transfer"`
|
||||
}
|
||||
|
||||
@@ -85,6 +86,11 @@ type PathMapping struct {
|
||||
Mode string `yaml:"mode"`
|
||||
}
|
||||
|
||||
type Links struct {
|
||||
BaseURL string `yaml:"base_url"`
|
||||
Primary string `yaml:"primary"`
|
||||
}
|
||||
|
||||
type TransferPolicy struct {
|
||||
OnDestinationSame string `yaml:"on_destination_same"`
|
||||
OnDestinationOlder string `yaml:"on_destination_older"`
|
||||
|
||||
@@ -30,6 +30,12 @@ const (
|
||||
PathMappingFixed = "fixed"
|
||||
)
|
||||
|
||||
const (
|
||||
LinkPrimaryAuto = "auto"
|
||||
LinkPrimaryHTML = "html"
|
||||
LinkPrimarySource = "source"
|
||||
)
|
||||
|
||||
const DefaultS3Region = "us-east-1"
|
||||
|
||||
func ApplyDefaults(cfg *Config) {
|
||||
@@ -51,6 +57,9 @@ func ApplyDefaults(cfg *Config) {
|
||||
if destination.PathMap.Mode == "" {
|
||||
destination.PathMap.Mode = PathMappingPreserveRelative
|
||||
}
|
||||
if destination.Links != nil && destination.Links.Primary == "" {
|
||||
destination.Links.Primary = LinkPrimaryAuto
|
||||
}
|
||||
if destination.Transfer.OnDestinationSame == "" {
|
||||
destination.Transfer.OnDestinationSame = TransferActionSkip
|
||||
}
|
||||
|
||||
@@ -184,6 +184,30 @@ pipelines:
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadFileDefaultsLinksPrimaryToAuto(t *testing.T) {
|
||||
cfg := loadConfig(t, `
|
||||
pipelines:
|
||||
- id: reports
|
||||
source:
|
||||
backend: local
|
||||
path: /source
|
||||
destinations:
|
||||
- id: web
|
||||
backend: local
|
||||
path: /destination
|
||||
links:
|
||||
base_url: https://reports.example.com/archive
|
||||
`)
|
||||
|
||||
links := cfg.Pipelines[0].Destinations[0].Links
|
||||
if links == nil {
|
||||
t.Fatal("links = nil, want config")
|
||||
}
|
||||
if links.BaseURL != "https://reports.example.com/archive" || links.Primary != LinkPrimaryAuto {
|
||||
t.Fatalf("links = %#v, want base URL with auto primary", links)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadFileValidBackendConfigs(t *testing.T) {
|
||||
tests := map[string]string{
|
||||
"local": `
|
||||
|
||||
@@ -2,6 +2,7 @@ package config
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/url"
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
@@ -59,6 +60,7 @@ func Validate(cfg Config) error {
|
||||
errs = validateDestinationBackend(errs, destinationContext, destination)
|
||||
errs = validatePublishTransformPolicy(errs, destinationContext, destination.Publish, destination.Transform)
|
||||
errs = validatePathMapping(errs, destinationContext+".path_mapping", destination.PathMap)
|
||||
errs = validateLinks(errs, destinationContext+".links", destination.Links)
|
||||
errs = validateTransferPolicy(errs, destinationContext+".transfer", destination.Transfer)
|
||||
}
|
||||
}
|
||||
@@ -179,6 +181,43 @@ func validatePathMapping(errs ValidationErrors, context string, mapping PathMapp
|
||||
return errs
|
||||
}
|
||||
|
||||
func validateLinks(errs ValidationErrors, context string, links *Links) ValidationErrors {
|
||||
if links == nil {
|
||||
return errs
|
||||
}
|
||||
if links.BaseURL == "" {
|
||||
errs = append(errs, context+".base_url is required")
|
||||
} else if err := validateLinkBaseURL(links.BaseURL); err != nil {
|
||||
errs = append(errs, context+".base_url "+err.Error())
|
||||
}
|
||||
switch links.Primary {
|
||||
case LinkPrimaryAuto, LinkPrimaryHTML, LinkPrimarySource:
|
||||
default:
|
||||
errs = append(errs, context+".primary must be "+LinkPrimaryAuto+", "+LinkPrimaryHTML+", or "+LinkPrimarySource)
|
||||
}
|
||||
return errs
|
||||
}
|
||||
|
||||
func validateLinkBaseURL(value string) error {
|
||||
parsed, err := url.Parse(value)
|
||||
if err != nil {
|
||||
return fmt.Errorf("must be a valid URL")
|
||||
}
|
||||
if parsed.Scheme != "http" && parsed.Scheme != "https" {
|
||||
return fmt.Errorf("must use http or https")
|
||||
}
|
||||
if parsed.Host == "" {
|
||||
return fmt.Errorf("must include a host")
|
||||
}
|
||||
if parsed.RawQuery != "" {
|
||||
return fmt.Errorf("must not include a query string")
|
||||
}
|
||||
if parsed.Fragment != "" {
|
||||
return fmt.Errorf("must not include a fragment")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateTransferPolicy(errs ValidationErrors, context string, policy TransferPolicy) ValidationErrors {
|
||||
if policy.OnDestinationSame != TransferActionSkip && policy.OnDestinationSame != TransferActionFail {
|
||||
errs = append(errs, context+".on_destination_same must be skip or fail")
|
||||
|
||||
@@ -104,6 +104,47 @@ func TestValidatePathMapping(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateLinks(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
links *Links
|
||||
wantErr bool
|
||||
}{
|
||||
{name: "absent links"},
|
||||
{name: "http", links: &Links{BaseURL: "http://reports.example.com/archive", Primary: LinkPrimaryAuto}},
|
||||
{name: "https", links: &Links{BaseURL: "https://reports.example.com/archive/", Primary: LinkPrimaryHTML}},
|
||||
{name: "source primary", links: &Links{BaseURL: "https://reports.example.com", Primary: LinkPrimarySource}},
|
||||
{name: "missing base", links: &Links{Primary: LinkPrimaryAuto}, wantErr: true},
|
||||
{name: "ftp scheme", links: &Links{BaseURL: "ftp://reports.example.com", Primary: LinkPrimaryAuto}, wantErr: true},
|
||||
{name: "missing host", links: &Links{BaseURL: "https:///archive", Primary: LinkPrimaryAuto}, wantErr: true},
|
||||
{name: "query", links: &Links{BaseURL: "https://reports.example.com/archive?preview=1", Primary: LinkPrimaryAuto}, wantErr: true},
|
||||
{name: "fragment", links: &Links{BaseURL: "https://reports.example.com/archive#top", Primary: LinkPrimaryAuto}, wantErr: true},
|
||||
{name: "invalid primary", links: &Links{BaseURL: "https://reports.example.com", Primary: "document"}, wantErr: true},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
cfg := Config{Pipelines: []Pipeline{{
|
||||
ID: "reports",
|
||||
Source: Backend{Backend: BackendLocal, Path: "/source"},
|
||||
Destinations: []Destination{{
|
||||
ID: "web",
|
||||
Backend: BackendLocal,
|
||||
Path: "/destination",
|
||||
Links: tt.links,
|
||||
}},
|
||||
}}}
|
||||
ApplyDefaults(&cfg)
|
||||
err := Validate(cfg)
|
||||
if tt.wantErr && err == nil {
|
||||
t.Fatal("Validate() error = nil, want error")
|
||||
}
|
||||
if !tt.wantErr && err != nil {
|
||||
t.Fatalf("Validate() error = %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
type publishTransformPolicyCase struct {
|
||||
name string
|
||||
publish PublishPolicy
|
||||
|
||||
Reference in New Issue
Block a user