Add destination link metadata

This commit is contained in:
2026-06-01 21:42:00 +00:00
parent a8564035d3
commit 980ae15249
24 changed files with 737 additions and 16 deletions

View File

@@ -33,6 +33,7 @@ type Destination struct {
Publish *PublishPolicy `yaml:"publish"`
Transform Transform `yaml:"transform"`
PathMap PathMapping `yaml:"path_mapping"`
Links *Links `yaml:"links"`
Transfer TransferPolicy `yaml:"transfer"`
}
@@ -85,6 +86,11 @@ type PathMapping struct {
Mode string `yaml:"mode"`
}
type Links struct {
BaseURL string `yaml:"base_url"`
Primary string `yaml:"primary"`
}
type TransferPolicy struct {
OnDestinationSame string `yaml:"on_destination_same"`
OnDestinationOlder string `yaml:"on_destination_older"`

View File

@@ -30,6 +30,12 @@ const (
PathMappingFixed = "fixed"
)
const (
LinkPrimaryAuto = "auto"
LinkPrimaryHTML = "html"
LinkPrimarySource = "source"
)
const DefaultS3Region = "us-east-1"
func ApplyDefaults(cfg *Config) {
@@ -51,6 +57,9 @@ func ApplyDefaults(cfg *Config) {
if destination.PathMap.Mode == "" {
destination.PathMap.Mode = PathMappingPreserveRelative
}
if destination.Links != nil && destination.Links.Primary == "" {
destination.Links.Primary = LinkPrimaryAuto
}
if destination.Transfer.OnDestinationSame == "" {
destination.Transfer.OnDestinationSame = TransferActionSkip
}

View File

@@ -184,6 +184,30 @@ pipelines:
}
}
func TestLoadFileDefaultsLinksPrimaryToAuto(t *testing.T) {
cfg := loadConfig(t, `
pipelines:
- id: reports
source:
backend: local
path: /source
destinations:
- id: web
backend: local
path: /destination
links:
base_url: https://reports.example.com/archive
`)
links := cfg.Pipelines[0].Destinations[0].Links
if links == nil {
t.Fatal("links = nil, want config")
}
if links.BaseURL != "https://reports.example.com/archive" || links.Primary != LinkPrimaryAuto {
t.Fatalf("links = %#v, want base URL with auto primary", links)
}
}
func TestLoadFileValidBackendConfigs(t *testing.T) {
tests := map[string]string{
"local": `

View File

@@ -2,6 +2,7 @@ package config
import (
"fmt"
"net/url"
"regexp"
"strings"
)
@@ -59,6 +60,7 @@ func Validate(cfg Config) error {
errs = validateDestinationBackend(errs, destinationContext, destination)
errs = validatePublishTransformPolicy(errs, destinationContext, destination.Publish, destination.Transform)
errs = validatePathMapping(errs, destinationContext+".path_mapping", destination.PathMap)
errs = validateLinks(errs, destinationContext+".links", destination.Links)
errs = validateTransferPolicy(errs, destinationContext+".transfer", destination.Transfer)
}
}
@@ -179,6 +181,43 @@ func validatePathMapping(errs ValidationErrors, context string, mapping PathMapp
return errs
}
func validateLinks(errs ValidationErrors, context string, links *Links) ValidationErrors {
if links == nil {
return errs
}
if links.BaseURL == "" {
errs = append(errs, context+".base_url is required")
} else if err := validateLinkBaseURL(links.BaseURL); err != nil {
errs = append(errs, context+".base_url "+err.Error())
}
switch links.Primary {
case LinkPrimaryAuto, LinkPrimaryHTML, LinkPrimarySource:
default:
errs = append(errs, context+".primary must be "+LinkPrimaryAuto+", "+LinkPrimaryHTML+", or "+LinkPrimarySource)
}
return errs
}
func validateLinkBaseURL(value string) error {
parsed, err := url.Parse(value)
if err != nil {
return fmt.Errorf("must be a valid URL")
}
if parsed.Scheme != "http" && parsed.Scheme != "https" {
return fmt.Errorf("must use http or https")
}
if parsed.Host == "" {
return fmt.Errorf("must include a host")
}
if parsed.RawQuery != "" {
return fmt.Errorf("must not include a query string")
}
if parsed.Fragment != "" {
return fmt.Errorf("must not include a fragment")
}
return nil
}
func validateTransferPolicy(errs ValidationErrors, context string, policy TransferPolicy) ValidationErrors {
if policy.OnDestinationSame != TransferActionSkip && policy.OnDestinationSame != TransferActionFail {
errs = append(errs, context+".on_destination_same must be skip or fail")

View File

@@ -104,6 +104,47 @@ func TestValidatePathMapping(t *testing.T) {
}
}
func TestValidateLinks(t *testing.T) {
tests := []struct {
name string
links *Links
wantErr bool
}{
{name: "absent links"},
{name: "http", links: &Links{BaseURL: "http://reports.example.com/archive", Primary: LinkPrimaryAuto}},
{name: "https", links: &Links{BaseURL: "https://reports.example.com/archive/", Primary: LinkPrimaryHTML}},
{name: "source primary", links: &Links{BaseURL: "https://reports.example.com", Primary: LinkPrimarySource}},
{name: "missing base", links: &Links{Primary: LinkPrimaryAuto}, wantErr: true},
{name: "ftp scheme", links: &Links{BaseURL: "ftp://reports.example.com", Primary: LinkPrimaryAuto}, wantErr: true},
{name: "missing host", links: &Links{BaseURL: "https:///archive", Primary: LinkPrimaryAuto}, wantErr: true},
{name: "query", links: &Links{BaseURL: "https://reports.example.com/archive?preview=1", Primary: LinkPrimaryAuto}, wantErr: true},
{name: "fragment", links: &Links{BaseURL: "https://reports.example.com/archive#top", Primary: LinkPrimaryAuto}, wantErr: true},
{name: "invalid primary", links: &Links{BaseURL: "https://reports.example.com", Primary: "document"}, wantErr: true},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
cfg := Config{Pipelines: []Pipeline{{
ID: "reports",
Source: Backend{Backend: BackendLocal, Path: "/source"},
Destinations: []Destination{{
ID: "web",
Backend: BackendLocal,
Path: "/destination",
Links: tt.links,
}},
}}}
ApplyDefaults(&cfg)
err := Validate(cfg)
if tt.wantErr && err == nil {
t.Fatal("Validate() error = nil, want error")
}
if !tt.wantErr && err != nil {
t.Fatalf("Validate() error = %v", err)
}
})
}
}
type publishTransformPolicyCase struct {
name string
publish PublishPolicy