feedkit: split the former maximumdirect.net/weatherd project in two.
feedkit now contains a reusable core, while weatherfeeder is a concrete implementation that includes weather-specific functions.
This commit is contained in:
139
config/config.go
Normal file
139
config/config.go
Normal file
@@ -0,0 +1,139 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
// Config is the top-level YAML configuration used by feedkit-style daemons.
|
||||
//
|
||||
// IMPORTANT: This package is intentionally domain-agnostic.
|
||||
// - We validate *shape* and internal consistency (required fields, uniqueness, etc.).
|
||||
// - We do NOT enforce domain rules (e.g., "observation|forecast|alert").
|
||||
// Domain modules (weatherfeeder/newsfeeder/...) can add their own validation layer.
|
||||
type Config struct {
|
||||
Sources []SourceConfig `yaml:"sources"`
|
||||
Sinks []SinkConfig `yaml:"sinks"`
|
||||
Routes []RouteConfig `yaml:"routes"`
|
||||
}
|
||||
|
||||
// SourceConfig describes one polling job.
|
||||
//
|
||||
// This is intentionally generic:
|
||||
// - driver-specific knobs belong in Params.
|
||||
// - "kind" is allowed (useful for safety checks / routing), but feedkit does not
|
||||
// restrict the allowed values.
|
||||
type SourceConfig struct {
|
||||
Name string `yaml:"name"`
|
||||
Driver string `yaml:"driver"` // e.g. "openmeteo_observation", "rss_feed", etc.
|
||||
|
||||
Every Duration `yaml:"every"` // "15m", "1m", etc.
|
||||
|
||||
// Kind is optional and domain-defined. If set, it should be a non-empty string.
|
||||
// Domains commonly use it to enforce "this source should only emit kind X".
|
||||
Kind string `yaml:"kind"`
|
||||
|
||||
// Params are driver-specific settings (URL, headers, station IDs, API keys, etc.).
|
||||
// The driver implementation is responsible for reading/validating these.
|
||||
Params map[string]any `yaml:"params"`
|
||||
}
|
||||
|
||||
// SinkConfig describes one output sink adapter.
|
||||
type SinkConfig struct {
|
||||
Name string `yaml:"name"`
|
||||
Driver string `yaml:"driver"` // "stdout", "file", "postgres", "rabbitmq", ...
|
||||
Params map[string]any `yaml:"params"` // sink-specific settings
|
||||
}
|
||||
|
||||
// RouteConfig describes a routing rule: which sink receives which kinds.
|
||||
type RouteConfig struct {
|
||||
Sink string `yaml:"sink"` // sink name
|
||||
|
||||
// Kinds is domain-defined. feedkit only enforces that each entry is non-empty.
|
||||
// Whether a given daemon "recognizes" a kind is domain-specific validation.
|
||||
Kinds []string `yaml:"kinds"`
|
||||
}
|
||||
|
||||
// Duration is a YAML-friendly duration wrapper.
|
||||
//
|
||||
// Supported YAML formats:
|
||||
//
|
||||
// every: 15m # string duration (recommended)
|
||||
// every: 30s
|
||||
// every: 1h
|
||||
//
|
||||
// Also supported for convenience (interpreted as minutes):
|
||||
//
|
||||
// every: 15 # integer minutes
|
||||
// every: "15" # string minutes
|
||||
type Duration struct {
|
||||
time.Duration
|
||||
}
|
||||
|
||||
func (d *Duration) UnmarshalYAML(value *yaml.Node) error {
|
||||
// We expect a scalar (string or int).
|
||||
if value.Kind != yaml.ScalarNode {
|
||||
return fmt.Errorf("duration must be a scalar (e.g. 15m), got %v", value.Kind)
|
||||
}
|
||||
|
||||
// Case 1: YAML integer -> interpret as minutes.
|
||||
if value.Tag == "!!int" {
|
||||
n, err := strconv.Atoi(value.Value)
|
||||
if err != nil {
|
||||
return fmt.Errorf("invalid integer duration %q: %w", value.Value, err)
|
||||
}
|
||||
if n <= 0 {
|
||||
return fmt.Errorf("duration must be > 0, got %d", n)
|
||||
}
|
||||
d.Duration = time.Duration(n) * time.Minute
|
||||
return nil
|
||||
}
|
||||
|
||||
// Case 2: YAML string.
|
||||
if value.Tag == "!!str" {
|
||||
s := strings.TrimSpace(value.Value)
|
||||
if s == "" {
|
||||
return fmt.Errorf("duration cannot be empty")
|
||||
}
|
||||
|
||||
// If it’s all digits, interpret as minutes (e.g. "15").
|
||||
if isAllDigits(s) {
|
||||
n, err := strconv.Atoi(s)
|
||||
if err != nil {
|
||||
return fmt.Errorf("invalid numeric duration %q: %w", s, err)
|
||||
}
|
||||
if n <= 0 {
|
||||
return fmt.Errorf("duration must be > 0, got %d", n)
|
||||
}
|
||||
d.Duration = time.Duration(n) * time.Minute
|
||||
return nil
|
||||
}
|
||||
|
||||
// Otherwise parse as Go duration string (15m, 30s, 1h, etc.).
|
||||
parsed, err := time.ParseDuration(s)
|
||||
if err != nil {
|
||||
return fmt.Errorf("invalid duration %q (expected e.g. 15m, 30s, 1h): %w", s, err)
|
||||
}
|
||||
if parsed <= 0 {
|
||||
return fmt.Errorf("duration must be > 0, got %s", parsed)
|
||||
}
|
||||
d.Duration = parsed
|
||||
return nil
|
||||
}
|
||||
|
||||
// Anything else: reject.
|
||||
return fmt.Errorf("duration must be a string like 15m or an integer minutes, got tag %s", value.Tag)
|
||||
}
|
||||
|
||||
func isAllDigits(s string) bool {
|
||||
for _, r := range s {
|
||||
if r < '0' || r > '9' {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return len(s) > 0
|
||||
}
|
||||
193
config/load.go
Normal file
193
config/load.go
Normal file
@@ -0,0 +1,193 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
// Load reads the YAML config file from disk, decodes it into Config,
|
||||
// and then validates it.
|
||||
//
|
||||
// Important behaviors:
|
||||
// - Uses yaml.Decoder with KnownFields(true) to catch typos like "srouces:".
|
||||
// - Returns a validation error that (usually) contains multiple issues at once.
|
||||
func Load(path string) (*Config, error) {
|
||||
if strings.TrimSpace(path) == "" {
|
||||
path = "config.yml"
|
||||
}
|
||||
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("config.Load: read %q: %w", path, err)
|
||||
}
|
||||
|
||||
var cfg Config
|
||||
|
||||
dec := yaml.NewDecoder(strings.NewReader(string(raw)))
|
||||
dec.KnownFields(true) // strict mode for struct fields
|
||||
|
||||
if err := dec.Decode(&cfg); err != nil {
|
||||
return nil, fmt.Errorf("config.Load: parse YAML %q: %w", path, err)
|
||||
}
|
||||
|
||||
// Optional: ensure there isn't a second YAML document accidentally appended.
|
||||
var extra any
|
||||
if err := dec.Decode(&extra); err == nil {
|
||||
return nil, fmt.Errorf("config.Load: %q contains multiple YAML documents; expected exactly one", path)
|
||||
}
|
||||
|
||||
if err := cfg.Validate(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &cfg, nil
|
||||
}
|
||||
|
||||
// Validate checks whether the config is internally consistent and safe to run.
|
||||
//
|
||||
// This is intentionally DOMAIN-AGNOSTIC validation:
|
||||
// - required fields are present
|
||||
// - names are unique
|
||||
// - durations are > 0
|
||||
// - routes reference defined sinks
|
||||
//
|
||||
// We DO NOT enforce domain-specific constraints like "allowed kinds" or
|
||||
// "NWS requires a user-agent". Those belong in the domain module (weatherfeeder).
|
||||
func (c *Config) Validate() error {
|
||||
var m multiError
|
||||
|
||||
// ---------- sources ----------
|
||||
if len(c.Sources) == 0 {
|
||||
m.Add(fieldErr("sources", "must contain at least one source"))
|
||||
} else {
|
||||
seen := map[string]bool{}
|
||||
for i, s := range c.Sources {
|
||||
path := fmt.Sprintf("sources[%d]", i)
|
||||
|
||||
// Name
|
||||
if strings.TrimSpace(s.Name) == "" {
|
||||
m.Add(fieldErr(path+".name", "is required"))
|
||||
} else {
|
||||
if seen[s.Name] {
|
||||
m.Add(fieldErr(path+".name", fmt.Sprintf("duplicate source name %q (source names must be unique)", s.Name)))
|
||||
}
|
||||
seen[s.Name] = true
|
||||
}
|
||||
|
||||
// Driver
|
||||
if strings.TrimSpace(s.Driver) == "" {
|
||||
m.Add(fieldErr(path+".driver", "is required (e.g. openmeteo_observation, rss_feed, ...)"))
|
||||
}
|
||||
|
||||
// Every
|
||||
if s.Every.Duration <= 0 {
|
||||
m.Add(fieldErr(path+".every", "must be a positive duration (e.g. 15m, 1m, 30s)"))
|
||||
}
|
||||
|
||||
// Kind (optional but if present must be non-empty after trimming)
|
||||
if s.Kind != "" && strings.TrimSpace(s.Kind) == "" {
|
||||
m.Add(fieldErr(path+".kind", "cannot be blank (omit it entirely, or provide a non-empty string)"))
|
||||
}
|
||||
|
||||
// Params can be nil; that's fine.
|
||||
}
|
||||
}
|
||||
|
||||
// ---------- sinks ----------
|
||||
sinkNames := map[string]bool{}
|
||||
if len(c.Sinks) == 0 {
|
||||
m.Add(fieldErr("sinks", "must contain at least one sink"))
|
||||
} else {
|
||||
for i, s := range c.Sinks {
|
||||
path := fmt.Sprintf("sinks[%d]", i)
|
||||
|
||||
if strings.TrimSpace(s.Name) == "" {
|
||||
m.Add(fieldErr(path+".name", "is required"))
|
||||
} else {
|
||||
if sinkNames[s.Name] {
|
||||
m.Add(fieldErr(path+".name", fmt.Sprintf("duplicate sink name %q (sink names must be unique)", s.Name)))
|
||||
}
|
||||
sinkNames[s.Name] = true
|
||||
}
|
||||
|
||||
if strings.TrimSpace(s.Driver) == "" {
|
||||
m.Add(fieldErr(path+".driver", "is required (stdout|file|postgres|rabbitmq|...)"))
|
||||
}
|
||||
|
||||
// Params can be nil; that's fine.
|
||||
}
|
||||
}
|
||||
|
||||
// ---------- routes ----------
|
||||
// Routes are optional. If provided, validate shape + references.
|
||||
for i, r := range c.Routes {
|
||||
path := fmt.Sprintf("routes[%d]", i)
|
||||
|
||||
if strings.TrimSpace(r.Sink) == "" {
|
||||
m.Add(fieldErr(path+".sink", "is required"))
|
||||
} else if !sinkNames[r.Sink] {
|
||||
m.Add(fieldErr(path+".sink", fmt.Sprintf("references unknown sink %q (define it under sinks:)", r.Sink)))
|
||||
}
|
||||
|
||||
if len(r.Kinds) == 0 {
|
||||
// You could relax this later (e.g. empty == "all kinds"), but for now
|
||||
// keeping it strict prevents accidental "route does nothing".
|
||||
m.Add(fieldErr(path+".kinds", "must contain at least one kind"))
|
||||
} else {
|
||||
for j, k := range r.Kinds {
|
||||
kpath := fmt.Sprintf("%s.kinds[%d]", path, j)
|
||||
if strings.TrimSpace(k) == "" {
|
||||
m.Add(fieldErr(kpath, "kind cannot be empty"))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return m.Err()
|
||||
}
|
||||
|
||||
// ---- error helpers ----
|
||||
|
||||
// fieldErr produces consistent "path: message" errors.
|
||||
func fieldErr(path, msg string) error {
|
||||
return fmt.Errorf("%s: %s", path, msg)
|
||||
}
|
||||
|
||||
// multiError collects many errors and returns them as one error.
|
||||
// This makes config iteration much nicer: you fix several things per run.
|
||||
type multiError struct {
|
||||
errs []error
|
||||
}
|
||||
|
||||
func (m *multiError) Add(err error) {
|
||||
if err == nil {
|
||||
return
|
||||
}
|
||||
m.errs = append(m.errs, err)
|
||||
}
|
||||
|
||||
func (m *multiError) Err() error {
|
||||
if len(m.errs) == 0 {
|
||||
return nil
|
||||
}
|
||||
// Sort for stable output (useful in tests and when iterating).
|
||||
sort.Slice(m.errs, func(i, j int) bool {
|
||||
return m.errs[i].Error() < m.errs[j].Error()
|
||||
})
|
||||
return m
|
||||
}
|
||||
|
||||
func (m *multiError) Error() string {
|
||||
var b strings.Builder
|
||||
b.WriteString("config validation failed:\n")
|
||||
for _, e := range m.errs {
|
||||
b.WriteString(" - ")
|
||||
b.WriteString(e.Error())
|
||||
b.WriteString("\n")
|
||||
}
|
||||
return strings.TrimRight(b.String(), "\n")
|
||||
}
|
||||
72
config/params.go
Normal file
72
config/params.go
Normal file
@@ -0,0 +1,72 @@
|
||||
// feedkit/config/params.go
|
||||
package config
|
||||
|
||||
import "strings"
|
||||
|
||||
// ParamString returns the first non-empty string found for any of the provided keys.
|
||||
// Values must actually be strings in the decoded config; other types are ignored.
|
||||
//
|
||||
// This keeps cfg.Params flexible (map[string]any) while letting callers stay type-safe.
|
||||
func (cfg SourceConfig) ParamString(keys ...string) (string, bool) {
|
||||
if cfg.Params == nil {
|
||||
return "", false
|
||||
}
|
||||
for _, k := range keys {
|
||||
v, ok := cfg.Params[k]
|
||||
if !ok || v == nil {
|
||||
continue
|
||||
}
|
||||
s, ok := v.(string)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
s = strings.TrimSpace(s)
|
||||
if s == "" {
|
||||
continue
|
||||
}
|
||||
return s, true
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
|
||||
// ParamStringDefault returns ParamString(keys...) if present; otherwise it returns def.
|
||||
// This is the “polite default” helper used by drivers for optional fields like user-agent.
|
||||
func (cfg SourceConfig) ParamStringDefault(def string, keys ...string) string {
|
||||
if s, ok := cfg.ParamString(keys...); ok {
|
||||
return s
|
||||
}
|
||||
return strings.TrimSpace(def)
|
||||
}
|
||||
|
||||
// ParamString returns the first non-empty string found for any of the provided keys
|
||||
// in SinkConfig.Params. (Same rationale as SourceConfig.ParamString.)
|
||||
func (cfg SinkConfig) ParamString(keys ...string) (string, bool) {
|
||||
if cfg.Params == nil {
|
||||
return "", false
|
||||
}
|
||||
for _, k := range keys {
|
||||
v, ok := cfg.Params[k]
|
||||
if !ok || v == nil {
|
||||
continue
|
||||
}
|
||||
s, ok := v.(string)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
s = strings.TrimSpace(s)
|
||||
if s == "" {
|
||||
continue
|
||||
}
|
||||
return s, true
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
|
||||
// ParamStringDefault returns ParamString(keys...) if present; otherwise it returns def.
|
||||
// Symmetric helper for sink implementations.
|
||||
func (cfg SinkConfig) ParamStringDefault(def string, keys ...string) string {
|
||||
if s, ok := cfg.ParamString(keys...); ok {
|
||||
return s
|
||||
}
|
||||
return strings.TrimSpace(def)
|
||||
}
|
||||
Reference in New Issue
Block a user