Remove global Postgres schema registration in favor of explicit schema-aware sink factory wiring, and update weatherfeeder to register the Postgres sink explicitly. Add optional per-source HTTP timeout and response body limit overrides while keeping feedkit defaults. Remove remaining legacy source/config compatibility surfaces, including singular kind support and old source registry/type aliases, and migrate weatherfeeder sources to plural `Kinds()` metadata. Clean up related docs, tests, and sample config to match the new Postgres, HTTP, and NATS configuration model.
182 lines
5.1 KiB
Go
182 lines
5.1 KiB
Go
package config
|
||
|
||
import (
|
||
"fmt"
|
||
"strconv"
|
||
"strings"
|
||
"time"
|
||
|
||
"gopkg.in/yaml.v3"
|
||
)
|
||
|
||
// Config is the top-level YAML configuration used by feedkit-style daemons.
|
||
//
|
||
// IMPORTANT: This package is intentionally domain-agnostic.
|
||
// - We validate *shape* and internal consistency (required fields, uniqueness, etc.).
|
||
// - We do NOT enforce domain rules (e.g., "observation|forecast|alert").
|
||
// Domain modules (weatherfeeder/newsfeeder/...) can add their own validation layer.
|
||
type Config struct {
|
||
Sources []SourceConfig `yaml:"sources"`
|
||
Sinks []SinkConfig `yaml:"sinks"`
|
||
Routes []RouteConfig `yaml:"routes"`
|
||
}
|
||
|
||
// SourceMode selects how a source receives upstream input.
|
||
//
|
||
// Empty mode means "auto": feedkit infers mode from the registered driver type.
|
||
type SourceMode string
|
||
|
||
const (
|
||
SourceModeAuto SourceMode = ""
|
||
SourceModePoll SourceMode = "poll"
|
||
SourceModeStream SourceMode = "stream"
|
||
)
|
||
|
||
// Normalize lowercases and trims the mode.
|
||
func (m SourceMode) Normalize() SourceMode {
|
||
switch strings.ToLower(strings.TrimSpace(string(m))) {
|
||
case "":
|
||
return SourceModeAuto
|
||
case string(SourceModePoll):
|
||
return SourceModePoll
|
||
case string(SourceModeStream):
|
||
return SourceModeStream
|
||
default:
|
||
return SourceMode(strings.ToLower(strings.TrimSpace(string(m))))
|
||
}
|
||
}
|
||
|
||
// SourceConfig describes one input source.
|
||
//
|
||
// This is intentionally generic:
|
||
// - driver-specific knobs belong in Params.
|
||
// - mode controls polling vs streaming behavior.
|
||
// - expected emitted kinds are optional and domain-defined.
|
||
type SourceConfig struct {
|
||
Name string `yaml:"name"`
|
||
Driver string `yaml:"driver"` // e.g. "openmeteo_observation", "rss_feed", etc.
|
||
|
||
// Mode is optional:
|
||
// - "poll": Every must be set (>0)
|
||
// - "stream": Every must be omitted/zero
|
||
// - empty: infer from driver registration type (poll vs stream)
|
||
Mode SourceMode `yaml:"mode"`
|
||
|
||
// Every is the poll cadence for poll-mode sources ("15m", "1m", etc.).
|
||
Every Duration `yaml:"every"`
|
||
|
||
// Kinds is optional and domain-defined.
|
||
// If set, it describes the expected emitted event kinds for this source.
|
||
Kinds []string `yaml:"kinds"`
|
||
|
||
// Params are driver-specific settings (URL, headers, station IDs, API keys, etc.).
|
||
// The driver implementation is responsible for reading/validating these.
|
||
Params map[string]any `yaml:"params"`
|
||
}
|
||
|
||
// ExpectedKinds returns normalized expected kinds from config.
|
||
func (cfg SourceConfig) ExpectedKinds() []string {
|
||
out := make([]string, 0, len(cfg.Kinds))
|
||
for _, k := range cfg.Kinds {
|
||
k = strings.TrimSpace(k)
|
||
if k == "" {
|
||
continue
|
||
}
|
||
out = append(out, k)
|
||
}
|
||
if len(out) == 0 {
|
||
return nil
|
||
}
|
||
return out
|
||
}
|
||
|
||
// SinkConfig describes one output sink adapter.
|
||
type SinkConfig struct {
|
||
Name string `yaml:"name"`
|
||
Driver string `yaml:"driver"` // "stdout", "nats", "postgres", ...
|
||
Params map[string]any `yaml:"params"` // sink-specific settings
|
||
}
|
||
|
||
// RouteConfig describes a routing rule: which sink receives which kinds.
|
||
type RouteConfig struct {
|
||
Sink string `yaml:"sink"` // sink name
|
||
|
||
// Kinds is domain-defined. feedkit only enforces that each entry is non-empty.
|
||
//
|
||
// If Kinds is omitted or empty, the route matches ALL kinds.
|
||
// This is useful when you want explicit per-sink routing rules even when a
|
||
// particular sink should receive everything.
|
||
Kinds []string `yaml:"kinds"`
|
||
}
|
||
|
||
// Duration is a YAML-friendly duration wrapper.
|
||
//
|
||
// Supported YAML formats:
|
||
//
|
||
// every: 15m # string duration (recommended)
|
||
// every: 30s
|
||
// every: 1h
|
||
//
|
||
// Also supported for convenience (interpreted as minutes):
|
||
//
|
||
// every: 15 # integer minutes
|
||
// every: "15" # string minutes
|
||
type Duration struct {
|
||
time.Duration
|
||
}
|
||
|
||
func (d *Duration) UnmarshalYAML(value *yaml.Node) error {
|
||
// We expect a scalar (string or int).
|
||
if value.Kind != yaml.ScalarNode {
|
||
return fmt.Errorf("duration must be a scalar (e.g. 15m), got %v", value.Kind)
|
||
}
|
||
|
||
// Case 1: YAML integer -> interpret as minutes.
|
||
if value.Tag == "!!int" {
|
||
n, err := strconv.Atoi(value.Value)
|
||
if err != nil {
|
||
return fmt.Errorf("invalid integer duration %q: %w", value.Value, err)
|
||
}
|
||
if n <= 0 {
|
||
return fmt.Errorf("duration must be > 0, got %d", n)
|
||
}
|
||
d.Duration = time.Duration(n) * time.Minute
|
||
return nil
|
||
}
|
||
|
||
// Case 2: YAML string.
|
||
if value.Tag == "!!str" {
|
||
s := strings.TrimSpace(value.Value)
|
||
if s == "" {
|
||
return fmt.Errorf("duration cannot be empty")
|
||
}
|
||
|
||
// If it’s all digits, interpret as minutes (e.g. "15").
|
||
if isAllDigits(s) {
|
||
n, err := strconv.Atoi(s)
|
||
if err != nil {
|
||
return fmt.Errorf("invalid numeric duration %q: %w", s, err)
|
||
}
|
||
if n <= 0 {
|
||
return fmt.Errorf("duration must be > 0, got %d", n)
|
||
}
|
||
d.Duration = time.Duration(n) * time.Minute
|
||
return nil
|
||
}
|
||
|
||
// Otherwise parse as Go duration string (15m, 30s, 1h, etc.).
|
||
parsed, err := time.ParseDuration(s)
|
||
if err != nil {
|
||
return fmt.Errorf("invalid duration %q (expected e.g. 15m, 30s, 1h): %w", s, err)
|
||
}
|
||
if parsed <= 0 {
|
||
return fmt.Errorf("duration must be > 0, got %s", parsed)
|
||
}
|
||
d.Duration = parsed
|
||
return nil
|
||
}
|
||
|
||
// Anything else: reject.
|
||
return fmt.Errorf("duration must be a string like 15m or an integer minutes, got tag %s", value.Tag)
|
||
}
|