feat(sources)!: split source contracts into PollSource/StreamSource and add mode-aware source config

- Introduce explicit source interfaces: sources.PollSource and sources.StreamSource, with shared sources.Input (Name() only).
- Remove mandatory Kind() from the base source contract to support sources that emit multiple kinds.
- Add config.SourceMode (poll, stream, or omitted/auto) and SourceConfig.Kinds (plural expected kinds), while keeping legacy SourceConfig.Kind for compatibility.
- Enforce mode semantics in config validation (poll requires every, stream forbids every) and detect mode/driver mismatches in sources.Registry.
- Update docs and tests for the new source model and config behavior.
This commit is contained in:
2026-03-15 19:19:19 -05:00
parent fafba0f01b
commit 6c5f95ad26
12 changed files with 591 additions and 542 deletions

View File

@@ -21,20 +21,56 @@ type Config struct {
Routes []RouteConfig `yaml:"routes"`
}
// SourceConfig describes one polling job.
// SourceMode selects how a source receives upstream input.
//
// Empty mode means "auto": feedkit infers mode from the registered driver type.
type SourceMode string
const (
SourceModeAuto SourceMode = ""
SourceModePoll SourceMode = "poll"
SourceModeStream SourceMode = "stream"
)
// Normalize lowercases and trims the mode.
func (m SourceMode) Normalize() SourceMode {
switch strings.ToLower(strings.TrimSpace(string(m))) {
case "":
return SourceModeAuto
case string(SourceModePoll):
return SourceModePoll
case string(SourceModeStream):
return SourceModeStream
default:
return SourceMode(strings.ToLower(strings.TrimSpace(string(m))))
}
}
// SourceConfig describes one input source.
//
// This is intentionally generic:
// - driver-specific knobs belong in Params.
// - "kind" is allowed (useful for safety checks / routing), but feedkit does not
// restrict the allowed values.
// - mode controls polling vs streaming behavior.
// - expected emitted kinds are optional and domain-defined.
type SourceConfig struct {
Name string `yaml:"name"`
Driver string `yaml:"driver"` // e.g. "openmeteo_observation", "rss_feed", etc.
Every Duration `yaml:"every"` // "15m", "1m", etc.
// Mode is optional:
// - "poll": Every must be set (>0)
// - "stream": Every must be omitted/zero
// - empty: infer from driver registration type (poll vs stream)
Mode SourceMode `yaml:"mode"`
// Kind is optional and domain-defined. If set, it should be a non-empty string.
// Domains commonly use it to enforce "this source should only emit kind X".
// Every is the poll cadence for poll-mode sources ("15m", "1m", etc.).
Every Duration `yaml:"every"`
// Kinds is optional and domain-defined.
// If set, it describes the expected emitted event kinds for this source.
Kinds []string `yaml:"kinds"`
// Kind is the legacy singular form. Prefer "kinds".
// If both kind and kinds are set, validation fails.
Kind string `yaml:"kind"`
// Params are driver-specific settings (URL, headers, station IDs, API keys, etc.).
@@ -42,6 +78,26 @@ type SourceConfig struct {
Params map[string]any `yaml:"params"`
}
// ExpectedKinds returns normalized expected kinds from config.
// "kinds" takes precedence; "kind" is used as a legacy fallback.
func (cfg SourceConfig) ExpectedKinds() []string {
if len(cfg.Kinds) > 0 {
out := make([]string, 0, len(cfg.Kinds))
for _, k := range cfg.Kinds {
k = strings.TrimSpace(k)
if k == "" {
continue
}
out = append(out, k)
}
return out
}
if k := strings.TrimSpace(cfg.Kind); k != "" {
return []string{k}
}
return nil
}
// SinkConfig describes one output sink adapter.
type SinkConfig struct {
Name string `yaml:"name"`

56
config/config_test.go Normal file
View File

@@ -0,0 +1,56 @@
package config
import (
"reflect"
"testing"
)
func TestSourceConfigExpectedKinds(t *testing.T) {
tests := []struct {
name string
cfg SourceConfig
want []string
}{
{
name: "plural kinds preferred",
cfg: SourceConfig{
Kinds: []string{" observation ", "forecast"},
Kind: "alert",
},
want: []string{"observation", "forecast"},
},
{
name: "legacy singular fallback",
cfg: SourceConfig{
Kind: " alert ",
},
want: []string{"alert"},
},
{
name: "empty kinds",
cfg: SourceConfig{},
want: nil,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := tt.cfg.ExpectedKinds()
if !reflect.DeepEqual(got, tt.want) {
t.Fatalf("ExpectedKinds() = %#v, want %#v", got, tt.want)
}
})
}
}
func TestSourceModeNormalize(t *testing.T) {
if got := SourceMode(" Poll ").Normalize(); got != SourceModePoll {
t.Fatalf("Normalize poll = %q, want %q", got, SourceModePoll)
}
if got := SourceMode("STREAM").Normalize(); got != SourceModeStream {
t.Fatalf("Normalize stream = %q, want %q", got, SourceModeStream)
}
if got := SourceMode("").Normalize(); got != SourceModeAuto {
t.Fatalf("Normalize auto = %q, want %q", got, SourceModeAuto)
}
}

View File

@@ -83,15 +83,41 @@ func (c *Config) Validate() error {
m.Add(fieldErr(path+".driver", "is required (e.g. openmeteo_observation, rss_feed, ...)"))
}
// Every (optional but if present must be >=0)
if s.Every.Duration < 0 {
m.Add(fieldErr(path+".every", "is optional, but must be a positive duration (e.g. 15m, 1m, 30s) if provided"))
// Mode
mode := s.Mode.Normalize()
if s.Mode != SourceModeAuto && mode != SourceModePoll && mode != SourceModeStream {
m.Add(fieldErr(path+".mode", `must be one of: "poll", "stream" (or omit for auto)`))
}
// Kind (optional but if present must be non-empty after trimming)
// Every
if s.Every.Duration < 0 {
m.Add(fieldErr(path+".every", "is optional, but must be a positive duration (e.g. 15m, 1m, 30s) if provided"))
} else {
switch mode {
case SourceModePoll:
if s.Every.Duration <= 0 {
m.Add(fieldErr(path+".every", `is required when mode="poll" (e.g. 15m, 1m, 30s)`))
}
case SourceModeStream:
if s.Every.Duration > 0 {
m.Add(fieldErr(path+".every", `must be omitted when mode="stream"`))
}
}
}
// Kind/Kinds (optional)
if s.Kind != "" && len(s.Kinds) > 0 {
m.Add(fieldErr(path+".kind", `cannot be set when "kinds" is provided (use only "kinds")`))
}
if s.Kind != "" && strings.TrimSpace(s.Kind) == "" {
m.Add(fieldErr(path+".kind", "cannot be blank (omit it entirely, or provide a non-empty string)"))
}
for j, k := range s.Kinds {
kpath := fmt.Sprintf("%s.kinds[%d]", path, j)
if strings.TrimSpace(k) == "" {
m.Add(fieldErr(kpath, "kind cannot be empty"))
}
}
// Params can be nil; that's fine.
}

View File

@@ -46,3 +46,119 @@ func TestValidate_RouteKindsRejectsBlankEntries(t *testing.T) {
t.Fatalf("expected error to mention blank kind entry, got: %v", err)
}
}
func TestValidate_SourceModePollRequiresEvery(t *testing.T) {
cfg := &Config{
Sources: []SourceConfig{
{Name: "src1", Driver: "driver1", Mode: SourceModePoll},
},
Sinks: []SinkConfig{
{Name: "sink1", Driver: "stdout"},
},
}
err := cfg.Validate()
if err == nil {
t.Fatalf("expected error, got nil")
}
if !strings.Contains(err.Error(), `sources[0].every`) {
t.Fatalf("expected error to mention sources[0].every, got: %v", err)
}
}
func TestValidate_SourceModeStreamRejectsEvery(t *testing.T) {
cfg := &Config{
Sources: []SourceConfig{
{
Name: "src1",
Driver: "driver1",
Mode: SourceModeStream,
Every: Duration{Duration: time.Minute},
},
},
Sinks: []SinkConfig{
{Name: "sink1", Driver: "stdout"},
},
}
err := cfg.Validate()
if err == nil {
t.Fatalf("expected error, got nil")
}
if !strings.Contains(err.Error(), `sources[0].every`) {
t.Fatalf("expected error to mention sources[0].every, got: %v", err)
}
}
func TestValidate_SourceModeRejectsUnknownValue(t *testing.T) {
cfg := &Config{
Sources: []SourceConfig{
{
Name: "src1",
Driver: "driver1",
Mode: SourceMode("batch"),
Every: Duration{Duration: time.Minute},
},
},
Sinks: []SinkConfig{
{Name: "sink1", Driver: "stdout"},
},
}
err := cfg.Validate()
if err == nil {
t.Fatalf("expected error, got nil")
}
if !strings.Contains(err.Error(), `sources[0].mode`) {
t.Fatalf("expected error to mention sources[0].mode, got: %v", err)
}
}
func TestValidate_SourceKindAndKindsConflict(t *testing.T) {
cfg := &Config{
Sources: []SourceConfig{
{
Name: "src1",
Driver: "driver1",
Every: Duration{Duration: time.Minute},
Kind: "observation",
Kinds: []string{"forecast"},
},
},
Sinks: []SinkConfig{
{Name: "sink1", Driver: "stdout"},
},
}
err := cfg.Validate()
if err == nil {
t.Fatalf("expected error, got nil")
}
if !strings.Contains(err.Error(), `sources[0].kind`) {
t.Fatalf("expected error to mention sources[0].kind, got: %v", err)
}
}
func TestValidate_SourceKindsRejectBlankEntries(t *testing.T) {
cfg := &Config{
Sources: []SourceConfig{
{
Name: "src1",
Driver: "driver1",
Every: Duration{Duration: time.Minute},
Kinds: []string{"observation", " "},
},
},
Sinks: []SinkConfig{
{Name: "sink1", Driver: "stdout"},
},
}
err := cfg.Validate()
if err == nil {
t.Fatalf("expected error, got nil")
}
if !strings.Contains(err.Error(), `sources[0].kinds[1]`) {
t.Fatalf("expected error to mention sources[0].kinds[1], got: %v", err)
}
}