feat(sources)!: split source contracts into PollSource/StreamSource and add mode-aware source config
- Introduce explicit source interfaces: sources.PollSource and sources.StreamSource, with shared sources.Input (Name() only). - Remove mandatory Kind() from the base source contract to support sources that emit multiple kinds. - Add config.SourceMode (poll, stream, or omitted/auto) and SourceConfig.Kinds (plural expected kinds), while keeping legacy SourceConfig.Kind for compatibility. - Enforce mode semantics in config validation (poll requires every, stream forbids every) and detect mode/driver mismatches in sources.Registry. - Update docs and tests for the new source model and config behavior.
This commit is contained in:
@@ -21,20 +21,56 @@ type Config struct {
|
||||
Routes []RouteConfig `yaml:"routes"`
|
||||
}
|
||||
|
||||
// SourceConfig describes one polling job.
|
||||
// SourceMode selects how a source receives upstream input.
|
||||
//
|
||||
// Empty mode means "auto": feedkit infers mode from the registered driver type.
|
||||
type SourceMode string
|
||||
|
||||
const (
|
||||
SourceModeAuto SourceMode = ""
|
||||
SourceModePoll SourceMode = "poll"
|
||||
SourceModeStream SourceMode = "stream"
|
||||
)
|
||||
|
||||
// Normalize lowercases and trims the mode.
|
||||
func (m SourceMode) Normalize() SourceMode {
|
||||
switch strings.ToLower(strings.TrimSpace(string(m))) {
|
||||
case "":
|
||||
return SourceModeAuto
|
||||
case string(SourceModePoll):
|
||||
return SourceModePoll
|
||||
case string(SourceModeStream):
|
||||
return SourceModeStream
|
||||
default:
|
||||
return SourceMode(strings.ToLower(strings.TrimSpace(string(m))))
|
||||
}
|
||||
}
|
||||
|
||||
// SourceConfig describes one input source.
|
||||
//
|
||||
// This is intentionally generic:
|
||||
// - driver-specific knobs belong in Params.
|
||||
// - "kind" is allowed (useful for safety checks / routing), but feedkit does not
|
||||
// restrict the allowed values.
|
||||
// - mode controls polling vs streaming behavior.
|
||||
// - expected emitted kinds are optional and domain-defined.
|
||||
type SourceConfig struct {
|
||||
Name string `yaml:"name"`
|
||||
Driver string `yaml:"driver"` // e.g. "openmeteo_observation", "rss_feed", etc.
|
||||
|
||||
Every Duration `yaml:"every"` // "15m", "1m", etc.
|
||||
// Mode is optional:
|
||||
// - "poll": Every must be set (>0)
|
||||
// - "stream": Every must be omitted/zero
|
||||
// - empty: infer from driver registration type (poll vs stream)
|
||||
Mode SourceMode `yaml:"mode"`
|
||||
|
||||
// Kind is optional and domain-defined. If set, it should be a non-empty string.
|
||||
// Domains commonly use it to enforce "this source should only emit kind X".
|
||||
// Every is the poll cadence for poll-mode sources ("15m", "1m", etc.).
|
||||
Every Duration `yaml:"every"`
|
||||
|
||||
// Kinds is optional and domain-defined.
|
||||
// If set, it describes the expected emitted event kinds for this source.
|
||||
Kinds []string `yaml:"kinds"`
|
||||
|
||||
// Kind is the legacy singular form. Prefer "kinds".
|
||||
// If both kind and kinds are set, validation fails.
|
||||
Kind string `yaml:"kind"`
|
||||
|
||||
// Params are driver-specific settings (URL, headers, station IDs, API keys, etc.).
|
||||
@@ -42,6 +78,26 @@ type SourceConfig struct {
|
||||
Params map[string]any `yaml:"params"`
|
||||
}
|
||||
|
||||
// ExpectedKinds returns normalized expected kinds from config.
|
||||
// "kinds" takes precedence; "kind" is used as a legacy fallback.
|
||||
func (cfg SourceConfig) ExpectedKinds() []string {
|
||||
if len(cfg.Kinds) > 0 {
|
||||
out := make([]string, 0, len(cfg.Kinds))
|
||||
for _, k := range cfg.Kinds {
|
||||
k = strings.TrimSpace(k)
|
||||
if k == "" {
|
||||
continue
|
||||
}
|
||||
out = append(out, k)
|
||||
}
|
||||
return out
|
||||
}
|
||||
if k := strings.TrimSpace(cfg.Kind); k != "" {
|
||||
return []string{k}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// SinkConfig describes one output sink adapter.
|
||||
type SinkConfig struct {
|
||||
Name string `yaml:"name"`
|
||||
|
||||
56
config/config_test.go
Normal file
56
config/config_test.go
Normal file
@@ -0,0 +1,56 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestSourceConfigExpectedKinds(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
cfg SourceConfig
|
||||
want []string
|
||||
}{
|
||||
{
|
||||
name: "plural kinds preferred",
|
||||
cfg: SourceConfig{
|
||||
Kinds: []string{" observation ", "forecast"},
|
||||
Kind: "alert",
|
||||
},
|
||||
want: []string{"observation", "forecast"},
|
||||
},
|
||||
{
|
||||
name: "legacy singular fallback",
|
||||
cfg: SourceConfig{
|
||||
Kind: " alert ",
|
||||
},
|
||||
want: []string{"alert"},
|
||||
},
|
||||
{
|
||||
name: "empty kinds",
|
||||
cfg: SourceConfig{},
|
||||
want: nil,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tt.cfg.ExpectedKinds()
|
||||
if !reflect.DeepEqual(got, tt.want) {
|
||||
t.Fatalf("ExpectedKinds() = %#v, want %#v", got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSourceModeNormalize(t *testing.T) {
|
||||
if got := SourceMode(" Poll ").Normalize(); got != SourceModePoll {
|
||||
t.Fatalf("Normalize poll = %q, want %q", got, SourceModePoll)
|
||||
}
|
||||
if got := SourceMode("STREAM").Normalize(); got != SourceModeStream {
|
||||
t.Fatalf("Normalize stream = %q, want %q", got, SourceModeStream)
|
||||
}
|
||||
if got := SourceMode("").Normalize(); got != SourceModeAuto {
|
||||
t.Fatalf("Normalize auto = %q, want %q", got, SourceModeAuto)
|
||||
}
|
||||
}
|
||||
@@ -83,15 +83,41 @@ func (c *Config) Validate() error {
|
||||
m.Add(fieldErr(path+".driver", "is required (e.g. openmeteo_observation, rss_feed, ...)"))
|
||||
}
|
||||
|
||||
// Every (optional but if present must be >=0)
|
||||
if s.Every.Duration < 0 {
|
||||
m.Add(fieldErr(path+".every", "is optional, but must be a positive duration (e.g. 15m, 1m, 30s) if provided"))
|
||||
// Mode
|
||||
mode := s.Mode.Normalize()
|
||||
if s.Mode != SourceModeAuto && mode != SourceModePoll && mode != SourceModeStream {
|
||||
m.Add(fieldErr(path+".mode", `must be one of: "poll", "stream" (or omit for auto)`))
|
||||
}
|
||||
|
||||
// Kind (optional but if present must be non-empty after trimming)
|
||||
// Every
|
||||
if s.Every.Duration < 0 {
|
||||
m.Add(fieldErr(path+".every", "is optional, but must be a positive duration (e.g. 15m, 1m, 30s) if provided"))
|
||||
} else {
|
||||
switch mode {
|
||||
case SourceModePoll:
|
||||
if s.Every.Duration <= 0 {
|
||||
m.Add(fieldErr(path+".every", `is required when mode="poll" (e.g. 15m, 1m, 30s)`))
|
||||
}
|
||||
case SourceModeStream:
|
||||
if s.Every.Duration > 0 {
|
||||
m.Add(fieldErr(path+".every", `must be omitted when mode="stream"`))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Kind/Kinds (optional)
|
||||
if s.Kind != "" && len(s.Kinds) > 0 {
|
||||
m.Add(fieldErr(path+".kind", `cannot be set when "kinds" is provided (use only "kinds")`))
|
||||
}
|
||||
if s.Kind != "" && strings.TrimSpace(s.Kind) == "" {
|
||||
m.Add(fieldErr(path+".kind", "cannot be blank (omit it entirely, or provide a non-empty string)"))
|
||||
}
|
||||
for j, k := range s.Kinds {
|
||||
kpath := fmt.Sprintf("%s.kinds[%d]", path, j)
|
||||
if strings.TrimSpace(k) == "" {
|
||||
m.Add(fieldErr(kpath, "kind cannot be empty"))
|
||||
}
|
||||
}
|
||||
|
||||
// Params can be nil; that's fine.
|
||||
}
|
||||
|
||||
@@ -46,3 +46,119 @@ func TestValidate_RouteKindsRejectsBlankEntries(t *testing.T) {
|
||||
t.Fatalf("expected error to mention blank kind entry, got: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidate_SourceModePollRequiresEvery(t *testing.T) {
|
||||
cfg := &Config{
|
||||
Sources: []SourceConfig{
|
||||
{Name: "src1", Driver: "driver1", Mode: SourceModePoll},
|
||||
},
|
||||
Sinks: []SinkConfig{
|
||||
{Name: "sink1", Driver: "stdout"},
|
||||
},
|
||||
}
|
||||
|
||||
err := cfg.Validate()
|
||||
if err == nil {
|
||||
t.Fatalf("expected error, got nil")
|
||||
}
|
||||
if !strings.Contains(err.Error(), `sources[0].every`) {
|
||||
t.Fatalf("expected error to mention sources[0].every, got: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidate_SourceModeStreamRejectsEvery(t *testing.T) {
|
||||
cfg := &Config{
|
||||
Sources: []SourceConfig{
|
||||
{
|
||||
Name: "src1",
|
||||
Driver: "driver1",
|
||||
Mode: SourceModeStream,
|
||||
Every: Duration{Duration: time.Minute},
|
||||
},
|
||||
},
|
||||
Sinks: []SinkConfig{
|
||||
{Name: "sink1", Driver: "stdout"},
|
||||
},
|
||||
}
|
||||
|
||||
err := cfg.Validate()
|
||||
if err == nil {
|
||||
t.Fatalf("expected error, got nil")
|
||||
}
|
||||
if !strings.Contains(err.Error(), `sources[0].every`) {
|
||||
t.Fatalf("expected error to mention sources[0].every, got: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidate_SourceModeRejectsUnknownValue(t *testing.T) {
|
||||
cfg := &Config{
|
||||
Sources: []SourceConfig{
|
||||
{
|
||||
Name: "src1",
|
||||
Driver: "driver1",
|
||||
Mode: SourceMode("batch"),
|
||||
Every: Duration{Duration: time.Minute},
|
||||
},
|
||||
},
|
||||
Sinks: []SinkConfig{
|
||||
{Name: "sink1", Driver: "stdout"},
|
||||
},
|
||||
}
|
||||
|
||||
err := cfg.Validate()
|
||||
if err == nil {
|
||||
t.Fatalf("expected error, got nil")
|
||||
}
|
||||
if !strings.Contains(err.Error(), `sources[0].mode`) {
|
||||
t.Fatalf("expected error to mention sources[0].mode, got: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidate_SourceKindAndKindsConflict(t *testing.T) {
|
||||
cfg := &Config{
|
||||
Sources: []SourceConfig{
|
||||
{
|
||||
Name: "src1",
|
||||
Driver: "driver1",
|
||||
Every: Duration{Duration: time.Minute},
|
||||
Kind: "observation",
|
||||
Kinds: []string{"forecast"},
|
||||
},
|
||||
},
|
||||
Sinks: []SinkConfig{
|
||||
{Name: "sink1", Driver: "stdout"},
|
||||
},
|
||||
}
|
||||
|
||||
err := cfg.Validate()
|
||||
if err == nil {
|
||||
t.Fatalf("expected error, got nil")
|
||||
}
|
||||
if !strings.Contains(err.Error(), `sources[0].kind`) {
|
||||
t.Fatalf("expected error to mention sources[0].kind, got: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidate_SourceKindsRejectBlankEntries(t *testing.T) {
|
||||
cfg := &Config{
|
||||
Sources: []SourceConfig{
|
||||
{
|
||||
Name: "src1",
|
||||
Driver: "driver1",
|
||||
Every: Duration{Duration: time.Minute},
|
||||
Kinds: []string{"observation", " "},
|
||||
},
|
||||
},
|
||||
Sinks: []SinkConfig{
|
||||
{Name: "sink1", Driver: "stdout"},
|
||||
},
|
||||
}
|
||||
|
||||
err := cfg.Validate()
|
||||
if err == nil {
|
||||
t.Fatalf("expected error, got nil")
|
||||
}
|
||||
if !strings.Contains(err.Error(), `sources[0].kinds[1]`) {
|
||||
t.Fatalf("expected error to mention sources[0].kinds[1], got: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user