Refactor feedkit boundaries ahead of v1

Remove global Postgres schema registration in favor of explicit schema-aware sink factory wiring, and update weatherfeeder to register the Postgres sink explicitly. Add optional per-source HTTP timeout and response body limit overrides while keeping feedkit defaults. Remove remaining legacy source/config compatibility surfaces, including singular kind support and old source registry/type aliases, and migrate weatherfeeder sources to plural `Kinds()` metadata. Clean up related docs, tests, and sample config to match the new Postgres, HTTP, and NATS configuration model.

sources/doc.go

@@ -25,6 +25,9 @@
 //   - params.url
 //   - params.user_agent
 //   - params.conditional (optional, default true)
+//   - params.http_timeout (optional, default transport.DefaultHTTPTimeout)
+//   - params.http_response_body_limit_bytes (optional, default
+//     transport.DefaultHTTPResponseBodyLimitBytes)
 //
 // When validators are available, NewHTTPSource prefers ETag/If-None-Match and
 // falls back to Last-Modified/If-Modified-Since. A 304 Not Modified response is

sources/helpers.go

@@ -127,11 +127,6 @@ func advertisedSourceKinds(in Input) map[event.Kind]bool {
 		return kinds
 	}
 
-	if ks, ok := in.(KindSource); ok {
-		kinds[ks.Kind()] = true
-		return kinds
-	}
-
 	return nil
 }
 

sources/helpers_test.go

@@ -15,13 +15,6 @@ type testInput struct {
 
 func (s testInput) Name() string { return s.name }
 
-type testKindSource struct {
-	testInput
-	kind event.Kind
-}
-
-func (s testKindSource) Kind() event.Kind { return s.kind }
-
 type testKindsSource struct {
 	testInput
 	kinds []event.Kind
@@ -29,18 +22,6 @@ type testKindsSource struct {
 
 func (s testKindsSource) Kinds() []event.Kind { return s.kinds }
 
-func TestValidateExpectedKindsLegacyKindFallback(t *testing.T) {
-	cfg := config.SourceConfig{Kind: "observation"}
-	in := testKindSource{
-		testInput: testInput{name: "test"},
-		kind:      event.Kind("observation"),
-	}
-
-	if err := ValidateExpectedKinds(cfg, in); err != nil {
-		t.Fatalf("ValidateExpectedKinds() unexpected error: %v", err)
-	}
-}
-
 func TestValidateExpectedKindsSubsetAllowed(t *testing.T) {
 	cfg := config.SourceConfig{Kinds: []string{"observation"}}
 	in := testKindsSource{

sources/http.go

@@ -19,13 +19,14 @@ import (
 // setup, and conditional GET validator handling. Concrete daemon sources remain
 // responsible for decoding the response body and constructing events.
 type HTTPSource struct {
-	Driver      string
-	Name        string
-	URL         string
-	UserAgent   string
-	Accept      string
-	Conditional bool
-	Client      *http.Client
+	Driver                 string
+	Name                   string
+	URL                    string
+	UserAgent              string
+	Accept                 string
+	Conditional            bool
+	ResponseBodyLimitBytes int64
+	Client                 *http.Client
 
 	mu         sync.Mutex
 	validators transport.HTTPValidators
@@ -68,14 +69,33 @@ func NewHTTPSource(driver string, cfg config.SourceConfig, accept string) (*HTTP
 		}
 	}
 
+	timeout := transport.DefaultHTTPTimeout
+	if _, exists := cfg.Params["http_timeout"]; exists {
+		var ok bool
+		timeout, ok = cfg.ParamDuration("http_timeout")
+		if !ok || timeout <= 0 {
+			return nil, fmt.Errorf("source %q: params.http_timeout must be a positive duration", cfg.Name)
+		}
+	}
+
+	bodyLimit := transport.DefaultHTTPResponseBodyLimitBytes
+	if _, exists := cfg.Params["http_response_body_limit_bytes"]; exists {
+		rawLimit, ok := cfg.ParamInt("http_response_body_limit_bytes")
+		if !ok || rawLimit <= 0 {
+			return nil, fmt.Errorf("source %q: params.http_response_body_limit_bytes must be a positive integer", cfg.Name)
+		}
+		bodyLimit = int64(rawLimit)
+	}
+
 	return &HTTPSource{
-		Driver:      driver,
-		Name:        name,
-		URL:         url,
-		UserAgent:   userAgent,
-		Accept:      accept,
-		Conditional: conditional,
-		Client:      transport.NewHTTPClient(transport.DefaultHTTPTimeout),
+		Driver:                 driver,
+		Name:                   name,
+		URL:                    url,
+		UserAgent:              userAgent,
+		Accept:                 accept,
+		Conditional:            conditional,
+		ResponseBodyLimitBytes: bodyLimit,
+		Client:                 transport.NewHTTPClient(timeout),
 	}, nil
 }
 
@@ -92,7 +112,12 @@ func (s *HTTPSource) FetchBytesIfChanged(ctx context.Context) ([]byte, bool, err
 	validators := s.validators
 	s.mu.Unlock()
 
-	body, changed, next, err := transport.FetchBodyIfChanged(
+	bodyLimit := s.ResponseBodyLimitBytes
+	if bodyLimit <= 0 {
+		bodyLimit = transport.DefaultHTTPResponseBodyLimitBytes
+	}
+
+	body, changed, next, err := transport.FetchBodyIfChangedWithLimit(
 		ctx,
 		client,
 		s.URL,
@@ -100,6 +125,7 @@ func (s *HTTPSource) FetchBytesIfChanged(ctx context.Context) ([]byte, bool, err
 		s.Accept,
 		s.Conditional,
 		validators,
+		bodyLimit,
 	)
 	if err != nil {
 		return nil, false, fmt.Errorf("%s %q: %w", s.Driver, s.Name, err)

sources/http_test.go

@@ -6,8 +6,10 @@ import (
 	"net/http/httptest"
 	"strings"
 	"testing"
+	"time"
 
 	"gitea.maximumdirect.net/ejr/feedkit/config"
+	"gitea.maximumdirect.net/ejr/feedkit/transport"
 )
 
 func TestNewHTTPSourceConditionalDefaultsTrue(t *testing.T) {
@@ -63,6 +65,81 @@ func TestNewHTTPSourceConditionalCanBeExplicitlyFalse(t *testing.T) {
 	}
 }
 
+func TestNewHTTPSourceHTTPTimeoutOverride(t *testing.T) {
+	src, err := NewHTTPSource("test_driver", config.SourceConfig{
+		Name:   "test-source",
+		Driver: "test_driver",
+		Params: map[string]any{
+			"url":          "https://example.invalid",
+			"user_agent":   "test-agent",
+			"http_timeout": "250ms",
+		},
+	}, "application/json")
+	if err != nil {
+		t.Fatalf("NewHTTPSource() error = %v", err)
+	}
+	if src.Client == nil {
+		t.Fatalf("Client = nil")
+	}
+	if src.Client.Timeout != 250*time.Millisecond {
+		t.Fatalf("Client.Timeout = %s, want 250ms", src.Client.Timeout)
+	}
+}
+
+func TestNewHTTPSourceBodyLimitOverride(t *testing.T) {
+	src, err := NewHTTPSource("test_driver", config.SourceConfig{
+		Name:   "test-source",
+		Driver: "test_driver",
+		Params: map[string]any{
+			"url":                            "https://example.invalid",
+			"user_agent":                     "test-agent",
+			"http_response_body_limit_bytes": 12345,
+		},
+	}, "application/json")
+	if err != nil {
+		t.Fatalf("NewHTTPSource() error = %v", err)
+	}
+	if src.ResponseBodyLimitBytes != 12345 {
+		t.Fatalf("ResponseBodyLimitBytes = %d, want 12345", src.ResponseBodyLimitBytes)
+	}
+}
+
+func TestNewHTTPSourceRejectsInvalidHTTPTimeout(t *testing.T) {
+	_, err := NewHTTPSource("test_driver", config.SourceConfig{
+		Name:   "test-source",
+		Driver: "test_driver",
+		Params: map[string]any{
+			"url":          "https://example.invalid",
+			"user_agent":   "test-agent",
+			"http_timeout": "soon",
+		},
+	}, "application/json")
+	if err == nil {
+		t.Fatalf("NewHTTPSource() error = nil, want error")
+	}
+	if !strings.Contains(err.Error(), "params.http_timeout must be a positive duration") {
+		t.Fatalf("NewHTTPSource() error = %q", err)
+	}
+}
+
+func TestNewHTTPSourceRejectsInvalidBodyLimit(t *testing.T) {
+	_, err := NewHTTPSource("test_driver", config.SourceConfig{
+		Name:   "test-source",
+		Driver: "test_driver",
+		Params: map[string]any{
+			"url":                            "https://example.invalid",
+			"user_agent":                     "test-agent",
+			"http_response_body_limit_bytes": "abc",
+		},
+	}, "application/json")
+	if err == nil {
+		t.Fatalf("NewHTTPSource() error = nil, want error")
+	}
+	if !strings.Contains(err.Error(), "params.http_response_body_limit_bytes must be a positive integer") {
+		t.Fatalf("NewHTTPSource() error = %q", err)
+	}
+}
+
 func TestHTTPSourceFetchJSONIfChanged(t *testing.T) {
 	var call int
 	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
@@ -116,3 +193,68 @@ func TestHTTPSourceFetchJSONIfChanged(t *testing.T) {
 		t.Fatalf("second FetchJSONIfChanged() body = %q, want nil", string(raw))
 	}
 }
+
+func TestHTTPSourceFetchJSONIfChangedHonorsBodyLimitOverride(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		_, _ = w.Write([]byte(`{"ok":true}`))
+	}))
+	defer srv.Close()
+
+	src, err := NewHTTPSource("test_driver", config.SourceConfig{
+		Name:   "test-source",
+		Driver: "test_driver",
+		Params: map[string]any{
+			"url":                            srv.URL,
+			"user_agent":                     "test-agent",
+			"http_response_body_limit_bytes": 4,
+		},
+	}, "application/json")
+	if err != nil {
+		t.Fatalf("NewHTTPSource() error = %v", err)
+	}
+
+	_, _, err = src.FetchJSONIfChanged(context.Background())
+	if err == nil {
+		t.Fatalf("FetchJSONIfChanged() error = nil, want limit error")
+	}
+	if !strings.Contains(err.Error(), "response body too large") {
+		t.Fatalf("FetchJSONIfChanged() error = %q", err)
+	}
+}
+
+func TestNewHTTPSourceUsesDefaultBodyLimitWhenUnset(t *testing.T) {
+	src, err := NewHTTPSource("test_driver", config.SourceConfig{
+		Name:   "test-source",
+		Driver: "test_driver",
+		Params: map[string]any{
+			"url":        "https://example.invalid",
+			"user_agent": "test-agent",
+		},
+	}, "application/json")
+	if err != nil {
+		t.Fatalf("NewHTTPSource() error = %v", err)
+	}
+	if src.ResponseBodyLimitBytes != transport.DefaultHTTPResponseBodyLimitBytes {
+		t.Fatalf("ResponseBodyLimitBytes = %d, want %d", src.ResponseBodyLimitBytes, transport.DefaultHTTPResponseBodyLimitBytes)
+	}
+}
+
+func TestNewHTTPSourceUsesDefaultTimeoutWhenUnset(t *testing.T) {
+	src, err := NewHTTPSource("test_driver", config.SourceConfig{
+		Name:   "test-source",
+		Driver: "test_driver",
+		Params: map[string]any{
+			"url":        "https://example.invalid",
+			"user_agent": "test-agent",
+		},
+	}, "application/json")
+	if err != nil {
+		t.Fatalf("NewHTTPSource() error = %v", err)
+	}
+	if src.Client == nil {
+		t.Fatalf("Client = nil")
+	}
+	if src.Client.Timeout != transport.DefaultHTTPTimeout {
+		t.Fatalf("Client.Timeout = %s, want %s", src.Client.Timeout, transport.DefaultHTTPTimeout)
+	}
+}

sources/registry.go

@@ -15,9 +15,6 @@ import (
 type PollFactory func(cfg config.SourceConfig) (PollSource, error)
 type StreamFactory func(cfg config.SourceConfig) (StreamSource, error)
 
-// Factory is the legacy alias for poll source factories.
-type Factory = PollFactory
-
 type Registry struct {
 	byPollDriver   map[string]PollFactory
 	byStreamDriver map[string]StreamFactory
@@ -30,13 +27,6 @@ func NewRegistry() *Registry {
 	}
 }
 
-// Register associates a driver name (e.g. "openmeteo_observation") with a factory.
-//
-// The driver string is the "lookup key" used by config.sources[].driver.
-func (r *Registry) Register(driver string, f PollFactory) {
-	r.RegisterPoll(driver, f)
-}
-
 // RegisterPoll associates a driver name with a polling-source factory.
 func (r *Registry) RegisterPoll(driver string, f PollFactory) {
 	driver = strings.TrimSpace(driver)
@@ -75,11 +65,6 @@ func (r *Registry) RegisterStream(driver string, f StreamFactory) {
 	r.byStreamDriver[driver] = f
 }
 
-// Build constructs a polling source from a SourceConfig by looking up cfg.Driver.
-func (r *Registry) Build(cfg config.SourceConfig) (PollSource, error) {
-	return r.BuildPoll(cfg)
-}
-
 // BuildPoll constructs a polling source from a SourceConfig by looking up cfg.Driver.
 func (r *Registry) BuildPoll(cfg config.SourceConfig) (PollSource, error) {
 	driver := strings.TrimSpace(cfg.Driver)

sources/source.go

@@ -31,9 +31,6 @@ type PollSource interface {
 	Poll(ctx context.Context) ([]event.Event, error)
 }
 
-// Source is a compatibility alias for the legacy polling-source name.
-type Source = PollSource
-
 // StreamSource is an event-driven source (NATS/RabbitMQ/MQTT/etc).
 //
 // Run should block, producing events into `out` until ctx is cancelled or a fatal error occurs.
@@ -43,12 +40,6 @@ type StreamSource interface {
 	Run(ctx context.Context, out chan<- event.Event) error
 }
 
-// KindSource is an optional interface for sources that advertise one "primary" kind.
-// This is legacy-friendly but no longer required.
-type KindSource interface {
-	Kind() event.Kind
-}
-
 // KindsSource is an optional interface for sources that advertise multiple kinds.
 type KindsSource interface {
 	Kinds() []event.Kind