Eric Rakestraw
eb9a7cb349
Remove global Postgres schema registration in favor of explicit schema-aware sink factory wiring, and update weatherfeeder to register the Postgres sink explicitly. Add optional per-source HTTP timeout and response body limit overrides while keeping feedkit defaults. Remove remaining legacy source/config compatibility surfaces, including singular kind support and old source registry/type aliases, and migrate weatherfeeder sources to plural `Kinds()` metadata. Clean up related docs, tests, and sample config to match the new Postgres, HTTP, and NATS configuration model.
261 lines
7.4 KiB
Go
261 lines
7.4 KiB
Go
package sources
|
|
|
|
import (
|
|
"context"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"gitea.maximumdirect.net/ejr/feedkit/config"
|
|
"gitea.maximumdirect.net/ejr/feedkit/transport"
|
|
)
|
|
|
|
func TestNewHTTPSourceConditionalDefaultsTrue(t *testing.T) {
|
|
src, err := NewHTTPSource("test_driver", config.SourceConfig{
|
|
Name: "test-source",
|
|
Driver: "test_driver",
|
|
Params: map[string]any{
|
|
"url": "https://example.invalid",
|
|
"user_agent": "test-agent",
|
|
},
|
|
}, "application/json")
|
|
if err != nil {
|
|
t.Fatalf("NewHTTPSource() error = %v", err)
|
|
}
|
|
if !src.Conditional {
|
|
t.Fatalf("Conditional = false, want true")
|
|
}
|
|
}
|
|
|
|
func TestNewHTTPSourceRejectsInvalidConditional(t *testing.T) {
|
|
_, err := NewHTTPSource("test_driver", config.SourceConfig{
|
|
Name: "test-source",
|
|
Driver: "test_driver",
|
|
Params: map[string]any{
|
|
"url": "https://example.invalid",
|
|
"user_agent": "test-agent",
|
|
"conditional": "sometimes",
|
|
},
|
|
}, "application/json")
|
|
if err == nil {
|
|
t.Fatalf("NewHTTPSource() error = nil, want error")
|
|
}
|
|
if !strings.Contains(err.Error(), "params.conditional must be a boolean") {
|
|
t.Fatalf("NewHTTPSource() error = %q, want params.conditional must be a boolean", err)
|
|
}
|
|
}
|
|
|
|
func TestNewHTTPSourceConditionalCanBeExplicitlyFalse(t *testing.T) {
|
|
src, err := NewHTTPSource("test_driver", config.SourceConfig{
|
|
Name: "test-source",
|
|
Driver: "test_driver",
|
|
Params: map[string]any{
|
|
"url": "https://example.invalid",
|
|
"user_agent": "test-agent",
|
|
"conditional": false,
|
|
},
|
|
}, "application/json")
|
|
if err != nil {
|
|
t.Fatalf("NewHTTPSource() error = %v", err)
|
|
}
|
|
if src.Conditional {
|
|
t.Fatalf("Conditional = true, want false")
|
|
}
|
|
}
|
|
|
|
func TestNewHTTPSourceHTTPTimeoutOverride(t *testing.T) {
|
|
src, err := NewHTTPSource("test_driver", config.SourceConfig{
|
|
Name: "test-source",
|
|
Driver: "test_driver",
|
|
Params: map[string]any{
|
|
"url": "https://example.invalid",
|
|
"user_agent": "test-agent",
|
|
"http_timeout": "250ms",
|
|
},
|
|
}, "application/json")
|
|
if err != nil {
|
|
t.Fatalf("NewHTTPSource() error = %v", err)
|
|
}
|
|
if src.Client == nil {
|
|
t.Fatalf("Client = nil")
|
|
}
|
|
if src.Client.Timeout != 250*time.Millisecond {
|
|
t.Fatalf("Client.Timeout = %s, want 250ms", src.Client.Timeout)
|
|
}
|
|
}
|
|
|
|
func TestNewHTTPSourceBodyLimitOverride(t *testing.T) {
|
|
src, err := NewHTTPSource("test_driver", config.SourceConfig{
|
|
Name: "test-source",
|
|
Driver: "test_driver",
|
|
Params: map[string]any{
|
|
"url": "https://example.invalid",
|
|
"user_agent": "test-agent",
|
|
"http_response_body_limit_bytes": 12345,
|
|
},
|
|
}, "application/json")
|
|
if err != nil {
|
|
t.Fatalf("NewHTTPSource() error = %v", err)
|
|
}
|
|
if src.ResponseBodyLimitBytes != 12345 {
|
|
t.Fatalf("ResponseBodyLimitBytes = %d, want 12345", src.ResponseBodyLimitBytes)
|
|
}
|
|
}
|
|
|
|
func TestNewHTTPSourceRejectsInvalidHTTPTimeout(t *testing.T) {
|
|
_, err := NewHTTPSource("test_driver", config.SourceConfig{
|
|
Name: "test-source",
|
|
Driver: "test_driver",
|
|
Params: map[string]any{
|
|
"url": "https://example.invalid",
|
|
"user_agent": "test-agent",
|
|
"http_timeout": "soon",
|
|
},
|
|
}, "application/json")
|
|
if err == nil {
|
|
t.Fatalf("NewHTTPSource() error = nil, want error")
|
|
}
|
|
if !strings.Contains(err.Error(), "params.http_timeout must be a positive duration") {
|
|
t.Fatalf("NewHTTPSource() error = %q", err)
|
|
}
|
|
}
|
|
|
|
func TestNewHTTPSourceRejectsInvalidBodyLimit(t *testing.T) {
|
|
_, err := NewHTTPSource("test_driver", config.SourceConfig{
|
|
Name: "test-source",
|
|
Driver: "test_driver",
|
|
Params: map[string]any{
|
|
"url": "https://example.invalid",
|
|
"user_agent": "test-agent",
|
|
"http_response_body_limit_bytes": "abc",
|
|
},
|
|
}, "application/json")
|
|
if err == nil {
|
|
t.Fatalf("NewHTTPSource() error = nil, want error")
|
|
}
|
|
if !strings.Contains(err.Error(), "params.http_response_body_limit_bytes must be a positive integer") {
|
|
t.Fatalf("NewHTTPSource() error = %q", err)
|
|
}
|
|
}
|
|
|
|
func TestHTTPSourceFetchJSONIfChanged(t *testing.T) {
|
|
var call int
|
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
call++
|
|
switch call {
|
|
case 1:
|
|
w.Header().Set("ETag", `"v1"`)
|
|
_, _ = w.Write([]byte(`{"ok":true}`))
|
|
case 2:
|
|
if got := r.Header.Get("If-None-Match"); got != `"v1"` {
|
|
t.Fatalf("second request If-None-Match = %q", got)
|
|
}
|
|
w.WriteHeader(http.StatusNotModified)
|
|
default:
|
|
t.Fatalf("unexpected call count %d", call)
|
|
}
|
|
}))
|
|
defer srv.Close()
|
|
|
|
src, err := NewHTTPSource("test_driver", config.SourceConfig{
|
|
Name: "test-source",
|
|
Driver: "test_driver",
|
|
Params: map[string]any{
|
|
"url": srv.URL,
|
|
"user_agent": "test-agent",
|
|
},
|
|
}, "application/json")
|
|
if err != nil {
|
|
t.Fatalf("NewHTTPSource() error = %v", err)
|
|
}
|
|
|
|
raw, changed, err := src.FetchJSONIfChanged(context.Background())
|
|
if err != nil {
|
|
t.Fatalf("first FetchJSONIfChanged() error = %v", err)
|
|
}
|
|
if !changed {
|
|
t.Fatalf("first FetchJSONIfChanged() changed = false, want true")
|
|
}
|
|
if got := string(raw); got != `{"ok":true}` {
|
|
t.Fatalf("first FetchJSONIfChanged() body = %q", got)
|
|
}
|
|
|
|
raw, changed, err = src.FetchJSONIfChanged(context.Background())
|
|
if err != nil {
|
|
t.Fatalf("second FetchJSONIfChanged() error = %v", err)
|
|
}
|
|
if changed {
|
|
t.Fatalf("second FetchJSONIfChanged() changed = true, want false")
|
|
}
|
|
if raw != nil {
|
|
t.Fatalf("second FetchJSONIfChanged() body = %q, want nil", string(raw))
|
|
}
|
|
}
|
|
|
|
func TestHTTPSourceFetchJSONIfChangedHonorsBodyLimitOverride(t *testing.T) {
|
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
_, _ = w.Write([]byte(`{"ok":true}`))
|
|
}))
|
|
defer srv.Close()
|
|
|
|
src, err := NewHTTPSource("test_driver", config.SourceConfig{
|
|
Name: "test-source",
|
|
Driver: "test_driver",
|
|
Params: map[string]any{
|
|
"url": srv.URL,
|
|
"user_agent": "test-agent",
|
|
"http_response_body_limit_bytes": 4,
|
|
},
|
|
}, "application/json")
|
|
if err != nil {
|
|
t.Fatalf("NewHTTPSource() error = %v", err)
|
|
}
|
|
|
|
_, _, err = src.FetchJSONIfChanged(context.Background())
|
|
if err == nil {
|
|
t.Fatalf("FetchJSONIfChanged() error = nil, want limit error")
|
|
}
|
|
if !strings.Contains(err.Error(), "response body too large") {
|
|
t.Fatalf("FetchJSONIfChanged() error = %q", err)
|
|
}
|
|
}
|
|
|
|
func TestNewHTTPSourceUsesDefaultBodyLimitWhenUnset(t *testing.T) {
|
|
src, err := NewHTTPSource("test_driver", config.SourceConfig{
|
|
Name: "test-source",
|
|
Driver: "test_driver",
|
|
Params: map[string]any{
|
|
"url": "https://example.invalid",
|
|
"user_agent": "test-agent",
|
|
},
|
|
}, "application/json")
|
|
if err != nil {
|
|
t.Fatalf("NewHTTPSource() error = %v", err)
|
|
}
|
|
if src.ResponseBodyLimitBytes != transport.DefaultHTTPResponseBodyLimitBytes {
|
|
t.Fatalf("ResponseBodyLimitBytes = %d, want %d", src.ResponseBodyLimitBytes, transport.DefaultHTTPResponseBodyLimitBytes)
|
|
}
|
|
}
|
|
|
|
func TestNewHTTPSourceUsesDefaultTimeoutWhenUnset(t *testing.T) {
|
|
src, err := NewHTTPSource("test_driver", config.SourceConfig{
|
|
Name: "test-source",
|
|
Driver: "test_driver",
|
|
Params: map[string]any{
|
|
"url": "https://example.invalid",
|
|
"user_agent": "test-agent",
|
|
},
|
|
}, "application/json")
|
|
if err != nil {
|
|
t.Fatalf("NewHTTPSource() error = %v", err)
|
|
}
|
|
if src.Client == nil {
|
|
t.Fatalf("Client = nil")
|
|
}
|
|
if src.Client.Timeout != transport.DefaultHTTPTimeout {
|
|
t.Fatalf("Client.Timeout = %s, want %s", src.Client.Timeout, transport.DefaultHTTPTimeout)
|
|
}
|
|
}
|