Refactor feedkit boundaries ahead of v1
Remove global Postgres schema registration in favor of explicit schema-aware sink factory wiring, and update weatherfeeder to register the Postgres sink explicitly. Add optional per-source HTTP timeout and response body limit overrides while keeping feedkit defaults. Remove remaining legacy source/config compatibility surfaces, including singular kind support and old source registry/type aliases, and migrate weatherfeeder sources to plural `Kinds()` metadata. Clean up related docs, tests, and sample config to match the new Postgres, HTTP, and NATS configuration model.
This commit is contained in:
@@ -19,13 +19,14 @@ import (
|
||||
// setup, and conditional GET validator handling. Concrete daemon sources remain
|
||||
// responsible for decoding the response body and constructing events.
|
||||
type HTTPSource struct {
|
||||
Driver string
|
||||
Name string
|
||||
URL string
|
||||
UserAgent string
|
||||
Accept string
|
||||
Conditional bool
|
||||
Client *http.Client
|
||||
Driver string
|
||||
Name string
|
||||
URL string
|
||||
UserAgent string
|
||||
Accept string
|
||||
Conditional bool
|
||||
ResponseBodyLimitBytes int64
|
||||
Client *http.Client
|
||||
|
||||
mu sync.Mutex
|
||||
validators transport.HTTPValidators
|
||||
@@ -68,14 +69,33 @@ func NewHTTPSource(driver string, cfg config.SourceConfig, accept string) (*HTTP
|
||||
}
|
||||
}
|
||||
|
||||
timeout := transport.DefaultHTTPTimeout
|
||||
if _, exists := cfg.Params["http_timeout"]; exists {
|
||||
var ok bool
|
||||
timeout, ok = cfg.ParamDuration("http_timeout")
|
||||
if !ok || timeout <= 0 {
|
||||
return nil, fmt.Errorf("source %q: params.http_timeout must be a positive duration", cfg.Name)
|
||||
}
|
||||
}
|
||||
|
||||
bodyLimit := transport.DefaultHTTPResponseBodyLimitBytes
|
||||
if _, exists := cfg.Params["http_response_body_limit_bytes"]; exists {
|
||||
rawLimit, ok := cfg.ParamInt("http_response_body_limit_bytes")
|
||||
if !ok || rawLimit <= 0 {
|
||||
return nil, fmt.Errorf("source %q: params.http_response_body_limit_bytes must be a positive integer", cfg.Name)
|
||||
}
|
||||
bodyLimit = int64(rawLimit)
|
||||
}
|
||||
|
||||
return &HTTPSource{
|
||||
Driver: driver,
|
||||
Name: name,
|
||||
URL: url,
|
||||
UserAgent: userAgent,
|
||||
Accept: accept,
|
||||
Conditional: conditional,
|
||||
Client: transport.NewHTTPClient(transport.DefaultHTTPTimeout),
|
||||
Driver: driver,
|
||||
Name: name,
|
||||
URL: url,
|
||||
UserAgent: userAgent,
|
||||
Accept: accept,
|
||||
Conditional: conditional,
|
||||
ResponseBodyLimitBytes: bodyLimit,
|
||||
Client: transport.NewHTTPClient(timeout),
|
||||
}, nil
|
||||
}
|
||||
|
||||
@@ -92,7 +112,12 @@ func (s *HTTPSource) FetchBytesIfChanged(ctx context.Context) ([]byte, bool, err
|
||||
validators := s.validators
|
||||
s.mu.Unlock()
|
||||
|
||||
body, changed, next, err := transport.FetchBodyIfChanged(
|
||||
bodyLimit := s.ResponseBodyLimitBytes
|
||||
if bodyLimit <= 0 {
|
||||
bodyLimit = transport.DefaultHTTPResponseBodyLimitBytes
|
||||
}
|
||||
|
||||
body, changed, next, err := transport.FetchBodyIfChangedWithLimit(
|
||||
ctx,
|
||||
client,
|
||||
s.URL,
|
||||
@@ -100,6 +125,7 @@ func (s *HTTPSource) FetchBytesIfChanged(ctx context.Context) ([]byte, bool, err
|
||||
s.Accept,
|
||||
s.Conditional,
|
||||
validators,
|
||||
bodyLimit,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, false, fmt.Errorf("%s %q: %w", s.Driver, s.Name, err)
|
||||
|
||||
Reference in New Issue
Block a user