v0.x: add reusable HTTP source spine; fix routing; upstream HTTP transport helper
- fix dispatch route compilation so empty Kinds matches all (nil), not none - introduce internal/sources/common/HTTPSource to centralize HTTP polling boilerplate: - standard cfg parsing (url + user_agent) - default HTTP client + Accept/User-Agent headers - consistent error wrapping - refactor observation sources (nws/openmeteo/openweather) to use HTTPSource - upstream generic HTTP fetch/limits/timeout helper from weatherfeeder to feedkit: - move internal/sources/common/http.go -> feedkit/transport/http.go - keep behavior: status checks, max-body limit, default timeout
This commit is contained in:
@@ -5,7 +5,6 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@@ -21,29 +20,21 @@ import (
|
||||
//
|
||||
// https://api.weather.gov/stations/KSTL/observations/latest
|
||||
type ObservationSource struct {
|
||||
name string
|
||||
url string
|
||||
userAgent string
|
||||
client *http.Client
|
||||
http *common.HTTPSource
|
||||
}
|
||||
|
||||
func NewObservationSource(cfg config.SourceConfig) (*ObservationSource, error) {
|
||||
const driver = "nws_observation"
|
||||
|
||||
c, err := common.RequireHTTPSourceConfig(driver, cfg)
|
||||
hs, err := common.NewHTTPSource(driver, cfg, "application/geo+json, application/json")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &ObservationSource{
|
||||
name: c.Name,
|
||||
url: c.URL,
|
||||
userAgent: c.UserAgent,
|
||||
client: common.NewHTTPClient(common.DefaultHTTPTimeout),
|
||||
}, nil
|
||||
return &ObservationSource{http: hs}, nil
|
||||
}
|
||||
|
||||
func (s *ObservationSource) Name() string { return s.name }
|
||||
func (s *ObservationSource) Name() string { return s.http.Name }
|
||||
|
||||
// Kind is used for routing/policy.
|
||||
// We keep Kind canonical (observation) even for raw events; Schema differentiates raw vs canonical.
|
||||
@@ -65,11 +56,13 @@ func (s *ObservationSource) Poll(ctx context.Context) ([]event.Event, error) {
|
||||
if ts.IsZero() {
|
||||
ts = time.Now().UTC()
|
||||
}
|
||||
|
||||
station := strings.TrimSpace(meta.StationID)
|
||||
if station == "" {
|
||||
station = "UNKNOWN"
|
||||
}
|
||||
eventID = fmt.Sprintf("nws:observation:%s:%s:%s", s.name, station, ts.UTC().Format(time.RFC3339Nano))
|
||||
|
||||
eventID = fmt.Sprintf("nws:observation:%s:%s:%s", s.http.Name, station, ts.UTC().Format(time.RFC3339Nano))
|
||||
}
|
||||
|
||||
// EffectiveAt is optional; for observations it’s naturally the observation timestamp.
|
||||
@@ -81,7 +74,7 @@ func (s *ObservationSource) Poll(ctx context.Context) ([]event.Event, error) {
|
||||
|
||||
return common.SingleRawEvent(
|
||||
s.Kind(),
|
||||
s.name,
|
||||
s.http.Name,
|
||||
standards.SchemaRawNWSObservationV1,
|
||||
eventID,
|
||||
effectiveAt,
|
||||
@@ -106,15 +99,13 @@ type observationMeta struct {
|
||||
}
|
||||
|
||||
func (s *ObservationSource) fetchRaw(ctx context.Context) (json.RawMessage, observationMeta, error) {
|
||||
b, err := common.FetchBody(ctx, s.client, s.url, s.userAgent, "application/geo+json, application/json")
|
||||
raw, err := s.http.FetchJSON(ctx)
|
||||
if err != nil {
|
||||
return nil, observationMeta{}, fmt.Errorf("nws_observation %q: %w", s.name, err)
|
||||
return nil, observationMeta{}, err
|
||||
}
|
||||
|
||||
raw := json.RawMessage(b)
|
||||
|
||||
var meta observationMeta
|
||||
if err := json.Unmarshal(b, &meta); err != nil {
|
||||
if err := json.Unmarshal(raw, &meta); err != nil {
|
||||
// If metadata decode fails, still return raw; envelope will fall back to computed ID.
|
||||
return raw, observationMeta{}, nil
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user