v0.x: add reusable HTTP source spine; fix routing; upstream HTTP transport helper

- fix dispatch route compilation so empty Kinds matches all (nil), not none
- introduce internal/sources/common/HTTPSource to centralize HTTP polling boilerplate:
  - standard cfg parsing (url + user_agent)
  - default HTTP client + Accept/User-Agent headers
  - consistent error wrapping
- refactor observation sources (nws/openmeteo/openweather) to use HTTPSource
- upstream generic HTTP fetch/limits/timeout helper from weatherfeeder to feedkit:
  - move internal/sources/common/http.go -> feedkit/transport/http.go
  - keep behavior: status checks, max-body limit, default timeout
This commit is contained in:
2026-01-15 19:11:58 -06:00
parent 1790218d38
commit d9474b5a5b
4 changed files with 91 additions and 116 deletions

View File

@@ -5,7 +5,6 @@ import (
"context"
"encoding/json"
"fmt"
"net/http"
"strings"
"time"
@@ -21,29 +20,21 @@ import (
//
// https://api.weather.gov/stations/KSTL/observations/latest
type ObservationSource struct {
name string
url string
userAgent string
client *http.Client
http *common.HTTPSource
}
func NewObservationSource(cfg config.SourceConfig) (*ObservationSource, error) {
const driver = "nws_observation"
c, err := common.RequireHTTPSourceConfig(driver, cfg)
hs, err := common.NewHTTPSource(driver, cfg, "application/geo+json, application/json")
if err != nil {
return nil, err
}
return &ObservationSource{
name: c.Name,
url: c.URL,
userAgent: c.UserAgent,
client: common.NewHTTPClient(common.DefaultHTTPTimeout),
}, nil
return &ObservationSource{http: hs}, nil
}
func (s *ObservationSource) Name() string { return s.name }
func (s *ObservationSource) Name() string { return s.http.Name }
// Kind is used for routing/policy.
// We keep Kind canonical (observation) even for raw events; Schema differentiates raw vs canonical.
@@ -65,11 +56,13 @@ func (s *ObservationSource) Poll(ctx context.Context) ([]event.Event, error) {
if ts.IsZero() {
ts = time.Now().UTC()
}
station := strings.TrimSpace(meta.StationID)
if station == "" {
station = "UNKNOWN"
}
eventID = fmt.Sprintf("nws:observation:%s:%s:%s", s.name, station, ts.UTC().Format(time.RFC3339Nano))
eventID = fmt.Sprintf("nws:observation:%s:%s:%s", s.http.Name, station, ts.UTC().Format(time.RFC3339Nano))
}
// EffectiveAt is optional; for observations its naturally the observation timestamp.
@@ -81,7 +74,7 @@ func (s *ObservationSource) Poll(ctx context.Context) ([]event.Event, error) {
return common.SingleRawEvent(
s.Kind(),
s.name,
s.http.Name,
standards.SchemaRawNWSObservationV1,
eventID,
effectiveAt,
@@ -106,15 +99,13 @@ type observationMeta struct {
}
func (s *ObservationSource) fetchRaw(ctx context.Context) (json.RawMessage, observationMeta, error) {
b, err := common.FetchBody(ctx, s.client, s.url, s.userAgent, "application/geo+json, application/json")
raw, err := s.http.FetchJSON(ctx)
if err != nil {
return nil, observationMeta{}, fmt.Errorf("nws_observation %q: %w", s.name, err)
return nil, observationMeta{}, err
}
raw := json.RawMessage(b)
var meta observationMeta
if err := json.Unmarshal(b, &meta); err != nil {
if err := json.Unmarshal(raw, &meta); err != nil {
// If metadata decode fails, still return raw; envelope will fall back to computed ID.
return raw, observationMeta{}, nil
}