ejr/weatherfeeder
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Moved common HTTP body fetch code into a shared helper function.

Browse Source
This commit is contained in:
Eric Rakestraw
2026-01-15 08:58:56 -06:00
parent b21ed856e9
commit e28ff49201
6 changed files with 277 additions and 74 deletions
Download Patch File Download Diff File Expand all files Collapse all files

55
internal/sources/common/http.go Normal file
View File

@@ -0,0 +1,55 @@
package common
import (
"context"
"fmt"
"io"
"net/http"
)
// maxResponseBodyBytes is a hard safety limit on HTTP response bodies.
// API responses should be small, so this protects us from accidental
// or malicious large responses.
const maxResponseBodyBytes = 2 << 21 // 4 MiB
func FetchBody(ctx context.Context, client *http.Client, url, userAgent, accept string) ([]byte, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
return nil, err
}
if userAgent != "" {
req.Header.Set("User-Agent", userAgent)
}
if accept != "" {
req.Header.Set("Accept", accept)
}
res, err := client.Do(req)
if err != nil {
return nil, err
}
defer res.Body.Close()
if res.StatusCode < 200 || res.StatusCode >= 300 {
return nil, fmt.Errorf("HTTP %s", res.Status)
}
// Read at most maxResponseBodyBytes + 1 so we can detect overflow.
limited := io.LimitReader(res.Body, maxResponseBodyBytes+1)
b, err := io.ReadAll(limited)
if err != nil {
return nil, err
}
if len(b) == 0 {
return nil, fmt.Errorf("empty response body")
}
if len(b) > maxResponseBodyBytes {
return nil, fmt.Errorf("response body too large (>%d bytes)", maxResponseBodyBytes)
}
return b, nil
}

27
internal/sources/nws/observation.go
View File

@@ -5,13 +5,13 @@ import (
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"strings"
"time"
"gitea.maximumdirect.net/ejr/feedkit/config"
"gitea.maximumdirect.net/ejr/feedkit/event"
"gitea.maximumdirect.net/ejr/weatherfeeder/internal/sources/common"
"gitea.maximumdirect.net/ejr/weatherfeeder/internal/standards"
)
@@ -133,30 +133,9 @@ type observationMeta struct {
}
func (s *ObservationSource) fetchRaw(ctx context.Context) (json.RawMessage, observationMeta, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, s.url, nil)
b, err := common.FetchBody(ctx, s.client, s.url, s.userAgent, "application/geo+json, application/json")
if err != nil {
return nil, observationMeta{}, err
}
req.Header.Set("User-Agent", s.userAgent)
req.Header.Set("Accept", "application/geo+json, application/json")
res, err := s.client.Do(req)
if err != nil {
return nil, observationMeta{}, err
}
defer res.Body.Close()
if res.StatusCode < 200 || res.StatusCode >= 300 {
return nil, observationMeta{}, fmt.Errorf("nws_observation %q: HTTP %s", s.name, res.Status)
}
b, err := io.ReadAll(res.Body)
if err != nil {
return nil, observationMeta{}, err
}
if len(b) == 0 {
return nil, observationMeta{}, fmt.Errorf("nws_observation %q: empty response body", s.name)
return nil, observationMeta{}, fmt.Errorf("nws_observation %q: %w", s.name, err)
}
raw := json.RawMessage(b)

27
internal/sources/openmeteo/observation.go
View File

@@ -5,13 +5,13 @@ import (
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"strings"
"time"
"gitea.maximumdirect.net/ejr/feedkit/config"
"gitea.maximumdirect.net/ejr/feedkit/event"
"gitea.maximumdirect.net/ejr/weatherfeeder/internal/sources/common"
"gitea.maximumdirect.net/ejr/weatherfeeder/internal/standards"
)
@@ -126,30 +126,9 @@ type openMeteoMeta struct {
}
func (s *ObservationSource) fetchRaw(ctx context.Context) (json.RawMessage, openMeteoMeta, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, s.url, nil)
b, err := common.FetchBody(ctx, s.client, s.url, s.userAgent, "application/json")
if err != nil {
return nil, openMeteoMeta{}, err
}
req.Header.Set("User-Agent", s.userAgent)
req.Header.Set("Accept", "application/json")
res, err := s.client.Do(req)
if err != nil {
return nil, openMeteoMeta{}, err
}
defer res.Body.Close()
if res.StatusCode < 200 || res.StatusCode >= 300 {
return nil, openMeteoMeta{}, fmt.Errorf("openmeteo_observation %q: HTTP %s", s.name, res.Status)
}
b, err := io.ReadAll(res.Body)
if err != nil {
return nil, openMeteoMeta{}, err
}
if len(b) == 0 {
return nil, openMeteoMeta{}, fmt.Errorf("openmeteo_observation %q: empty response body", s.name)
return nil, openMeteoMeta{}, fmt.Errorf("openmeteo_observation %q: %w", s.name, err)
}
raw := json.RawMessage(b)

27
internal/sources/openweather/observation.go
View File

@@ -5,7 +5,6 @@ import (
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"net/url"
"strings"
@@ -13,6 +12,7 @@ import (
"gitea.maximumdirect.net/ejr/feedkit/config"
"gitea.maximumdirect.net/ejr/feedkit/event"
"gitea.maximumdirect.net/ejr/weatherfeeder/internal/sources/common"
"gitea.maximumdirect.net/ejr/weatherfeeder/internal/standards"
)
@@ -139,30 +139,9 @@ type openWeatherMeta struct {
}
func (s *ObservationSource) fetchRaw(ctx context.Context) (json.RawMessage, openWeatherMeta, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, s.url, nil)
b, err := common.FetchBody(ctx, s.client, s.url, s.userAgent, "application/json")
if err != nil {
return nil, openWeatherMeta{}, err
}
req.Header.Set("User-Agent", s.userAgent)
req.Header.Set("Accept", "application/json")
res, err := s.client.Do(req)
if err != nil {
return nil, openWeatherMeta{}, err
}
defer res.Body.Close()
if res.StatusCode < 200 || res.StatusCode >= 300 {
return nil, openWeatherMeta{}, fmt.Errorf("openweather_observation %q: HTTP %s", s.name, res.Status)
}
b, err := io.ReadAll(res.Body)
if err != nil {
return nil, openWeatherMeta{}, err
}
if len(b) == 0 {
return nil, openWeatherMeta{}, fmt.Errorf("openweather_observation %q: empty response body", s.name)
return nil, openWeatherMeta{}, fmt.Errorf("openweather_observation %q: %w", s.name, err)
}
raw := json.RawMessage(b)