Compare commits
53 Commits
002f9d0ba6
...
v0.12.2
| Author | SHA1 | Date | |
|---|---|---|---|
| 9b88be4dd2 | |||
| 456a46e01b | |||
| 3740c779eb | |||
| 6943a5ebc9 | |||
| 9a09454621 | |||
| b8c6708439 | |||
| a62cb87b78 | |||
| 0b5eaf46f4 | |||
| 882059014c | |||
| 0a3e52d0e5 | |||
| 2b19a121fa | |||
| 29c65971eb | |||
| 3ecf4c5b7f | |||
| f402e27542 | |||
| f720b6cdc0 | |||
| 50215d2105 | |||
| 74411e3f54 | |||
| 4358a7cdce | |||
| dcea5261ab | |||
| 97141c7a9b | |||
| 2e2d36024e | |||
| 4d2cddf801 | |||
| 21a35a5205 | |||
| 435d1ade07 | |||
| 819ac24aed | |||
| 2c472449e8 | |||
| 5d7f604a2c | |||
| 8041f99782 | |||
| c417c892d9 | |||
| 481215c5db | |||
| 5d94d3f32d | |||
| f8f1b8d4a5 | |||
| 06d5973746 | |||
| 8045b27173 | |||
| 985468c1b9 | |||
| 6a0b30b7c7 | |||
| 86ce4eb68c | |||
| 33541a71fc | |||
| b7277e0c02 | |||
| dec05821bf | |||
| 1a9f462fbf | |||
| a990da957b | |||
| a4cd63ca4e | |||
| fba519cab0 | |||
| da8ff81692 | |||
| f91a185f9d | |||
| e966276c40 | |||
| 1e2db468ea | |||
| cefd4dfc7c | |||
| 42c646c328 | |||
| b2c429983c | |||
| 0f20d1e4cb | |||
| 979d754d18 |
@@ -1,7 +1,8 @@
|
|||||||
.git
|
.git
|
||||||
.gitignore
|
.gitignore
|
||||||
**/*.md
|
**/*.md
|
||||||
|
!docs/*.md
|
||||||
|
!docs/**/*.md
|
||||||
dist/
|
dist/
|
||||||
tmp/
|
tmp/
|
||||||
.DS_Store
|
.DS_Store
|
||||||
|
|
||||||
|
|||||||
4
AGENTS.md
Normal file
4
AGENTS.md
Normal file
@@ -0,0 +1,4 @@
|
|||||||
|
Please carefully review the documents in `docs/policy` before making any changes to this repository.
|
||||||
|
- `architecture.md` provides the canonical high-level architecture policy for this repository.
|
||||||
|
- `development.md` provides more granular development policy for this repository.
|
||||||
|
- `documentation.md` provides the canonical documentation policy for this repository.
|
||||||
4
API.md
4
API.md
@@ -1,4 +0,0 @@
|
|||||||
# Event Wire Contract
|
|
||||||
|
|
||||||
The canonical event wire contract has moved to
|
|
||||||
[docs/integrations/events.md](docs/integrations/events.md).
|
|
||||||
@@ -5,9 +5,9 @@ provider-specific responses into canonical weather events, and routes those
|
|||||||
events to configured sinks.
|
events to configured sinks.
|
||||||
|
|
||||||
It currently supports NWS observations, alerts, hourly forecasts, narrative
|
It currently supports NWS observations, alerts, hourly forecasts, narrative
|
||||||
forecasts, forecast discussions, and weather stories; Open-Meteo observations
|
forecasts, forecast discussions, and weather stories; SPC Day 1-3 convective
|
||||||
and hourly forecasts; and OpenWeather observations. Implemented sinks are
|
outlooks; Open-Meteo observations and hourly forecasts; and OpenWeather
|
||||||
stdout, NATS, and Postgres.
|
observations. Implemented sinks are stdout, NATS, and Postgres.
|
||||||
|
|
||||||
## Quickstart
|
## Quickstart
|
||||||
|
|
||||||
@@ -29,9 +29,12 @@ current working directory.
|
|||||||
- [Operations guide](docs/operations.md)
|
- [Operations guide](docs/operations.md)
|
||||||
- [Troubleshooting guide](docs/troubleshooting.md)
|
- [Troubleshooting guide](docs/troubleshooting.md)
|
||||||
- [Example configs](examples/)
|
- [Example configs](examples/)
|
||||||
|
- [Go consumer guide](docs/consumers/api.md)
|
||||||
- [Event wire contract](docs/integrations/events.md)
|
- [Event wire contract](docs/integrations/events.md)
|
||||||
- [Postgres table contract](docs/integrations/postgres.md)
|
- [Postgres table contract](docs/integrations/postgres.md)
|
||||||
|
- [Feedkit integration notes](docs/integrations/feedkit.md)
|
||||||
- [NWS integration notes](docs/integrations/nws.md)
|
- [NWS integration notes](docs/integrations/nws.md)
|
||||||
|
- [SPC integration notes](docs/integrations/spc.md)
|
||||||
- [Open-Meteo integration notes](docs/integrations/openmeteo.md)
|
- [Open-Meteo integration notes](docs/integrations/openmeteo.md)
|
||||||
- [OpenWeather integration notes](docs/integrations/openweather.md)
|
- [OpenWeather integration notes](docs/integrations/openweather.md)
|
||||||
- [Architecture policy](docs/policy/architecture.md)
|
- [Architecture policy](docs/policy/architecture.md)
|
||||||
|
|||||||
@@ -99,6 +99,18 @@ sources:
|
|||||||
url: "https://api.weather.gov/alerts?point=38.6239,-90.3571&limit=20"
|
url: "https://api.weather.gov/alerts?point=38.6239,-90.3571&limit=20"
|
||||||
user_agent: "HomeOps (eric@maximumdirect.net)"
|
user_agent: "HomeOps (eric@maximumdirect.net)"
|
||||||
|
|
||||||
|
- name: SPCConvectiveOutlookSTL
|
||||||
|
mode: poll
|
||||||
|
kinds: ["outlook"]
|
||||||
|
driver: spc_convective_outlook
|
||||||
|
every: 30m
|
||||||
|
params:
|
||||||
|
latitude: 38.6239
|
||||||
|
longitude: -90.3571
|
||||||
|
location_id: "stl"
|
||||||
|
location_name: "St. Louis, MO"
|
||||||
|
user_agent: "HomeOps (eric@maximumdirect.net)"
|
||||||
|
|
||||||
sinks:
|
sinks:
|
||||||
- name: stdout
|
- name: stdout
|
||||||
driver: stdout
|
driver: stdout
|
||||||
@@ -126,13 +138,13 @@ sinks:
|
|||||||
|
|
||||||
routes:
|
routes:
|
||||||
- sink: stdout
|
- sink: stdout
|
||||||
kinds: ["observation", "forecast", "forecast_discussion", "weather_story", "alert"]
|
kinds: ["observation", "forecast", "forecast_discussion", "weather_story", "alert", "outlook"]
|
||||||
|
|
||||||
- sink: nats_weatherfeeder
|
- sink: nats_weatherfeeder
|
||||||
kinds: ["observation", "forecast", "forecast_discussion", "weather_story", "alert"]
|
kinds: ["observation", "forecast", "forecast_discussion", "weather_story", "alert", "outlook"]
|
||||||
|
|
||||||
# - sink: pg_weatherfeeder
|
# - sink: pg_weatherfeeder
|
||||||
# kinds: ["observation", "forecast", "forecast_discussion", "weather_story", "alert"]
|
# kinds: ["observation", "forecast", "forecast_discussion", "weather_story", "alert", "outlook"]
|
||||||
|
|
||||||
# - sink: logfile
|
# - sink: logfile
|
||||||
# kinds: ["observation", "alert", "forecast", "forecast_discussion", "weather_story"]
|
# kinds: ["observation", "alert", "forecast", "forecast_discussion", "weather_story", "outlook"]
|
||||||
|
|||||||
@@ -20,6 +20,7 @@ import (
|
|||||||
|
|
||||||
wfnormalizers "gitea.maximumdirect.net/ejr/weatherfeeder/internal/normalizers"
|
wfnormalizers "gitea.maximumdirect.net/ejr/weatherfeeder/internal/normalizers"
|
||||||
wfsources "gitea.maximumdirect.net/ejr/weatherfeeder/internal/sources"
|
wfsources "gitea.maximumdirect.net/ejr/weatherfeeder/internal/sources"
|
||||||
|
"gitea.maximumdirect.net/ejr/weatherfeeder/standards"
|
||||||
)
|
)
|
||||||
|
|
||||||
type testInput struct {
|
type testInput struct {
|
||||||
@@ -36,10 +37,10 @@ type testKindsSource struct {
|
|||||||
func (s testKindsSource) Kinds() []fkevent.Kind { return s.kinds }
|
func (s testKindsSource) Kinds() []fkevent.Kind { return s.kinds }
|
||||||
|
|
||||||
func TestValidateSourceExpectedKindsSubsetAllowed(t *testing.T) {
|
func TestValidateSourceExpectedKindsSubsetAllowed(t *testing.T) {
|
||||||
sc := config.SourceConfig{Kinds: []string{"observation"}}
|
sc := config.SourceConfig{Kinds: []string{standards.KindObservation}}
|
||||||
in := testKindsSource{
|
in := testKindsSource{
|
||||||
testInput: testInput{name: "test"},
|
testInput: testInput{name: "test"},
|
||||||
kinds: []fkevent.Kind{"observation", "forecast"},
|
kinds: []fkevent.Kind{fkevent.Kind(standards.KindObservation), fkevent.Kind(standards.KindForecast)},
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := fksources.ValidateExpectedKinds(sc, in); err != nil {
|
if err := fksources.ValidateExpectedKinds(sc, in); err != nil {
|
||||||
@@ -48,10 +49,10 @@ func TestValidateSourceExpectedKindsSubsetAllowed(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestValidateSourceExpectedKindsMismatchFails(t *testing.T) {
|
func TestValidateSourceExpectedKindsMismatchFails(t *testing.T) {
|
||||||
sc := config.SourceConfig{Kinds: []string{"alert"}}
|
sc := config.SourceConfig{Kinds: []string{standards.KindAlert}}
|
||||||
in := testKindsSource{
|
in := testKindsSource{
|
||||||
testInput: testInput{name: "test"},
|
testInput: testInput{name: "test"},
|
||||||
kinds: []fkevent.Kind{"observation", "forecast"},
|
kinds: []fkevent.Kind{fkevent.Kind(standards.KindObservation), fkevent.Kind(standards.KindForecast)},
|
||||||
}
|
}
|
||||||
|
|
||||||
err := fksources.ValidateExpectedKinds(sc, in)
|
err := fksources.ValidateExpectedKinds(sc, in)
|
||||||
@@ -64,7 +65,7 @@ func TestValidateSourceExpectedKindsMismatchFails(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestValidateSourceExpectedKindsNoMetadataSkipsCheck(t *testing.T) {
|
func TestValidateSourceExpectedKindsNoMetadataSkipsCheck(t *testing.T) {
|
||||||
sc := config.SourceConfig{Kinds: []string{"alert"}}
|
sc := config.SourceConfig{Kinds: []string{standards.KindAlert}}
|
||||||
in := testInput{name: "test"}
|
in := testInput{name: "test"}
|
||||||
|
|
||||||
if err := fksources.ValidateExpectedKinds(sc, in); err != nil {
|
if err := fksources.ValidateExpectedKinds(sc, in); err != nil {
|
||||||
@@ -111,6 +112,10 @@ func TestMaintainedConfigExamplesLoad(t *testing.T) {
|
|||||||
func assertConfigSourcesBuildSchedulerJobs(t *testing.T, cfg *config.Config) {
|
func assertConfigSourcesBuildSchedulerJobs(t *testing.T, cfg *config.Config) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
|
if len(cfg.Sources) == 0 {
|
||||||
|
t.Fatalf("config has no sources")
|
||||||
|
}
|
||||||
|
|
||||||
reg := fksources.NewRegistry()
|
reg := fksources.NewRegistry()
|
||||||
wfsources.RegisterBuiltins(reg)
|
wfsources.RegisterBuiltins(reg)
|
||||||
|
|
||||||
@@ -158,7 +163,7 @@ func TestNormalizeNoMatchPassThrough(t *testing.T) {
|
|||||||
pl := &fkpipeline.Pipeline{Processors: chain}
|
pl := &fkpipeline.Pipeline{Processors: chain}
|
||||||
in := fkevent.Event{
|
in := fkevent.Event{
|
||||||
ID: "evt-no-match",
|
ID: "evt-no-match",
|
||||||
Kind: fkevent.Kind("observation"),
|
Kind: fkevent.Kind(standards.KindObservation),
|
||||||
Source: "test",
|
Source: "test",
|
||||||
EmittedAt: time.Now().UTC(),
|
EmittedAt: time.Now().UTC(),
|
||||||
Schema: "raw.weatherfeeder.unknown.v1",
|
Schema: "raw.weatherfeeder.unknown.v1",
|
||||||
@@ -188,7 +193,7 @@ func TestDedupeDropsSecondEventWithSameID(t *testing.T) {
|
|||||||
pl := &fkpipeline.Pipeline{Processors: chain}
|
pl := &fkpipeline.Pipeline{Processors: chain}
|
||||||
in := fkevent.Event{
|
in := fkevent.Event{
|
||||||
ID: "evt-dedupe-1",
|
ID: "evt-dedupe-1",
|
||||||
Kind: fkevent.Kind("observation"),
|
Kind: fkevent.Kind(standards.KindObservation),
|
||||||
Source: "test",
|
Source: "test",
|
||||||
EmittedAt: time.Now().UTC(),
|
EmittedAt: time.Now().UTC(),
|
||||||
Schema: "raw.weatherfeeder.unknown.v1",
|
Schema: "raw.weatherfeeder.unknown.v1",
|
||||||
|
|||||||
@@ -52,6 +52,18 @@ sources:
|
|||||||
url: "https://api.weather.gov/alerts?point=38.6239,-90.3571&limit=20"
|
url: "https://api.weather.gov/alerts?point=38.6239,-90.3571&limit=20"
|
||||||
user_agent: "Example weatherfeeder operator (ops@example.com)"
|
user_agent: "Example weatherfeeder operator (ops@example.com)"
|
||||||
|
|
||||||
|
- name: SPCConvectiveOutlookLocal
|
||||||
|
mode: poll
|
||||||
|
driver: spc_convective_outlook
|
||||||
|
every: 30m
|
||||||
|
kinds: ["outlook"]
|
||||||
|
params:
|
||||||
|
latitude: 38.6239
|
||||||
|
longitude: -90.3571
|
||||||
|
location_id: "local"
|
||||||
|
location_name: "Configured point"
|
||||||
|
user_agent: "Example weatherfeeder operator (ops@example.com)"
|
||||||
|
|
||||||
sinks:
|
sinks:
|
||||||
- name: nats_weather
|
- name: nats_weather
|
||||||
driver: nats
|
driver: nats
|
||||||
@@ -69,10 +81,10 @@ sinks:
|
|||||||
|
|
||||||
routes:
|
routes:
|
||||||
- sink: nats_weather
|
- sink: nats_weather
|
||||||
kinds: ["observation", "forecast", "forecast_discussion", "weather_story", "alert"]
|
kinds: ["observation", "forecast", "forecast_discussion", "weather_story", "alert", "outlook"]
|
||||||
|
|
||||||
- sink: pg_weather
|
- sink: pg_weather
|
||||||
kinds: ["observation", "forecast", "forecast_discussion", "weather_story", "alert"]
|
kinds: ["observation", "forecast", "forecast_discussion", "weather_story", "alert", "outlook"]
|
||||||
```
|
```
|
||||||
|
|
||||||
Do not commit real API keys, database passwords, or personal contact addresses in
|
Do not commit real API keys, database passwords, or personal contact addresses in
|
||||||
@@ -95,10 +107,10 @@ copyable configs.
|
|||||||
| `mode` | no | `poll`, `stream`, or omitted for auto. Current weatherfeeder drivers are polling drivers. |
|
| `mode` | no | `poll`, `stream`, or omitted for auto. Current weatherfeeder drivers are polling drivers. |
|
||||||
| `every` | yes | Poll interval for current weatherfeeder source drivers. |
|
| `every` | yes | Poll interval for current weatherfeeder source drivers. |
|
||||||
| `kinds` | no | Expected event kinds. If present, startup verifies they match the source driver. |
|
| `kinds` | no | Expected event kinds. If present, startup verifies they match the source driver. |
|
||||||
| `params` | driver-specific | Driver parameters. Current source drivers require HTTP params. |
|
| `params` | driver-specific | Driver parameters. See the source-specific sections below. |
|
||||||
|
|
||||||
Current event kinds are `observation`, `forecast`, `forecast_discussion`,
|
Current event kinds are `observation`, `forecast`, `forecast_discussion`,
|
||||||
`weather_story`, and `alert`.
|
`weather_story`, `alert`, and `outlook`.
|
||||||
|
|
||||||
## Source Drivers
|
## Source Drivers
|
||||||
|
|
||||||
@@ -113,10 +125,11 @@ Current event kinds are `observation`, `forecast`, `forecast_discussion`,
|
|||||||
| `openmeteo_observation` | `observation` | Open-Meteo current conditions. |
|
| `openmeteo_observation` | `observation` | Open-Meteo current conditions. |
|
||||||
| `openmeteo_forecast` | `forecast` | Open-Meteo hourly forecast. |
|
| `openmeteo_forecast` | `forecast` | Open-Meteo hourly forecast. |
|
||||||
| `openweather_observation` | `observation` | OpenWeather current weather. |
|
| `openweather_observation` | `observation` | OpenWeather current weather. |
|
||||||
|
| `spc_convective_outlook` | `outlook` | SPC Day 1-3 convective outlooks. |
|
||||||
|
|
||||||
## HTTP Source Params
|
## HTTP Source Params
|
||||||
|
|
||||||
All current source drivers use the shared HTTP polling helper.
|
Most source drivers use the shared HTTP polling helper.
|
||||||
|
|
||||||
| Param | Required | Description |
|
| Param | Required | Description |
|
||||||
|---|:---:|---|
|
|---|:---:|---|
|
||||||
@@ -132,6 +145,44 @@ source emits no events for that poll.
|
|||||||
OpenWeather observation URLs must include `units=metric`. Startup fails if the
|
OpenWeather observation URLs must include `units=metric`. Startup fails if the
|
||||||
URL omits it or sets another unit system.
|
URL omits it or sets another unit system.
|
||||||
|
|
||||||
|
## SPC Convective Outlook Params
|
||||||
|
|
||||||
|
`spc_convective_outlook` fetches the nine required Day 1-3 GeoJSON outlook
|
||||||
|
products and the three required Day 1-3 print pages as one atomic bundle.
|
||||||
|
|
||||||
|
| Param | Required | Description |
|
||||||
|
|---|:---:|---|
|
||||||
|
| `latitude` | yes | Location latitude in decimal degrees. |
|
||||||
|
| `longitude` | yes | Location longitude in decimal degrees. |
|
||||||
|
| `user_agent` | yes | User-Agent sent to SPC. `userAgent` is also accepted. |
|
||||||
|
| `location_id` | no | Operator-defined location identifier copied into canonical outlook runs. |
|
||||||
|
| `location_name` | no | Human location label copied into canonical outlook runs. |
|
||||||
|
| `geojson_urls` | no | Map of product key to override URL. Used for tests and upstream URL changes. |
|
||||||
|
| `discussion_urls` | no | Map of day key to override print-page URL. Used for tests and upstream URL changes. |
|
||||||
|
| `rss_url` | no | Optional RSS URL. RSS is not fetched unless this is configured. |
|
||||||
|
| `http_timeout` | no | Positive duration for the HTTP client timeout. |
|
||||||
|
| `http_response_body_limit_bytes` | no | Positive integer response body limit in bytes. |
|
||||||
|
|
||||||
|
GeoJSON product keys are `day1_categorical`, `day1_tornado`, `day1_hail`,
|
||||||
|
`day1_wind`, `day2_categorical`, `day2_tornado`, `day2_hail`, `day2_wind`,
|
||||||
|
and `day3_categorical`. SPC does not provide Day 3 tornado, hail, or wind
|
||||||
|
GeoJSON products. Discussion keys are `day1`, `day2`, and `day3`.
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
sources:
|
||||||
|
- name: SPCConvectiveOutlookSTL
|
||||||
|
mode: poll
|
||||||
|
kinds: ["outlook"]
|
||||||
|
driver: spc_convective_outlook
|
||||||
|
every: 30m
|
||||||
|
params:
|
||||||
|
latitude: 38.6239
|
||||||
|
longitude: -90.3571
|
||||||
|
location_id: "stl"
|
||||||
|
location_name: "St. Louis, MO"
|
||||||
|
user_agent: "Example weatherfeeder operator (ops@example.com)"
|
||||||
|
```
|
||||||
|
|
||||||
## Sink Fields
|
## Sink Fields
|
||||||
|
|
||||||
| Field | Required | Description |
|
| Field | Required | Description |
|
||||||
|
|||||||
92
docs/consumers/api.md
Normal file
92
docs/consumers/api.md
Normal file
@@ -0,0 +1,92 @@
|
|||||||
|
# Consumer API Guide
|
||||||
|
|
||||||
|
## Purpose
|
||||||
|
|
||||||
|
This guide is for developers and LLM coding agents integrating `weatherfeeder`
|
||||||
|
from another Go codebase.
|
||||||
|
|
||||||
|
`weatherfeeder` is primarily a daemon, not an SDK. Its public integration
|
||||||
|
surface is intentionally narrow:
|
||||||
|
|
||||||
|
- `model`: canonical weather payload structs.
|
||||||
|
- `standards`: schema strings, event kind strings, and shared WMO constants.
|
||||||
|
- JSON event output from stdout and NATS sinks.
|
||||||
|
- Postgres tables written by the optional Postgres sink.
|
||||||
|
|
||||||
|
Packages under `internal/` are implementation details and are not public
|
||||||
|
integration surfaces.
|
||||||
|
|
||||||
|
## Recommended Workflow
|
||||||
|
|
||||||
|
Consumers should switch on the event `schema` value and decode `payload` into
|
||||||
|
the matching `model` type.
|
||||||
|
|
||||||
|
Minimal example:
|
||||||
|
|
||||||
|
```go
|
||||||
|
package consumer
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/ejr/weatherfeeder/model"
|
||||||
|
"gitea.maximumdirect.net/ejr/weatherfeeder/standards"
|
||||||
|
)
|
||||||
|
|
||||||
|
type Event struct {
|
||||||
|
ID string `json:"id"`
|
||||||
|
Kind string `json:"kind"`
|
||||||
|
Schema string `json:"schema"`
|
||||||
|
Payload json.RawMessage `json:"payload"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func Decode(payload []byte) (any, error) {
|
||||||
|
var evt Event
|
||||||
|
if err := json.Unmarshal(payload, &evt); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
switch evt.Schema {
|
||||||
|
case standards.SchemaWeatherObservationV1:
|
||||||
|
var out model.WeatherObservation
|
||||||
|
return &out, json.Unmarshal(evt.Payload, &out)
|
||||||
|
case standards.SchemaWeatherForecastV1:
|
||||||
|
var out model.WeatherForecastRun
|
||||||
|
return &out, json.Unmarshal(evt.Payload, &out)
|
||||||
|
case standards.SchemaWeatherForecastDiscussionV1:
|
||||||
|
var out model.WeatherForecastDiscussion
|
||||||
|
return &out, json.Unmarshal(evt.Payload, &out)
|
||||||
|
case standards.SchemaWeatherStoryV1:
|
||||||
|
var out model.WeatherStoryRun
|
||||||
|
return &out, json.Unmarshal(evt.Payload, &out)
|
||||||
|
case standards.SchemaWeatherAlertV1:
|
||||||
|
var out model.WeatherAlertRun
|
||||||
|
return &out, json.Unmarshal(evt.Payload, &out)
|
||||||
|
case standards.SchemaWeatherOutlookV2:
|
||||||
|
var out model.WeatherOutlookRun
|
||||||
|
return &out, json.Unmarshal(evt.Payload, &out)
|
||||||
|
default:
|
||||||
|
return nil, fmt.Errorf("unsupported weatherfeeder schema %q", evt.Schema)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
## Consumer Responsibilities
|
||||||
|
|
||||||
|
- Treat event IDs as opaque.
|
||||||
|
- Treat absent `omitempty` fields as unknown, not zero.
|
||||||
|
- Prefer schema constants from `standards` over string literals in Go code.
|
||||||
|
- Expect canonical numeric measurements to use metric units.
|
||||||
|
- Expect canonical timestamps from normalizers to be UTC unless a field-specific
|
||||||
|
contract says otherwise.
|
||||||
|
- Handle additive fields within the same schema version.
|
||||||
|
- Do not import `internal/...` packages.
|
||||||
|
|
||||||
|
## Canonical References
|
||||||
|
|
||||||
|
- Public payload package: [`pkg-model.md`](pkg-model.md).
|
||||||
|
- Public constants package: [`pkg-standards.md`](pkg-standards.md).
|
||||||
|
- JSON event wire contract: [`../integrations/events.md`](../integrations/events.md).
|
||||||
|
- Postgres table contract: [`../integrations/postgres.md`](../integrations/postgres.md).
|
||||||
|
- Runtime and adapter architecture: [`../policy/architecture.md`](../policy/architecture.md).
|
||||||
64
docs/consumers/pkg-model.md
Normal file
64
docs/consumers/pkg-model.md
Normal file
@@ -0,0 +1,64 @@
|
|||||||
|
# Package `model`
|
||||||
|
|
||||||
|
## Import Path
|
||||||
|
|
||||||
|
```go
|
||||||
|
import "gitea.maximumdirect.net/ejr/weatherfeeder/model"
|
||||||
|
```
|
||||||
|
|
||||||
|
## Purpose
|
||||||
|
|
||||||
|
Package `model` defines `weatherfeeder`'s canonical weather payload structs.
|
||||||
|
These structs are emitted as the `payload` of canonical `weather.*` events and
|
||||||
|
are also the domain types consumed by downstream applications such as
|
||||||
|
`weatherapi`.
|
||||||
|
|
||||||
|
The JSON field tags on these structs are part of the wire contract. For the full
|
||||||
|
field-by-field JSON contract, use the [event wire contract](../integrations/events.md).
|
||||||
|
|
||||||
|
## Payload Types
|
||||||
|
|
||||||
|
Current canonical schema families map to these public types:
|
||||||
|
|
||||||
|
| Schema | Primary type |
|
||||||
|
|---|---|
|
||||||
|
| `weather.observation.v1` | `WeatherObservation` |
|
||||||
|
| `weather.forecast.v1` | `WeatherForecastRun` |
|
||||||
|
| `weather.forecast_discussion.v1` | `WeatherForecastDiscussion` |
|
||||||
|
| `weather.weather_story.v1` | `WeatherStoryRun` |
|
||||||
|
| `weather.alert.v1` | `WeatherAlertRun` |
|
||||||
|
| `weather.outlook.v2` | `WeatherOutlookRun` |
|
||||||
|
|
||||||
|
Related child types include:
|
||||||
|
|
||||||
|
- `WeatherObservationPresentWeather`
|
||||||
|
- `WeatherForecastPeriod`
|
||||||
|
- `WeatherForecastDiscussionSection`
|
||||||
|
- `WeatherStory`
|
||||||
|
- `WeatherAlert`
|
||||||
|
- `WeatherAlertReference`
|
||||||
|
- `WeatherOutlook`
|
||||||
|
- `WeatherOutlookDiscussion`
|
||||||
|
- `WMOCode`
|
||||||
|
|
||||||
|
## Wire And Compatibility Rules
|
||||||
|
|
||||||
|
- JSON tags define canonical payload field names.
|
||||||
|
- Pointer fields and fields tagged `omitempty` are optional on the wire.
|
||||||
|
- Missing optional fields mean unknown or not applicable.
|
||||||
|
- Canonical measurements use metric units.
|
||||||
|
- Canonical timestamps are `time.Time` values encoded by Go's JSON encoder.
|
||||||
|
- Normalized canonical timestamps are UTC unless a field-specific contract says
|
||||||
|
otherwise.
|
||||||
|
- Additive fields are compatible within a schema version.
|
||||||
|
- Removing, renaming, or changing the meaning of a field requires a new schema
|
||||||
|
identifier.
|
||||||
|
|
||||||
|
## Boundaries
|
||||||
|
|
||||||
|
`model` should not depend on source adapters, sinks, SQL column names, provider
|
||||||
|
HTTP shapes, or runtime configuration.
|
||||||
|
|
||||||
|
Consumers should not rely on packages under `internal/...`. Use `model` with
|
||||||
|
schema constants from [`standards`](pkg-standards.md) and the JSON contract in
|
||||||
|
[`docs/integrations/events.md`](../integrations/events.md).
|
||||||
96
docs/consumers/pkg-standards.md
Normal file
96
docs/consumers/pkg-standards.md
Normal file
@@ -0,0 +1,96 @@
|
|||||||
|
# Package `standards`
|
||||||
|
|
||||||
|
## Import Path
|
||||||
|
|
||||||
|
```go
|
||||||
|
import "gitea.maximumdirect.net/ejr/weatherfeeder/standards"
|
||||||
|
```
|
||||||
|
|
||||||
|
## Purpose
|
||||||
|
|
||||||
|
Package `standards` defines stable identifiers and shared weather constants used
|
||||||
|
by `weatherfeeder` producers and Go consumers.
|
||||||
|
|
||||||
|
Use this package when switching on event schemas, comparing event kinds, or
|
||||||
|
working with canonical WMO condition codes.
|
||||||
|
|
||||||
|
## Event Kind Constants
|
||||||
|
|
||||||
|
Current event kind constants are:
|
||||||
|
|
||||||
|
| Constant | Value |
|
||||||
|
|---|---|
|
||||||
|
| `KindObservation` | `observation` |
|
||||||
|
| `KindForecast` | `forecast` |
|
||||||
|
| `KindForecastDiscussion` | `forecast_discussion` |
|
||||||
|
| `KindWeatherStory` | `weather_story` |
|
||||||
|
| `KindAlert` | `alert` |
|
||||||
|
| `KindOutlook` | `outlook` |
|
||||||
|
|
||||||
|
These are plain string constants. Convert them at adapter boundaries when using
|
||||||
|
feedkit's `event.Kind` type.
|
||||||
|
|
||||||
|
## Canonical Schema Constants
|
||||||
|
|
||||||
|
Canonical schemas emitted after normalization:
|
||||||
|
|
||||||
|
| Constant | Value |
|
||||||
|
|---|---|
|
||||||
|
| `SchemaWeatherObservationV1` | `weather.observation.v1` |
|
||||||
|
| `SchemaWeatherForecastV1` | `weather.forecast.v1` |
|
||||||
|
| `SchemaWeatherForecastDiscussionV1` | `weather.forecast_discussion.v1` |
|
||||||
|
| `SchemaWeatherStoryV1` | `weather.weather_story.v1` |
|
||||||
|
| `SchemaWeatherAlertV1` | `weather.alert.v1` |
|
||||||
|
| `SchemaWeatherOutlookV2` | `weather.outlook.v2` |
|
||||||
|
|
||||||
|
Historical canonical schema constant:
|
||||||
|
|
||||||
|
| Constant | Value |
|
||||||
|
|---|---|
|
||||||
|
| `SchemaWeatherOutlookV1` | `weather.outlook.v1` |
|
||||||
|
|
||||||
|
## Raw Schema Constants
|
||||||
|
|
||||||
|
Raw source schemas emitted by current registered sources:
|
||||||
|
|
||||||
|
| Constant | Value |
|
||||||
|
|---|---|
|
||||||
|
| `SchemaRawNWSObservationV1` | `raw.nws.observation.v1` |
|
||||||
|
| `SchemaRawOpenMeteoCurrentV1` | `raw.openmeteo.current.v1` |
|
||||||
|
| `SchemaRawOpenWeatherCurrentV1` | `raw.openweather.current.v1` |
|
||||||
|
| `SchemaRawNWSHourlyForecastV1` | `raw.nws.hourly.forecast.v1` |
|
||||||
|
| `SchemaRawNWSNarrativeForecastV1` | `raw.nws.narrative.forecast.v1` |
|
||||||
|
| `SchemaRawNWSForecastDiscussionV1` | `raw.nws.forecast_discussion.v1` |
|
||||||
|
| `SchemaRawNWSWeatherStoriesV1` | `raw.nws.weatherstories.v1` |
|
||||||
|
| `SchemaRawOpenMeteoHourlyForecastV1` | `raw.openmeteo.hourly.forecast.v1` |
|
||||||
|
| `SchemaRawNWSAlertsV1` | `raw.nws.alerts.v1` |
|
||||||
|
| `SchemaRawSPCConvectiveOutlookV1` | `raw.spc.convective_outlook.v1` |
|
||||||
|
|
||||||
|
Additional raw schema constant:
|
||||||
|
|
||||||
|
| Constant | Value |
|
||||||
|
|---|---|
|
||||||
|
| `SchemaRawOpenWeatherHourlyForecastV1` | `raw.openweather.hourly.forecast.v1` |
|
||||||
|
|
||||||
|
`SchemaRawOpenWeatherHourlyForecastV1` exists in code, but no current registered
|
||||||
|
source emits it. Consumers should not expect that raw schema unless a later
|
||||||
|
registered source documents it as part of the current event contract.
|
||||||
|
|
||||||
|
## WMO Constants And Text
|
||||||
|
|
||||||
|
`standards` also defines the canonical `WMOCode` vocabulary and text helpers
|
||||||
|
used by normalized observations and forecasts.
|
||||||
|
|
||||||
|
Consumer guidance:
|
||||||
|
|
||||||
|
- Treat `WMOUnknown` as unknown condition data.
|
||||||
|
- Observation `conditionCode` is required in the current event contract.
|
||||||
|
- Forecast period `conditionCode` is optional because some forecast products do
|
||||||
|
not provide a meaningful WMO condition.
|
||||||
|
- Prefer WMO constants and helper functions from this package instead of
|
||||||
|
duplicating code tables in consumers.
|
||||||
|
|
||||||
|
## Boundaries
|
||||||
|
|
||||||
|
`standards` is provider-agnostic. Provider-specific parsing belongs in
|
||||||
|
`weatherfeeder` internals, not in this package and not in consumers.
|
||||||
@@ -37,6 +37,7 @@ Canonical schemas emitted after normalization:
|
|||||||
| `forecast_discussion` | `weather.forecast_discussion.v1` | `WeatherForecastDiscussion` |
|
| `forecast_discussion` | `weather.forecast_discussion.v1` | `WeatherForecastDiscussion` |
|
||||||
| `weather_story` | `weather.weather_story.v1` | `WeatherStoryRun` |
|
| `weather_story` | `weather.weather_story.v1` | `WeatherStoryRun` |
|
||||||
| `alert` | `weather.alert.v1` | `WeatherAlertRun` |
|
| `alert` | `weather.alert.v1` | `WeatherAlertRun` |
|
||||||
|
| `outlook` | `weather.outlook.v2` | `WeatherOutlookRun` |
|
||||||
|
|
||||||
Raw upstream schemas emitted by current sources:
|
Raw upstream schemas emitted by current sources:
|
||||||
|
|
||||||
@@ -51,6 +52,7 @@ Raw upstream schemas emitted by current sources:
|
|||||||
| `weather_story` | `raw.nws.weatherstories.v1` | NWS weather stories JSON |
|
| `weather_story` | `raw.nws.weatherstories.v1` | NWS weather stories JSON |
|
||||||
| `forecast` | `raw.openmeteo.hourly.forecast.v1` | Open-Meteo hourly forecast JSON |
|
| `forecast` | `raw.openmeteo.hourly.forecast.v1` | Open-Meteo hourly forecast JSON |
|
||||||
| `alert` | `raw.nws.alerts.v1` | NWS alerts JSON |
|
| `alert` | `raw.nws.alerts.v1` | NWS alerts JSON |
|
||||||
|
| `outlook` | `raw.spc.convective_outlook.v1` | SPC convective outlook raw bundle |
|
||||||
|
|
||||||
`standards.SchemaRawOpenWeatherHourlyForecastV1` exists in code, but no current
|
`standards.SchemaRawOpenWeatherHourlyForecastV1` exists in code, but no current
|
||||||
registered source emits it.
|
registered source emits it.
|
||||||
@@ -209,8 +211,9 @@ Payload type: `WeatherAlertRun`.
|
|||||||
| `instruction` | string | no | Alert instruction. |
|
| `instruction` | string | no | Alert instruction. |
|
||||||
| `sent` | timestamp | no | Provider sent time. |
|
| `sent` | timestamp | no | Provider sent time. |
|
||||||
| `effective` | timestamp | no | Effective time. |
|
| `effective` | timestamp | no | Effective time. |
|
||||||
| `onset` | timestamp | no | Onset time. |
|
| `onset` | timestamp | no | Alert period start. |
|
||||||
| `expires` | timestamp | no | Expiration time. |
|
| `ends` | timestamp | no | Alert period end. |
|
||||||
|
| `expires` | timestamp | no | Provider expiration metadata; not necessarily the alert period end. |
|
||||||
| `areaDescription` | string | no | Affected area description. |
|
| `areaDescription` | string | no | Affected area description. |
|
||||||
| `senderName` | string | no | Provider sender name. |
|
| `senderName` | string | no | Provider sender name. |
|
||||||
| `references` | array | no | Related alerts. |
|
| `references` | array | no | Related alerts. |
|
||||||
@@ -218,6 +221,92 @@ Payload type: `WeatherAlertRun`.
|
|||||||
`references[]` entries contain optional `id`, `identifier`, `sender`, and
|
`references[]` entries contain optional `id`, `identifier`, `sender`, and
|
||||||
`sent` fields.
|
`sent` fields.
|
||||||
|
|
||||||
|
## `weather.outlook.v2`
|
||||||
|
|
||||||
|
Payload type: `WeatherOutlookRun`.
|
||||||
|
|
||||||
|
The current producer is the SPC convective outlook normalizer. It emits Day 1-3
|
||||||
|
convective outlook snapshots for categorical, tornado, hail, and wind products
|
||||||
|
that apply to the configured forecast point. Raw SPC bundles remain complete;
|
||||||
|
canonical outlook payloads are filtered to local polygons. All timestamps are
|
||||||
|
UTC.
|
||||||
|
|
||||||
|
| Field | Type | Required | Notes |
|
||||||
|
|---|---|:---:|---|
|
||||||
|
| `locationId` | string | no | Operator-configured location identifier. |
|
||||||
|
| `locationName` | string | no | Operator-configured location label. |
|
||||||
|
| `latitude` | number | no | Configured point latitude in decimal degrees. |
|
||||||
|
| `longitude` | number | no | Configured point longitude in decimal degrees. |
|
||||||
|
| `asOf` | timestamp | yes | Snapshot time. For SPC, this is the latest outlook issue time when available. |
|
||||||
|
| `issuedAt` | timestamp | no | Latest issue time across outlook features when any feature exists. |
|
||||||
|
| `outlooks` | array | yes | Ordered outlook polygons containing the configured point. |
|
||||||
|
| `discussions` | array | yes | Run-level day discussions for retained outlook days. |
|
||||||
|
|
||||||
|
`outlooks[]` entries:
|
||||||
|
|
||||||
|
| Field | Type | Required | Notes |
|
||||||
|
|---|---|:---:|---|
|
||||||
|
| `id` | string | yes | Deterministic weatherfeeder outlook identifier. |
|
||||||
|
| `provider` | string | yes | Current value is `spc`. |
|
||||||
|
| `product` | string | yes | Current value is `convective`. |
|
||||||
|
| `day` | integer | yes | SPC outlook day, currently `1`, `2`, or `3`. |
|
||||||
|
| `outlookType` | string | yes | `categorical`, `tornado`, `hail`, or `wind`. |
|
||||||
|
| `label` | string | yes | SPC outlook label such as `SLGT` or `15`. |
|
||||||
|
| `labelText` | string | no | Human label text from SPC, when present. |
|
||||||
|
| `severityRank` | integer | no | SPC `DN` value, when present. |
|
||||||
|
| `validFrom` | timestamp | yes | Valid period start. |
|
||||||
|
| `validTo` | timestamp | yes | Valid period end. |
|
||||||
|
| `issuedAt` | timestamp | yes | Feature issue time. |
|
||||||
|
| `expiresAt` | timestamp | yes | Expiration time; currently equal to `validTo`. |
|
||||||
|
| `forecaster` | string | no | SPC forecaster text, when present. |
|
||||||
|
| `sourceUrl` | string | no | GeoJSON product URL for this outlook feature. |
|
||||||
|
| `imageUrl` | string | no | Reserved for provider image URLs; currently empty. |
|
||||||
|
| `containsLocation` | boolean | yes | Always `true` for emitted v2 outlooks. |
|
||||||
|
| `geometry` | object | yes | Compact GeoJSON `Polygon` or `MultiPolygon` geometry. |
|
||||||
|
|
||||||
|
`geometry` preserves the SPC feature geometry as compact GeoJSON using
|
||||||
|
`[longitude, latitude]` coordinate order. `containsLocation` is computed with
|
||||||
|
that geometry and the configured source `latitude`/`longitude`; boundary points
|
||||||
|
count as contained. Polygons that do not contain the configured point are not
|
||||||
|
included in canonical v2 payloads.
|
||||||
|
|
||||||
|
`discussions[]` entries:
|
||||||
|
|
||||||
|
| Field | Type | Required | Notes |
|
||||||
|
|---|---|:---:|---|
|
||||||
|
| `day` | integer | yes | SPC outlook day, currently `1`, `2`, or `3`. |
|
||||||
|
| `headline` | string | no | Matching Day 1-3 print-page product title. |
|
||||||
|
| `summary` | string | no | Text from the print-page `...SUMMARY...` section. |
|
||||||
|
| `discussion` | string | no | Cleaned full print-page product text. |
|
||||||
|
| `updatedAt` | timestamp | no | Print-page update time, when present. |
|
||||||
|
|
||||||
|
When no SPC polygons apply locally, the run is still emitted with `outlooks: []`
|
||||||
|
and `discussions: []`. Discussions are included only for days represented by at
|
||||||
|
least one retained outlook, and multiple retained outlook types for the same day
|
||||||
|
share one discussion entry.
|
||||||
|
|
||||||
|
### SPC Outlook Supersession
|
||||||
|
|
||||||
|
Consumers should prefer latest-run semantics for current conditions: read the
|
||||||
|
latest `WeatherOutlookRun` for the configured location and use its `outlooks`
|
||||||
|
and `discussions` arrays together.
|
||||||
|
|
||||||
|
Historical SQL consumers that collapse older rows should identify superseded
|
||||||
|
outlooks by `provider`, `product`, `outlookType`, `validFrom`, and `validTo`,
|
||||||
|
then keep rows with the greatest `issuedAt`. `day` and `label` are not identity
|
||||||
|
fields. When multiple retained polygons share that latest `issuedAt`, preserve
|
||||||
|
the full group.
|
||||||
|
|
||||||
|
## Legacy `weather.outlook.v1`
|
||||||
|
|
||||||
|
`weather.outlook.v1` is a historical canonical schema retained as a standards
|
||||||
|
constant for older data and consumers. Current SPC normalization emits
|
||||||
|
`weather.outlook.v2`.
|
||||||
|
|
||||||
|
The v1 payload used `WeatherOutlookRun` and placed `headline`, `summary`, and
|
||||||
|
`discussion` on each `outlooks[]` polygon. It also represented the pre-v2 SPC
|
||||||
|
canonical behavior, where national polygons were preserved in canonical output.
|
||||||
|
|
||||||
## Compact Example
|
## Compact Example
|
||||||
|
|
||||||
```json
|
```json
|
||||||
|
|||||||
106
docs/integrations/feedkit.md
Normal file
106
docs/integrations/feedkit.md
Normal file
@@ -0,0 +1,106 @@
|
|||||||
|
# Feedkit Integration
|
||||||
|
|
||||||
|
## Purpose
|
||||||
|
|
||||||
|
This document describes the feedkit runtime behavior that `weatherfeeder`
|
||||||
|
currently depends on. It is for maintainers and LLM coding agents changing
|
||||||
|
runtime wiring, config behavior, source construction, processing, routing, or
|
||||||
|
sink behavior.
|
||||||
|
|
||||||
|
Weather-domain behavior belongs in `weatherfeeder`. Generic daemon mechanics
|
||||||
|
belong to feedkit.
|
||||||
|
|
||||||
|
## Current Dependency
|
||||||
|
|
||||||
|
`weatherfeeder` imports feedkit as its daemon framework dependency. The exact
|
||||||
|
module version is declared in `go.mod`.
|
||||||
|
|
||||||
|
Feedkit provides:
|
||||||
|
|
||||||
|
- YAML config loading and validation.
|
||||||
|
- Source, processor, and sink registries.
|
||||||
|
- HTTP source helper behavior.
|
||||||
|
- Scheduler polling.
|
||||||
|
- Normalize and dedupe processors.
|
||||||
|
- Route compilation and sink dispatch.
|
||||||
|
- Built-in stdout, NATS, and Postgres sink mechanics.
|
||||||
|
|
||||||
|
## Config Contract
|
||||||
|
|
||||||
|
`cmd/weatherfeeder` calls feedkit config loading for `config.yml` in the current
|
||||||
|
working directory.
|
||||||
|
|
||||||
|
Implemented behavior relied on by weatherfeeder docs and tests:
|
||||||
|
|
||||||
|
- Top-level config contains `sources`, `sinks`, and optional `routes`.
|
||||||
|
- Config struct fields are decoded strictly, so misspelled struct fields fail
|
||||||
|
startup.
|
||||||
|
- Driver-specific `params` maps are decoded generically and validated by the
|
||||||
|
source or sink constructor that consumes them.
|
||||||
|
- Source `kinds` can be validated against a source's advertised `Kinds()`.
|
||||||
|
|
||||||
|
## Source And HTTP Contract
|
||||||
|
|
||||||
|
Most weatherfeeder sources use feedkit's single-document HTTP source helper for:
|
||||||
|
|
||||||
|
- request construction;
|
||||||
|
- `User-Agent` and `Accept` headers;
|
||||||
|
- optional conditional GET validators;
|
||||||
|
- response body size limits;
|
||||||
|
- context-aware HTTP work;
|
||||||
|
- unchanged `304 Not Modified` responses that emit no events.
|
||||||
|
|
||||||
|
The SPC convective outlook source fetches multiple documents itself, but it uses
|
||||||
|
feedkit transport helpers for HTTP clients and response body limits.
|
||||||
|
|
||||||
|
## Scheduler And Processing Contract
|
||||||
|
|
||||||
|
Weatherfeeder builds feedkit scheduler jobs from source configs. Current source
|
||||||
|
drivers are polling drivers and use the configured `every` interval.
|
||||||
|
|
||||||
|
Events flow through a feedkit pipeline in this order:
|
||||||
|
|
||||||
|
1. normalize processor;
|
||||||
|
2. dedupe processor.
|
||||||
|
|
||||||
|
The normalize processor is configured with `RequireMatch=false`, so unmatched
|
||||||
|
schemas pass through unchanged. Weatherfeeder registers its built-in normalizers
|
||||||
|
and owns the provider-to-canonical mapping.
|
||||||
|
|
||||||
|
The dedupe processor stores a bounded in-memory set of recent event IDs. The
|
||||||
|
bound is configured in `cmd/weatherfeeder`.
|
||||||
|
|
||||||
|
## Dispatch And Sink Contract
|
||||||
|
|
||||||
|
Feedkit compiles routes from config and dispatches processed events to matching
|
||||||
|
sinks. If `routes` is omitted, every configured sink receives every event kind.
|
||||||
|
|
||||||
|
Feedkit owns sink fanout mechanics, per-sink workers, queueing, context-aware
|
||||||
|
shutdown, and sink error logging. Weatherfeeder owns the event kinds and schemas
|
||||||
|
that make routes meaningful.
|
||||||
|
|
||||||
|
Built-in feedkit sinks used by weatherfeeder:
|
||||||
|
|
||||||
|
- `stdout`: validates and writes JSON events to stdout.
|
||||||
|
- `nats`: publishes JSON events to a configured subject.
|
||||||
|
- generic `postgres` sink factory: opens the database, ensures tables/indexes,
|
||||||
|
runs transactions, inserts mapped rows, and prunes when configured.
|
||||||
|
|
||||||
|
Weatherfeeder supplies its Postgres table schema and event mapper to feedkit's
|
||||||
|
Postgres sink factory. The table contract is documented in
|
||||||
|
[`postgres.md`](postgres.md).
|
||||||
|
|
||||||
|
## Boundaries
|
||||||
|
|
||||||
|
Do not move weather-domain policy into feedkit. Weatherfeeder owns:
|
||||||
|
|
||||||
|
- provider source drivers;
|
||||||
|
- raw and canonical schema constants;
|
||||||
|
- event kind meaning;
|
||||||
|
- canonical payload structs;
|
||||||
|
- normalizers;
|
||||||
|
- Postgres table shape and row mapping.
|
||||||
|
|
||||||
|
Do not duplicate generic feedkit mechanics in weatherfeeder unless there is a
|
||||||
|
narrow weather-specific reason. Runtime composition details are documented in
|
||||||
|
[`../internal/runtime.md`](../internal/runtime.md).
|
||||||
@@ -44,6 +44,9 @@ normalizer uses fields under `properties` such as `stationId`, `stationName`,
|
|||||||
`nws_alerts` expects an alerts FeatureCollection. The normalizer uses the
|
`nws_alerts` expects an alerts FeatureCollection. The normalizer uses the
|
||||||
collection `updated` timestamp, `title`, each feature ID, alert classification
|
collection `updated` timestamp, `title`, each feature ID, alert classification
|
||||||
fields, narrative fields, timing fields, sender fields, and references.
|
fields, narrative fields, timing fields, sender fields, and references.
|
||||||
|
`properties.onset` and `properties.ends` map to the canonical alert period
|
||||||
|
start and end. `properties.expires` maps only to canonical `expires` provider
|
||||||
|
metadata and is not treated as the alert period end.
|
||||||
|
|
||||||
`nws_forecast_hourly` and `nws_forecast_narrative` expect gridpoint forecast
|
`nws_forecast_hourly` and `nws_forecast_narrative` expect gridpoint forecast
|
||||||
GeoJSON with `properties.generatedAt`, `properties.updateTime`, elevation,
|
GeoJSON with `properties.generatedAt`, `properties.updateTime`, elevation,
|
||||||
@@ -98,8 +101,9 @@ unset. Forecast temperatures are converted to Celsius when NWS supplies
|
|||||||
Fahrenheit, and wind speed strings are converted to kilometers per hour.
|
Fahrenheit, and wind speed strings are converted to kilometers per hour.
|
||||||
|
|
||||||
Alert timing fields are parsed best-effort. Invalid per-alert timestamps are
|
Alert timing fields are parsed best-effort. Invalid per-alert timestamps are
|
||||||
left unset rather than failing the whole alert run. Missing alert IDs are
|
left unset rather than failing the whole alert run. NWS `ends` is preserved
|
||||||
synthesized from the run snapshot time and array position.
|
separately from `expires`; `expires` does not fall back to `ends`. Missing alert
|
||||||
|
IDs are synthesized from the run snapshot time and array position.
|
||||||
|
|
||||||
Forecast discussion parsing requires an issue time. Weather story entries require
|
Forecast discussion parsing requires an issue time. Weather story entries require
|
||||||
start time, end time, and update time.
|
start time, end time, and update time.
|
||||||
|
|||||||
@@ -23,6 +23,7 @@ Events are mapped only for canonical weather schemas:
|
|||||||
- `weather.forecast_discussion.v1`
|
- `weather.forecast_discussion.v1`
|
||||||
- `weather.weather_story.v1`
|
- `weather.weather_story.v1`
|
||||||
- `weather.alert.v1`
|
- `weather.alert.v1`
|
||||||
|
- `weather.outlook.v2`
|
||||||
|
|
||||||
Unsupported schemas produce no writes for this sink. Mapped events are inserted
|
Unsupported schemas produce no writes for this sink. Mapped events are inserted
|
||||||
transactionally. Inserts use ordinary `INSERT`; duplicate primary keys fail the
|
transactionally. Inserts use ordinary `INSERT`; duplicate primary keys fail the
|
||||||
@@ -56,6 +57,9 @@ Parent tables store the feed event envelope:
|
|||||||
| `alert_runs` | `event_id` | `as_of` |
|
| `alert_runs` | `event_id` | `as_of` |
|
||||||
| `alerts` | `run_event_id`, `alert_index` | `as_of` |
|
| `alerts` | `run_event_id`, `alert_index` | `as_of` |
|
||||||
| `alert_references` | `run_event_id`, `alert_index`, `reference_index` | `as_of` |
|
| `alert_references` | `run_event_id`, `alert_index`, `reference_index` | `as_of` |
|
||||||
|
| `outlook_runs` | `event_id` | `as_of` |
|
||||||
|
| `outlooks` | `run_event_id`, `outlook_index` | `as_of` |
|
||||||
|
| `outlook_discussions` | `run_event_id`, `discussion_index` | `as_of` |
|
||||||
|
|
||||||
## Table Contract
|
## Table Contract
|
||||||
|
|
||||||
@@ -349,6 +353,7 @@ Indexes:
|
|||||||
| `sent` | `TIMESTAMPTZ` | yes | `payload.alerts[].sent` |
|
| `sent` | `TIMESTAMPTZ` | yes | `payload.alerts[].sent` |
|
||||||
| `effective` | `TIMESTAMPTZ` | yes | `payload.alerts[].effective` |
|
| `effective` | `TIMESTAMPTZ` | yes | `payload.alerts[].effective` |
|
||||||
| `onset` | `TIMESTAMPTZ` | yes | `payload.alerts[].onset` |
|
| `onset` | `TIMESTAMPTZ` | yes | `payload.alerts[].onset` |
|
||||||
|
| `ends` | `TIMESTAMPTZ` | yes | `payload.alerts[].ends` |
|
||||||
| `expires` | `TIMESTAMPTZ` | yes | `payload.alerts[].expires` |
|
| `expires` | `TIMESTAMPTZ` | yes | `payload.alerts[].expires` |
|
||||||
| `area_description` | `TEXT` | yes | `payload.alerts[].areaDescription` |
|
| `area_description` | `TEXT` | yes | `payload.alerts[].areaDescription` |
|
||||||
| `sender_name` | `TEXT` | yes | `payload.alerts[].senderName` |
|
| `sender_name` | `TEXT` | yes | `payload.alerts[].senderName` |
|
||||||
@@ -379,6 +384,97 @@ Indexes:
|
|||||||
| `sender` | `TEXT` | yes | `payload.alerts[].references[].sender` |
|
| `sender` | `TEXT` | yes | `payload.alerts[].references[].sender` |
|
||||||
| `sent` | `TIMESTAMPTZ` | yes | `payload.alerts[].references[].sent` |
|
| `sent` | `TIMESTAMPTZ` | yes | `payload.alerts[].references[].sent` |
|
||||||
|
|
||||||
|
### `outlook_runs`
|
||||||
|
|
||||||
|
Primary key: `event_id`
|
||||||
|
|
||||||
|
Prune column: `as_of`
|
||||||
|
|
||||||
|
Indexes:
|
||||||
|
|
||||||
|
- `idx_wf_outlook_run_location_as_of` on `location_id`, `as_of`
|
||||||
|
- `idx_wf_outlook_run_as_of` on `as_of`
|
||||||
|
|
||||||
|
| Column | Type | Null | Source |
|
||||||
|
|---|---|:---:|---|
|
||||||
|
| `event_id` | `TEXT` | no | `event.id` |
|
||||||
|
| `event_kind` | `TEXT` | no | `event.kind` |
|
||||||
|
| `event_source` | `TEXT` | no | `event.source` |
|
||||||
|
| `event_schema` | `TEXT` | no | `event.schema` |
|
||||||
|
| `event_emitted_at` | `TIMESTAMPTZ` | no | `event.emitted_at` |
|
||||||
|
| `event_effective_at` | `TIMESTAMPTZ` | yes | `event.effective_at` |
|
||||||
|
| `location_id` | `TEXT` | yes | `payload.locationId` |
|
||||||
|
| `location_name` | `TEXT` | yes | `payload.locationName` |
|
||||||
|
| `latitude` | `DOUBLE PRECISION` | yes | `payload.latitude` |
|
||||||
|
| `longitude` | `DOUBLE PRECISION` | yes | `payload.longitude` |
|
||||||
|
| `as_of` | `TIMESTAMPTZ` | no | `payload.asOf` |
|
||||||
|
| `issued_at` | `TIMESTAMPTZ` | yes | `payload.issuedAt` |
|
||||||
|
| `outlook_count` | `INTEGER` | no | `len(payload.outlooks)` |
|
||||||
|
| `discussion_count` | `INTEGER` | no | `len(payload.discussions)` |
|
||||||
|
|
||||||
|
### `outlooks`
|
||||||
|
|
||||||
|
Primary key: `run_event_id`, `outlook_index`
|
||||||
|
|
||||||
|
Prune column: `as_of`
|
||||||
|
|
||||||
|
Foreign key: `run_event_id` references `outlook_runs(event_id)` with cascade
|
||||||
|
delete.
|
||||||
|
|
||||||
|
Indexes:
|
||||||
|
|
||||||
|
- `idx_wf_outlooks_contains_valid` on `contains_location`, `valid_from`, `valid_to`
|
||||||
|
- `idx_wf_outlooks_day_type_label` on `day`, `outlook_type`, `label`
|
||||||
|
- `idx_wf_outlooks_valid` on `valid_from`, `valid_to`
|
||||||
|
|
||||||
|
| Column | Type | Null | Source |
|
||||||
|
|---|---|:---:|---|
|
||||||
|
| `run_event_id` | `TEXT REFERENCES outlook_runs(event_id) ON DELETE CASCADE` | no | Parent event ID. |
|
||||||
|
| `outlook_index` | `INTEGER` | no | `payload.outlooks[]` index. |
|
||||||
|
| `as_of` | `TIMESTAMPTZ` | no | Parent `payload.asOf` |
|
||||||
|
| `outlook_id` | `TEXT` | no | `payload.outlooks[].id` |
|
||||||
|
| `provider` | `TEXT` | no | `payload.outlooks[].provider` |
|
||||||
|
| `product` | `TEXT` | no | `payload.outlooks[].product` |
|
||||||
|
| `day` | `INTEGER` | no | `payload.outlooks[].day` |
|
||||||
|
| `outlook_type` | `TEXT` | no | `payload.outlooks[].outlookType` |
|
||||||
|
| `label` | `TEXT` | no | `payload.outlooks[].label` |
|
||||||
|
| `label_text` | `TEXT` | yes | `payload.outlooks[].labelText` |
|
||||||
|
| `severity_rank` | `INTEGER` | yes | `payload.outlooks[].severityRank` |
|
||||||
|
| `valid_from` | `TIMESTAMPTZ` | no | `payload.outlooks[].validFrom` |
|
||||||
|
| `valid_to` | `TIMESTAMPTZ` | no | `payload.outlooks[].validTo` |
|
||||||
|
| `issued_at` | `TIMESTAMPTZ` | no | `payload.outlooks[].issuedAt` |
|
||||||
|
| `expires_at` | `TIMESTAMPTZ` | no | `payload.outlooks[].expiresAt` |
|
||||||
|
| `forecaster` | `TEXT` | yes | `payload.outlooks[].forecaster` |
|
||||||
|
| `source_url` | `TEXT` | yes | `payload.outlooks[].sourceUrl` |
|
||||||
|
| `image_url` | `TEXT` | yes | `payload.outlooks[].imageUrl` |
|
||||||
|
| `contains_location` | `BOOLEAN` | no | `payload.outlooks[].containsLocation` |
|
||||||
|
| `geometry_json` | `TEXT` | no | Compact JSON from `payload.outlooks[].geometry` |
|
||||||
|
|
||||||
|
### `outlook_discussions`
|
||||||
|
|
||||||
|
Primary key: `run_event_id`, `discussion_index`
|
||||||
|
|
||||||
|
Prune column: `as_of`
|
||||||
|
|
||||||
|
Foreign key: `run_event_id` references `outlook_runs(event_id)` with cascade
|
||||||
|
delete.
|
||||||
|
|
||||||
|
Indexes:
|
||||||
|
|
||||||
|
- `idx_wf_outlook_discussions_day_as_of` on `day`, `as_of`
|
||||||
|
- unique `idx_wf_outlook_discussions_run_day` on `run_event_id`, `day`
|
||||||
|
|
||||||
|
| Column | Type | Null | Source |
|
||||||
|
|---|---|:---:|---|
|
||||||
|
| `run_event_id` | `TEXT REFERENCES outlook_runs(event_id) ON DELETE CASCADE` | no | Parent event ID. |
|
||||||
|
| `discussion_index` | `INTEGER` | no | `payload.discussions[]` index. |
|
||||||
|
| `as_of` | `TIMESTAMPTZ` | no | Parent `payload.asOf` |
|
||||||
|
| `day` | `INTEGER` | no | `payload.discussions[].day` |
|
||||||
|
| `headline` | `TEXT` | yes | `payload.discussions[].headline` |
|
||||||
|
| `summary` | `TEXT` | yes | `payload.discussions[].summary` |
|
||||||
|
| `discussion` | `TEXT` | yes | `payload.discussions[].discussion` |
|
||||||
|
| `updated_at` | `TIMESTAMPTZ` | yes | `payload.discussions[].updatedAt` |
|
||||||
|
|
||||||
## Retention
|
## Retention
|
||||||
|
|
||||||
When sink param `prune` is set, every successful write transaction deletes rows
|
When sink param `prune` is set, every successful write transaction deletes rows
|
||||||
@@ -401,3 +497,6 @@ binary does not provide CLI commands for them.
|
|||||||
- `WeatherAlertRun`: read `alert_runs`, join `alerts` by `run_event_id` ordered
|
- `WeatherAlertRun`: read `alert_runs`, join `alerts` by `run_event_id` ordered
|
||||||
by `alert_index`, then join `alert_references` by `run_event_id` and
|
by `alert_index`, then join `alert_references` by `run_event_id` and
|
||||||
`alert_index` ordered by `reference_index`.
|
`alert_index` ordered by `reference_index`.
|
||||||
|
- `WeatherOutlookRun`: read `outlook_runs`, join `outlooks` by `run_event_id`
|
||||||
|
ordered by `outlook_index`, then join `outlook_discussions` by
|
||||||
|
`run_event_id` ordered by `discussion_index`.
|
||||||
|
|||||||
144
docs/integrations/spc.md
Normal file
144
docs/integrations/spc.md
Normal file
@@ -0,0 +1,144 @@
|
|||||||
|
# SPC Integration Notes
|
||||||
|
|
||||||
|
## Purpose
|
||||||
|
|
||||||
|
This document describes the Storm Prediction Center convective outlook usage
|
||||||
|
implemented by `weatherfeeder`. It is for developers and operators maintaining
|
||||||
|
SPC source configuration, provider helpers, fixtures, and tests.
|
||||||
|
|
||||||
|
General config syntax belongs in [configuration](../config.md). Emitted JSON
|
||||||
|
events are documented in [event wire contract](events.md).
|
||||||
|
|
||||||
|
## Implemented Driver
|
||||||
|
|
||||||
|
| Driver | Kind | Raw schema | Canonical schema |
|
||||||
|
| --- | --- | --- | --- |
|
||||||
|
| `spc_convective_outlook` | `outlook` | `raw.spc.convective_outlook.v1` | `weather.outlook.v2` |
|
||||||
|
|
||||||
|
## Config Requirements
|
||||||
|
|
||||||
|
The driver requires:
|
||||||
|
|
||||||
|
- `latitude`
|
||||||
|
- `longitude`
|
||||||
|
- `user_agent`
|
||||||
|
|
||||||
|
Optional params are:
|
||||||
|
|
||||||
|
- `location_id`
|
||||||
|
- `location_name`
|
||||||
|
- `geojson_urls`
|
||||||
|
- `discussion_urls`
|
||||||
|
- `rss_url`
|
||||||
|
- `http_timeout`
|
||||||
|
- `http_response_body_limit_bytes`
|
||||||
|
|
||||||
|
RSS is not fetched unless `rss_url` is configured. URL override maps are intended
|
||||||
|
for tests and upstream URL changes; the default driver configuration owns the
|
||||||
|
current Day 1-3 SPC product URLs.
|
||||||
|
|
||||||
|
## Upstream Products Used
|
||||||
|
|
||||||
|
The source fetches nine required GeoJSON products every poll:
|
||||||
|
|
||||||
|
- Day 1 categorical, tornado, hail, and wind
|
||||||
|
- Day 2 categorical, tornado, hail, and wind
|
||||||
|
- Day 3 categorical
|
||||||
|
|
||||||
|
It also fetches three required print pages:
|
||||||
|
|
||||||
|
- Day 1 convective outlook print page
|
||||||
|
- Day 2 convective outlook print page
|
||||||
|
- Day 3 convective outlook print page
|
||||||
|
|
||||||
|
GeoJSON products are authoritative for outlook polygons, valid windows, issue
|
||||||
|
times, labels, and severity rank. Print pages are authoritative for run-level
|
||||||
|
day discussion headline, summary, and discussion text.
|
||||||
|
|
||||||
|
## Accept Headers
|
||||||
|
|
||||||
|
GeoJSON requests use:
|
||||||
|
|
||||||
|
```text
|
||||||
|
application/geo+json, application/json
|
||||||
|
```
|
||||||
|
|
||||||
|
Print-page requests use:
|
||||||
|
|
||||||
|
```text
|
||||||
|
text/html, application/xhtml+xml
|
||||||
|
```
|
||||||
|
|
||||||
|
RSS requests, when configured, use:
|
||||||
|
|
||||||
|
```text
|
||||||
|
application/rss+xml, application/xml, text/xml
|
||||||
|
```
|
||||||
|
|
||||||
|
## Polling And Raw Events
|
||||||
|
|
||||||
|
The source polls all required GeoJSON and print-page products as one bundle. If
|
||||||
|
any required request fails or returns a non-2xx response, the poll returns an
|
||||||
|
error and emits no partial event.
|
||||||
|
|
||||||
|
The raw payload contains fetched bodies plus configured location metadata and
|
||||||
|
per-product metadata. The source parses only the timestamp metadata needed for
|
||||||
|
event effective time selection; canonical mapping belongs to the normalizer.
|
||||||
|
|
||||||
|
The source emits no event when a complete fetched bundle is unchanged from the
|
||||||
|
previous successful poll. It does this with a source-local hash of the fetched
|
||||||
|
document bodies.
|
||||||
|
|
||||||
|
## Time Handling
|
||||||
|
|
||||||
|
Raw source `effective_at` prefers:
|
||||||
|
|
||||||
|
1. the latest valid GeoJSON `ISSUE_ISO`;
|
||||||
|
2. the latest print-page `Updated:` timestamp;
|
||||||
|
3. RSS `lastBuildDate` when RSS was fetched and parseable;
|
||||||
|
4. fetch time.
|
||||||
|
|
||||||
|
The normalizer sets canonical `asOf` and normalized event `effective_at` from
|
||||||
|
the latest valid GeoJSON issue time across the complete raw bundle, including
|
||||||
|
empty no-risk placeholders, with fallback to print-page update time and then the
|
||||||
|
incoming event envelope.
|
||||||
|
|
||||||
|
## Mapping Notes
|
||||||
|
|
||||||
|
The raw source fetches and envelopes the complete SPC bundle. The normalizer
|
||||||
|
decodes every configured GeoJSON product, skips empty no-risk
|
||||||
|
`GeometryCollection` placeholders, and emits canonical outlooks only when the
|
||||||
|
configured point is inside or on the boundary of a real feature. Products are
|
||||||
|
ordered by day, then categorical, tornado, hail, and wind. Retained feature order
|
||||||
|
is preserved within each product.
|
||||||
|
|
||||||
|
The normalizer computes `containsLocation` with the configured latitude and
|
||||||
|
longitude against compact GeoJSON `Polygon` or `MultiPolygon` geometry.
|
||||||
|
Coordinates use GeoJSON order, `[longitude, latitude]`, and boundary points
|
||||||
|
count as contained.
|
||||||
|
|
||||||
|
Canonical outlook runs are emitted even when no polygons apply locally. In that
|
||||||
|
case the payload contains empty `outlooks` and `discussions` arrays.
|
||||||
|
|
||||||
|
Print-page prose is represented as run-level day discussions. Discussions are
|
||||||
|
included only for days represented by at least one retained outlook. Multiple
|
||||||
|
retained outlook types for the same day share one discussion entry.
|
||||||
|
|
||||||
|
For downstream current-state and historical supersession guidance, see the
|
||||||
|
[event wire contract](events.md#spc-outlook-supersession).
|
||||||
|
|
||||||
|
## Failure Behavior
|
||||||
|
|
||||||
|
Constructor validation failures stop daemon startup. Polling failures are
|
||||||
|
returned to the scheduler and emit no event for that poll.
|
||||||
|
|
||||||
|
Normalization fails when required GeoJSON timestamps, labels, geometry, or
|
||||||
|
configured coordinates are missing or invalid. Print-page extraction failures
|
||||||
|
also fail normalization because print pages are required inputs.
|
||||||
|
|
||||||
|
## Tests To Inspect
|
||||||
|
|
||||||
|
- `internal/providers/spc/*_test.go`
|
||||||
|
- `internal/sources/spc/*_test.go`
|
||||||
|
- `internal/normalizers/spc/*_test.go`
|
||||||
|
- fixtures under `internal/providers/spc/testdata`
|
||||||
@@ -27,6 +27,7 @@ Current mappings:
|
|||||||
| `raw.nws.forecast_discussion.v1` | `weather.forecast_discussion.v1` |
|
| `raw.nws.forecast_discussion.v1` | `weather.forecast_discussion.v1` |
|
||||||
| `raw.nws.weatherstories.v1` | `weather.weather_story.v1` |
|
| `raw.nws.weatherstories.v1` | `weather.weather_story.v1` |
|
||||||
| `raw.nws.alerts.v1` | `weather.alert.v1` |
|
| `raw.nws.alerts.v1` | `weather.alert.v1` |
|
||||||
|
| `raw.spc.convective_outlook.v1` | `weather.outlook.v2` |
|
||||||
|
|
||||||
## Boundaries
|
## Boundaries
|
||||||
|
|
||||||
@@ -54,9 +55,18 @@ Weatherfeeder registers normalizers in a stable order:
|
|||||||
1. NWS
|
1. NWS
|
||||||
2. Open-Meteo
|
2. Open-Meteo
|
||||||
3. OpenWeather
|
3. OpenWeather
|
||||||
|
4. SPC
|
||||||
|
|
||||||
The current normalizers avoid ambiguous matches by using schema equality.
|
The current normalizers avoid ambiguous matches by using schema equality.
|
||||||
|
|
||||||
|
The SPC outlook normalizer decodes the raw multi-document bundle, maps
|
||||||
|
location-containing GeoJSON features to canonical outlooks, and adds one
|
||||||
|
run-level print-page discussion per retained outlook day. It preserves compact
|
||||||
|
GeoJSON feature geometry and computes `containsLocation` with
|
||||||
|
`internal/geo.ContainsPoint` using the source-configured point. Boundary points
|
||||||
|
count as contained. Polygons that do not contain the point are omitted from the
|
||||||
|
canonical run.
|
||||||
|
|
||||||
## State
|
## State
|
||||||
|
|
||||||
Normalizers should be stateless. Shared helpers should be deterministic and free
|
Normalizers should be stateless. Shared helpers should be deterministic and free
|
||||||
@@ -78,6 +88,7 @@ to four digits after the decimal point.
|
|||||||
- provider normalizer tests under `internal/normalizers/nws`
|
- provider normalizer tests under `internal/normalizers/nws`
|
||||||
- provider normalizer tests under `internal/normalizers/openmeteo`
|
- provider normalizer tests under `internal/normalizers/openmeteo`
|
||||||
- provider normalizer tests under `internal/normalizers/openweather`
|
- provider normalizer tests under `internal/normalizers/openweather`
|
||||||
|
- provider normalizer tests under `internal/normalizers/spc`
|
||||||
- common helper tests under `internal/normalizers/common`
|
- common helper tests under `internal/normalizers/common`
|
||||||
|
|
||||||
## Invariants
|
## Invariants
|
||||||
|
|||||||
@@ -19,10 +19,16 @@ Inputs are canonical feed events. The mapper currently handles these schemas:
|
|||||||
- `weather.forecast_discussion.v1`
|
- `weather.forecast_discussion.v1`
|
||||||
- `weather.weather_story.v1`
|
- `weather.weather_story.v1`
|
||||||
- `weather.alert.v1`
|
- `weather.alert.v1`
|
||||||
|
- `weather.outlook.v2`
|
||||||
|
|
||||||
Outputs are feedkit `PostgresWrite` values for weatherfeeder-owned tables.
|
Outputs are feedkit `PostgresWrite` values for weatherfeeder-owned tables.
|
||||||
Unsupported schemas produce no writes and no error.
|
Unsupported schemas produce no writes and no error.
|
||||||
|
|
||||||
|
Outlook runs are written to `outlook_runs`, retained local polygons are written
|
||||||
|
to `outlooks`, and run-level day discussions are written to
|
||||||
|
`outlook_discussions`. The parent run row stores `outlook_count` and
|
||||||
|
`discussion_count`.
|
||||||
|
|
||||||
## Boundaries
|
## Boundaries
|
||||||
|
|
||||||
- Weatherfeeder owns table definitions in `schema.go`.
|
- Weatherfeeder owns table definitions in `schema.go`.
|
||||||
@@ -80,6 +86,8 @@ Child rows use positional indexes to preserve canonical array order:
|
|||||||
- `story_index`
|
- `story_index`
|
||||||
- `alert_index`
|
- `alert_index`
|
||||||
- `reference_index`
|
- `reference_index`
|
||||||
|
- `outlook_index`
|
||||||
|
- `discussion_index`
|
||||||
|
|
||||||
Required canonical fields are validated before writes are returned:
|
Required canonical fields are validated before writes are returned:
|
||||||
|
|
||||||
@@ -89,10 +97,17 @@ Required canonical fields are validated before writes are returned:
|
|||||||
- forecast discussions require `issuedAt` and `product`;
|
- forecast discussions require `issuedAt` and `product`;
|
||||||
- weather story runs require `asOf`, and each story requires `startTime`,
|
- weather story runs require `asOf`, and each story requires `startTime`,
|
||||||
`endTime`, and `updatedAt`;
|
`endTime`, and `updatedAt`;
|
||||||
- alert runs require `asOf`, and each alert requires `id`.
|
- alert runs require `asOf`, and each alert requires `id`;
|
||||||
|
- outlook runs require `asOf`, and each outlook requires `id`, `provider`,
|
||||||
|
`product`, `day`, `outlookType`, `label`, `validFrom`, `validTo`, `issuedAt`,
|
||||||
|
`expiresAt`, `containsLocation: true`, and `geometry`;
|
||||||
|
- outlook discussions require day `1`, `2`, or `3`; at least one of
|
||||||
|
`headline`, `summary`, or `discussion`; and no duplicate discussion day in the
|
||||||
|
same run.
|
||||||
|
|
||||||
Nullable canonical values are converted to SQL nulls by mapper helpers.
|
Nullable canonical values are converted to SQL nulls by mapper helpers.
|
||||||
Observation present-weather raw values are stored as compact JSON text.
|
Observation present-weather raw values and outlook geometry values are stored as
|
||||||
|
compact JSON text.
|
||||||
|
|
||||||
## Failure Behavior
|
## Failure Behavior
|
||||||
|
|
||||||
|
|||||||
@@ -68,13 +68,28 @@ Runtime composition uses feedkit for:
|
|||||||
|
|
||||||
Weatherfeeder registers its own source drivers and its Postgres schema mapper.
|
Weatherfeeder registers its own source drivers and its Postgres schema mapper.
|
||||||
|
|
||||||
|
Responsibility split:
|
||||||
|
|
||||||
|
| Runtime concern | Owner |
|
||||||
|
| --- | --- |
|
||||||
|
| Config loading and generic validation | feedkit |
|
||||||
|
| Source, processor, and sink registries | feedkit mechanics; weatherfeeder registrations |
|
||||||
|
| Source polling and stream supervision | feedkit scheduler |
|
||||||
|
| Raw weather data fetching | weatherfeeder source adapters |
|
||||||
|
| Normalizer execution order and pass-through behavior | feedkit normalize processor |
|
||||||
|
| Weather raw-to-canonical mapping | weatherfeeder normalizers |
|
||||||
|
| Dedupe mechanics | feedkit dedupe processor |
|
||||||
|
| Route compilation and sink fanout | feedkit dispatch |
|
||||||
|
| Weather Postgres table shape and row mapping | weatherfeeder Postgres adapter |
|
||||||
|
| Postgres connection, DDL, inserts, transactions, and pruning | feedkit Postgres sink |
|
||||||
|
|
||||||
## State
|
## State
|
||||||
|
|
||||||
Weatherfeeder-owned runtime state is in process:
|
Weatherfeeder-owned runtime state is in process:
|
||||||
|
|
||||||
- event channel contents;
|
- event channel contents;
|
||||||
- the bounded dedupe key set;
|
- the bounded dedupe key set;
|
||||||
- source instances and their HTTP conditional validators;
|
- source instances and their in-memory unchanged-content state;
|
||||||
- scheduler and dispatcher goroutines.
|
- scheduler and dispatcher goroutines.
|
||||||
|
|
||||||
There is no weatherfeeder-owned durable scheduler state, checkpoint, replay log,
|
There is no weatherfeeder-owned durable scheduler state, checkpoint, replay log,
|
||||||
|
|||||||
@@ -28,6 +28,7 @@ Current drivers:
|
|||||||
| `openmeteo_observation` | `observation` | `raw.openmeteo.current.v1` |
|
| `openmeteo_observation` | `observation` | `raw.openmeteo.current.v1` |
|
||||||
| `openmeteo_forecast` | `forecast` | `raw.openmeteo.hourly.forecast.v1` |
|
| `openmeteo_forecast` | `forecast` | `raw.openmeteo.hourly.forecast.v1` |
|
||||||
| `openweather_observation` | `observation` | `raw.openweather.current.v1` |
|
| `openweather_observation` | `observation` | `raw.openweather.current.v1` |
|
||||||
|
| `spc_convective_outlook` | `outlook` | `raw.spc.convective_outlook.v1` |
|
||||||
|
|
||||||
## Boundaries
|
## Boundaries
|
||||||
|
|
||||||
@@ -40,7 +41,7 @@ Current drivers:
|
|||||||
|
|
||||||
## Config Fields Used
|
## Config Fields Used
|
||||||
|
|
||||||
All current source drivers use feedkit `HTTPSource`.
|
Most source drivers use feedkit `HTTPSource`.
|
||||||
|
|
||||||
Required params:
|
Required params:
|
||||||
|
|
||||||
@@ -56,12 +57,18 @@ Optional params:
|
|||||||
OpenWeather observation sources additionally require the configured URL to use
|
OpenWeather observation sources additionally require the configured URL to use
|
||||||
metric units. This is enforced by `internal/providers/openweather`.
|
metric units. This is enforced by `internal/providers/openweather`.
|
||||||
|
|
||||||
|
The SPC convective outlook source is a multi-document poll source rather than a
|
||||||
|
single-URL `HTTPSource`. It requires `latitude`, `longitude`, and `user_agent`;
|
||||||
|
accepts optional `location_id`, `location_name`, `geojson_urls`,
|
||||||
|
`discussion_urls`, and `rss_url`; and supports `http_timeout` and
|
||||||
|
`http_response_body_limit_bytes`.
|
||||||
|
|
||||||
Source-level `kinds`, when configured, are validated against the source's
|
Source-level `kinds`, when configured, are validated against the source's
|
||||||
advertised `Kinds()`.
|
advertised `Kinds()`.
|
||||||
|
|
||||||
## External Adapters Used
|
## External Adapters Used
|
||||||
|
|
||||||
Sources use feedkit's HTTP helper for:
|
Most sources use feedkit's HTTP helper for:
|
||||||
|
|
||||||
- request construction;
|
- request construction;
|
||||||
- `User-Agent` and `Accept` headers;
|
- `User-Agent` and `Accept` headers;
|
||||||
@@ -71,11 +78,15 @@ Sources use feedkit's HTTP helper for:
|
|||||||
|
|
||||||
NWS helpers parse NWS timestamps. Open-Meteo helpers parse provider-local times
|
NWS helpers parse NWS timestamps. Open-Meteo helpers parse provider-local times
|
||||||
with timezone or UTC-offset data. OpenWeather helpers enforce metric-unit URLs.
|
with timezone or UTC-offset data. OpenWeather helpers enforce metric-unit URLs.
|
||||||
|
SPC helpers define Day 1-3 product metadata, parse GeoJSON timestamps, extract
|
||||||
|
cleaned print-page discussion text, and parse optional RSS metadata.
|
||||||
|
|
||||||
## State
|
## State
|
||||||
|
|
||||||
HTTP conditional validators are held in each source instance. They are not
|
HTTP conditional validators are held in each single-document HTTP source
|
||||||
persisted across process restarts.
|
instance. They are not persisted across process restarts. The SPC outlook source
|
||||||
|
keeps only a source-local hash of the most recent complete required product
|
||||||
|
bundle and emits no event when a later complete bundle is unchanged.
|
||||||
|
|
||||||
## Failure Behavior
|
## Failure Behavior
|
||||||
|
|
||||||
@@ -88,12 +99,18 @@ to default ID/effective-time behavior from feedkit source helpers.
|
|||||||
|
|
||||||
Unchanged conditional responses return no events and no error.
|
Unchanged conditional responses return no events and no error.
|
||||||
|
|
||||||
|
The SPC outlook source fetches all configured Day 1-3 GeoJSON products and print
|
||||||
|
pages atomically. If any required GeoJSON or print-page request fails, the poll
|
||||||
|
returns an error and emits no partial bundle. RSS is fetched only when `rss_url`
|
||||||
|
is configured.
|
||||||
|
|
||||||
## Tests To Inspect
|
## Tests To Inspect
|
||||||
|
|
||||||
- `internal/sources/builtins_test.go`
|
- `internal/sources/builtins_test.go`
|
||||||
- provider source tests under `internal/sources/nws`
|
- provider source tests under `internal/sources/nws`
|
||||||
- provider source tests under `internal/sources/openmeteo`
|
- provider source tests under `internal/sources/openmeteo`
|
||||||
- provider source tests under `internal/sources/openweather`
|
- provider source tests under `internal/sources/openweather`
|
||||||
|
- provider source tests under `internal/sources/spc`
|
||||||
- provider helper tests under `internal/providers`
|
- provider helper tests under `internal/providers`
|
||||||
|
|
||||||
## Invariants
|
## Invariants
|
||||||
|
|||||||
@@ -64,21 +64,25 @@ and before each interval tick. If no jitter is configured in code, feedkit uses
|
|||||||
Poll failures are logged and do not stop the daemon. A failed poll emits no
|
Poll failures are logged and do not stop the daemon. A failed poll emits no
|
||||||
events for that source until a subsequent poll succeeds.
|
events for that source until a subsequent poll succeeds.
|
||||||
|
|
||||||
## Conditional HTTP Fetches
|
## Unchanged Upstream Content
|
||||||
|
|
||||||
All current sources use feedkit's HTTP polling helper. By default,
|
Most sources use feedkit's single-document HTTP polling helper. By default,
|
||||||
`params.conditional` is `true`, so the helper keeps ETag and Last-Modified
|
`params.conditional` is `true` for those sources, so the helper keeps ETag and
|
||||||
validators in memory for each source instance.
|
Last-Modified validators in memory for each source instance.
|
||||||
|
|
||||||
If the upstream returns `304 Not Modified`, the source emits no events for that
|
If the upstream returns `304 Not Modified`, the source emits no events for that
|
||||||
poll. Validator state is in memory only; restarting the process starts with no
|
poll. Validator state is in memory only; restarting the process starts with no
|
||||||
cached validators.
|
cached validators.
|
||||||
|
|
||||||
|
The SPC convective outlook source polls multiple required documents as one
|
||||||
|
bundle. It emits no event when a later complete bundle has the same fetched
|
||||||
|
document bodies as the previous successful poll.
|
||||||
|
|
||||||
## Processing And Dedupe
|
## Processing And Dedupe
|
||||||
|
|
||||||
Every event passes through normalization first and dedupe second.
|
Every event passes through normalization first and dedupe second.
|
||||||
|
|
||||||
Normalizers match raw source schemas and produce canonical `weather.*.v1`
|
Normalizers match raw source schemas and produce versioned canonical `weather.*`
|
||||||
payloads. If an event has no matching normalizer, the normalize processor passes
|
payloads. If an event has no matching normalizer, the normalize processor passes
|
||||||
it through unchanged.
|
it through unchanged.
|
||||||
|
|
||||||
@@ -128,6 +132,7 @@ table in that same transaction. See the
|
|||||||
|
|
||||||
- scheduler goroutines and timers;
|
- scheduler goroutines and timers;
|
||||||
- HTTP conditional request validators;
|
- HTTP conditional request validators;
|
||||||
|
- source-local unchanged-content state;
|
||||||
- event channel buffers;
|
- event channel buffers;
|
||||||
- per-sink fanout queues;
|
- per-sink fanout queues;
|
||||||
- the dedupe ID set.
|
- the dedupe ID set.
|
||||||
|
|||||||
@@ -23,10 +23,32 @@ The implemented runtime flow is:
|
|||||||
|
|
||||||
Canonical payload structs live in `model`. Schema identifiers and cross-provider wire conventions live in `standards`. Source adapters live under `internal/sources`. Normalizers live under `internal/normalizers`. Provider-specific parsing helpers shared by sources and normalizers live under `internal/providers`. Sink-specific persistence mapping lives under `internal/sinks`.
|
Canonical payload structs live in `model`. Schema identifiers and cross-provider wire conventions live in `standards`. Source adapters live under `internal/sources`. Normalizers live under `internal/normalizers`. Provider-specific parsing helpers shared by sources and normalizers live under `internal/providers`. Sink-specific persistence mapping lives under `internal/sinks`.
|
||||||
|
|
||||||
|
## Architecture Style
|
||||||
|
|
||||||
|
`weatherfeeder` uses a pragmatic ports-and-adapters architecture rather than a
|
||||||
|
formal framework. Provider APIs, config loading, scheduling, dispatch, and sinks
|
||||||
|
sit outside the weather domain model and normalization rules.
|
||||||
|
|
||||||
|
The implementation style is:
|
||||||
|
|
||||||
|
- Pipeline-oriented: events flow from source polling through normalization,
|
||||||
|
dedupe, routing, and sink fanout.
|
||||||
|
- Schema-routed: normalizers select raw payloads by explicit schema strings, not
|
||||||
|
source names or configured routes.
|
||||||
|
- Provider-isolated: NWS, Open-Meteo, OpenWeather, and SPC quirks stay in
|
||||||
|
provider-specific source, provider-helper, and normalizer packages.
|
||||||
|
- Registry-based: built-in source drivers, normalizers, processors, and sinks
|
||||||
|
are assembled explicitly through registries instead of dynamic plugin loading.
|
||||||
|
- Adapter-clean: persistence and external-system details stay behind source and
|
||||||
|
sink adapters, not in `model` or normalizers.
|
||||||
|
- Direct Go: prefer small package-level constructors and straightforward code
|
||||||
|
over broad abstractions.
|
||||||
|
|
||||||
## Core Design Principles
|
## Core Design Principles
|
||||||
|
|
||||||
- Hexagonal boundaries: provider APIs, config loading, scheduling, dispatch, and sinks are external mechanisms around the weather domain model and normalization logic.
|
- Hexagonal boundaries: provider APIs, config loading, scheduling, dispatch, and sinks are external mechanisms around the weather domain model and normalization logic.
|
||||||
- Raw-to-canonical flow: sources should fetch and envelope raw provider payloads; normalizers should own provider-to-canonical mapping.
|
- Raw-to-canonical flow: sources should fetch and envelope raw provider payloads; normalizers should own provider-to-canonical mapping.
|
||||||
|
- Location-focused canonical data: canonical weather events represent data relevant to the configured forecast location or configured provider object; `weatherfeeder` is not a national provider-data archive.
|
||||||
- Schema-based routing: normalizers match on event schema, not source name or event kind.
|
- Schema-based routing: normalizers match on event schema, not source name or event kind.
|
||||||
- Composable registries: source drivers, normalizers, processors, and sinks are assembled explicitly through registries.
|
- Composable registries: source drivers, normalizers, processors, and sinks are assembled explicitly through registries.
|
||||||
- Bounded concurrency: scheduling and sink fanout are concurrent, but the application should keep queues, goroutine ownership, logging, and cancellation behavior visible.
|
- Bounded concurrency: scheduling and sink fanout are concurrent, but the application should keep queues, goroutine ownership, logging, and cancellation behavior visible.
|
||||||
@@ -57,6 +79,23 @@ Tests and examples:
|
|||||||
- The sample `cmd/weatherfeeder/config.yml` is executable test input and is load-tested.
|
- The sample `cmd/weatherfeeder/config.yml` is executable test input and is load-tested.
|
||||||
- Tests should keep exercising package contracts directly rather than relying only on full-daemon execution.
|
- Tests should keep exercising package contracts directly rather than relying only on full-daemon execution.
|
||||||
|
|
||||||
|
## Feedkit Boundary
|
||||||
|
|
||||||
|
`feedkit` provides reusable daemon infrastructure. `weatherfeeder` provides the
|
||||||
|
weather-domain adapters, models, schemas, and mapping policy.
|
||||||
|
|
||||||
|
| Area | Feedkit owns | Weatherfeeder owns |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| Config | Generic YAML shape: sources, sinks, routes, modes, cadence, and params. | Driver-specific config rules such as NWS `user_agent`, OpenWeather `units=metric`, and SPC coordinates. |
|
||||||
|
| Events | Domain-agnostic event envelope: ID, kind, source, emitted/effective times, schema, and payload. | Event kind meaning, schema strings, and canonical weather payloads. |
|
||||||
|
| Sources | Source interfaces, registry, expected-kind validation, HTTP helper, and default event ID helper. | NWS/Open-Meteo/OpenWeather/SPC source drivers and raw schema emission. |
|
||||||
|
| Processing | Processor registry, normalize processor, dedupe processor, and pipeline execution. | Weather normalizers and schema-specific raw-to-canonical mapping. |
|
||||||
|
| Dispatch | Route compilation and sink fanout mechanics. | Which weather event kinds are configured and meaningful. |
|
||||||
|
| Sinks | Generic stdout, NATS, and Postgres sink mechanics. | Weather-specific Postgres schema and canonical event-to-row mapping. |
|
||||||
|
|
||||||
|
Do not move weather-domain policy into `feedkit`, and do not duplicate generic
|
||||||
|
daemon mechanics in `weatherfeeder` when feedkit already provides the boundary.
|
||||||
|
|
||||||
## Modules Or Processing Steps
|
## Modules Or Processing Steps
|
||||||
|
|
||||||
The implemented processing steps are source polling, normalization, dedupe, and sink dispatch.
|
The implemented processing steps are source polling, normalization, dedupe, and sink dispatch.
|
||||||
|
|||||||
@@ -24,7 +24,8 @@ docs, not here.
|
|||||||
normalizers.
|
normalizers.
|
||||||
- `internal/sinks/postgres/`: weatherfeeder-owned Postgres schema and canonical
|
- `internal/sinks/postgres/`: weatherfeeder-owned Postgres schema and canonical
|
||||||
event mapper.
|
event mapper.
|
||||||
- `docs/`: current behavior, policies, integration contracts, and roadmap files.
|
- `docs/`: current behavior, consumer guides, integration contracts, policies,
|
||||||
|
and roadmap files.
|
||||||
- `examples/`: maintained, copyable configuration examples.
|
- `examples/`: maintained, copyable configuration examples.
|
||||||
|
|
||||||
## Build And Test
|
## Build And Test
|
||||||
@@ -70,6 +71,32 @@ Use fixtures, local test servers, and package-level tests.
|
|||||||
- Prefer explicit registries and small package-level constructors over hidden
|
- Prefer explicit registries and small package-level constructors over hidden
|
||||||
global behavior.
|
global behavior.
|
||||||
|
|
||||||
|
## Architecture-Preserving Changes
|
||||||
|
|
||||||
|
When changing `weatherfeeder`, preserve the split between feedkit
|
||||||
|
infrastructure and weather-domain behavior.
|
||||||
|
|
||||||
|
Do:
|
||||||
|
|
||||||
|
- keep generic scheduling, dispatch, processor, config, and sink mechanics in
|
||||||
|
feedkit;
|
||||||
|
- keep weather provider rules in source adapters, provider helpers, and
|
||||||
|
normalizers;
|
||||||
|
- keep canonical weather payloads in `model` and schema/wire identifiers in
|
||||||
|
`standards`;
|
||||||
|
- keep Postgres table and row mapping under `internal/sinks/postgres`;
|
||||||
|
- use explicit registries for built-in sources and normalizers.
|
||||||
|
|
||||||
|
Do not:
|
||||||
|
|
||||||
|
- move provider parsing, WMO mapping, or canonical weather policy into
|
||||||
|
`cmd/weatherfeeder`;
|
||||||
|
- move weather-specific constants, schemas, or validation rules into feedkit;
|
||||||
|
- put database column metadata or sink-specific tags on canonical model structs;
|
||||||
|
- replace explicit registries with dynamic plugin loading;
|
||||||
|
- introduce broad abstractions when a small provider-specific helper preserves
|
||||||
|
clarity.
|
||||||
|
|
||||||
## Dependency Policy
|
## Dependency Policy
|
||||||
|
|
||||||
Prefer the Go standard library unless a dependency materially improves
|
Prefer the Go standard library unless a dependency materially improves
|
||||||
@@ -183,6 +210,7 @@ When behavior changes, update the canonical docs in the same change:
|
|||||||
- CLI behavior: `docs/cli.md`;
|
- CLI behavior: `docs/cli.md`;
|
||||||
- operations and recovery: `docs/operations.md`;
|
- operations and recovery: `docs/operations.md`;
|
||||||
- troubleshooting: `docs/troubleshooting.md`;
|
- troubleshooting: `docs/troubleshooting.md`;
|
||||||
|
- public Go package consumption: `docs/consumers/`;
|
||||||
- external contracts: `docs/integrations/`;
|
- external contracts: `docs/integrations/`;
|
||||||
- internal component behavior: `docs/internal/`;
|
- internal component behavior: `docs/internal/`;
|
||||||
- copyable configs: `examples/`.
|
- copyable configs: `examples/`.
|
||||||
|
|||||||
@@ -2,12 +2,13 @@
|
|||||||
|
|
||||||
## Purpose
|
## Purpose
|
||||||
|
|
||||||
Project documentation must help four audiences:
|
Project documentation must help five audiences:
|
||||||
|
|
||||||
1. users who need to run the application;
|
1. users who need to run the application;
|
||||||
2. administrators/operators who need to configure and operate it;
|
2. administrators/operators who need to configure and operate it;
|
||||||
3. developers who need to understand and change it safely;
|
3. developers who need to understand and change it safely;
|
||||||
4. LLM coding agents that need clear scope, boundaries, and invariants.
|
4. LLM coding agents that need clear scope, boundaries, and invariants;
|
||||||
|
5. developers and LLM coding agents integrating this project from another codebase.
|
||||||
|
|
||||||
Docs should be accurate, concise, task-oriented, and organized by audience. Prefer links to canonical docs over repetition.
|
Docs should be accurate, concise, task-oriented, and organized by audience. Prefer links to canonical docs over repetition.
|
||||||
|
|
||||||
@@ -42,11 +43,14 @@ Canonical homes:
|
|||||||
|
|
||||||
- project purpose and quickstart: `README.md`
|
- project purpose and quickstart: `README.md`
|
||||||
- development principles: `docs/policy/architecture.md`
|
- development principles: `docs/policy/architecture.md`
|
||||||
|
- public HTTP API reference: `docs/api.md`
|
||||||
- configuration reference: `docs/config.md`
|
- configuration reference: `docs/config.md`
|
||||||
- CLI reference: `docs/cli.md`
|
- CLI reference: `docs/cli.md`
|
||||||
- operations and recovery: `docs/operations.md`
|
- operations and recovery: `docs/operations.md`
|
||||||
- troubleshooting: `docs/troubleshooting.md`
|
- troubleshooting: `docs/troubleshooting.md`
|
||||||
|
- public API/package consumer guidance: `docs/consumers/`
|
||||||
- implemented internals: `docs/internal/`
|
- implemented internals: `docs/internal/`
|
||||||
|
- external protocol, service, and file-format contracts: `docs/integrations/`
|
||||||
- future work: `docs/roadmap/`
|
- future work: `docs/roadmap/`
|
||||||
- contributor workflow: `docs/policy/development.md`
|
- contributor workflow: `docs/policy/development.md`
|
||||||
- copyable examples: `examples/`
|
- copyable examples: `examples/`
|
||||||
@@ -106,7 +110,7 @@ Recommended:
|
|||||||
- `examples/`
|
- `examples/`
|
||||||
- `docs/policy/development.md`
|
- `docs/policy/development.md`
|
||||||
|
|
||||||
### Modular, staged, service-oriented, or orchestration application
|
### Modular, service-oriented, or orchestration application
|
||||||
|
|
||||||
Required:
|
Required:
|
||||||
- `docs/cli.md`, if CLI-based
|
- `docs/cli.md`, if CLI-based
|
||||||
@@ -119,6 +123,31 @@ Recommended:
|
|||||||
- `docs/troubleshooting.md`
|
- `docs/troubleshooting.md`
|
||||||
- validated examples under `examples/`
|
- validated examples under `examples/`
|
||||||
|
|
||||||
|
### Public HTTP API service
|
||||||
|
|
||||||
|
Required:
|
||||||
|
- `docs/api.md`
|
||||||
|
- `docs/cli.md`, if CLI-based
|
||||||
|
- `docs/config.md`, if config-driven
|
||||||
|
- `docs/operations.md`
|
||||||
|
- `docs/internal/`
|
||||||
|
- `docs/policy/development.md`
|
||||||
|
|
||||||
|
Recommended:
|
||||||
|
- `docs/troubleshooting.md`
|
||||||
|
- `docs/consumers/`, for task-oriented client integration guides
|
||||||
|
- `docs/integrations/`, for upstream/downstream service contracts
|
||||||
|
- validated examples under `examples/`
|
||||||
|
|
||||||
|
### Project with public packages or consumer APIs
|
||||||
|
|
||||||
|
Required:
|
||||||
|
- `docs/consumers/api.md`
|
||||||
|
- one `docs/consumers/pkg-<name>.md` file per public package, if public packages exist
|
||||||
|
|
||||||
|
Recommended:
|
||||||
|
- copyable consumer examples under `examples/`, if practical
|
||||||
|
|
||||||
## Required Documents
|
## Required Documents
|
||||||
|
|
||||||
### README.md
|
### README.md
|
||||||
@@ -161,6 +190,32 @@ It should include:
|
|||||||
|
|
||||||
For small projects, this file may be brief. It may simply state that the project is intentionally narrow, monolithic, and dependency-light.
|
For small projects, this file may be brief. It may simply state that the project is intentionally narrow, monolithic, and dependency-light.
|
||||||
|
|
||||||
|
### docs/api.md
|
||||||
|
|
||||||
|
**Audience:** external HTTP API consumers, developers, LLM coding agents integrating by HTTP
|
||||||
|
|
||||||
|
Required for projects whose primary public interface is HTTP.
|
||||||
|
|
||||||
|
`docs/api.md` is the canonical public HTTP API contract. It should be normative for external consumers and should not be duplicated by README, operations docs, consumer guides, or integration docs.
|
||||||
|
|
||||||
|
It should include:
|
||||||
|
|
||||||
|
1. base URL conventions;
|
||||||
|
2. authentication and authorization behavior, if implemented;
|
||||||
|
3. response envelope;
|
||||||
|
4. supported media types and content negotiation behavior;
|
||||||
|
5. shared query parameters;
|
||||||
|
6. endpoint reference grouped by route family;
|
||||||
|
7. request parameters and validation rules;
|
||||||
|
8. response fields, units, nullability, and optionality;
|
||||||
|
9. error response shape and status codes;
|
||||||
|
10. pagination, caching, rate-limit, idempotency, and retry behavior, if implemented;
|
||||||
|
11. compact request and response examples.
|
||||||
|
|
||||||
|
It must document only implemented endpoints and behavior. Planned endpoints, proposed fields, future filters, and experimental response shapes belong only under `docs/roadmap/`.
|
||||||
|
|
||||||
|
For HTTP API projects, `docs/consumers/` may provide task-oriented client integration guides, but those guides should link to `docs/api.md` for the authoritative endpoint contract.
|
||||||
|
|
||||||
### docs/policy/development.md
|
### docs/policy/development.md
|
||||||
|
|
||||||
**Audience:** developers, LLM coding agents
|
**Audience:** developers, LLM coding agents
|
||||||
@@ -175,7 +230,7 @@ It should include:
|
|||||||
- dependency policy;
|
- dependency policy;
|
||||||
- how to add config fields;
|
- how to add config fields;
|
||||||
- how to add CLI flags;
|
- how to add CLI flags;
|
||||||
- how to add stages/modules/adapters, if applicable;
|
- how to add modules or adapters, if applicable;
|
||||||
- how to update examples;
|
- how to update examples;
|
||||||
- documentation update expectations.
|
- documentation update expectations.
|
||||||
|
|
||||||
@@ -216,7 +271,7 @@ Explain when commands are useful, not just their syntax.
|
|||||||
|
|
||||||
**Audience:** administrators, operators
|
**Audience:** administrators, operators
|
||||||
|
|
||||||
Required for applications that maintain state, support resume behavior, run multiple stages, write durable artifacts, use remote storage, or require recovery procedures.
|
Required for applications that maintain state, support resume behavior, run multi-step workflows, write durable artifacts, use remote storage, or require recovery procedures.
|
||||||
|
|
||||||
It should cover:
|
It should cover:
|
||||||
|
|
||||||
@@ -244,11 +299,40 @@ Each entry should include:
|
|||||||
- safe fix;
|
- safe fix;
|
||||||
- relevant links.
|
- relevant links.
|
||||||
|
|
||||||
|
### docs/consumers/
|
||||||
|
|
||||||
|
**Audience:** developers and LLM coding agents integrating this project from another codebase
|
||||||
|
|
||||||
|
Required for projects with public packages, SDKs, client APIs, plugin APIs, or other application-facing integration surfaces.
|
||||||
|
|
||||||
|
This directory describes how an external codebase should consume the project's public API. It should be task-oriented and copyable where useful. It is not the place for internal implementation details or operator procedures.
|
||||||
|
|
||||||
|
For projects whose public API is HTTP, `docs/consumers/` is not required, and it should not duplicate the endpoint reference in `docs/api.md`. If present, it may provide practical integration workflows, client-specific examples, or migration notes that link back to `docs/api.md`.
|
||||||
|
|
||||||
|
`docs/consumers/api.md` should provide the consumer-facing overview and primary implementation workflow. It should include:
|
||||||
|
|
||||||
|
1. intended consumer audience and use cases;
|
||||||
|
2. required inputs supplied by operators or deployment configuration;
|
||||||
|
3. recommended public package or API workflow;
|
||||||
|
4. minimal copyable example;
|
||||||
|
5. consumer responsibilities and boundaries;
|
||||||
|
6. retry, idempotency, or status behavior, if applicable;
|
||||||
|
7. links to package-specific docs and canonical integration contracts.
|
||||||
|
|
||||||
|
Package-specific docs should be named `pkg-<name>.md` and should include:
|
||||||
|
|
||||||
|
1. import path;
|
||||||
|
2. intended use cases;
|
||||||
|
3. primary types and functions needed by consumers;
|
||||||
|
4. minimal examples;
|
||||||
|
5. validation, error, retry, and boundary behavior;
|
||||||
|
6. links to canonical file-format or wire-protocol contracts.
|
||||||
|
|
||||||
### docs/internal/
|
### docs/internal/
|
||||||
|
|
||||||
**Audience:** developers, LLM coding agents
|
**Audience:** developers, LLM coding agents
|
||||||
|
|
||||||
Required for modular, staged, service-oriented, or orchestration projects.
|
Required for modular, service-oriented, or orchestration projects.
|
||||||
|
|
||||||
This directory describes implemented internal components. It is not the roadmap.
|
This directory describes implemented internal components. It is not the roadmap.
|
||||||
|
|
||||||
@@ -289,7 +373,9 @@ Roadmap docs should not be confused with current behavior.
|
|||||||
|
|
||||||
Required for projects that depend on external CLIs, APIs, services, protocols, or file formats where the integration contract is important to maintain.
|
Required for projects that depend on external CLIs, APIs, services, protocols, or file formats where the integration contract is important to maintain.
|
||||||
|
|
||||||
This directory contains concise, versioned reference notes for external integration contracts. It should document only the parts of the external system that this project actually uses.
|
This directory contains concise, versioned reference notes for external integration contracts. It should document only the parts of the external system that this project actually uses or exposes.
|
||||||
|
|
||||||
|
For public HTTP API services, `docs/integrations/` should document upstream, downstream, storage, protocol, or runtime contracts that the service depends on or bridges. It should not become a second copy of the public HTTP endpoint reference; that belongs in `docs/api.md`.
|
||||||
|
|
||||||
Use one file per integration where useful.
|
Use one file per integration where useful.
|
||||||
|
|
||||||
@@ -346,8 +432,10 @@ Before merging documentation changes, verify:
|
|||||||
|
|
||||||
- README is concise and orientation-focused.
|
- README is concise and orientation-focused.
|
||||||
- `docs/policy/architecture.md` describes development principles.
|
- `docs/policy/architecture.md` describes development principles.
|
||||||
|
- `docs/api.md` is the canonical HTTP contract for HTTP API services.
|
||||||
- Future work appears only under `docs/roadmap/`.
|
- Future work appears only under `docs/roadmap/`.
|
||||||
- User-facing docs avoid unnecessary internals.
|
- User-facing docs avoid unnecessary internals.
|
||||||
|
- Consumer-facing docs explain public APIs without duplicating HTTP endpoint or integration contracts.
|
||||||
- Developer-facing docs preserve boundaries and invariants.
|
- Developer-facing docs preserve boundaries and invariants.
|
||||||
- Config examples match the schema.
|
- Config examples match the schema.
|
||||||
- CLI examples match real commands and flags.
|
- CLI examples match real commands and flags.
|
||||||
|
|||||||
@@ -1,491 +0,0 @@
|
|||||||
# Documentation Roadmap
|
|
||||||
|
|
||||||
## Purpose
|
|
||||||
|
|
||||||
This roadmap defines the work required to bring `weatherfeeder` documentation into compliance with [`docs/policy/documentation.md`](../policy/documentation.md) and the current implementation described by [`docs/policy/architecture.md`](../policy/architecture.md).
|
|
||||||
|
|
||||||
This is a planning document only. Implementation stages must update current-behavior documentation so it reflects implemented code, and keep proposed or unimplemented work under `docs/roadmap/`.
|
|
||||||
|
|
||||||
## Repository Documentation Inventory
|
|
||||||
|
|
||||||
- `README.md`: keep and rewrite. It is the correct project orientation file, but it is stale: it says stdout is the only implemented sink, uses “current MVP” framing, and lacks a shortest useful run command and links to targeted docs.
|
|
||||||
- `API.md`: move and rewrite. It currently acts as the event wire contract, but external integration contracts belong under `docs/integrations/`. It is also stale: it omits the emitted `kind` and `emitted_at` envelope fields, documents `effectiveAt` instead of the actual `effective_at` JSON field, lists only some raw schemas, and has duplicate alert `response` rows.
|
|
||||||
- `docs/policy/documentation.md`: keep and lightly update only if needed. It is the controlling documentation policy and should not be duplicated elsewhere.
|
|
||||||
- `docs/policy/architecture.md`: keep and lightly update after the documentation tree is migrated. Its architecture content is current, but its link to `API.md` should be updated when the wire contract moves to `docs/integrations/events.md`.
|
|
||||||
- `docs/roadmap/spc.md`: keep. It is correctly located under roadmap because it describes unimplemented SPC outlook support.
|
|
||||||
- `docs/roadmap/documentation.md`: create. This file is the current deliverable.
|
|
||||||
- `cmd/weatherfeeder/config.yml`: keep and treat as an executable sample, but do not use it as the only operator-facing config reference. It includes live endpoints and a commented OpenWeather URL with an apparent real API key; future cleanup should remove or replace that secret-like value.
|
|
||||||
- `internal/normalizers/doc.go`: keep and lightly update. It contains implemented developer guidance but incorrectly references `internal/standards/schema.go`; the actual package is `standards` at repository root.
|
|
||||||
- `internal/normalizers/common/doc.go`: keep and lightly update if needed. It is concise and current.
|
|
||||||
- `internal/providers/nws/doc.go`: keep and lightly update if needed. It documents implemented provider-helper boundaries.
|
|
||||||
- `internal/providers/openweather/doc.go`: keep and lightly update. It references future forecasts/alerts in a comment; outside roadmap docs this should be rewritten to current behavior only.
|
|
||||||
- `standards/doc.go`: keep and lightly update. It incorrectly says `internal/standards/doc.go`; actual path is `standards/doc.go`.
|
|
||||||
- `model/doc.go`: keep and lightly update. It says JSON tags are wire contract for “stdout today; others later,” which is stale because NATS and Postgres are implemented.
|
|
||||||
- `internal/sinks/postgres/doc.go`: keep and lightly update. It is the authoritative implemented Postgres table contract, but should be linked from `docs/integrations/postgres.md` instead of being the only discoverable operator/developer reference.
|
|
||||||
- `examples/`: create new. No examples directory exists. Current example material is embedded in `cmd/weatherfeeder/config.yml` only.
|
|
||||||
- `docs/config.md`: create new. Required because `weatherfeeder` is config-driven.
|
|
||||||
- `docs/cli.md`: create new. Required because `weatherfeeder` is a command-line executable, even though it currently has no flags.
|
|
||||||
- `docs/operations.md`: create new. Required because `weatherfeeder` is an operator-facing daemon with polling, concurrency, external sinks, retention/pruning, and shutdown behavior.
|
|
||||||
- `docs/troubleshooting.md`: create new. Recommended and useful because recurring operator failures are visible in constructors and config validation.
|
|
||||||
- `docs/policy/development.md`: create new. Required for a modular project maintained by humans and coding agents.
|
|
||||||
- `docs/internal/`: create new. Required because the project is modular and staged.
|
|
||||||
- `docs/integrations/`: create new. Required because the project depends on external weather APIs, NATS, Postgres, and an emitted event contract.
|
|
||||||
|
|
||||||
## Policy Compliance Assessment
|
|
||||||
|
|
||||||
Required missing documents:
|
|
||||||
|
|
||||||
- `docs/cli.md`
|
|
||||||
- `docs/config.md`
|
|
||||||
- `docs/operations.md`
|
|
||||||
- `docs/internal/`
|
|
||||||
- `docs/policy/development.md`
|
|
||||||
- `docs/integrations/` entries for important external contracts
|
|
||||||
|
|
||||||
Recommended missing documents:
|
|
||||||
|
|
||||||
- `docs/troubleshooting.md`
|
|
||||||
- `examples/` with maintained config examples
|
|
||||||
|
|
||||||
Existing stale or misplaced content:
|
|
||||||
|
|
||||||
- `README.md` is not concise enough as a current quickstart and contains stale sink status.
|
|
||||||
- `API.md` is in the wrong canonical home for an external integration contract and does not match `feedkit/event.Event` JSON tags.
|
|
||||||
- Some package comments include stale path references or future-looking language outside roadmap files.
|
|
||||||
- The sample config contains live endpoint examples and an OpenWeather URL with an apparent credential-like `appid` value in a commented block; examples should be secret-free.
|
|
||||||
|
|
||||||
Unimplemented, historical, or roadmap-only content outside roadmap:
|
|
||||||
|
|
||||||
- README “current MVP” phrasing is development-history framing and should be removed.
|
|
||||||
- Comments such as “others later,” “forecasts/alerts later,” and “future expensive steps” should be rewritten or removed unless they describe an implemented boundary.
|
|
||||||
- Do not move SPC outlook content out of `docs/roadmap/spc.md` until it is implemented.
|
|
||||||
|
|
||||||
Examples needing work:
|
|
||||||
|
|
||||||
- There is no `examples/` directory.
|
|
||||||
- `cmd/weatherfeeder/config.yml` is load-tested by `cmd/weatherfeeder/main_test.go`, but it is not an ideal copyable public example because it includes local/operator-specific values and a commented API key-like OpenWeather URL.
|
|
||||||
- Future examples should be checked with `feedkit/config.Load` or a weatherfeeder-specific test.
|
|
||||||
|
|
||||||
Links needing verification:
|
|
||||||
|
|
||||||
- Links from `README.md` to `API.md` should be changed to the new canonical event contract path if `API.md` is moved.
|
|
||||||
- `docs/policy/architecture.md` links to `../../API.md`; update this after moving the contract.
|
|
||||||
- Internal package comments referring to `internal/standards` need correction to `standards`.
|
|
||||||
|
|
||||||
## Target Documentation Set
|
|
||||||
|
|
||||||
### `README.md`
|
|
||||||
|
|
||||||
- Audience: users, administrators, operators.
|
|
||||||
- Purpose: orient readers and get them to the shortest useful run path.
|
|
||||||
- Canonical scope: project purpose, elevator pitch, shortest useful command, and links to deeper docs.
|
|
||||||
- Recommended outline: description; what it does; shortest useful command; emitted products and providers summary; links to config, CLI, operations, event contract, development policy.
|
|
||||||
- Source of truth: `cmd/weatherfeeder/main.go`, `cmd/weatherfeeder/config.yml`, `internal/sources/builtins.go`, `internal/normalizers/builtins.go`, `standards/schema.go`, `README.md` stale claims.
|
|
||||||
- Acceptance criteria: concise, no stale stdout-only claim, no full config reference, no development-history framing, links target canonical docs.
|
|
||||||
|
|
||||||
### `docs/cli.md`
|
|
||||||
|
|
||||||
- Audience: users, administrators, operators.
|
|
||||||
- Purpose: document the executable interface.
|
|
||||||
- Canonical scope: how to run `weatherfeeder` and what CLI flags exist.
|
|
||||||
- Recommended outline: shortest useful command; command overview; complete flag reference stating there are currently no flags; working-directory requirement for `config.yml`; common local/container workflows; shutdown signal behavior.
|
|
||||||
- Source of truth: `cmd/weatherfeeder/main.go`, `Dockerfile`, `cmd/weatherfeeder/main_test.go`.
|
|
||||||
- Acceptance criteria: accurately states that `config.yml` is loaded from current working directory and no config path flag exists; does not invent flags.
|
|
||||||
|
|
||||||
### `docs/config.md`
|
|
||||||
|
|
||||||
- Audience: administrators, operators, advanced users.
|
|
||||||
- Purpose: canonical configuration reference.
|
|
||||||
- Canonical scope: YAML shape, field rules, source/sink/route definitions, driver params, and defaults.
|
|
||||||
- Recommended outline: config file location; minimal config; production-oriented config; top-level reference; source drivers; source params; sink drivers; sink params; routes; duration formats; secrets handling; links to examples.
|
|
||||||
- Source of truth: `cmd/weatherfeeder/main.go`, `cmd/weatherfeeder/config.yml`, `../feedkit/config/config.go`, `../feedkit/config/load.go`, `../feedkit/sources/http.go`, `../feedkit/sinks/*.go`, `internal/sources/builtins.go`, source constructors.
|
|
||||||
- Acceptance criteria: documents strict YAML known-field behavior, required `sources` and `sinks`, optional `routes`, `mode`, `every`, `kinds`, HTTP params including `url`, `user_agent`, `conditional`, `http_timeout`, and `http_response_body_limit_bytes`; documents OpenWeather `units=metric` requirement; avoids embedding secrets.
|
|
||||||
|
|
||||||
### `docs/operations.md`
|
|
||||||
|
|
||||||
- Audience: administrators and operators.
|
|
||||||
- Purpose: explain how to run, observe, shut down, and recover the daemon.
|
|
||||||
- Canonical scope: runtime behavior, logs, scheduling, sink behavior, Postgres initialization/pruning, and operational caveats.
|
|
||||||
- Recommended outline: normal workflow; runtime lifecycle; logs; scheduler and polling behavior; conditional HTTP fetches; routing and sink fanout; Postgres table creation and retention; shutdown; recovery; caveats.
|
|
||||||
- Source of truth: `cmd/weatherfeeder/main.go`, `../feedkit/scheduler`, `../feedkit/dispatch`, `../feedkit/sources/http.go`, `../feedkit/sinks/postgres.go`, `internal/sinks/postgres/schema.go`, `Dockerfile`.
|
|
||||||
- Acceptance criteria: describes implemented behavior only; no unsupported admin commands; clearly states durable state is external sink state.
|
|
||||||
|
|
||||||
### `docs/troubleshooting.md`
|
|
||||||
|
|
||||||
- Audience: administrators and operators.
|
|
||||||
- Purpose: provide safe diagnosis for common implemented failure modes.
|
|
||||||
- Canonical scope: symptoms, likely causes, checks, fixes, and links.
|
|
||||||
- Recommended outline: missing `config.yml`; YAML parse or unknown-field errors; unknown source/sink driver; source kind mismatch; missing `params.url` or `params.user_agent`; OpenWeather missing `units=metric`; NATS connection failure; Postgres connection/table/credential failure; no events due to 304 unchanged responses; route sends no events.
|
|
||||||
- Source of truth: constructors and validation in feedkit/weatherfeeder, `cmd/weatherfeeder/main.go`, source tests, sink tests.
|
|
||||||
- Acceptance criteria: every entry has symptom, likely cause, diagnostic step, safe fix, and link to config/operations where useful.
|
|
||||||
|
|
||||||
### `docs/policy/development.md`
|
|
||||||
|
|
||||||
- Audience: developers and LLM coding agents.
|
|
||||||
- Purpose: contributor workflow and safe-change guidance.
|
|
||||||
- Canonical scope: repository layout, build/test commands, coding conventions, dependency policy, adding drivers/normalizers/schemas/config fields/docs.
|
|
||||||
- Recommended outline: layout; Go/test commands; style; dependency policy; adding a source; adding a normalizer; adding a canonical model/schema; adding Postgres mapping; updating sample config/examples/docs; review checklist.
|
|
||||||
- Source of truth: `docs/policy/architecture.md`, `go.mod`, `cmd/weatherfeeder/main_test.go`, package docs, builtins registries, tests.
|
|
||||||
- Acceptance criteria: no user/operator reference duplication; links to config/CLI docs; gives enough steps for coding agents to preserve architecture.
|
|
||||||
|
|
||||||
### `docs/internal/runtime.md`
|
|
||||||
|
|
||||||
- Audience: developers and LLM coding agents.
|
|
||||||
- Purpose: describe implemented runtime composition.
|
|
||||||
- Canonical scope: config load, registries, scheduler, processor chain, dispatcher, sink fanout, shutdown.
|
|
||||||
- Recommended outline: purpose; inputs/outputs; boundaries; config fields used; adapters used; state; failure behavior; tests; invariants.
|
|
||||||
- Source of truth: `cmd/weatherfeeder/main.go`, `cmd/weatherfeeder/main_test.go`, `../feedkit/scheduler`, `../feedkit/dispatch`, `../feedkit/processors`.
|
|
||||||
- Acceptance criteria: accurately describes normalize then dedupe, `dedupeMaxEntries = 2048`, event channel buffer 256, and context shutdown behavior.
|
|
||||||
|
|
||||||
### `docs/internal/sources.md`
|
|
||||||
|
|
||||||
- Audience: developers and LLM coding agents.
|
|
||||||
- Purpose: document implemented source drivers and source contracts.
|
|
||||||
- Canonical scope: source boundaries, registered drivers, kinds, raw schemas, params, effective-time policies at a summary level.
|
|
||||||
- Recommended outline: source contract; common HTTP behavior; driver table; provider notes; failure behavior; tests; invariants.
|
|
||||||
- Source of truth: `internal/sources/builtins.go`, `internal/sources/*`, `../feedkit/sources/http.go`, source tests, `standards/schema.go`.
|
|
||||||
- Acceptance criteria: includes all current drivers and no legacy `nws_forecast`; documents raw payload preservation and conditional GET behavior without duplicating full config reference.
|
|
||||||
|
|
||||||
### `docs/internal/normalizers.md`
|
|
||||||
|
|
||||||
- Audience: developers and LLM coding agents.
|
|
||||||
- Purpose: document raw-to-canonical mapping boundaries and registration.
|
|
||||||
- Canonical scope: normalizer contract, schema matching, provider packages, output schemas, tests.
|
|
||||||
- Recommended outline: contract; registration/order; mapping table from raw schemas to canonical schemas; common helpers; failure behavior; tests; invariants.
|
|
||||||
- Source of truth: `internal/normalizers/doc.go`, `internal/normalizers/builtins.go`, provider normalizer packages, `standards/schema.go`, normalizer tests.
|
|
||||||
- Acceptance criteria: fixes stale `internal/standards` references; avoids duplicating every payload field from event contract.
|
|
||||||
|
|
||||||
### `docs/internal/postgres-sink.md`
|
|
||||||
|
|
||||||
- Audience: developers and LLM coding agents.
|
|
||||||
- Purpose: explain weatherfeeder-owned Postgres mapping internals.
|
|
||||||
- Canonical scope: schema registration, mapper behavior, required-field validation, table contract link.
|
|
||||||
- Recommended outline: purpose; input schemas; writes; parent/child tables; nullable rules; pruning columns; tests; invariants.
|
|
||||||
- Source of truth: `internal/sinks/postgres/doc.go`, `schema.go`, `map.go`, sink tests, feedkit Postgres sink.
|
|
||||||
- Acceptance criteria: links to `docs/integrations/postgres.md` for operator-facing table contract; does not duplicate full table definitions unless that integration doc remains intentionally concise.
|
|
||||||
|
|
||||||
### `docs/integrations/events.md`
|
|
||||||
|
|
||||||
- Audience: downstream consumers and developers.
|
|
||||||
- Purpose: canonical emitted event wire contract.
|
|
||||||
- Canonical scope: feedkit event envelope JSON and canonical weather payload schemas.
|
|
||||||
- Recommended outline: envelope; timestamp rules; kinds and schemas; raw schema note; shared conventions; canonical payloads; compatibility rules; compact examples.
|
|
||||||
- Source of truth: `../feedkit/event/event.go`, `standards/schema.go`, `model/*.go`, `internal/normalizers/common/round.go`, normalizer tests.
|
|
||||||
- Acceptance criteria: documents actual JSON field names (`id`, `kind`, `source`, `emitted_at`, `effective_at`, `schema`, `payload`); includes all current raw and canonical schemas; states forecast period `conditionCode` is optional; examples validate against current structs.
|
|
||||||
|
|
||||||
### `docs/integrations/postgres.md`
|
|
||||||
|
|
||||||
- Audience: downstream SQL consumers, administrators, developers.
|
|
||||||
- Purpose: canonical Postgres table contract for weatherfeeder writes.
|
|
||||||
- Canonical scope: tables, columns, keys, indexes, pruning, reconstruction notes, migration caveats.
|
|
||||||
- Recommended outline: scope; sink config link; initialization behavior; table overview; full table contract; pruning; migrations; reconstruction of canonical payloads.
|
|
||||||
- Source of truth: `internal/sinks/postgres/doc.go`, `schema.go`, `map.go`, `../feedkit/sinks/postgres.go`, schema/map tests.
|
|
||||||
- Acceptance criteria: matches generated schema exactly; notes `CREATE TABLE IF NOT EXISTS` does not alter existing schemas; documents `forecast_periods.condition_code` nullable.
|
|
||||||
|
|
||||||
### `docs/integrations/nws.md`
|
|
||||||
|
|
||||||
- Audience: developers and operators maintaining NWS integrations.
|
|
||||||
- Purpose: concise notes for implemented NWS endpoint usage.
|
|
||||||
- Canonical scope: only endpoints and fields weatherfeeder currently uses.
|
|
||||||
- Recommended outline: supported drivers; endpoint shapes; required user agent; accept headers; effective-time policies; parser caveats; tests.
|
|
||||||
- Source of truth: `internal/sources/nws`, `internal/normalizers/nws`, `internal/providers/nws`, NWS tests and fixtures.
|
|
||||||
- Acceptance criteria: no undocumented SPC outlook content; weather stories and forecast discussions included as implemented.
|
|
||||||
|
|
||||||
### `docs/integrations/openmeteo.md`
|
|
||||||
|
|
||||||
- Audience: developers and operators maintaining Open-Meteo integrations.
|
|
||||||
- Purpose: concise notes for implemented Open-Meteo observation and hourly forecast usage.
|
|
||||||
- Canonical scope: current endpoint usage, time parsing, fields mapped, effective time.
|
|
||||||
- Source of truth: `internal/sources/openmeteo`, `internal/normalizers/openmeteo`, `internal/providers/openmeteo`, tests.
|
|
||||||
- Acceptance criteria: no daily forecast or unsupported products documented as current.
|
|
||||||
|
|
||||||
### `docs/integrations/openweather.md`
|
|
||||||
|
|
||||||
- Audience: developers and operators maintaining OpenWeather integration.
|
|
||||||
- Purpose: concise notes for implemented OpenWeather observation usage.
|
|
||||||
- Canonical scope: current observation driver, `units=metric` requirement, timestamp behavior, mapped fields.
|
|
||||||
- Source of truth: `internal/sources/openweather`, `internal/normalizers/openweather`, `internal/providers/openweather`, tests.
|
|
||||||
- Acceptance criteria: no forecast/alert OpenWeather support documented as current.
|
|
||||||
|
|
||||||
### `examples/config.minimal.yml`
|
|
||||||
|
|
||||||
- Audience: users and operators.
|
|
||||||
- Purpose: copyable minimal config.
|
|
||||||
- Canonical scope: one source, one stdout sink, one route.
|
|
||||||
- Source of truth: `cmd/weatherfeeder/config.yml`, config tests, source constructors.
|
|
||||||
- Acceptance criteria: loads with `config.Load`; no secrets; uses placeholder-safe or public endpoints as appropriate.
|
|
||||||
|
|
||||||
### `examples/config.nats.yml`
|
|
||||||
|
|
||||||
- Audience: operators.
|
|
||||||
- Purpose: copyable NATS publishing config.
|
|
||||||
- Canonical scope: selected sources, NATS sink, route examples.
|
|
||||||
- Source of truth: sample config, feedkit NATS sink.
|
|
||||||
- Acceptance criteria: loads with `config.Load`; no secrets; clearly uses replaceable NATS URL/subject.
|
|
||||||
|
|
||||||
### `examples/config.postgres.yml`
|
|
||||||
|
|
||||||
- Audience: operators.
|
|
||||||
- Purpose: copyable Postgres persistence config.
|
|
||||||
- Canonical scope: selected sources, Postgres sink params, pruning example, routes.
|
|
||||||
- Source of truth: sample config, feedkit Postgres sink, weatherfeeder Postgres schema.
|
|
||||||
- Acceptance criteria: loads with `config.Load`; uses placeholders for username/password/URI; links from config and operations docs.
|
|
||||||
|
|
||||||
## File-by-File Rewrite Guidance
|
|
||||||
|
|
||||||
### `README.md`
|
|
||||||
|
|
||||||
Cover what the daemon does, the shortest useful command, and where to go next. Use implemented provider and product summaries from source/normalizer registries. Avoid full config tables, full wire schemas, history, “MVP” language, and stale sink limitations. Link to `docs/cli.md`, `docs/config.md`, `docs/operations.md`, `docs/integrations/events.md`, and `docs/policy/development.md`.
|
|
||||||
|
|
||||||
Do not carry forward the claim that stdout is the only implemented sink.
|
|
||||||
|
|
||||||
### `API.md`
|
|
||||||
|
|
||||||
Move its canonical content to `docs/integrations/events.md` and either delete `API.md` or replace it with a short pointer only if backward-compatible repository links are required. The rewritten contract must inspect `../feedkit/event/event.go` before documenting envelope fields. Do not preserve camelCase envelope examples unless the code changes first.
|
|
||||||
|
|
||||||
### `docs/config.md`
|
|
||||||
|
|
||||||
Use feedkit config structs for generic fields and weatherfeeder source/sink constructors for driver-specific params. Link to examples instead of embedding all large configs. Avoid documenting feedkit stream-only params as actively used by weatherfeeder unless clearly marked as generic feedkit config accepted by the current config model; all current weatherfeeder sources are poll sources.
|
|
||||||
|
|
||||||
### `docs/cli.md`
|
|
||||||
|
|
||||||
State the current no-flags behavior plainly. Include that `weatherfeeder` expects `config.yml` in the current working directory. Do not document environment variables, subcommands, or alternate config paths unless implemented.
|
|
||||||
|
|
||||||
### `docs/operations.md`
|
|
||||||
|
|
||||||
Focus on running the daemon and operating sinks. Include signal shutdown, log behavior, conditional HTTP fetches, Postgres table creation, retention pruning, and no internal durable scheduler state. Avoid unsupported recovery commands.
|
|
||||||
|
|
||||||
### `docs/troubleshooting.md`
|
|
||||||
|
|
||||||
Base entries on actual validation and constructor errors. Link to `docs/config.md` for fixes. Avoid speculative upstream outages beyond generic HTTP/source errors unless tests or code reveal specific behavior.
|
|
||||||
|
|
||||||
### `docs/policy/development.md`
|
|
||||||
|
|
||||||
Use `docs/policy/architecture.md` as the authority. Give concrete safe-change steps for adding source drivers, normalizers, schemas, Postgres tables, config examples, and docs. Avoid repeating the architecture policy wholesale.
|
|
||||||
|
|
||||||
### `docs/internal/*.md`
|
|
||||||
|
|
||||||
Create concise component docs. They should guide changes, not become manuals. Each should include tests to inspect before changing. Link to package docs when those are more precise.
|
|
||||||
|
|
||||||
### `docs/integrations/*.md`
|
|
||||||
|
|
||||||
Document only external contracts actually used by the implementation. Keep provider docs narrow and maintenance-oriented. Do not describe unimplemented SPC support outside `docs/roadmap/spc.md`.
|
|
||||||
|
|
||||||
### Package comments
|
|
||||||
|
|
||||||
Fix stale path and future-language comments during the documentation migration stage that touches developer docs. Specific likely fixes:
|
|
||||||
|
|
||||||
- `model/doc.go`: remove “stdout today; others later.”
|
|
||||||
- `standards/doc.go`: correct `internal/standards/doc.go` path.
|
|
||||||
- `internal/normalizers/doc.go`: correct `internal/standards/schema.go` path.
|
|
||||||
- `internal/providers/openweather/doc.go`: remove “forecasts/alerts later.”
|
|
||||||
|
|
||||||
## Examples Plan
|
|
||||||
|
|
||||||
No `examples/` directory currently exists. Create one because the project has non-trivial configuration and operator workflows.
|
|
||||||
|
|
||||||
Recommended examples:
|
|
||||||
|
|
||||||
- `examples/config.minimal.yml`: one public HTTP source, stdout sink, route. Validity check: `config.Load` in a new or existing config/example test. Link from README, config, CLI.
|
|
||||||
- `examples/config.nats.yml`: NATS sink publishing example. Validity check: `config.Load`; do not require a live NATS server. Link from config and operations.
|
|
||||||
- `examples/config.postgres.yml`: Postgres sink with placeholder credentials and optional `prune`. Validity check: `config.Load`; do not require a live Postgres server. Link from config, operations, Postgres integration docs.
|
|
||||||
- `examples/config.full.yml`: optional, only if the team wants one annotated config covering all implemented drivers. Validity check: `config.Load`; must use placeholders for secrets and avoid real OpenWeather API keys. Link from config docs only.
|
|
||||||
|
|
||||||
Do not add SPC examples until SPC support is implemented.
|
|
||||||
|
|
||||||
## Internal Documentation Plan
|
|
||||||
|
|
||||||
### Runtime Pipeline
|
|
||||||
|
|
||||||
- Path: `docs/internal/runtime.md`
|
|
||||||
- Purpose: explain daemon composition.
|
|
||||||
- Inputs and outputs: YAML config in; feed events through scheduler/pipeline/dispatcher to sinks out.
|
|
||||||
- Boundaries: composition belongs in `cmd/weatherfeeder`; domain mapping belongs in normalizers/model.
|
|
||||||
- Config fields used: sources, sinks, routes, source `every`, mode/kinds, sink params.
|
|
||||||
- Adapters used: feedkit scheduler, processors, dispatch, sinks.
|
|
||||||
- Failure behavior: fatal startup errors; runtime errors logged and cancel context; interrupt/SIGTERM shutdown.
|
|
||||||
- Tests to inspect: `cmd/weatherfeeder/main_test.go`, feedkit scheduler/dispatch tests.
|
|
||||||
- Invariants: normalize before dedupe; bounded event channel; context-aware shutdown.
|
|
||||||
|
|
||||||
### Sources
|
|
||||||
|
|
||||||
- Path: `docs/internal/sources.md`
|
|
||||||
- Purpose: document source driver architecture.
|
|
||||||
- Inputs and outputs: `config.SourceConfig` and upstream HTTP responses in; raw feed events out.
|
|
||||||
- Boundaries: sources fetch and envelope raw payloads; normalizers decode full payloads.
|
|
||||||
- Config fields used: name, driver, mode, every, kinds, params URL/user agent/HTTP options.
|
|
||||||
- Adapters used: feedkit source registry and HTTP helper.
|
|
||||||
- Failure behavior: constructor validation, HTTP errors, unchanged 304 returns no events, some malformed metadata still emits raw events.
|
|
||||||
- Tests to inspect: `internal/sources/builtins_test.go`, provider source tests.
|
|
||||||
- Invariants: drivers registered in one place; raw schemas from `standards`; no canonical mapping in sources.
|
|
||||||
|
|
||||||
### Normalizers
|
|
||||||
|
|
||||||
- Path: `docs/internal/normalizers.md`
|
|
||||||
- Purpose: document raw-to-canonical mapping architecture.
|
|
||||||
- Inputs and outputs: raw events matched by schema in; canonical model payload events out.
|
|
||||||
- Boundaries: no HTTP, no sink persistence, no CLI/config behavior.
|
|
||||||
- Config fields used: none directly.
|
|
||||||
- Adapters used: feedkit normalize processor.
|
|
||||||
- Failure behavior: decode/parse/missing required fields return contextual errors; no-match passthrough configured in main.
|
|
||||||
- Tests to inspect: `internal/normalizers/builtins_test.go`, provider normalizer tests, `internal/normalizers/common` tests.
|
|
||||||
- Invariants: schema matching only, stable registration order, canonical schema constants.
|
|
||||||
|
|
||||||
### Postgres Sink Mapping
|
|
||||||
|
|
||||||
- Path: `docs/internal/postgres-sink.md`
|
|
||||||
- Purpose: document weatherfeeder-specific Postgres mapping internals.
|
|
||||||
- Inputs and outputs: canonical events in; feedkit `PostgresWrite` rows out.
|
|
||||||
- Boundaries: mapper validates and maps; feedkit owns DB connection, DDL, transactions, pruning.
|
|
||||||
- Config fields used: sink `uri`, `username`, `password`, optional `prune` via feedkit.
|
|
||||||
- Adapters used: feedkit Postgres sink.
|
|
||||||
- Failure behavior: unsupported schemas map to no writes or errors per mapper behavior; required missing fields fail before write.
|
|
||||||
- Tests to inspect: `internal/sinks/postgres/map_test.go`, `schema_test.go`, feedkit Postgres tests.
|
|
||||||
- Invariants: consume canonical schemas only; preserve envelope columns; keep parent/child order indexes.
|
|
||||||
|
|
||||||
## Integration Documentation Plan
|
|
||||||
|
|
||||||
### Event Wire Contract
|
|
||||||
|
|
||||||
- Path: `docs/integrations/events.md`
|
|
||||||
- External system or contract: JSON events emitted to stdout/NATS and represented in Postgres parent envelope columns.
|
|
||||||
- Current usage: sinks marshal feedkit events or map canonical events to tables.
|
|
||||||
- Version notes: schema identifiers use `raw.*.v1` and `weather.*.v1`; JSON tags are compatibility contract.
|
|
||||||
- Document: actual envelope fields, schema list, payload fields, units, optionality, examples.
|
|
||||||
- Do not document: weatherapi HTTP endpoints, unimplemented outlooks, or future schemas.
|
|
||||||
|
|
||||||
### Postgres
|
|
||||||
|
|
||||||
- Path: `docs/integrations/postgres.md`
|
|
||||||
- External system or contract: PostgreSQL schema created/written by feedkit Postgres sink using weatherfeeder schema definition.
|
|
||||||
- Current usage: optional configured sink; create-if-missing tables and indexes; transactional writes; optional pruning.
|
|
||||||
- Version notes: no migration framework is implemented; existing DBs may need manual schema changes when table definitions change.
|
|
||||||
- Document: params, table contract, pruning, reconstruction, migration caveats.
|
|
||||||
- Do not document: weatherapi query behavior except as a downstream consumer link if needed.
|
|
||||||
|
|
||||||
### NWS
|
|
||||||
|
|
||||||
- Path: `docs/integrations/nws.md`
|
|
||||||
- External system or contract: NWS API and NWS forecast discussion/weather story endpoints used by current drivers.
|
|
||||||
- Current usage: observations, alerts, hourly forecast, narrative forecast, forecast discussion, weather stories.
|
|
||||||
- Version notes: no explicit upstream API version in code; weatherfeeder pins its own raw schema names.
|
|
||||||
- Document: endpoint shapes used, required User-Agent, accept headers, effective time, raw schemas.
|
|
||||||
- Do not document: SPC outlooks or unsupported NWS products.
|
|
||||||
|
|
||||||
### Open-Meteo
|
|
||||||
|
|
||||||
- Path: `docs/integrations/openmeteo.md`
|
|
||||||
- External system or contract: Open-Meteo forecast API for current observations and hourly forecasts.
|
|
||||||
- Current usage: JSON HTTP source, time parsing from timezone/UTC offset, hourly fields mapped to canonical forecast.
|
|
||||||
- Version notes: no explicit upstream API version in code.
|
|
||||||
- Document: required URL/user agent params, expected current/hourly response pieces, effective time behavior.
|
|
||||||
- Do not document: daily forecast unless implemented.
|
|
||||||
|
|
||||||
### OpenWeather
|
|
||||||
|
|
||||||
- Path: `docs/integrations/openweather.md`
|
|
||||||
- External system or contract: OpenWeather current weather endpoint.
|
|
||||||
- Current usage: observation source/normalizer only.
|
|
||||||
- Version notes: source requires `units=metric` in URL.
|
|
||||||
- Document: metric units requirement, timestamp behavior, mapped fields, secret handling for API key.
|
|
||||||
- Do not document: OpenWeather forecasts or alerts.
|
|
||||||
|
|
||||||
### NATS
|
|
||||||
|
|
||||||
- Path: include in `docs/operations.md` and `docs/config.md`; create `docs/integrations/nats.md` only if NATS-specific maintenance grows.
|
|
||||||
- External system or contract: NATS publish subject configured by feedkit sink.
|
|
||||||
- Current usage: optional sink publishes each event as JSON.
|
|
||||||
- Document: `url`, `subject`, JSON event payload, connection failure behavior.
|
|
||||||
- Do not document: subscriptions or server administration beyond this sink contract.
|
|
||||||
|
|
||||||
## Recommended Implementation Sequence
|
|
||||||
|
|
||||||
### Stage 1: Current-State README, CLI, and Config Docs
|
|
||||||
|
|
||||||
- Goal: establish accurate user/operator entry points.
|
|
||||||
- Files to create/update/delete/move: rewrite `README.md`; create `docs/cli.md`; create `docs/config.md`.
|
|
||||||
- Repository areas to inspect: `cmd/weatherfeeder/main.go`, `cmd/weatherfeeder/config.yml`, `../feedkit/config`, `../feedkit/sources/http.go`, `../feedkit/sinks`, `internal/sources/builtins.go`, source constructors.
|
|
||||||
- Acceptance criteria: README concise and current; CLI doc states no flags and cwd `config.yml`; config doc includes complete implemented config reference and no secrets.
|
|
||||||
- Suggested validation commands: `go test ./cmd/weatherfeeder`; `rg -n "only implemented sink|current MVP|--config|API.md" README.md docs/cli.md docs/config.md`.
|
|
||||||
- Size: small enough for one implementation prompt.
|
|
||||||
|
|
||||||
### Stage 2: Event and Postgres Integration Contracts
|
|
||||||
|
|
||||||
- Goal: move and correct external contracts.
|
|
||||||
- Files to create/update/delete/move: create `docs/integrations/events.md`; create `docs/integrations/postgres.md`; delete or replace `API.md` with a pointer; update links in `README.md` and `docs/policy/architecture.md`.
|
|
||||||
- Repository areas to inspect: `../feedkit/event/event.go`, `standards/schema.go`, `model/*.go`, `internal/sinks/postgres/doc.go`, `schema.go`, `map.go`, normalizer tests, sink tests.
|
|
||||||
- Acceptance criteria: event envelope field names match code; all current schemas listed; Postgres contract matches schema definition; no duplicate or stale API rows.
|
|
||||||
- Suggested validation commands: `go test ./internal/sinks/postgres ./internal/normalizers/...`; `rg -n "effectiveAt|emittedAt|API.md|response \| string.*response" README.md docs API.md`.
|
|
||||||
- Size: one implementation prompt if kept focused; split if examples are expanded heavily.
|
|
||||||
|
|
||||||
### Stage 3: Operations and Troubleshooting Docs
|
|
||||||
|
|
||||||
- Goal: document running and recovering the daemon.
|
|
||||||
- Files to create/update/delete/move: create `docs/operations.md`; create `docs/troubleshooting.md`.
|
|
||||||
- Repository areas to inspect: `cmd/weatherfeeder/main.go`, `Dockerfile`, feedkit scheduler/dispatch/source/sink implementations, config validation, source/sink constructors.
|
|
||||||
- Acceptance criteria: operational docs describe logs, shutdown, scheduling, conditional HTTP, fanout, Postgres create/prune behavior, and common failures without inventing commands.
|
|
||||||
- Suggested validation commands: `go test ./cmd/weatherfeeder`; `rg -n "TODO|future|planned|unimplemented" docs/operations.md docs/troubleshooting.md`.
|
|
||||||
- Size: small enough for one implementation prompt.
|
|
||||||
|
|
||||||
### Stage 4: Examples Directory and Example Validation
|
|
||||||
|
|
||||||
- Goal: add maintained copyable configs.
|
|
||||||
- Files to create/update/delete/move: create `examples/config.minimal.yml`, `examples/config.nats.yml`, `examples/config.postgres.yml`; optionally create `examples/config.full.yml`; update tests to load examples; update links in README/config/operations.
|
|
||||||
- Repository areas to inspect: `cmd/weatherfeeder/config.yml`, `cmd/weatherfeeder/main_test.go`, feedkit config validation, source/sink params.
|
|
||||||
- Acceptance criteria: examples contain no secrets, load successfully, and are linked from canonical docs.
|
|
||||||
- Suggested validation commands: `go test ./cmd/weatherfeeder`; `rg -n "appid=|password: [^<]|token|secret" examples cmd/weatherfeeder/config.yml docs`.
|
|
||||||
- Size: one implementation prompt.
|
|
||||||
|
|
||||||
### Stage 5: Developer and Internal Docs
|
|
||||||
|
|
||||||
- Goal: document safe change workflow and internal component boundaries.
|
|
||||||
- Files to create/update/delete/move: create `docs/policy/development.md`; create `docs/internal/runtime.md`; create `docs/internal/sources.md`; create `docs/internal/normalizers.md`; create `docs/internal/postgres-sink.md`; lightly update package comments with stale paths/future language.
|
|
||||||
- Repository areas to inspect: architecture policy, package docs, registries, source/normalizer/sink tests, `go.mod`.
|
|
||||||
- Acceptance criteria: docs are inward-facing, concise, and action-oriented; package comments no longer contain stale path or future-support language outside roadmap.
|
|
||||||
- Suggested validation commands: `go test ./...`; `rg -n "internal/standards|others later|forecasts/alerts later|future expensive steps" . -g '*.go' -g '*.md'`.
|
|
||||||
- Size: likely one implementation prompt, but split package-comment cleanup if code comment edits are considered too broad.
|
|
||||||
|
|
||||||
### Stage 6: Provider Integration Notes
|
|
||||||
|
|
||||||
- Goal: document external provider contracts actually used by implemented source/normalizer pairs.
|
|
||||||
- Files to create/update/delete/move: create `docs/integrations/nws.md`, `docs/integrations/openmeteo.md`, `docs/integrations/openweather.md`; optionally create `docs/integrations/nats.md` only if NATS content should stand alone.
|
|
||||||
- Repository areas to inspect: provider source packages, provider normalizer packages, `internal/providers`, provider tests/fixtures.
|
|
||||||
- Acceptance criteria: each integration doc is narrow, current, and excludes unimplemented products; NWS doc does not include SPC outlooks except linking to roadmap if needed.
|
|
||||||
- Suggested validation commands: `go test ./internal/sources/... ./internal/normalizers/... ./internal/providers/...`; `rg -n "SPC|outlook|daily forecast|OpenWeather forecast|OpenWeather alert" docs/integrations`.
|
|
||||||
- Size: one implementation prompt.
|
|
||||||
|
|
||||||
### Stage 7: Final Documentation Consistency Pass
|
|
||||||
|
|
||||||
- Goal: remove stale references and verify policy compliance.
|
|
||||||
- Files to create/update/delete/move: any docs touched in prior stages; remove obsolete root `API.md` if not kept as pointer.
|
|
||||||
- Repository areas to inspect: whole documentation tree, package comments, tests.
|
|
||||||
- Acceptance criteria: every non-roadmap doc describes implemented behavior only; every doc has clear audience/scope; roadmap content remains only in `docs/roadmap`; links resolve by inspection.
|
|
||||||
- Suggested validation commands: `go test ./...`; `find docs -type f -name '*.md' -print | sort`; `rg -n "MVP|only implemented sink|planned|future|later|deprecated|experimental|TODO|API.md|effectiveAt|emittedAt|internal/standards|appid=" README.md docs internal model standards cmd examples`.
|
|
||||||
- Size: small enough for one implementation prompt.
|
|
||||||
|
|
||||||
## Validation Plan
|
|
||||||
|
|
||||||
Automated checks available now:
|
|
||||||
|
|
||||||
- `go test ./...` verifies code behavior and existing load-tested sample config.
|
|
||||||
- `go test ./cmd/weatherfeeder` verifies `cmd/weatherfeeder/config.yml` loads and sources build scheduler jobs.
|
|
||||||
- Focused package tests verify source drivers, normalizers, and Postgres schema/mapping.
|
|
||||||
|
|
||||||
Recommended new checks during implementation:
|
|
||||||
|
|
||||||
- Add tests that load all `examples/*.yml` with `feedkit/config.Load`.
|
|
||||||
- Add or extend tests to ensure documented source drivers in config examples build through `internal/sources.RegisterBuiltins`.
|
|
||||||
- Use `rg` checks for stale or prohibited terms: `only implemented sink`, `current MVP`, `future`, `later`, `planned`, `deprecated`, `experimental`, `API.md`, `effectiveAt`, `emittedAt`, `internal/standards`, and secret-like strings such as `appid=`.
|
|
||||||
- Manually verify Markdown links because no dedicated Markdown/link checker is currently present.
|
|
||||||
- Manually compare `docs/integrations/events.md` against `../feedkit/event/event.go`, `standards/schema.go`, and `model/*.go` before accepting.
|
|
||||||
- Manually compare `docs/integrations/postgres.md` against `internal/sinks/postgres/schema.go` before accepting.
|
|
||||||
|
|
||||||
No repository-local Markdown formatter, Markdown linter, Makefile, justfile, Taskfile, or `package.json` docs tooling was found during this planning pass.
|
|
||||||
|
|
||||||
## Open Questions
|
|
||||||
|
|
||||||
No question blocks implementation of this documentation migration. Recommended decisions:
|
|
||||||
|
|
||||||
- Move the canonical event contract from `API.md` to `docs/integrations/events.md` to comply with the documentation policy.
|
|
||||||
- Replace root `API.md` with a short pointer only if backward-compatible links are considered important; otherwise delete it during the migration.
|
|
||||||
- Keep `cmd/weatherfeeder/config.yml` as the executable in-repo sample used by tests, but create public copyable examples under `examples/` and remove secret-like values from examples.
|
|
||||||
99
docs/roadmap/future.md
Normal file
99
docs/roadmap/future.md
Normal file
@@ -0,0 +1,99 @@
|
|||||||
|
# Future Work
|
||||||
|
|
||||||
|
## Purpose
|
||||||
|
|
||||||
|
This document is the catch-all roadmap for planned, deferred, aspirational, experimental, or unimplemented weatherfeeder work. Current behavior belongs in the canonical docs outside `docs/roadmap/`.
|
||||||
|
|
||||||
|
## NWS AFD Parsing Resilience
|
||||||
|
|
||||||
|
The current parser handles the concrete RAH, LWX, and MFR variants that
|
||||||
|
motivated these ideas. Future work should keep those extension points
|
||||||
|
maintainable as additional evidence appears.
|
||||||
|
|
||||||
|
### Generalize Wrapper-Scoped Embedded Sections
|
||||||
|
|
||||||
|
The scanner currently permits undotted nested headings only inside `PREV
|
||||||
|
DISCUSSION`. If other wrapper identities are observed, replace the single
|
||||||
|
wrapper check with a small explicit provider-local registry and add a fixture
|
||||||
|
for each wrapper family. Do not make the leading dot globally optional: wrapper
|
||||||
|
scope is the safeguard against classifying uppercase prose as a section.
|
||||||
|
|
||||||
|
### Extend Conservative Preamble Classification
|
||||||
|
|
||||||
|
Leading key-message metadata currently supports validated `Issued at`, `Updated
|
||||||
|
at`, and `As of <clock> <weekday>...` forms. Add future wording variants as
|
||||||
|
small, ordered classifiers with strict label boundaries and value grammars.
|
||||||
|
Every addition should include collision tests proving that similar message prose
|
||||||
|
and malformed metadata remain canonical content.
|
||||||
|
|
||||||
|
### Keep List-Marker Recognition Extensible
|
||||||
|
|
||||||
|
The marker parser currently supports hyphens, asterisks, `N)`, `N.`, `(N)`, and
|
||||||
|
composite forms such as `- (N)`. If new decorators appear, evolve the helper
|
||||||
|
toward an explicit marker grammar or typed classification result rather than a
|
||||||
|
broad punctuation heuristic. Preserve positive-number and whitespace-boundary
|
||||||
|
checks so ordinary prose is not stripped.
|
||||||
|
|
||||||
|
### Maintain a Cross-Office Fixture Corpus
|
||||||
|
|
||||||
|
The compact RAH, LWX, and current MFR fixtures seed regression coverage for the
|
||||||
|
observed layouts. Future parser changes should add concise, deterministic HTML
|
||||||
|
fixtures for materially distinct office formats and exercise them through both
|
||||||
|
the provider parser and normalizer. Fixture comments should identify the format
|
||||||
|
family and state that edited prose is not an archived product; tests must remain
|
||||||
|
offline and assert both intended extraction and adjacent-section isolation.
|
||||||
|
|
||||||
|
## SPC Convective Outlook Follow-Ups
|
||||||
|
|
||||||
|
### Weatherapi Outlook Endpoints
|
||||||
|
|
||||||
|
Expose persisted SPC convective outlooks through `weatherapi` after the weatherfeeder storage contract is stable.
|
||||||
|
|
||||||
|
Likely endpoints:
|
||||||
|
|
||||||
|
- `GET /outlooks/convective`
|
||||||
|
- `GET /outlooks/convective/active`
|
||||||
|
- `GET /outlooks/convective/location`
|
||||||
|
|
||||||
|
Recommended behavior:
|
||||||
|
|
||||||
|
- Return the latest outlook run by default.
|
||||||
|
- Support active outlook filtering by current time and `containsLocation=true`.
|
||||||
|
- Consider optional query filters for `day`, `outlookType`, and `containsLocation`.
|
||||||
|
- Preserve canonical outlook geometry for downstream display and audit use.
|
||||||
|
|
||||||
|
### SPC Day 4-8 Outlooks
|
||||||
|
|
||||||
|
Add SPC Day 4-8 convective outlook support as a schema-compatible extension only after Day 1-3 operation is proven.
|
||||||
|
|
||||||
|
Notes:
|
||||||
|
|
||||||
|
- Day 4-8 products have different semantics from Day 1-3 categorical/tornado/hail/wind products.
|
||||||
|
- Avoid forcing Day 4-8 assumptions into the current Day 1-3 model until the source shapes and consumer needs are reviewed.
|
||||||
|
- Prefer reusing `weather.outlook.v2` if the fields remain accurate; otherwise write a separate roadmap before changing the canonical contract.
|
||||||
|
|
||||||
|
### Degraded SPC Bundle Mode
|
||||||
|
|
||||||
|
Evaluate whether the SPC source should support degraded partial bundles when one required upstream product fails.
|
||||||
|
|
||||||
|
Current behavior should remain atomic: if a required GeoJSON or print-page fetch fails, emit no event for that poll.
|
||||||
|
|
||||||
|
Future degraded mode would need a clear contract for:
|
||||||
|
|
||||||
|
- distinguishing "no risk polygon" from "product missing";
|
||||||
|
- exposing per-product fetch errors without leaking raw provider internals into canonical events;
|
||||||
|
- deciding whether downstream sinks and APIs should store or serve partial snapshots.
|
||||||
|
|
||||||
|
### Richer SPC Page Assets And Tables
|
||||||
|
|
||||||
|
Evaluate whether to parse additional SPC print-page metadata beyond the current discussion text.
|
||||||
|
|
||||||
|
Possible additions:
|
||||||
|
|
||||||
|
- archive GeoJSON/shapefile/KML links;
|
||||||
|
- image URLs;
|
||||||
|
- page risk tables;
|
||||||
|
- city tables;
|
||||||
|
- richer discussion section metadata.
|
||||||
|
|
||||||
|
Keep GeoJSON products authoritative for polygons, validity windows, and point matching unless a future roadmap explicitly changes that contract.
|
||||||
87
docs/roadmap/outlook-schema-transition.md
Normal file
87
docs/roadmap/outlook-schema-transition.md
Normal file
@@ -0,0 +1,87 @@
|
|||||||
|
# SPC Outlook Postgres Schema Transition
|
||||||
|
|
||||||
|
## Purpose
|
||||||
|
|
||||||
|
This document describes how to reset existing Postgres outlook tables from the
|
||||||
|
`weather.outlook.v1` storage shape to the `weather.outlook.v2` compatible
|
||||||
|
storage shape.
|
||||||
|
|
||||||
|
Updated `weatherfeeder` versions create outlook tables with run-level
|
||||||
|
discussion storage. Existing databases that already contain the old outlook
|
||||||
|
table family need a manual reset because the Postgres sink creates tables with
|
||||||
|
`CREATE TABLE IF NOT EXISTS`.
|
||||||
|
|
||||||
|
## Scope
|
||||||
|
|
||||||
|
This reset drops only the outlook table family and lets updated `weatherfeeder`
|
||||||
|
recreate it:
|
||||||
|
|
||||||
|
- `outlook_discussions`
|
||||||
|
- `outlooks`
|
||||||
|
- `outlook_runs`
|
||||||
|
|
||||||
|
Other weather tables are not affected.
|
||||||
|
|
||||||
|
## Warning
|
||||||
|
|
||||||
|
These commands delete stored SPC outlook history. Existing `weather.outlook.v1`
|
||||||
|
outlook rows are intentionally removed. Downstream readers should be updated
|
||||||
|
intentionally for the new outlook shape.
|
||||||
|
|
||||||
|
## Deployment Order
|
||||||
|
|
||||||
|
1. Stop `weatherfeeder`.
|
||||||
|
2. Drop the existing outlook tables.
|
||||||
|
3. Deploy updated `weatherfeeder`.
|
||||||
|
4. Start `weatherfeeder` so the Postgres sink recreates the new outlook tables.
|
||||||
|
5. Deploy updated downstream consumers such as `weatherapi`.
|
||||||
|
|
||||||
|
## Reset SQL
|
||||||
|
|
||||||
|
```sql
|
||||||
|
DROP TABLE IF EXISTS outlook_discussions;
|
||||||
|
DROP TABLE IF EXISTS outlooks;
|
||||||
|
DROP TABLE IF EXISTS outlook_runs;
|
||||||
|
```
|
||||||
|
|
||||||
|
## Verification SQL
|
||||||
|
|
||||||
|
Before or after the updated daemon starts, this query shows which outlook tables
|
||||||
|
exist:
|
||||||
|
|
||||||
|
```sql
|
||||||
|
SELECT table_name
|
||||||
|
FROM information_schema.tables
|
||||||
|
WHERE table_name IN ('outlook_runs', 'outlooks', 'outlook_discussions')
|
||||||
|
ORDER BY table_name;
|
||||||
|
```
|
||||||
|
|
||||||
|
After the updated daemon has started and recreated the tables, verify the new
|
||||||
|
run column:
|
||||||
|
|
||||||
|
```sql
|
||||||
|
SELECT column_name, is_nullable, data_type
|
||||||
|
FROM information_schema.columns
|
||||||
|
WHERE table_name = 'outlook_runs'
|
||||||
|
AND column_name = 'discussion_count';
|
||||||
|
```
|
||||||
|
|
||||||
|
Verify the discussion table indexes:
|
||||||
|
|
||||||
|
```sql
|
||||||
|
SELECT indexname
|
||||||
|
FROM pg_indexes
|
||||||
|
WHERE tablename = 'outlook_discussions'
|
||||||
|
ORDER BY indexname;
|
||||||
|
```
|
||||||
|
|
||||||
|
Verify that legacy polygon-level prose columns are gone from `outlooks`:
|
||||||
|
|
||||||
|
```sql
|
||||||
|
SELECT column_name
|
||||||
|
FROM information_schema.columns
|
||||||
|
WHERE table_name = 'outlooks'
|
||||||
|
AND column_name IN ('headline', 'summary', 'discussion');
|
||||||
|
```
|
||||||
|
|
||||||
|
The final query should return zero rows.
|
||||||
@@ -1,323 +0,0 @@
|
|||||||
# SPC Convective Outlook Support Roadmap
|
|
||||||
|
|
||||||
## Summary
|
|
||||||
|
|
||||||
Add `weatherfeeder` support for Storm Prediction Center convective outlooks as a new canonical outlook domain. The feature should poll SPC GeoJSON outlook products, optionally enrich them with RSS discussion metadata, compute whether the configured forecast point is inside each active outlook polygon, normalize the result into a provider-independent schema, and persist it through existing sinks.
|
|
||||||
|
|
||||||
This is a new domain, not an extension of `weather.alert.v1`. SPC outlooks describe probabilistic/categorical risk areas over a validity window; alerts describe active actionable hazard messages.
|
|
||||||
|
|
||||||
## Public Contract
|
|
||||||
|
|
||||||
Add schema constants:
|
|
||||||
|
|
||||||
- Raw schema: `raw.spc.convective_outlook.v1`
|
|
||||||
- Canonical schema: `weather.outlook.v1`
|
|
||||||
|
|
||||||
Add source driver:
|
|
||||||
|
|
||||||
- `spc_convective_outlook`
|
|
||||||
|
|
||||||
Add event kind:
|
|
||||||
|
|
||||||
- `outlook`
|
|
||||||
|
|
||||||
Add canonical model types:
|
|
||||||
|
|
||||||
- `model.WeatherOutlookRun`
|
|
||||||
- `model.WeatherOutlook`
|
|
||||||
|
|
||||||
Recommended canonical run fields:
|
|
||||||
|
|
||||||
- `locationId`, `locationName`
|
|
||||||
- `latitude`, `longitude`
|
|
||||||
- `asOf`
|
|
||||||
- `issuedAt`
|
|
||||||
- `outlooks`
|
|
||||||
|
|
||||||
Recommended canonical outlook fields:
|
|
||||||
|
|
||||||
- `id`
|
|
||||||
- `provider`
|
|
||||||
- `product`
|
|
||||||
- `day`
|
|
||||||
- `outlookType`
|
|
||||||
- `label`
|
|
||||||
- `labelText`
|
|
||||||
- `severityRank`
|
|
||||||
- `validFrom`
|
|
||||||
- `validTo`
|
|
||||||
- `issuedAt`
|
|
||||||
- `expiresAt`
|
|
||||||
- `forecaster`
|
|
||||||
- `headline`
|
|
||||||
- `summary`
|
|
||||||
- `discussion`
|
|
||||||
- `sourceUrl`
|
|
||||||
- `imageUrl`
|
|
||||||
- `containsLocation`
|
|
||||||
- `geometry`
|
|
||||||
|
|
||||||
Contract defaults:
|
|
||||||
|
|
||||||
- `product` should be `convective`.
|
|
||||||
- `outlookType` should be one of `categorical`, `tornado`, `hail`, `wind`.
|
|
||||||
- `day` should be `1`, `2`, or `3` for this first implementation.
|
|
||||||
- `containsLocation` is computed against configured forecast coordinates.
|
|
||||||
- `geometry` should preserve compact GeoJSON geometry for auditability and future API use.
|
|
||||||
|
|
||||||
## Source Scope
|
|
||||||
|
|
||||||
The source should fetch a bundle of SPC products in one poll cycle and emit one raw event containing the fetched RSS metadata, GeoJSON products, configured point, and per-product fetch metadata.
|
|
||||||
|
|
||||||
Poll these GeoJSON URLs:
|
|
||||||
|
|
||||||
- `https://www.spc.noaa.gov/products/outlook/day1otlk_cat.nolyr.geojson`
|
|
||||||
- `https://www.spc.noaa.gov/products/outlook/day1otlk_torn.nolyr.geojson`
|
|
||||||
- `https://www.spc.noaa.gov/products/outlook/day1otlk_hail.nolyr.geojson`
|
|
||||||
- `https://www.spc.noaa.gov/products/outlook/day1otlk_wind.nolyr.geojson`
|
|
||||||
- `https://www.spc.noaa.gov/products/outlook/day2otlk_cat.nolyr.geojson`
|
|
||||||
- `https://www.spc.noaa.gov/products/outlook/day2otlk_torn.nolyr.geojson`
|
|
||||||
- `https://www.spc.noaa.gov/products/outlook/day2otlk_hail.nolyr.geojson`
|
|
||||||
- `https://www.spc.noaa.gov/products/outlook/day2otlk_wind.nolyr.geojson`
|
|
||||||
- `https://www.spc.noaa.gov/products/outlook/day3otlk_cat.nolyr.geojson`
|
|
||||||
- `https://www.spc.noaa.gov/products/outlook/day3otlk_torn.nolyr.geojson`
|
|
||||||
- `https://www.spc.noaa.gov/products/outlook/day3otlk_hail.nolyr.geojson`
|
|
||||||
- `https://www.spc.noaa.gov/products/outlook/day3otlk_wind.nolyr.geojson`
|
|
||||||
|
|
||||||
Note: the initial candidate list duplicated Day 3 wind and omitted Day 2 wind. Use the corrected 12-product matrix above.
|
|
||||||
|
|
||||||
Also poll:
|
|
||||||
|
|
||||||
- `https://www.spc.noaa.gov/products/spcacrss.xml`
|
|
||||||
|
|
||||||
RSS usage:
|
|
||||||
|
|
||||||
- Use RSS as discussion/discovery metadata, not as the geometry source.
|
|
||||||
- Preserve item title, link, description text, pubDate, and guid where useful.
|
|
||||||
- Extract the narrative outlook text from the RSS item description when practical.
|
|
||||||
- Do not attempt to parse polygons from RSS HTML.
|
|
||||||
|
|
||||||
Polling cadence:
|
|
||||||
|
|
||||||
- Use a default cadence of `30m`, not daily/twice-daily. SPC current outlook files can update several times per day, and a 30-minute poll with HTTP caching is low cost and less likely to miss updates.
|
|
||||||
- Rely on ETag/Last-Modified handling from the HTTP source layer where available.
|
|
||||||
- Keep the cadence configurable via `every`.
|
|
||||||
|
|
||||||
Recommended config shape:
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
- name: SPCConvectiveOutlookSTL
|
|
||||||
mode: poll
|
|
||||||
kinds: ["outlook"]
|
|
||||||
driver: spc_convective_outlook
|
|
||||||
every: 30m
|
|
||||||
params:
|
|
||||||
latitude: 38.6239
|
|
||||||
longitude: -90.3571
|
|
||||||
location_id: "stl"
|
|
||||||
location_name: "St. Louis, MO"
|
|
||||||
user_agent: "HomeOps (eric@maximumdirect.net)"
|
|
||||||
```
|
|
||||||
|
|
||||||
The source should own default SPC URLs, with optional params to override the RSS URL and product URLs for tests or future provider changes.
|
|
||||||
|
|
||||||
## Implementation Stages
|
|
||||||
|
|
||||||
### Stage 1: Raw Source and Schema
|
|
||||||
|
|
||||||
- Add schema constants and model placeholders.
|
|
||||||
- Add `internal/sources/spc` package.
|
|
||||||
- Implement `spc_convective_outlook` as a poll source.
|
|
||||||
- Fetch all configured GeoJSON products and RSS during a poll cycle.
|
|
||||||
- Emit one raw event with kind `outlook` and schema `raw.spc.convective_outlook.v1`.
|
|
||||||
- Use the latest valid `ISSUE_ISO`, RSS `lastBuildDate`, or fetch time for `effectiveAt`, in that order.
|
|
||||||
- Preserve partial fetch errors as source errors for the first implementation; do not emit incomplete outlook bundles unless a later explicit degraded-mode policy is added.
|
|
||||||
|
|
||||||
Tests:
|
|
||||||
|
|
||||||
- source driver builds as a `PollSource`
|
|
||||||
- source advertises kind `outlook`
|
|
||||||
- source emits one raw bundled event
|
|
||||||
- source chooses expected effective time
|
|
||||||
- source uses configured location metadata
|
|
||||||
- source fails clearly on missing latitude/longitude
|
|
||||||
|
|
||||||
### Stage 2: GeoJSON and Spatial Matching
|
|
||||||
|
|
||||||
- Add an internal geometry helper package, for example `internal/geo`.
|
|
||||||
- Support GeoJSON `Polygon` and `MultiPolygon`.
|
|
||||||
- Implement point-in-polygon with ring support:
|
|
||||||
- first ring is the exterior
|
|
||||||
- subsequent rings are holes
|
|
||||||
- boundary counts as inside
|
|
||||||
- GeoJSON coordinate order is `[longitude, latitude]`
|
|
||||||
- Use planar ray casting for this operational point-in-polygon check.
|
|
||||||
|
|
||||||
Tests:
|
|
||||||
|
|
||||||
- point inside polygon
|
|
||||||
- point outside polygon
|
|
||||||
- point on polygon boundary
|
|
||||||
- point inside a hole is outside
|
|
||||||
- point inside one multipolygon member is inside
|
|
||||||
- longitude/latitude order regression test
|
|
||||||
|
|
||||||
### Stage 3: Normalizer and Canonical Mapping
|
|
||||||
|
|
||||||
- Add `SPCConvectiveOutlookNormalizer`.
|
|
||||||
- Match only `raw.spc.convective_outlook.v1`.
|
|
||||||
- Decode the raw source bundle.
|
|
||||||
- Map each GeoJSON feature to one `WeatherOutlook`.
|
|
||||||
- Preserve SPC feature order within each product, then order products by day and type.
|
|
||||||
- Map properties:
|
|
||||||
- `VALID_ISO` -> `validFrom`
|
|
||||||
- `EXPIRE_ISO` -> `validTo` and `expiresAt`
|
|
||||||
- `ISSUE_ISO` -> `issuedAt`
|
|
||||||
- `FORECASTER` -> `forecaster`
|
|
||||||
- `LABEL` -> `label`
|
|
||||||
- `LABEL2` -> `labelText`
|
|
||||||
- `DN` -> `severityRank`
|
|
||||||
- Derive:
|
|
||||||
- `day` from product key or URL
|
|
||||||
- `outlookType` from product key or URL
|
|
||||||
- `id` from day, type, label, issuedAt, validFrom, and feature index
|
|
||||||
- `containsLocation` from configured point and GeoJSON geometry
|
|
||||||
- Enrich narrative fields from RSS where a matching day/product item can be determined.
|
|
||||||
- Set run `asOf` to the latest valid `issuedAt` across features, falling back to RSS `lastBuildDate`, then input event time.
|
|
||||||
- Set run `issuedAt` to the latest valid `issuedAt` across features.
|
|
||||||
- Set output event `effectiveAt` to run `asOf`.
|
|
||||||
|
|
||||||
Tests:
|
|
||||||
|
|
||||||
- normalizer routes only raw SPC schema
|
|
||||||
- categorical and probabilistic products map expected fields
|
|
||||||
- `containsLocation` is true for a known point inside a fixture polygon
|
|
||||||
- `containsLocation` is false outside
|
|
||||||
- missing optional RSS still permits GeoJSON normalization
|
|
||||||
- malformed required GeoJSON timestamps fail with useful context
|
|
||||||
- canonical JSON shape does not expose raw-provider-only bundle internals
|
|
||||||
|
|
||||||
### Stage 4: Postgres Sink
|
|
||||||
|
|
||||||
Add tables:
|
|
||||||
|
|
||||||
- `outlook_runs`
|
|
||||||
- `outlooks`
|
|
||||||
|
|
||||||
Suggested `outlook_runs` columns:
|
|
||||||
|
|
||||||
- event envelope columns
|
|
||||||
- `location_id`
|
|
||||||
- `location_name`
|
|
||||||
- `latitude`
|
|
||||||
- `longitude`
|
|
||||||
- `as_of`
|
|
||||||
- `issued_at`
|
|
||||||
- `outlook_count`
|
|
||||||
|
|
||||||
Suggested `outlooks` columns:
|
|
||||||
|
|
||||||
- `run_event_id`
|
|
||||||
- `outlook_index`
|
|
||||||
- `as_of`
|
|
||||||
- `product`
|
|
||||||
- `day`
|
|
||||||
- `outlook_type`
|
|
||||||
- `label`
|
|
||||||
- `label_text`
|
|
||||||
- `severity_rank`
|
|
||||||
- `valid_from`
|
|
||||||
- `valid_to`
|
|
||||||
- `issued_at`
|
|
||||||
- `expires_at`
|
|
||||||
- `forecaster`
|
|
||||||
- `headline`
|
|
||||||
- `summary`
|
|
||||||
- `discussion`
|
|
||||||
- `source_url`
|
|
||||||
- `image_url`
|
|
||||||
- `contains_location`
|
|
||||||
- `geometry_json`
|
|
||||||
|
|
||||||
Suggested indexes:
|
|
||||||
|
|
||||||
- `outlook_runs(location_id, as_of)`
|
|
||||||
- `outlooks(contains_location, valid_from, valid_to)`
|
|
||||||
- `outlooks(day, outlook_type, label)`
|
|
||||||
- `outlooks(valid_from, valid_to)`
|
|
||||||
|
|
||||||
Mapping rules:
|
|
||||||
|
|
||||||
- Store `geometry` as compact GeoJSON text in `geometry_json`.
|
|
||||||
- Require run `asOf`.
|
|
||||||
- Require outlook `validFrom`, `validTo`, `issuedAt`, `day`, `outlookType`, and `label`.
|
|
||||||
- Preserve all outlook polygons, not only polygons containing the configured point.
|
|
||||||
|
|
||||||
Tests:
|
|
||||||
|
|
||||||
- schema includes outlook tables and indexes
|
|
||||||
- mapper writes one run row plus one row per outlook
|
|
||||||
- mapper stores compact geometry JSON
|
|
||||||
- mapper rejects missing required run/outlook fields
|
|
||||||
|
|
||||||
### Stage 5: Config and Documentation
|
|
||||||
|
|
||||||
- Update sample config with `SPCConvectiveOutlookSTL`.
|
|
||||||
- Add `outlook` to route examples.
|
|
||||||
- Update README provider capabilities.
|
|
||||||
- Update `API.md` with `weather.outlook.v1`.
|
|
||||||
- Update Postgres sink docs with outlook table contract.
|
|
||||||
|
|
||||||
Docs should clearly state:
|
|
||||||
|
|
||||||
- RSS is used for narrative/discussion metadata.
|
|
||||||
- GeoJSON is used for polygons and point matching.
|
|
||||||
- `containsLocation` is computed by weatherfeeder at ingestion time.
|
|
||||||
- Geometry is stored for downstream audit/display.
|
|
||||||
|
|
||||||
### Stage 6: Weatherapi Follow-Up
|
|
||||||
|
|
||||||
Do not include weatherapi changes in the first weatherfeeder implementation unless explicitly requested.
|
|
||||||
|
|
||||||
Likely future weatherapi endpoints:
|
|
||||||
|
|
||||||
- `GET /outlooks/convective`
|
|
||||||
- `GET /outlooks/convective/active`
|
|
||||||
- `GET /outlooks/convective/location`
|
|
||||||
|
|
||||||
Recommended API behavior:
|
|
||||||
|
|
||||||
- latest run by default
|
|
||||||
- active outlooks filtered by current time and `containsLocation=true`
|
|
||||||
- optional filter query params for `day`, `outlookType`, and `containsLocation`
|
|
||||||
|
|
||||||
## Open Questions for Implementation
|
|
||||||
|
|
||||||
- Whether to include Day 4-8 probabilistic outlooks in a later version. Exclude them from v1.
|
|
||||||
- Whether to make partial source bundles acceptable if one product URL fails. Default v1 behavior should fail the poll and retry.
|
|
||||||
- Whether to parse detailed discussion text from RSS only, HTML pages, or both. Default v1 should use RSS only.
|
|
||||||
- Whether to keep `geometry` in canonical JSON permanently. Default v1 should include it because it preserves source context and enables downstream display.
|
|
||||||
|
|
||||||
## Verification Commands
|
|
||||||
|
|
||||||
Run focused tests:
|
|
||||||
|
|
||||||
```sh
|
|
||||||
go test ./internal/sources ./internal/normalizers/... ./internal/sinks/postgres ./model
|
|
||||||
```
|
|
||||||
|
|
||||||
Run full weatherfeeder tests:
|
|
||||||
|
|
||||||
```sh
|
|
||||||
go test ./...
|
|
||||||
```
|
|
||||||
|
|
||||||
## Acceptance Criteria
|
|
||||||
|
|
||||||
- A configured SPC source emits raw outlook bundles.
|
|
||||||
- Normalization produces `weather.outlook.v1` events.
|
|
||||||
- Each outlook indicates whether the configured forecast point is inside its polygon.
|
|
||||||
- All current Day 1-3 categorical/tornado/hail/wind products are represented.
|
|
||||||
- RSS discussion metadata is preserved where available.
|
|
||||||
- Postgres sink persists outlook runs and outlook rows.
|
|
||||||
- Sample config and public docs describe the new kind, driver, schema, and storage contract.
|
|
||||||
@@ -58,7 +58,8 @@ Diagnostic: compare the configured driver with the source driver table in
|
|||||||
Safe fix: correct the driver name. Current drivers include `nws_observation`,
|
Safe fix: correct the driver name. Current drivers include `nws_observation`,
|
||||||
`nws_alerts`, `nws_forecast_hourly`, `nws_forecast_narrative`,
|
`nws_alerts`, `nws_forecast_hourly`, `nws_forecast_narrative`,
|
||||||
`nws_forecast_discussion`, `nws_weatherstories`, `openmeteo_observation`,
|
`nws_forecast_discussion`, `nws_weatherstories`, `openmeteo_observation`,
|
||||||
`openmeteo_forecast`, and `openweather_observation`.
|
`openmeteo_forecast`, `openweather_observation`, and
|
||||||
|
`spc_convective_outlook`.
|
||||||
|
|
||||||
## `unknown sink driver`
|
## `unknown sink driver`
|
||||||
|
|
||||||
@@ -94,6 +95,20 @@ Diagnostic: inspect the named source in the error and check its `params`.
|
|||||||
Safe fix: add non-empty `url` and `user_agent` values. See
|
Safe fix: add non-empty `url` and `user_agent` values. See
|
||||||
[HTTP source params](config.md#http-source-params).
|
[HTTP source params](config.md#http-source-params).
|
||||||
|
|
||||||
|
SPC convective outlook sources do not use `params.url`; they require
|
||||||
|
`latitude`, `longitude`, and `user_agent`. See
|
||||||
|
[SPC convective outlook params](config.md#spc-convective-outlook-params).
|
||||||
|
|
||||||
|
## `params.latitude is required` Or `params.longitude is required`
|
||||||
|
|
||||||
|
Symptom: startup exits for an `spc_convective_outlook` source.
|
||||||
|
|
||||||
|
Likely cause: the source is missing one of the configured point coordinates.
|
||||||
|
|
||||||
|
Diagnostic: inspect the named SPC source in the error and check its `params`.
|
||||||
|
|
||||||
|
Safe fix: add numeric `latitude` and `longitude` values in decimal degrees.
|
||||||
|
|
||||||
## `url must include units=metric`
|
## `url must include units=metric`
|
||||||
|
|
||||||
Symptom: startup exits for an `openweather_observation` source.
|
Symptom: startup exits for an `openweather_observation` source.
|
||||||
|
|||||||
@@ -18,6 +18,18 @@ sources:
|
|||||||
url: "https://api.weather.gov/alerts?point=38.6239,-90.3571&limit=20"
|
url: "https://api.weather.gov/alerts?point=38.6239,-90.3571&limit=20"
|
||||||
user_agent: "weatherfeeder example (operator@example.com)"
|
user_agent: "weatherfeeder example (operator@example.com)"
|
||||||
|
|
||||||
|
- name: SPCConvectiveOutlookSTL
|
||||||
|
mode: poll
|
||||||
|
kinds: ["outlook"]
|
||||||
|
driver: spc_convective_outlook
|
||||||
|
every: 30m
|
||||||
|
params:
|
||||||
|
latitude: 38.6239
|
||||||
|
longitude: -90.3571
|
||||||
|
location_id: "stl"
|
||||||
|
location_name: "St. Louis, MO"
|
||||||
|
user_agent: "weatherfeeder example (operator@example.com)"
|
||||||
|
|
||||||
sinks:
|
sinks:
|
||||||
- name: nats_weather
|
- name: nats_weather
|
||||||
driver: nats
|
driver: nats
|
||||||
@@ -27,4 +39,4 @@ sinks:
|
|||||||
|
|
||||||
routes:
|
routes:
|
||||||
- sink: nats_weather
|
- sink: nats_weather
|
||||||
kinds: ["observation", "alert"]
|
kinds: ["observation", "alert", "outlook"]
|
||||||
|
|||||||
@@ -18,6 +18,18 @@ sources:
|
|||||||
url: "https://api.open-meteo.com/v1/forecast?latitude=38.6239&longitude=-90.3571&hourly=temperature_2m,relative_humidity_2m,dew_point_2m,apparent_temperature,precipitation_probability,precipitation,snowfall,weather_code,surface_pressure,wind_speed_10m,wind_direction_10m&forecast_days=3"
|
url: "https://api.open-meteo.com/v1/forecast?latitude=38.6239&longitude=-90.3571&hourly=temperature_2m,relative_humidity_2m,dew_point_2m,apparent_temperature,precipitation_probability,precipitation,snowfall,weather_code,surface_pressure,wind_speed_10m,wind_direction_10m&forecast_days=3"
|
||||||
user_agent: "weatherfeeder example (operator@example.com)"
|
user_agent: "weatherfeeder example (operator@example.com)"
|
||||||
|
|
||||||
|
- name: SPCConvectiveOutlookSTL
|
||||||
|
mode: poll
|
||||||
|
kinds: ["outlook"]
|
||||||
|
driver: spc_convective_outlook
|
||||||
|
every: 30m
|
||||||
|
params:
|
||||||
|
latitude: 38.6239
|
||||||
|
longitude: -90.3571
|
||||||
|
location_id: "stl"
|
||||||
|
location_name: "St. Louis, MO"
|
||||||
|
user_agent: "weatherfeeder example (operator@example.com)"
|
||||||
|
|
||||||
sinks:
|
sinks:
|
||||||
- name: pg_weather
|
- name: pg_weather
|
||||||
driver: postgres
|
driver: postgres
|
||||||
@@ -29,4 +41,4 @@ sinks:
|
|||||||
|
|
||||||
routes:
|
routes:
|
||||||
- sink: pg_weather
|
- sink: pg_weather
|
||||||
kinds: ["observation", "forecast"]
|
kinds: ["observation", "forecast", "outlook"]
|
||||||
|
|||||||
107
internal/geo/geojson.go
Normal file
107
internal/geo/geojson.go
Normal file
@@ -0,0 +1,107 @@
|
|||||||
|
package geo
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
)
|
||||||
|
|
||||||
|
type geometry struct {
|
||||||
|
Type string `json:"type"`
|
||||||
|
Coordinates json.RawMessage `json:"coordinates"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// ContainsPoint reports whether a GeoJSON Polygon or MultiPolygon contains p.
|
||||||
|
// GeoJSON coordinate order is [longitude, latitude].
|
||||||
|
func ContainsPoint(raw []byte, p Point) (bool, error) {
|
||||||
|
if len(raw) == 0 {
|
||||||
|
return false, fmt.Errorf("geojson geometry is empty")
|
||||||
|
}
|
||||||
|
|
||||||
|
var geom geometry
|
||||||
|
if err := json.Unmarshal(raw, &geom); err != nil {
|
||||||
|
return false, fmt.Errorf("decode geojson geometry: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
switch geom.Type {
|
||||||
|
case "Polygon":
|
||||||
|
polygon, err := decodePolygon(geom.Coordinates)
|
||||||
|
if err != nil {
|
||||||
|
return false, fmt.Errorf("decode polygon: %w", err)
|
||||||
|
}
|
||||||
|
return polygonContainsPoint(polygon, p), nil
|
||||||
|
case "MultiPolygon":
|
||||||
|
multiPolygon, err := decodeMultiPolygon(geom.Coordinates)
|
||||||
|
if err != nil {
|
||||||
|
return false, fmt.Errorf("decode multipolygon: %w", err)
|
||||||
|
}
|
||||||
|
for _, polygon := range multiPolygon {
|
||||||
|
if polygonContainsPoint(polygon, p) {
|
||||||
|
return true, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false, nil
|
||||||
|
case "":
|
||||||
|
return false, fmt.Errorf("geojson geometry type is required")
|
||||||
|
default:
|
||||||
|
return false, fmt.Errorf("unsupported geojson geometry type %q", geom.Type)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func decodePolygon(raw json.RawMessage) (Polygon, error) {
|
||||||
|
var coords [][][]float64
|
||||||
|
if err := json.Unmarshal(raw, &coords); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return polygonFromCoordinates(coords)
|
||||||
|
}
|
||||||
|
|
||||||
|
func decodeMultiPolygon(raw json.RawMessage) ([]Polygon, error) {
|
||||||
|
var coords [][][][]float64
|
||||||
|
if err := json.Unmarshal(raw, &coords); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if len(coords) == 0 {
|
||||||
|
return nil, fmt.Errorf("multipolygon has no polygons")
|
||||||
|
}
|
||||||
|
|
||||||
|
out := make([]Polygon, 0, len(coords))
|
||||||
|
for i, polygonCoords := range coords {
|
||||||
|
polygon, err := polygonFromCoordinates(polygonCoords)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("polygons[%d]: %w", i, err)
|
||||||
|
}
|
||||||
|
out = append(out, polygon)
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func polygonFromCoordinates(coords [][][]float64) (Polygon, error) {
|
||||||
|
if len(coords) == 0 {
|
||||||
|
return nil, fmt.Errorf("polygon has no rings")
|
||||||
|
}
|
||||||
|
|
||||||
|
polygon := make(Polygon, 0, len(coords))
|
||||||
|
for i, ringCoords := range coords {
|
||||||
|
ring, err := ringFromCoordinates(ringCoords)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("rings[%d]: %w", i, err)
|
||||||
|
}
|
||||||
|
polygon = append(polygon, ring)
|
||||||
|
}
|
||||||
|
return polygon, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func ringFromCoordinates(coords [][]float64) (Ring, error) {
|
||||||
|
if len(coords) == 0 {
|
||||||
|
return nil, fmt.Errorf("ring has no points")
|
||||||
|
}
|
||||||
|
|
||||||
|
ring := make(Ring, 0, len(coords))
|
||||||
|
for i, pair := range coords {
|
||||||
|
if len(pair) < 2 {
|
||||||
|
return nil, fmt.Errorf("points[%d] has %d values, need longitude and latitude", i, len(pair))
|
||||||
|
}
|
||||||
|
ring = append(ring, Point{Longitude: pair[0], Latitude: pair[1]})
|
||||||
|
}
|
||||||
|
return ring, nil
|
||||||
|
}
|
||||||
105
internal/geo/point.go
Normal file
105
internal/geo/point.go
Normal file
@@ -0,0 +1,105 @@
|
|||||||
|
package geo
|
||||||
|
|
||||||
|
import "math"
|
||||||
|
|
||||||
|
const epsilon = 1e-9
|
||||||
|
|
||||||
|
// Point is a geographic coordinate in decimal degrees.
|
||||||
|
type Point struct {
|
||||||
|
Longitude float64
|
||||||
|
Latitude float64
|
||||||
|
}
|
||||||
|
|
||||||
|
// Ring is one GeoJSON linear ring.
|
||||||
|
type Ring []Point
|
||||||
|
|
||||||
|
// Polygon is a GeoJSON polygon. The first ring is the exterior ring; subsequent
|
||||||
|
// rings are holes.
|
||||||
|
type Polygon []Ring
|
||||||
|
|
||||||
|
func polygonContainsPoint(polygon Polygon, p Point) bool {
|
||||||
|
if len(polygon) == 0 {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
if pointOnRing(polygon[0], p) {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
if !ringContainsPoint(polygon[0], p) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
for _, hole := range polygon[1:] {
|
||||||
|
if pointOnRing(hole, p) {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
if ringContainsPoint(hole, p) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
func ringContainsPoint(ring Ring, p Point) bool {
|
||||||
|
inside := false
|
||||||
|
n := len(ring)
|
||||||
|
if n == 0 {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
for i, j := 0, n-1; i < n; j, i = i, i+1 {
|
||||||
|
a := ring[j]
|
||||||
|
b := ring[i]
|
||||||
|
if pointOnSegment(p, a, b) {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
intersects := (a.Latitude > p.Latitude) != (b.Latitude > p.Latitude)
|
||||||
|
if intersects {
|
||||||
|
x := (b.Longitude-a.Longitude)*(p.Latitude-a.Latitude)/(b.Latitude-a.Latitude) + a.Longitude
|
||||||
|
if almostEqual(x, p.Longitude) {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
if x > p.Longitude {
|
||||||
|
inside = !inside
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return inside
|
||||||
|
}
|
||||||
|
|
||||||
|
func pointOnRing(ring Ring, p Point) bool {
|
||||||
|
n := len(ring)
|
||||||
|
if n == 0 {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
for i, j := 0, n-1; i < n; j, i = i, i+1 {
|
||||||
|
if pointOnSegment(p, ring[j], ring[i]) {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
func pointOnSegment(p, a, b Point) bool {
|
||||||
|
cross := (p.Latitude-a.Latitude)*(b.Longitude-a.Longitude) - (p.Longitude-a.Longitude)*(b.Latitude-a.Latitude)
|
||||||
|
if math.Abs(cross) > epsilon {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
minLon, maxLon := minMax(a.Longitude, b.Longitude)
|
||||||
|
minLat, maxLat := minMax(a.Latitude, b.Latitude)
|
||||||
|
return p.Longitude >= minLon-epsilon &&
|
||||||
|
p.Longitude <= maxLon+epsilon &&
|
||||||
|
p.Latitude >= minLat-epsilon &&
|
||||||
|
p.Latitude <= maxLat+epsilon
|
||||||
|
}
|
||||||
|
|
||||||
|
func minMax(a, b float64) (float64, float64) {
|
||||||
|
if a < b {
|
||||||
|
return a, b
|
||||||
|
}
|
||||||
|
return b, a
|
||||||
|
}
|
||||||
|
|
||||||
|
func almostEqual(a, b float64) bool {
|
||||||
|
return math.Abs(a-b) <= epsilon
|
||||||
|
}
|
||||||
181
internal/geo/point_test.go
Normal file
181
internal/geo/point_test.go
Normal file
@@ -0,0 +1,181 @@
|
|||||||
|
package geo
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
const squarePolygon = `{
|
||||||
|
"type": "Polygon",
|
||||||
|
"coordinates": [[
|
||||||
|
[-91.0, 38.0],
|
||||||
|
[-90.0, 38.0],
|
||||||
|
[-90.0, 39.0],
|
||||||
|
[-91.0, 39.0],
|
||||||
|
[-91.0, 38.0]
|
||||||
|
]]
|
||||||
|
}`
|
||||||
|
|
||||||
|
func TestContainsPointInsideSimplePolygon(t *testing.T) {
|
||||||
|
got, err := ContainsPoint([]byte(squarePolygon), Point{Longitude: -90.5, Latitude: 38.5})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ContainsPoint() error = %v", err)
|
||||||
|
}
|
||||||
|
if !got {
|
||||||
|
t.Fatalf("ContainsPoint() = false, want true")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestContainsPointAcceptsRawMessage(t *testing.T) {
|
||||||
|
got, err := ContainsPoint(json.RawMessage(squarePolygon), Point{Longitude: -90.5, Latitude: 38.5})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ContainsPoint() error = %v", err)
|
||||||
|
}
|
||||||
|
if !got {
|
||||||
|
t.Fatalf("ContainsPoint() = false, want true")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestContainsPointOutsideSimplePolygon(t *testing.T) {
|
||||||
|
got, err := ContainsPoint([]byte(squarePolygon), Point{Longitude: -89.5, Latitude: 38.5})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ContainsPoint() error = %v", err)
|
||||||
|
}
|
||||||
|
if got {
|
||||||
|
t.Fatalf("ContainsPoint() = true, want false")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestContainsPointOnBoundary(t *testing.T) {
|
||||||
|
got, err := ContainsPoint([]byte(squarePolygon), Point{Longitude: -91.0, Latitude: 38.5})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ContainsPoint() error = %v", err)
|
||||||
|
}
|
||||||
|
if !got {
|
||||||
|
t.Fatalf("ContainsPoint() = false, want true")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestContainsPointInHoleReturnsFalse(t *testing.T) {
|
||||||
|
const polygonWithHole = `{
|
||||||
|
"type": "Polygon",
|
||||||
|
"coordinates": [
|
||||||
|
[[0,0],[10,0],[10,10],[0,10],[0,0]],
|
||||||
|
[[4,4],[6,4],[6,6],[4,6],[4,4]]
|
||||||
|
]
|
||||||
|
}`
|
||||||
|
|
||||||
|
got, err := ContainsPoint([]byte(polygonWithHole), Point{Longitude: 5, Latitude: 5})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ContainsPoint() error = %v", err)
|
||||||
|
}
|
||||||
|
if got {
|
||||||
|
t.Fatalf("ContainsPoint() = true, want false")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestContainsPointOnHoleBoundaryReturnsTrue(t *testing.T) {
|
||||||
|
const polygonWithHole = `{
|
||||||
|
"type": "Polygon",
|
||||||
|
"coordinates": [
|
||||||
|
[[0,0],[10,0],[10,10],[0,10],[0,0]],
|
||||||
|
[[4,4],[6,4],[6,6],[4,6],[4,4]]
|
||||||
|
]
|
||||||
|
}`
|
||||||
|
|
||||||
|
got, err := ContainsPoint([]byte(polygonWithHole), Point{Longitude: 4, Latitude: 5})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ContainsPoint() error = %v", err)
|
||||||
|
}
|
||||||
|
if !got {
|
||||||
|
t.Fatalf("ContainsPoint() = false, want true")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestContainsPointInsideOneMultiPolygonMember(t *testing.T) {
|
||||||
|
const multiPolygon = `{
|
||||||
|
"type": "MultiPolygon",
|
||||||
|
"coordinates": [
|
||||||
|
[[[0,0],[1,0],[1,1],[0,1],[0,0]]],
|
||||||
|
[[[10,10],[12,10],[12,12],[10,12],[10,10]]]
|
||||||
|
]
|
||||||
|
}`
|
||||||
|
|
||||||
|
got, err := ContainsPoint([]byte(multiPolygon), Point{Longitude: 11, Latitude: 11})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ContainsPoint() error = %v", err)
|
||||||
|
}
|
||||||
|
if !got {
|
||||||
|
t.Fatalf("ContainsPoint() = false, want true")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestContainsPointUsesLongitudeLatitudeOrder(t *testing.T) {
|
||||||
|
const narrowPolygon = `{
|
||||||
|
"type": "Polygon",
|
||||||
|
"coordinates": [[
|
||||||
|
[-91.0, 38.0],
|
||||||
|
[-90.0, 38.0],
|
||||||
|
[-90.0, 39.0],
|
||||||
|
[-91.0, 39.0],
|
||||||
|
[-91.0, 38.0]
|
||||||
|
]]
|
||||||
|
}`
|
||||||
|
|
||||||
|
got, err := ContainsPoint([]byte(narrowPolygon), Point{Longitude: -90.5, Latitude: 38.5})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ContainsPoint() error = %v", err)
|
||||||
|
}
|
||||||
|
if !got {
|
||||||
|
t.Fatalf("ContainsPoint() = false, want true")
|
||||||
|
}
|
||||||
|
|
||||||
|
got, err = ContainsPoint([]byte(narrowPolygon), Point{Longitude: 38.5, Latitude: -90.5})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ContainsPoint() reversed error = %v", err)
|
||||||
|
}
|
||||||
|
if got {
|
||||||
|
t.Fatalf("ContainsPoint() with reversed coordinate values = true, want false")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestContainsPointUnsupportedGeometryError(t *testing.T) {
|
||||||
|
_, err := ContainsPoint([]byte(`{"type":"Point","coordinates":[-90,38]}`), Point{Longitude: -90, Latitude: 38})
|
||||||
|
if err == nil {
|
||||||
|
t.Fatalf("ContainsPoint() error = nil, want error")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), `unsupported geojson geometry type "Point"`) {
|
||||||
|
t.Fatalf("ContainsPoint() error = %q", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestContainsPointInvalidJSONError(t *testing.T) {
|
||||||
|
_, err := ContainsPoint([]byte(`{"type":"Polygon"`), Point{Longitude: -90, Latitude: 38})
|
||||||
|
if err == nil {
|
||||||
|
t.Fatalf("ContainsPoint() error = nil, want error")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "decode geojson geometry") {
|
||||||
|
t.Fatalf("ContainsPoint() error = %q", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestContainsPointMalformedCoordinatesError(t *testing.T) {
|
||||||
|
_, err := ContainsPoint([]byte(`{"type":"Polygon","coordinates":[[[1]]]}`), Point{Longitude: 1, Latitude: 1})
|
||||||
|
if err == nil {
|
||||||
|
t.Fatalf("ContainsPoint() error = nil, want error")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "need longitude and latitude") {
|
||||||
|
t.Fatalf("ContainsPoint() error = %q", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestContainsPointEmptyRingError(t *testing.T) {
|
||||||
|
_, err := ContainsPoint([]byte(`{"type":"Polygon","coordinates":[[]]}`), Point{Longitude: 1, Latitude: 1})
|
||||||
|
if err == nil {
|
||||||
|
t.Fatalf("ContainsPoint() error = nil, want error")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "ring has no points") {
|
||||||
|
t.Fatalf("ContainsPoint() error = %q", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -7,12 +7,14 @@ import (
|
|||||||
"gitea.maximumdirect.net/ejr/weatherfeeder/internal/normalizers/nws"
|
"gitea.maximumdirect.net/ejr/weatherfeeder/internal/normalizers/nws"
|
||||||
"gitea.maximumdirect.net/ejr/weatherfeeder/internal/normalizers/openmeteo"
|
"gitea.maximumdirect.net/ejr/weatherfeeder/internal/normalizers/openmeteo"
|
||||||
"gitea.maximumdirect.net/ejr/weatherfeeder/internal/normalizers/openweather"
|
"gitea.maximumdirect.net/ejr/weatherfeeder/internal/normalizers/openweather"
|
||||||
|
"gitea.maximumdirect.net/ejr/weatherfeeder/internal/normalizers/spc"
|
||||||
)
|
)
|
||||||
|
|
||||||
var builtinRegistrations = []func([]fknormalize.Normalizer) []fknormalize.Normalizer{
|
var builtinRegistrations = []func([]fknormalize.Normalizer) []fknormalize.Normalizer{
|
||||||
nws.Register,
|
nws.Register,
|
||||||
openmeteo.Register,
|
openmeteo.Register,
|
||||||
openweather.Register,
|
openweather.Register,
|
||||||
|
spc.Register,
|
||||||
}
|
}
|
||||||
|
|
||||||
// RegisterBuiltins registers all normalizers shipped with this binary.
|
// RegisterBuiltins registers all normalizers shipped with this binary.
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ import (
|
|||||||
"gitea.maximumdirect.net/ejr/weatherfeeder/internal/normalizers/nws"
|
"gitea.maximumdirect.net/ejr/weatherfeeder/internal/normalizers/nws"
|
||||||
"gitea.maximumdirect.net/ejr/weatherfeeder/internal/normalizers/openmeteo"
|
"gitea.maximumdirect.net/ejr/weatherfeeder/internal/normalizers/openmeteo"
|
||||||
"gitea.maximumdirect.net/ejr/weatherfeeder/internal/normalizers/openweather"
|
"gitea.maximumdirect.net/ejr/weatherfeeder/internal/normalizers/openweather"
|
||||||
|
"gitea.maximumdirect.net/ejr/weatherfeeder/internal/normalizers/spc"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestRegisterBuiltinsOrder(t *testing.T) {
|
func TestRegisterBuiltinsOrder(t *testing.T) {
|
||||||
@@ -25,6 +26,7 @@ func TestRegisterBuiltinsOrder(t *testing.T) {
|
|||||||
openmeteo.ObservationNormalizer{},
|
openmeteo.ObservationNormalizer{},
|
||||||
openmeteo.ForecastNormalizer{},
|
openmeteo.ForecastNormalizer{},
|
||||||
openweather.ObservationNormalizer{},
|
openweather.ObservationNormalizer{},
|
||||||
|
spc.ConvectiveOutlookNormalizer{},
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(got) != len(want) {
|
if len(got) != len(want) {
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/ejr/feedkit/event"
|
"gitea.maximumdirect.net/ejr/feedkit/event"
|
||||||
|
"gitea.maximumdirect.net/ejr/weatherfeeder/standards"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestFinalizeRoundsWeatherPayloadFloats(t *testing.T) {
|
func TestFinalizeRoundsWeatherPayloadFloats(t *testing.T) {
|
||||||
@@ -14,7 +15,7 @@ func TestFinalizeRoundsWeatherPayloadFloats(t *testing.T) {
|
|||||||
|
|
||||||
in := event.Event{
|
in := event.Event{
|
||||||
ID: "evt-1",
|
ID: "evt-1",
|
||||||
Kind: event.Kind("observation"),
|
Kind: event.Kind(standards.KindObservation),
|
||||||
Source: "source-a",
|
Source: "source-a",
|
||||||
EmittedAt: time.Date(2026, 3, 28, 12, 0, 0, 0, time.UTC),
|
EmittedAt: time.Date(2026, 3, 28, 12, 0, 0, 0, time.UTC),
|
||||||
Schema: "raw.example.v1",
|
Schema: "raw.example.v1",
|
||||||
|
|||||||
@@ -18,6 +18,18 @@ import (
|
|||||||
// Errors include a small amount of operation context ("extract payload", "decode raw payload").
|
// Errors include a small amount of operation context ("extract payload", "decode raw payload").
|
||||||
// Callers typically wrap these with a provider/kind label.
|
// Callers typically wrap these with a provider/kind label.
|
||||||
func DecodeJSONPayload[T any](in event.Event) (T, error) {
|
func DecodeJSONPayload[T any](in event.Event) (T, error) {
|
||||||
|
var zero T
|
||||||
|
|
||||||
|
if typed, ok := in.Payload.(T); ok {
|
||||||
|
return typed, nil
|
||||||
|
}
|
||||||
|
if ptr, ok := in.Payload.(*T); ok {
|
||||||
|
if ptr == nil {
|
||||||
|
return zero, fmt.Errorf("extract payload: payload pointer is nil")
|
||||||
|
}
|
||||||
|
return *ptr, nil
|
||||||
|
}
|
||||||
|
|
||||||
return fknormalize.DecodeJSONPayload[T](in)
|
return fknormalize.DecodeJSONPayload[T](in)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
39
internal/normalizers/common/json_test.go
Normal file
39
internal/normalizers/common/json_test.go
Normal file
@@ -0,0 +1,39 @@
|
|||||||
|
package common
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/ejr/feedkit/event"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestDecodeJSONPayloadAcceptsTypedPayload(t *testing.T) {
|
||||||
|
type rawPayload struct {
|
||||||
|
Value string `json:"value"`
|
||||||
|
}
|
||||||
|
|
||||||
|
got, err := DecodeJSONPayload[rawPayload](event.Event{
|
||||||
|
Payload: rawPayload{Value: "ok"},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("DecodeJSONPayload() error = %v", err)
|
||||||
|
}
|
||||||
|
if got.Value != "ok" {
|
||||||
|
t.Fatalf("Value = %q, want ok", got.Value)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDecodeJSONPayloadAcceptsTypedPointerPayload(t *testing.T) {
|
||||||
|
type rawPayload struct {
|
||||||
|
Value string `json:"value"`
|
||||||
|
}
|
||||||
|
|
||||||
|
got, err := DecodeJSONPayload[rawPayload](event.Event{
|
||||||
|
Payload: &rawPayload{Value: "ok"},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("DecodeJSONPayload() error = %v", err)
|
||||||
|
}
|
||||||
|
if got.Value != "ok" {
|
||||||
|
t.Fatalf("Value = %q, want ok", got.Value)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -93,12 +93,8 @@ func buildAlerts(parsed nwsAlertsResponse, fallbackAsOf time.Time) (model.Weathe
|
|||||||
sent := nwscommon.ParseTimePtr(p.Sent)
|
sent := nwscommon.ParseTimePtr(p.Sent)
|
||||||
effective := nwscommon.ParseTimePtr(p.Effective)
|
effective := nwscommon.ParseTimePtr(p.Effective)
|
||||||
onset := nwscommon.ParseTimePtr(p.Onset)
|
onset := nwscommon.ParseTimePtr(p.Onset)
|
||||||
|
ends := nwscommon.ParseTimePtr(p.Ends)
|
||||||
// Expires: prefer "expires"; fall back to "ends" if present.
|
|
||||||
expires := nwscommon.ParseTimePtr(p.Expires)
|
expires := nwscommon.ParseTimePtr(p.Expires)
|
||||||
if expires == nil {
|
|
||||||
expires = nwscommon.ParseTimePtr(p.Ends)
|
|
||||||
}
|
|
||||||
|
|
||||||
refs := parseNWSAlertReferences(p.References)
|
refs := parseNWSAlertReferences(p.References)
|
||||||
|
|
||||||
@@ -123,6 +119,7 @@ func buildAlerts(parsed nwsAlertsResponse, fallbackAsOf time.Time) (model.Weathe
|
|||||||
Sent: sent,
|
Sent: sent,
|
||||||
Effective: effective,
|
Effective: effective,
|
||||||
Onset: onset,
|
Onset: onset,
|
||||||
|
Ends: ends,
|
||||||
Expires: expires,
|
Expires: expires,
|
||||||
|
|
||||||
AreaDescription: strings.TrimSpace(p.AreaDesc),
|
AreaDescription: strings.TrimSpace(p.AreaDesc),
|
||||||
|
|||||||
136
internal/normalizers/nws/alerts_test.go
Normal file
136
internal/normalizers/nws/alerts_test.go
Normal file
@@ -0,0 +1,136 @@
|
|||||||
|
package nws
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/ejr/feedkit/event"
|
||||||
|
"gitea.maximumdirect.net/ejr/weatherfeeder/model"
|
||||||
|
"gitea.maximumdirect.net/ejr/weatherfeeder/standards"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestAlertsNormalizerMapsEndsSeparatelyFromExpires(t *testing.T) {
|
||||||
|
raw := []byte(`{
|
||||||
|
"updated":"2026-06-16T10:00:00+00:00",
|
||||||
|
"title":"Current watches, warnings, and advisories for St. Louis",
|
||||||
|
"features":[{
|
||||||
|
"id":"https://api.weather.gov/alerts/alert-1",
|
||||||
|
"properties":{
|
||||||
|
"event":"Flood Warning",
|
||||||
|
"headline":"Flood Warning issued",
|
||||||
|
"sent":"2026-06-16T09:55:00+00:00",
|
||||||
|
"effective":"2026-06-16T10:00:00+00:00",
|
||||||
|
"onset":"2026-06-16T10:15:00+00:00",
|
||||||
|
"ends":"2026-06-16T14:00:00+00:00",
|
||||||
|
"expires":"2026-06-16T11:00:00+00:00"
|
||||||
|
}
|
||||||
|
}]
|
||||||
|
}`)
|
||||||
|
|
||||||
|
out, err := AlertsNormalizer{}.Normalize(context.Background(), alertRawEvent(raw))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Normalize() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
run := decodeAlertRun(t, out)
|
||||||
|
if len(run.Alerts) != 1 {
|
||||||
|
t.Fatalf("expected 1 alert, got %d", len(run.Alerts))
|
||||||
|
}
|
||||||
|
alert := run.Alerts[0]
|
||||||
|
wantEnds := time.Date(2026, 6, 16, 14, 0, 0, 0, time.UTC)
|
||||||
|
wantExpires := time.Date(2026, 6, 16, 11, 0, 0, 0, time.UTC)
|
||||||
|
if alert.Ends == nil || !alert.Ends.Equal(wantEnds) {
|
||||||
|
t.Fatalf("ends = %v, want %s", alert.Ends, wantEnds)
|
||||||
|
}
|
||||||
|
if alert.Expires == nil || !alert.Expires.Equal(wantExpires) {
|
||||||
|
t.Fatalf("expires = %v, want %s", alert.Expires, wantExpires)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAlertsNormalizerDoesNotFallbackExpiresToEnds(t *testing.T) {
|
||||||
|
raw := []byte(`{
|
||||||
|
"updated":"2026-06-16T10:00:00+00:00",
|
||||||
|
"features":[{
|
||||||
|
"id":"alert-ends-only",
|
||||||
|
"properties":{
|
||||||
|
"event":"Heat Advisory",
|
||||||
|
"ends":"2026-06-16T22:00:00+00:00"
|
||||||
|
}
|
||||||
|
}]
|
||||||
|
}`)
|
||||||
|
|
||||||
|
out, err := AlertsNormalizer{}.Normalize(context.Background(), alertRawEvent(raw))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Normalize() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
run := decodeAlertRun(t, out)
|
||||||
|
alert := run.Alerts[0]
|
||||||
|
if alert.Ends == nil {
|
||||||
|
t.Fatal("expected ends to be populated")
|
||||||
|
}
|
||||||
|
if alert.Expires != nil {
|
||||||
|
t.Fatalf("expected expires nil when upstream expires is absent, got %v", alert.Expires)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAlertsNormalizerIgnoresInvalidEnds(t *testing.T) {
|
||||||
|
raw := []byte(`{
|
||||||
|
"updated":"2026-06-16T10:00:00+00:00",
|
||||||
|
"features":[{
|
||||||
|
"id":"alert-invalid-ends",
|
||||||
|
"properties":{
|
||||||
|
"event":"Special Weather Statement",
|
||||||
|
"ends":"not-a-time",
|
||||||
|
"expires":"2026-06-16T11:00:00+00:00"
|
||||||
|
}
|
||||||
|
}]
|
||||||
|
}`)
|
||||||
|
|
||||||
|
out, err := AlertsNormalizer{}.Normalize(context.Background(), alertRawEvent(raw))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Normalize() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
run := decodeAlertRun(t, out)
|
||||||
|
alert := run.Alerts[0]
|
||||||
|
if alert.Ends != nil {
|
||||||
|
t.Fatalf("expected invalid ends to map nil, got %v", alert.Ends)
|
||||||
|
}
|
||||||
|
if alert.Expires == nil {
|
||||||
|
t.Fatal("expected expires to remain populated")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func alertRawEvent(raw []byte) event.Event {
|
||||||
|
emittedAt := time.Date(2026, 6, 16, 10, 5, 0, 0, time.UTC)
|
||||||
|
return event.Event{
|
||||||
|
ID: "raw-alerts",
|
||||||
|
Kind: event.Kind(standards.KindAlert),
|
||||||
|
Source: "NWSAlerts",
|
||||||
|
Schema: standards.SchemaRawNWSAlertsV1,
|
||||||
|
EmittedAt: emittedAt,
|
||||||
|
Payload: json.RawMessage(raw),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func decodeAlertRun(t *testing.T, e *event.Event) model.WeatherAlertRun {
|
||||||
|
t.Helper()
|
||||||
|
if e == nil {
|
||||||
|
t.Fatal("expected normalized event")
|
||||||
|
}
|
||||||
|
if e.Schema != standards.SchemaWeatherAlertV1 {
|
||||||
|
t.Fatalf("schema = %q, want %q", e.Schema, standards.SchemaWeatherAlertV1)
|
||||||
|
}
|
||||||
|
var run model.WeatherAlertRun
|
||||||
|
raw, err := json.Marshal(e.Payload)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("marshal alert payload: %v", err)
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(raw, &run); err != nil {
|
||||||
|
t.Fatalf("decode alert payload: %v", err)
|
||||||
|
}
|
||||||
|
return run
|
||||||
|
}
|
||||||
@@ -4,6 +4,7 @@ import (
|
|||||||
"encoding/json"
|
"encoding/json"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
"reflect"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
@@ -18,7 +19,7 @@ func TestForecastDiscussionNormalizerProducesCanonicalSchema(t *testing.T) {
|
|||||||
|
|
||||||
out, err := (ForecastDiscussionNormalizer{}).Normalize(nil, event.Event{
|
out, err := (ForecastDiscussionNormalizer{}).Normalize(nil, event.Event{
|
||||||
ID: "evt-discussion-1",
|
ID: "evt-discussion-1",
|
||||||
Kind: event.Kind("forecast_discussion"),
|
Kind: event.Kind(standards.KindForecastDiscussion),
|
||||||
Source: "nws-discussion-test",
|
Source: "nws-discussion-test",
|
||||||
EmittedAt: time.Date(2026, 3, 28, 19, 25, 0, 0, time.UTC),
|
EmittedAt: time.Date(2026, 3, 28, 19, 25, 0, 0, time.UTC),
|
||||||
Schema: standards.SchemaRawNWSForecastDiscussionV1,
|
Schema: standards.SchemaRawNWSForecastDiscussionV1,
|
||||||
@@ -33,7 +34,7 @@ func TestForecastDiscussionNormalizerProducesCanonicalSchema(t *testing.T) {
|
|||||||
if out.Schema != standards.SchemaWeatherForecastDiscussionV1 {
|
if out.Schema != standards.SchemaWeatherForecastDiscussionV1 {
|
||||||
t.Fatalf("Schema = %q, want %q", out.Schema, standards.SchemaWeatherForecastDiscussionV1)
|
t.Fatalf("Schema = %q, want %q", out.Schema, standards.SchemaWeatherForecastDiscussionV1)
|
||||||
}
|
}
|
||||||
if out.Kind != event.Kind("forecast_discussion") {
|
if out.Kind != event.Kind(standards.KindForecastDiscussion) {
|
||||||
t.Fatalf("Kind = %q, want forecast_discussion", out.Kind)
|
t.Fatalf("Kind = %q, want forecast_discussion", out.Kind)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -71,10 +72,281 @@ func TestForecastDiscussionNormalizerProducesCanonicalSchema(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestForecastDiscussionNormalizerSupportsMixedHeadingFormats(t *testing.T) {
|
||||||
|
out, err := (ForecastDiscussionNormalizer{}).Normalize(nil, event.Event{
|
||||||
|
ID: "evt-discussion-mixed-format",
|
||||||
|
Kind: event.Kind(standards.KindForecastDiscussion),
|
||||||
|
Source: "nws-discussion-test",
|
||||||
|
EmittedAt: time.Date(2026, 3, 28, 19, 25, 0, 0, time.UTC),
|
||||||
|
Schema: standards.SchemaRawNWSForecastDiscussionV1,
|
||||||
|
Payload: loadMixedFormatForecastDiscussionSampleHTML(t),
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Normalize() error = %v", err)
|
||||||
|
}
|
||||||
|
if out == nil {
|
||||||
|
t.Fatalf("Normalize() returned nil output")
|
||||||
|
}
|
||||||
|
if out.Kind != event.Kind(standards.KindForecastDiscussion) {
|
||||||
|
t.Fatalf("Kind = %q, want forecast_discussion", out.Kind)
|
||||||
|
}
|
||||||
|
if out.Schema != standards.SchemaWeatherForecastDiscussionV1 {
|
||||||
|
t.Fatalf("Schema = %q, want %q", out.Schema, standards.SchemaWeatherForecastDiscussionV1)
|
||||||
|
}
|
||||||
|
wantEffectiveAt := time.Date(2026, 3, 28, 19, 24, 0, 0, time.UTC)
|
||||||
|
if out.EffectiveAt == nil || !out.EffectiveAt.Equal(wantEffectiveAt) {
|
||||||
|
t.Fatalf("EffectiveAt = %v, want %s", out.EffectiveAt, wantEffectiveAt.Format(time.RFC3339))
|
||||||
|
}
|
||||||
|
|
||||||
|
payload, ok := out.Payload.(model.WeatherForecastDiscussion)
|
||||||
|
if !ok {
|
||||||
|
t.Fatalf("Payload type = %T, want model.WeatherForecastDiscussion", out.Payload)
|
||||||
|
}
|
||||||
|
if payload.ShortTerm == nil || payload.LongTerm == nil {
|
||||||
|
t.Fatalf("ShortTerm=%v LongTerm=%v, want both populated", payload.ShortTerm, payload.LongTerm)
|
||||||
|
}
|
||||||
|
if payload.ShortTerm.Qualifier != "Through Late Sunday Night" {
|
||||||
|
t.Fatalf("ShortTerm.Qualifier = %q", payload.ShortTerm.Qualifier)
|
||||||
|
}
|
||||||
|
if !strings.Contains(payload.ShortTerm.Text, "After a chilly morning") {
|
||||||
|
t.Fatalf("ShortTerm.Text missing expected prose: %q", payload.ShortTerm.Text)
|
||||||
|
}
|
||||||
|
if payload.LongTerm.Qualifier != "Monday through Next Saturday" {
|
||||||
|
t.Fatalf("LongTerm.Qualifier = %q", payload.LongTerm.Qualifier)
|
||||||
|
}
|
||||||
|
if !strings.Contains(payload.LongTerm.Text, "The peak of the warmth arrives Monday and Tuesday") {
|
||||||
|
t.Fatalf("LongTerm.Text missing expected prose: %q", payload.LongTerm.Text)
|
||||||
|
}
|
||||||
|
if strings.Contains(payload.LongTerm.Text, "AVIATION") || strings.Contains(payload.LongTerm.Text, "VFR conditions are expected") {
|
||||||
|
t.Fatalf("LongTerm.Text includes aviation content: %q", payload.LongTerm.Text)
|
||||||
|
}
|
||||||
|
|
||||||
|
b, err := json.Marshal(out.Payload)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("json.Marshal(payload) error = %v", err)
|
||||||
|
}
|
||||||
|
var fields map[string]any
|
||||||
|
if err := json.Unmarshal(b, &fields); err != nil {
|
||||||
|
t.Fatalf("json.Unmarshal(payload) error = %v", err)
|
||||||
|
}
|
||||||
|
for _, key := range []string{"aviation", "sections"} {
|
||||||
|
if _, ok := fields[key]; ok {
|
||||||
|
t.Fatalf("unexpected key %q in canonical payload", key)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestForecastDiscussionNormalizerSupportsCrossOfficeLayout(t *testing.T) {
|
||||||
|
in := event.Event{
|
||||||
|
ID: "evt-discussion-bou",
|
||||||
|
Kind: event.Kind(standards.KindForecastDiscussion),
|
||||||
|
Source: "nws-discussion-bou-test",
|
||||||
|
EmittedAt: time.Date(2026, 4, 7, 19, 1, 0, 0, time.UTC),
|
||||||
|
Schema: standards.SchemaRawNWSForecastDiscussionV1,
|
||||||
|
Payload: loadForecastDiscussionBOUSampleHTML(t),
|
||||||
|
}
|
||||||
|
|
||||||
|
out, err := (ForecastDiscussionNormalizer{}).Normalize(nil, in)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Normalize() error = %v", err)
|
||||||
|
}
|
||||||
|
if out == nil {
|
||||||
|
t.Fatalf("Normalize() returned nil output")
|
||||||
|
}
|
||||||
|
if out.ID != in.ID || out.Source != in.Source || !out.EmittedAt.Equal(in.EmittedAt) {
|
||||||
|
t.Fatalf("envelope = %#v, want ID/source/emittedAt from input", out)
|
||||||
|
}
|
||||||
|
if out.Kind != event.Kind(standards.KindForecastDiscussion) {
|
||||||
|
t.Fatalf("Kind = %q, want forecast_discussion", out.Kind)
|
||||||
|
}
|
||||||
|
if out.Schema != standards.SchemaWeatherForecastDiscussionV1 {
|
||||||
|
t.Fatalf("Schema = %q, want %q", out.Schema, standards.SchemaWeatherForecastDiscussionV1)
|
||||||
|
}
|
||||||
|
wantEffectiveAt := time.Date(2026, 4, 7, 19, 0, 0, 0, time.UTC)
|
||||||
|
if out.EffectiveAt == nil || !out.EffectiveAt.Equal(wantEffectiveAt) {
|
||||||
|
t.Fatalf("EffectiveAt = %v, want %s", out.EffectiveAt, wantEffectiveAt.Format(time.RFC3339))
|
||||||
|
}
|
||||||
|
|
||||||
|
payload, ok := out.Payload.(model.WeatherForecastDiscussion)
|
||||||
|
if !ok {
|
||||||
|
t.Fatalf("Payload type = %T, want model.WeatherForecastDiscussion", out.Payload)
|
||||||
|
}
|
||||||
|
if payload.OfficeID != "BOU" || payload.OfficeName != "National Weather Service Denver CO" {
|
||||||
|
t.Fatalf("OfficeID=%q OfficeName=%q", payload.OfficeID, payload.OfficeName)
|
||||||
|
}
|
||||||
|
wantMessages := []string{
|
||||||
|
"Strong winds are expected along the Front Range this evening.",
|
||||||
|
"Cooler temperatures arrive on Wednesday.",
|
||||||
|
}
|
||||||
|
if len(payload.KeyMessages) != len(wantMessages) {
|
||||||
|
t.Fatalf("KeyMessages = %#v, want %#v", payload.KeyMessages, wantMessages)
|
||||||
|
}
|
||||||
|
for i := range wantMessages {
|
||||||
|
if payload.KeyMessages[i] != wantMessages[i] {
|
||||||
|
t.Fatalf("KeyMessages[%d] = %q, want %q", i, payload.KeyMessages[i], wantMessages[i])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if payload.ShortTerm == nil || payload.LongTerm == nil {
|
||||||
|
t.Fatalf("ShortTerm=%v LongTerm=%v, want both populated", payload.ShortTerm, payload.LongTerm)
|
||||||
|
}
|
||||||
|
if payload.ShortTerm.Qualifier != "(Tonight through Wednesday)" || payload.ShortTerm.Text != "Gusty west winds will continue through the evening before decreasing overnight." {
|
||||||
|
t.Fatalf("ShortTerm = %#v", payload.ShortTerm)
|
||||||
|
}
|
||||||
|
if payload.LongTerm.Qualifier != "(Thursday through Saturday)" || payload.LongTerm.Text != "Warmer and drier conditions return Thursday, followed by a chance of showers Friday." {
|
||||||
|
t.Fatalf("LongTerm = %#v", payload.LongTerm)
|
||||||
|
}
|
||||||
|
if payload.ShortTerm.IssuedAt == nil || payload.LongTerm.IssuedAt == nil ||
|
||||||
|
!payload.ShortTerm.IssuedAt.Equal(wantEffectiveAt) || !payload.LongTerm.IssuedAt.Equal(wantEffectiveAt) {
|
||||||
|
t.Fatalf("section issue times = short %v long %v, want %s", payload.ShortTerm.IssuedAt, payload.LongTerm.IssuedAt, wantEffectiveAt.Format(time.RFC3339))
|
||||||
|
}
|
||||||
|
|
||||||
|
b, err := json.Marshal(out.Payload)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("json.Marshal(payload) error = %v", err)
|
||||||
|
}
|
||||||
|
var fields map[string]any
|
||||||
|
if err := json.Unmarshal(b, &fields); err != nil {
|
||||||
|
t.Fatalf("json.Unmarshal(payload) error = %v", err)
|
||||||
|
}
|
||||||
|
for _, key := range []string{"aviation", "discussion", "sections"} {
|
||||||
|
if _, ok := fields[key]; ok {
|
||||||
|
t.Fatalf("unexpected key %q in canonical payload", key)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestForecastDiscussionNormalizerSupportsCrossOfficeKeyMessageFixtures(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
filename string
|
||||||
|
id string
|
||||||
|
source string
|
||||||
|
emittedAt time.Time
|
||||||
|
effectiveAt time.Time
|
||||||
|
messages []string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "numbered key messages",
|
||||||
|
filename: "forecast_discussion_bgm_numbered_sample.html",
|
||||||
|
id: "evt-discussion-bgm",
|
||||||
|
source: "nws-discussion-bgm-test",
|
||||||
|
emittedAt: time.Date(2026, 4, 10, 17, 31, 0, 0, time.UTC),
|
||||||
|
effectiveAt: time.Date(2026, 4, 10, 17, 30, 0, 0, time.UTC),
|
||||||
|
messages: []string{
|
||||||
|
"Periods of rain are expected through Saturday, with locally heavier amounts possible.",
|
||||||
|
"Cooler temperatures return late this weekend.",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "key points alias",
|
||||||
|
filename: "forecast_discussion_mfr_key_points_sample.html",
|
||||||
|
id: "evt-discussion-mfr",
|
||||||
|
source: "nws-discussion-mfr-test",
|
||||||
|
emittedAt: time.Date(2026, 4, 10, 19, 46, 0, 0, time.UTC),
|
||||||
|
effectiveAt: time.Date(2026, 4, 10, 19, 45, 0, 0, time.UTC),
|
||||||
|
messages: []string{
|
||||||
|
"Gusty winds will develop over exposed ridges, especially during the afternoon.",
|
||||||
|
"Inland valleys remain dry through Saturday.",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "as of preamble",
|
||||||
|
filename: "forecast_discussion_rah_as_of_sample.html",
|
||||||
|
id: "evt-discussion-rah",
|
||||||
|
source: "nws-discussion-rah-test",
|
||||||
|
emittedAt: time.Date(2026, 8, 2, 16, 36, 0, 0, time.UTC),
|
||||||
|
effectiveAt: time.Date(2026, 8, 2, 16, 35, 0, 0, time.UTC),
|
||||||
|
messages: []string{
|
||||||
|
"Scattered storms may produce locally heavy rain this afternoon.",
|
||||||
|
"Drier weather arrives Monday.",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "parenthesized numeric markers",
|
||||||
|
filename: "forecast_discussion_lwx_parenthesized_number_sample.html",
|
||||||
|
id: "evt-discussion-lwx",
|
||||||
|
source: "nws-discussion-lwx-test",
|
||||||
|
emittedAt: time.Date(2026, 8, 2, 18, 1, 0, 0, time.UTC),
|
||||||
|
effectiveAt: time.Date(2026, 8, 2, 18, 0, 0, 0, time.UTC),
|
||||||
|
messages: []string{
|
||||||
|
"Thunderstorms remain possible near the Blue Ridge this evening.",
|
||||||
|
"Seasonably warm conditions continue Monday.",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "embedded key messages in previous discussion",
|
||||||
|
filename: "forecast_discussion_mfr_prev_discussion_sample.html",
|
||||||
|
id: "evt-discussion-mfr-previous",
|
||||||
|
source: "nws-discussion-mfr-previous-test",
|
||||||
|
emittedAt: time.Date(2026, 8, 2, 22, 20, 0, 0, time.UTC),
|
||||||
|
effectiveAt: time.Date(2026, 8, 2, 22, 19, 0, 0, time.UTC),
|
||||||
|
messages: []string{
|
||||||
|
"Heat returns to inland valleys Monday.",
|
||||||
|
"Gusty afternoon winds develop east of the Cascades.",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
in := event.Event{
|
||||||
|
ID: tt.id,
|
||||||
|
Kind: event.Kind(standards.KindForecastDiscussion),
|
||||||
|
Source: tt.source,
|
||||||
|
EmittedAt: tt.emittedAt,
|
||||||
|
Schema: standards.SchemaRawNWSForecastDiscussionV1,
|
||||||
|
Payload: loadForecastDiscussionFixtureHTML(t, tt.filename),
|
||||||
|
}
|
||||||
|
|
||||||
|
out, err := (ForecastDiscussionNormalizer{}).Normalize(nil, in)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Normalize() error = %v", err)
|
||||||
|
}
|
||||||
|
if out == nil {
|
||||||
|
t.Fatalf("Normalize() returned nil output")
|
||||||
|
}
|
||||||
|
if out.ID != in.ID || out.Source != in.Source || !out.EmittedAt.Equal(in.EmittedAt) {
|
||||||
|
t.Fatalf("envelope = %#v, want ID/source/emittedAt from input", out)
|
||||||
|
}
|
||||||
|
if out.Kind != event.Kind(standards.KindForecastDiscussion) {
|
||||||
|
t.Fatalf("Kind = %q, want forecast_discussion", out.Kind)
|
||||||
|
}
|
||||||
|
if out.Schema != standards.SchemaWeatherForecastDiscussionV1 {
|
||||||
|
t.Fatalf("Schema = %q, want %q", out.Schema, standards.SchemaWeatherForecastDiscussionV1)
|
||||||
|
}
|
||||||
|
if out.EffectiveAt == nil || !out.EffectiveAt.Equal(tt.effectiveAt) {
|
||||||
|
t.Fatalf("EffectiveAt = %v, want %s", out.EffectiveAt, tt.effectiveAt.Format(time.RFC3339))
|
||||||
|
}
|
||||||
|
|
||||||
|
payload, ok := out.Payload.(model.WeatherForecastDiscussion)
|
||||||
|
if !ok {
|
||||||
|
t.Fatalf("Payload type = %T, want model.WeatherForecastDiscussion", out.Payload)
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(payload.KeyMessages, tt.messages) {
|
||||||
|
t.Fatalf("KeyMessages = %#v, want %#v", payload.KeyMessages, tt.messages)
|
||||||
|
}
|
||||||
|
|
||||||
|
b, err := json.Marshal(payload)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("json.Marshal(payload) error = %v", err)
|
||||||
|
}
|
||||||
|
var fields map[string]any
|
||||||
|
if err := json.Unmarshal(b, &fields); err != nil {
|
||||||
|
t.Fatalf("json.Unmarshal(payload) error = %v", err)
|
||||||
|
}
|
||||||
|
for _, key := range []string{"nearTerm", "discussion", "aviation", "sections"} {
|
||||||
|
if _, ok := fields[key]; ok {
|
||||||
|
t.Fatalf("unexpected key %q in canonical payload", key)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestForecastDiscussionNormalizerRejectsMissingIssueTime(t *testing.T) {
|
func TestForecastDiscussionNormalizerRejectsMissingIssueTime(t *testing.T) {
|
||||||
_, err := (ForecastDiscussionNormalizer{}).Normalize(nil, event.Event{
|
_, err := (ForecastDiscussionNormalizer{}).Normalize(nil, event.Event{
|
||||||
ID: "evt-discussion-bad",
|
ID: "evt-discussion-bad",
|
||||||
Kind: event.Kind("forecast_discussion"),
|
Kind: event.Kind(standards.KindForecastDiscussion),
|
||||||
Source: "nws-discussion-test",
|
Source: "nws-discussion-test",
|
||||||
EmittedAt: time.Date(2026, 3, 28, 19, 25, 0, 0, time.UTC),
|
EmittedAt: time.Date(2026, 3, 28, 19, 25, 0, 0, time.UTC),
|
||||||
Schema: standards.SchemaRawNWSForecastDiscussionV1,
|
Schema: standards.SchemaRawNWSForecastDiscussionV1,
|
||||||
@@ -93,7 +365,7 @@ func TestForecastDiscussionNormalizerWireShapeHasNoUnexpectedKeys(t *testing.T)
|
|||||||
|
|
||||||
out, err := (ForecastDiscussionNormalizer{}).Normalize(nil, event.Event{
|
out, err := (ForecastDiscussionNormalizer{}).Normalize(nil, event.Event{
|
||||||
ID: "evt-discussion-2",
|
ID: "evt-discussion-2",
|
||||||
Kind: event.Kind("forecast_discussion"),
|
Kind: event.Kind(standards.KindForecastDiscussion),
|
||||||
Source: "nws-discussion-test",
|
Source: "nws-discussion-test",
|
||||||
EmittedAt: time.Date(2026, 3, 28, 19, 25, 0, 0, time.UTC),
|
EmittedAt: time.Date(2026, 3, 28, 19, 25, 0, 0, time.UTC),
|
||||||
Schema: standards.SchemaRawNWSForecastDiscussionV1,
|
Schema: standards.SchemaRawNWSForecastDiscussionV1,
|
||||||
@@ -128,3 +400,56 @@ func loadForecastDiscussionSampleHTML(t *testing.T) string {
|
|||||||
}
|
}
|
||||||
return string(b)
|
return string(b)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func loadForecastDiscussionBOUSampleHTML(t *testing.T) string {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
path := filepath.Join("..", "..", "providers", "nws", "testdata", "forecast_discussion_bou_sample.html")
|
||||||
|
b, err := os.ReadFile(path)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("os.ReadFile(%q) error = %v", path, err)
|
||||||
|
}
|
||||||
|
return string(b)
|
||||||
|
}
|
||||||
|
|
||||||
|
func loadForecastDiscussionFixtureHTML(t *testing.T, filename string) string {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
path := filepath.Join("..", "..", "providers", "nws", "testdata", filename)
|
||||||
|
b, err := os.ReadFile(path)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("os.ReadFile(%q) error = %v", path, err)
|
||||||
|
}
|
||||||
|
return string(b)
|
||||||
|
}
|
||||||
|
|
||||||
|
func loadMixedFormatForecastDiscussionSampleHTML(t *testing.T) string {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
raw := loadForecastDiscussionSampleHTML(t)
|
||||||
|
replacements := []struct {
|
||||||
|
original string
|
||||||
|
replacement string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
original: ".SHORT TERM... (Through Late Sunday Night)",
|
||||||
|
replacement: ".SHORT TERM /Through Late Sunday Night/...",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
original: ".LONG TERM... (Monday through Next Saturday)",
|
||||||
|
replacement: ".LONG TERM /Monday through Next Saturday/...",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
original: ".AVIATION... (For the 18z TAFs through 18z Sunday Afternoon)",
|
||||||
|
replacement: ".AVIATION /For the 18z TAFs through 18z Sunday Afternoon/...",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
for _, replacement := range replacements {
|
||||||
|
if !strings.Contains(raw, replacement.original) {
|
||||||
|
t.Fatalf("fixture missing heading %q", replacement.original)
|
||||||
|
}
|
||||||
|
raw = strings.Replace(raw, replacement.original, replacement.replacement, 1)
|
||||||
|
}
|
||||||
|
|
||||||
|
return raw
|
||||||
|
}
|
||||||
|
|||||||
@@ -182,7 +182,7 @@ func TestNormalizeForecastEventBySchemaProducesCanonicalWeatherForecastSchema(t
|
|||||||
t.Run(tt.name, func(t *testing.T) {
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
out, err := normalizeForecastEventBySchema(event.Event{
|
out, err := normalizeForecastEventBySchema(event.Event{
|
||||||
ID: "evt-1",
|
ID: "evt-1",
|
||||||
Kind: event.Kind("forecast"),
|
Kind: event.Kind(standards.KindForecast),
|
||||||
Source: "nws-test",
|
Source: "nws-test",
|
||||||
EmittedAt: time.Date(2026, 3, 16, 18, 0, 0, 0, time.UTC),
|
EmittedAt: time.Date(2026, 3, 16, 18, 0, 0, 0, time.UTC),
|
||||||
Schema: tt.schema,
|
Schema: tt.schema,
|
||||||
|
|||||||
@@ -22,7 +22,7 @@ func TestWeatherStoriesNormalizerProducesCanonicalSchemaAndMapsSample(t *testing
|
|||||||
if out.Schema != standards.SchemaWeatherStoryV1 {
|
if out.Schema != standards.SchemaWeatherStoryV1 {
|
||||||
t.Fatalf("Schema = %q, want %q", out.Schema, standards.SchemaWeatherStoryV1)
|
t.Fatalf("Schema = %q, want %q", out.Schema, standards.SchemaWeatherStoryV1)
|
||||||
}
|
}
|
||||||
if out.Kind != event.Kind("weather_story") {
|
if out.Kind != event.Kind(standards.KindWeatherStory) {
|
||||||
t.Fatalf("Kind = %q, want weather_story", out.Kind)
|
t.Fatalf("Kind = %q, want weather_story", out.Kind)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -118,7 +118,7 @@ func TestWeatherStoriesNormalizerMatch(t *testing.T) {
|
|||||||
func weatherStoriesRawEvent(payload string) event.Event {
|
func weatherStoriesRawEvent(payload string) event.Event {
|
||||||
return event.Event{
|
return event.Event{
|
||||||
ID: "evt-weatherstories-1",
|
ID: "evt-weatherstories-1",
|
||||||
Kind: event.Kind("weather_story"),
|
Kind: event.Kind(standards.KindWeatherStory),
|
||||||
Source: "nws-weatherstories-test",
|
Source: "nws-weatherstories-test",
|
||||||
EmittedAt: time.Date(2026, 5, 30, 9, 5, 0, 0, time.UTC),
|
EmittedAt: time.Date(2026, 5, 30, 9, 5, 0, 0, time.UTC),
|
||||||
Schema: standards.SchemaRawNWSWeatherStoriesV1,
|
Schema: standards.SchemaRawNWSWeatherStoriesV1,
|
||||||
|
|||||||
365
internal/normalizers/spc/convective_outlook.go
Normal file
365
internal/normalizers/spc/convective_outlook.go
Normal file
@@ -0,0 +1,365 @@
|
|||||||
|
package spc
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"math"
|
||||||
|
"regexp"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/ejr/feedkit/event"
|
||||||
|
"gitea.maximumdirect.net/ejr/weatherfeeder/internal/geo"
|
||||||
|
normcommon "gitea.maximumdirect.net/ejr/weatherfeeder/internal/normalizers/common"
|
||||||
|
spcprovider "gitea.maximumdirect.net/ejr/weatherfeeder/internal/providers/spc"
|
||||||
|
"gitea.maximumdirect.net/ejr/weatherfeeder/model"
|
||||||
|
"gitea.maximumdirect.net/ejr/weatherfeeder/standards"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
providerSPC = "spc"
|
||||||
|
productConvective = "convective"
|
||||||
|
outlookNormalizer = "spc convective outlook"
|
||||||
|
outlookTypeUnknown = 99
|
||||||
|
)
|
||||||
|
|
||||||
|
var idTokenRE = regexp.MustCompile(`[^a-z0-9]+`)
|
||||||
|
|
||||||
|
// ConvectiveOutlookNormalizer converts:
|
||||||
|
//
|
||||||
|
// standards.SchemaRawSPCConvectiveOutlookV1 -> standards.SchemaWeatherOutlookV2
|
||||||
|
//
|
||||||
|
// It maps SPC GeoJSON outlook features containing the configured location into
|
||||||
|
// canonical outlook polygons and adds matching day-level print-page discussions.
|
||||||
|
type ConvectiveOutlookNormalizer struct{}
|
||||||
|
|
||||||
|
func (ConvectiveOutlookNormalizer) Match(e event.Event) bool {
|
||||||
|
return strings.TrimSpace(e.Schema) == standards.SchemaRawSPCConvectiveOutlookV1
|
||||||
|
}
|
||||||
|
|
||||||
|
func (ConvectiveOutlookNormalizer) Normalize(ctx context.Context, in event.Event) (*event.Event, error) {
|
||||||
|
_ = ctx
|
||||||
|
|
||||||
|
fallbackAsOf := in.EmittedAt.UTC()
|
||||||
|
if in.EffectiveAt != nil && !in.EffectiveAt.IsZero() {
|
||||||
|
fallbackAsOf = in.EffectiveAt.UTC()
|
||||||
|
}
|
||||||
|
|
||||||
|
return normcommon.NormalizeJSON(
|
||||||
|
in,
|
||||||
|
outlookNormalizer,
|
||||||
|
standards.SchemaWeatherOutlookV2,
|
||||||
|
func(parsed spcprovider.RawConvectiveOutlookBundle) (model.WeatherOutlookRun, time.Time, error) {
|
||||||
|
return buildConvectiveOutlook(parsed, fallbackAsOf)
|
||||||
|
},
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
func buildConvectiveOutlook(bundle spcprovider.RawConvectiveOutlookBundle, fallbackAsOf time.Time) (model.WeatherOutlookRun, time.Time, error) {
|
||||||
|
if err := validateCoordinates(bundle.Latitude, bundle.Longitude); err != nil {
|
||||||
|
return model.WeatherOutlookRun{}, time.Time{}, err
|
||||||
|
}
|
||||||
|
|
||||||
|
discussions, latestDiscussionUpdated, err := parseDiscussions(bundle.Discussions)
|
||||||
|
if err != nil {
|
||||||
|
return model.WeatherOutlookRun{}, time.Time{}, err
|
||||||
|
}
|
||||||
|
|
||||||
|
products := orderedProducts(bundle.Products)
|
||||||
|
point := geo.Point{Latitude: bundle.Latitude, Longitude: bundle.Longitude}
|
||||||
|
outlooks := make([]model.WeatherOutlook, 0)
|
||||||
|
var latestIssue time.Time
|
||||||
|
|
||||||
|
for _, product := range products {
|
||||||
|
if err := validateProductMetadata(product); err != nil {
|
||||||
|
return model.WeatherOutlookRun{}, time.Time{}, err
|
||||||
|
}
|
||||||
|
if _, ok := discussions[product.Day]; !ok {
|
||||||
|
return model.WeatherOutlookRun{}, time.Time{}, fmt.Errorf("product %s: discussion for day %d is required", product.Key, product.Day)
|
||||||
|
}
|
||||||
|
|
||||||
|
collection, err := spcprovider.DecodeGeoJSON(product.Body)
|
||||||
|
if err != nil {
|
||||||
|
return model.WeatherOutlookRun{}, time.Time{}, fmt.Errorf("product %s: %w", product.Key, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
for i, feature := range collection.Features {
|
||||||
|
if spcprovider.IsEmptyGeometryCollection(feature.Geometry) {
|
||||||
|
issuedAt, err := parseRequiredSPCTime(feature.Properties.IssueISO, fmt.Sprintf("product %s feature %d.ISSUE_ISO", product.Key, i))
|
||||||
|
if err != nil {
|
||||||
|
return model.WeatherOutlookRun{}, time.Time{}, err
|
||||||
|
}
|
||||||
|
if latestIssue.IsZero() || issuedAt.After(latestIssue) {
|
||||||
|
latestIssue = issuedAt
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
outlook, err := mapFeature(product, feature, i, point)
|
||||||
|
if err != nil {
|
||||||
|
return model.WeatherOutlookRun{}, time.Time{}, err
|
||||||
|
}
|
||||||
|
if latestIssue.IsZero() || outlook.IssuedAt.After(latestIssue) {
|
||||||
|
latestIssue = outlook.IssuedAt
|
||||||
|
}
|
||||||
|
if !outlook.ContainsLocation {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
outlooks = append(outlooks, outlook)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
runDiscussions, err := buildOutlookDiscussions(outlooks, discussions)
|
||||||
|
if err != nil {
|
||||||
|
return model.WeatherOutlookRun{}, time.Time{}, err
|
||||||
|
}
|
||||||
|
|
||||||
|
asOf := latestIssue
|
||||||
|
if asOf.IsZero() {
|
||||||
|
asOf = latestDiscussionUpdated
|
||||||
|
}
|
||||||
|
if asOf.IsZero() {
|
||||||
|
asOf = fallbackAsOf.UTC()
|
||||||
|
}
|
||||||
|
|
||||||
|
var issuedAt *time.Time
|
||||||
|
if !latestIssue.IsZero() {
|
||||||
|
t := latestIssue.UTC()
|
||||||
|
issuedAt = &t
|
||||||
|
}
|
||||||
|
|
||||||
|
lat := bundle.Latitude
|
||||||
|
lon := bundle.Longitude
|
||||||
|
run := model.WeatherOutlookRun{
|
||||||
|
LocationID: strings.TrimSpace(bundle.LocationID),
|
||||||
|
LocationName: strings.TrimSpace(bundle.LocationName),
|
||||||
|
Latitude: &lat,
|
||||||
|
Longitude: &lon,
|
||||||
|
AsOf: asOf.UTC(),
|
||||||
|
IssuedAt: issuedAt,
|
||||||
|
Outlooks: outlooks,
|
||||||
|
Discussions: runDiscussions,
|
||||||
|
}
|
||||||
|
return run, run.AsOf, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
type parsedDiscussion struct {
|
||||||
|
Headline string
|
||||||
|
Summary string
|
||||||
|
Discussion string
|
||||||
|
UpdatedAt *time.Time
|
||||||
|
}
|
||||||
|
|
||||||
|
func parseDiscussions(pages []spcprovider.RawDiscussionPage) (map[int]parsedDiscussion, time.Time, error) {
|
||||||
|
out := map[int]parsedDiscussion{}
|
||||||
|
var latestUpdated time.Time
|
||||||
|
for _, page := range pages {
|
||||||
|
parsed, err := spcprovider.ParseDiscussionHTML(page.Body)
|
||||||
|
if err != nil {
|
||||||
|
return nil, time.Time{}, fmt.Errorf("discussion %s: %w", page.Key, err)
|
||||||
|
}
|
||||||
|
day := page.Day
|
||||||
|
if day == 0 {
|
||||||
|
if meta, ok := spcprovider.DiscussionProductByKey(page.Key); ok {
|
||||||
|
day = meta.Day
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if day < 1 || day > 3 {
|
||||||
|
return nil, time.Time{}, fmt.Errorf("discussion %s: day must be 1, 2, or 3, got %d", page.Key, page.Day)
|
||||||
|
}
|
||||||
|
disc := parsedDiscussion{
|
||||||
|
Headline: strings.TrimSpace(parsed.Headline),
|
||||||
|
Summary: strings.TrimSpace(parsed.Summary),
|
||||||
|
Discussion: strings.TrimSpace(parsed.Discussion),
|
||||||
|
UpdatedAt: parsed.UpdatedAt,
|
||||||
|
}
|
||||||
|
out[day] = disc
|
||||||
|
if parsed.UpdatedAt != nil && (latestUpdated.IsZero() || parsed.UpdatedAt.After(latestUpdated)) {
|
||||||
|
latestUpdated = parsed.UpdatedAt.UTC()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out, latestUpdated, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func buildOutlookDiscussions(outlooks []model.WeatherOutlook, discussions map[int]parsedDiscussion) ([]model.WeatherOutlookDiscussion, error) {
|
||||||
|
daysWithOutlooks := map[int]bool{}
|
||||||
|
for _, outlook := range outlooks {
|
||||||
|
daysWithOutlooks[outlook.Day] = true
|
||||||
|
}
|
||||||
|
|
||||||
|
days := make([]int, 0, len(daysWithOutlooks))
|
||||||
|
for day := range daysWithOutlooks {
|
||||||
|
days = append(days, day)
|
||||||
|
}
|
||||||
|
sort.Ints(days)
|
||||||
|
|
||||||
|
out := make([]model.WeatherOutlookDiscussion, 0, len(days))
|
||||||
|
for _, day := range days {
|
||||||
|
disc, ok := discussions[day]
|
||||||
|
if !ok {
|
||||||
|
return nil, fmt.Errorf("discussion for retained day %d is required", day)
|
||||||
|
}
|
||||||
|
out = append(out, model.WeatherOutlookDiscussion{
|
||||||
|
Day: day,
|
||||||
|
Headline: disc.Headline,
|
||||||
|
Summary: disc.Summary,
|
||||||
|
Discussion: disc.Discussion,
|
||||||
|
UpdatedAt: utcTimePtr(disc.UpdatedAt),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func utcTimePtr(t *time.Time) *time.Time {
|
||||||
|
if t == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
tt := t.UTC()
|
||||||
|
return &tt
|
||||||
|
}
|
||||||
|
|
||||||
|
func orderedProducts(products []spcprovider.RawOutlookProduct) []spcprovider.RawOutlookProduct {
|
||||||
|
out := make([]spcprovider.RawOutlookProduct, len(products))
|
||||||
|
copy(out, products)
|
||||||
|
sort.SliceStable(out, func(i, j int) bool {
|
||||||
|
if out[i].Day != out[j].Day {
|
||||||
|
return out[i].Day < out[j].Day
|
||||||
|
}
|
||||||
|
left := outlookTypeOrder(out[i].OutlookType)
|
||||||
|
right := outlookTypeOrder(out[j].OutlookType)
|
||||||
|
if left != right {
|
||||||
|
return left < right
|
||||||
|
}
|
||||||
|
return out[i].Key < out[j].Key
|
||||||
|
})
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func outlookTypeOrder(outlookType string) int {
|
||||||
|
switch strings.TrimSpace(outlookType) {
|
||||||
|
case spcprovider.OutlookTypeCategorical:
|
||||||
|
return 0
|
||||||
|
case spcprovider.OutlookTypeTornado:
|
||||||
|
return 1
|
||||||
|
case spcprovider.OutlookTypeHail:
|
||||||
|
return 2
|
||||||
|
case spcprovider.OutlookTypeWind:
|
||||||
|
return 3
|
||||||
|
default:
|
||||||
|
return outlookTypeUnknown
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func validateProductMetadata(product spcprovider.RawOutlookProduct) error {
|
||||||
|
if product.Day < 1 || product.Day > 3 {
|
||||||
|
return fmt.Errorf("product %s: day must be 1, 2, or 3, got %d", product.Key, product.Day)
|
||||||
|
}
|
||||||
|
switch strings.TrimSpace(product.OutlookType) {
|
||||||
|
case spcprovider.OutlookTypeCategorical, spcprovider.OutlookTypeTornado, spcprovider.OutlookTypeHail, spcprovider.OutlookTypeWind:
|
||||||
|
return nil
|
||||||
|
default:
|
||||||
|
return fmt.Errorf("product %s: unsupported outlook type %q", product.Key, product.OutlookType)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func mapFeature(product spcprovider.RawOutlookProduct, feature spcprovider.GeoJSONFeature, index int, point geo.Point) (model.WeatherOutlook, error) {
|
||||||
|
fieldPrefix := fmt.Sprintf("product %s feature %d", product.Key, index)
|
||||||
|
props := feature.Properties
|
||||||
|
|
||||||
|
validFrom, err := parseRequiredSPCTime(props.ValidISO, fieldPrefix+".VALID_ISO")
|
||||||
|
if err != nil {
|
||||||
|
return model.WeatherOutlook{}, err
|
||||||
|
}
|
||||||
|
validTo, err := parseRequiredSPCTime(props.ExpireISO, fieldPrefix+".EXPIRE_ISO")
|
||||||
|
if err != nil {
|
||||||
|
return model.WeatherOutlook{}, err
|
||||||
|
}
|
||||||
|
issuedAt, err := parseRequiredSPCTime(props.IssueISO, fieldPrefix+".ISSUE_ISO")
|
||||||
|
if err != nil {
|
||||||
|
return model.WeatherOutlook{}, err
|
||||||
|
}
|
||||||
|
label := strings.TrimSpace(props.Label)
|
||||||
|
if label == "" {
|
||||||
|
return model.WeatherOutlook{}, fmt.Errorf("%s.LABEL is required", fieldPrefix)
|
||||||
|
}
|
||||||
|
if len(feature.Geometry) == 0 {
|
||||||
|
return model.WeatherOutlook{}, fmt.Errorf("%s.geometry is required", fieldPrefix)
|
||||||
|
}
|
||||||
|
containsLocation, err := geo.ContainsPoint(feature.Geometry, point)
|
||||||
|
if err != nil {
|
||||||
|
return model.WeatherOutlook{}, fmt.Errorf("%s.geometry: %w", fieldPrefix, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
geometry := make(json.RawMessage, len(feature.Geometry))
|
||||||
|
copy(geometry, feature.Geometry)
|
||||||
|
|
||||||
|
return model.WeatherOutlook{
|
||||||
|
ID: outlookID(product.Day, product.OutlookType, label, issuedAt, validFrom, index),
|
||||||
|
Provider: providerSPC,
|
||||||
|
Product: productConvective,
|
||||||
|
Day: product.Day,
|
||||||
|
OutlookType: strings.TrimSpace(product.OutlookType),
|
||||||
|
Label: label,
|
||||||
|
LabelText: strings.TrimSpace(props.Label2),
|
||||||
|
SeverityRank: props.DN,
|
||||||
|
ValidFrom: validFrom,
|
||||||
|
ValidTo: validTo,
|
||||||
|
IssuedAt: issuedAt,
|
||||||
|
ExpiresAt: validTo,
|
||||||
|
Forecaster: strings.TrimSpace(props.Forecaster),
|
||||||
|
SourceURL: strings.TrimSpace(product.URL),
|
||||||
|
ImageURL: "",
|
||||||
|
ContainsLocation: containsLocation,
|
||||||
|
Geometry: geometry,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func parseRequiredSPCTime(value, field string) (time.Time, error) {
|
||||||
|
if strings.TrimSpace(value) == "" {
|
||||||
|
return time.Time{}, fmt.Errorf("%s is required", field)
|
||||||
|
}
|
||||||
|
t, err := spcprovider.ParseISOTimestamp(value)
|
||||||
|
if err != nil {
|
||||||
|
return time.Time{}, fmt.Errorf("%s: %w", field, err)
|
||||||
|
}
|
||||||
|
return t.UTC(), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func outlookID(day int, outlookType, label string, issuedAt time.Time, validFrom time.Time, index int) string {
|
||||||
|
return fmt.Sprintf(
|
||||||
|
"spc-convective-day%d-%s-%s-%s-%s-%d",
|
||||||
|
day,
|
||||||
|
safeIDToken(outlookType),
|
||||||
|
safeIDToken(label),
|
||||||
|
issuedAt.UTC().Format(time.RFC3339),
|
||||||
|
validFrom.UTC().Format(time.RFC3339),
|
||||||
|
index,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
func safeIDToken(value string) string {
|
||||||
|
value = strings.ToLower(strings.TrimSpace(value))
|
||||||
|
value = idTokenRE.ReplaceAllString(value, "-")
|
||||||
|
value = strings.Trim(value, "-")
|
||||||
|
if value == "" {
|
||||||
|
return "unknown"
|
||||||
|
}
|
||||||
|
return value
|
||||||
|
}
|
||||||
|
|
||||||
|
func validateCoordinates(latitude, longitude float64) error {
|
||||||
|
switch {
|
||||||
|
case math.IsNaN(latitude) || math.IsInf(latitude, 0):
|
||||||
|
return fmt.Errorf("latitude must be finite")
|
||||||
|
case math.IsNaN(longitude) || math.IsInf(longitude, 0):
|
||||||
|
return fmt.Errorf("longitude must be finite")
|
||||||
|
case latitude < -90 || latitude > 90:
|
||||||
|
return fmt.Errorf("latitude must be between -90 and 90, got %v", latitude)
|
||||||
|
case longitude < -180 || longitude > 180:
|
||||||
|
return fmt.Errorf("longitude must be between -180 and 180, got %v", longitude)
|
||||||
|
default:
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
}
|
||||||
526
internal/normalizers/spc/convective_outlook_test.go
Normal file
526
internal/normalizers/spc/convective_outlook_test.go
Normal file
@@ -0,0 +1,526 @@
|
|||||||
|
package spc
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/ejr/feedkit/event"
|
||||||
|
spcprovider "gitea.maximumdirect.net/ejr/weatherfeeder/internal/providers/spc"
|
||||||
|
"gitea.maximumdirect.net/ejr/weatherfeeder/model"
|
||||||
|
"gitea.maximumdirect.net/ejr/weatherfeeder/standards"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestConvectiveOutlookNormalizerMatch(t *testing.T) {
|
||||||
|
n := ConvectiveOutlookNormalizer{}
|
||||||
|
if !n.Match(event.Event{Schema: standards.SchemaRawSPCConvectiveOutlookV1}) {
|
||||||
|
t.Fatalf("Match(raw SPC outlook) = false, want true")
|
||||||
|
}
|
||||||
|
if n.Match(event.Event{Schema: standards.SchemaRawNWSAlertsV1}) {
|
||||||
|
t.Fatalf("Match(raw NWS alerts) = true, want false")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestConvectiveOutlookNormalizerProducesCanonicalSchemaAndMapsSample(t *testing.T) {
|
||||||
|
out, err := (ConvectiveOutlookNormalizer{}).Normalize(nil, spcRawEvent(t, spcBundle(t, 38.5, -90.5)))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Normalize() error = %v", err)
|
||||||
|
}
|
||||||
|
if out.Schema != standards.SchemaWeatherOutlookV2 {
|
||||||
|
t.Fatalf("Schema = %q, want %q", out.Schema, standards.SchemaWeatherOutlookV2)
|
||||||
|
}
|
||||||
|
if out.Kind != event.Kind(standards.KindOutlook) {
|
||||||
|
t.Fatalf("Kind = %q, want outlook", out.Kind)
|
||||||
|
}
|
||||||
|
|
||||||
|
run, ok := out.Payload.(model.WeatherOutlookRun)
|
||||||
|
if !ok {
|
||||||
|
t.Fatalf("Payload type = %T, want model.WeatherOutlookRun", out.Payload)
|
||||||
|
}
|
||||||
|
wantAsOf := time.Date(2026, 6, 11, 19, 45, 0, 0, time.UTC)
|
||||||
|
if !run.AsOf.Equal(wantAsOf) {
|
||||||
|
t.Fatalf("AsOf = %s, want %s", run.AsOf, wantAsOf)
|
||||||
|
}
|
||||||
|
if run.IssuedAt == nil || !run.IssuedAt.Equal(wantAsOf) {
|
||||||
|
t.Fatalf("IssuedAt = %v, want %s", run.IssuedAt, wantAsOf)
|
||||||
|
}
|
||||||
|
if out.EffectiveAt == nil || !out.EffectiveAt.Equal(wantAsOf) {
|
||||||
|
t.Fatalf("EffectiveAt = %v, want %s", out.EffectiveAt, wantAsOf)
|
||||||
|
}
|
||||||
|
if run.LocationID != "stl" || run.LocationName != "St. Louis, MO" {
|
||||||
|
t.Fatalf("location metadata = %q/%q", run.LocationID, run.LocationName)
|
||||||
|
}
|
||||||
|
if run.Latitude == nil || *run.Latitude != 38.5 || run.Longitude == nil || *run.Longitude != -90.5 {
|
||||||
|
t.Fatalf("coordinates = %v,%v", run.Latitude, run.Longitude)
|
||||||
|
}
|
||||||
|
if len(run.Outlooks) != 4 {
|
||||||
|
t.Fatalf("Outlooks length = %d, want 4", len(run.Outlooks))
|
||||||
|
}
|
||||||
|
assertAllOutlooksContainLocation(t, run.Outlooks)
|
||||||
|
assertDiscussionDays(t, run.Discussions, 1)
|
||||||
|
day1Discussion := run.Discussions[0]
|
||||||
|
if day1Discussion.Headline != "Day 1 Convective Outlook" {
|
||||||
|
t.Fatalf("day 1 Headline = %q", day1Discussion.Headline)
|
||||||
|
}
|
||||||
|
if !strings.Contains(day1Discussion.Summary, "central Plains") {
|
||||||
|
t.Fatalf("day 1 Summary = %q", day1Discussion.Summary)
|
||||||
|
}
|
||||||
|
if !strings.Contains(day1Discussion.Discussion, "...DISCUSSION...") {
|
||||||
|
t.Fatalf("day 1 Discussion missing product text: %q", day1Discussion.Discussion)
|
||||||
|
}
|
||||||
|
if !strings.HasPrefix(day1Discussion.Discussion, "SPC AC 111234") {
|
||||||
|
t.Fatalf("day 1 Discussion = %q, want SPC product code prefix", day1Discussion.Discussion)
|
||||||
|
}
|
||||||
|
|
||||||
|
got := run.Outlooks[0]
|
||||||
|
if got.Provider != "spc" || got.Product != "convective" {
|
||||||
|
t.Fatalf("provider/product = %q/%q", got.Provider, got.Product)
|
||||||
|
}
|
||||||
|
if got.Day != 1 || got.OutlookType != spcprovider.OutlookTypeCategorical {
|
||||||
|
t.Fatalf("day/type = %d/%q", got.Day, got.OutlookType)
|
||||||
|
}
|
||||||
|
if got.Label != "SLGT" || got.LabelText != "Slight Risk" {
|
||||||
|
t.Fatalf("label fields = %q/%q", got.Label, got.LabelText)
|
||||||
|
}
|
||||||
|
if got.SeverityRank == nil || *got.SeverityRank != 3 {
|
||||||
|
t.Fatalf("SeverityRank = %v, want 3", got.SeverityRank)
|
||||||
|
}
|
||||||
|
assertTime(t, "ValidFrom", got.ValidFrom, 2026, 6, 11, 13, 0, 0)
|
||||||
|
assertTime(t, "ValidTo", got.ValidTo, 2026, 6, 12, 12, 0, 0)
|
||||||
|
assertTime(t, "IssuedAt", got.IssuedAt, 2026, 6, 11, 12, 34, 56)
|
||||||
|
assertTime(t, "ExpiresAt", got.ExpiresAt, 2026, 6, 12, 12, 0, 0)
|
||||||
|
if got.Forecaster != "SMITH" {
|
||||||
|
t.Fatalf("Forecaster = %q, want SMITH", got.Forecaster)
|
||||||
|
}
|
||||||
|
if got.SourceURL != "https://example.invalid/day1_categorical.geojson" {
|
||||||
|
t.Fatalf("SourceURL = %q", got.SourceURL)
|
||||||
|
}
|
||||||
|
wantGeometry := `{"type":"Polygon","coordinates":[[[-91.0,38.0],[-90.0,38.0],[-90.0,39.0],[-91.0,39.0],[-91.0,38.0]]]}`
|
||||||
|
if string(got.Geometry) != wantGeometry {
|
||||||
|
t.Fatalf("Geometry = %s, want %s", got.Geometry, wantGeometry)
|
||||||
|
}
|
||||||
|
if !got.ContainsLocation {
|
||||||
|
t.Fatalf("ContainsLocation = false, want true")
|
||||||
|
}
|
||||||
|
if got.ID != "spc-convective-day1-categorical-slgt-2026-06-11T12:34:56Z-2026-06-11T13:00:00Z-0" {
|
||||||
|
t.Fatalf("ID = %q", got.ID)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestConvectiveOutlookNormalizerAcceptsTypedSourcePayload(t *testing.T) {
|
||||||
|
bundle := spcBundle(t, 38.5, -90.5)
|
||||||
|
in := spcRawEvent(t, bundle)
|
||||||
|
in.Payload = bundle
|
||||||
|
|
||||||
|
out, err := (ConvectiveOutlookNormalizer{}).Normalize(nil, in)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Normalize() error = %v", err)
|
||||||
|
}
|
||||||
|
run := out.Payload.(model.WeatherOutlookRun)
|
||||||
|
if len(run.Outlooks) != 4 {
|
||||||
|
t.Fatalf("Outlooks length = %d, want 4", len(run.Outlooks))
|
||||||
|
}
|
||||||
|
assertAllOutlooksContainLocation(t, run.Outlooks)
|
||||||
|
assertDiscussionDays(t, run.Discussions, 1)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestConvectiveOutlookNormalizerEmitsEmptyLocalRunOutsidePolygons(t *testing.T) {
|
||||||
|
out, err := (ConvectiveOutlookNormalizer{}).Normalize(nil, spcRawEvent(t, spcBundle(t, 0, 0)))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Normalize() error = %v", err)
|
||||||
|
}
|
||||||
|
run := out.Payload.(model.WeatherOutlookRun)
|
||||||
|
if len(run.Outlooks) != 0 {
|
||||||
|
t.Fatalf("Outlooks length = %d, want 0", len(run.Outlooks))
|
||||||
|
}
|
||||||
|
if len(run.Discussions) != 0 {
|
||||||
|
t.Fatalf("Discussions length = %d, want 0", len(run.Discussions))
|
||||||
|
}
|
||||||
|
wantAsOf := time.Date(2026, 6, 11, 19, 45, 0, 0, time.UTC)
|
||||||
|
if !run.AsOf.Equal(wantAsOf) {
|
||||||
|
t.Fatalf("AsOf = %s, want latest product issue time %s", run.AsOf, wantAsOf)
|
||||||
|
}
|
||||||
|
if run.IssuedAt == nil || !run.IssuedAt.Equal(wantAsOf) {
|
||||||
|
t.Fatalf("IssuedAt = %v, want latest product issue time %s", run.IssuedAt, wantAsOf)
|
||||||
|
}
|
||||||
|
if out.EffectiveAt == nil || !out.EffectiveAt.Equal(run.AsOf) {
|
||||||
|
t.Fatalf("EffectiveAt = %v, want run AsOf %s", out.EffectiveAt, run.AsOf)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestConvectiveOutlookNormalizerOrdersProductsByDayAndType(t *testing.T) {
|
||||||
|
bundle := spcBundle(t, 38.5, -90.5)
|
||||||
|
for i, j := 0, len(bundle.Products)-1; i < j; i, j = i+1, j-1 {
|
||||||
|
bundle.Products[i], bundle.Products[j] = bundle.Products[j], bundle.Products[i]
|
||||||
|
}
|
||||||
|
|
||||||
|
out, err := (ConvectiveOutlookNormalizer{}).Normalize(nil, spcRawEvent(t, bundle))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Normalize() error = %v", err)
|
||||||
|
}
|
||||||
|
run := out.Payload.(model.WeatherOutlookRun)
|
||||||
|
got := []string{
|
||||||
|
run.Outlooks[0].OutlookType,
|
||||||
|
run.Outlooks[1].OutlookType,
|
||||||
|
run.Outlooks[2].OutlookType,
|
||||||
|
run.Outlooks[3].OutlookType,
|
||||||
|
}
|
||||||
|
want := []string{
|
||||||
|
spcprovider.OutlookTypeCategorical,
|
||||||
|
spcprovider.OutlookTypeTornado,
|
||||||
|
spcprovider.OutlookTypeHail,
|
||||||
|
spcprovider.OutlookTypeWind,
|
||||||
|
}
|
||||||
|
for i := range want {
|
||||||
|
if got[i] != want[i] || run.Outlooks[i].Day != 1 {
|
||||||
|
t.Fatalf("outlook[%d] = day %d type %q, want day 1 type %q", i, run.Outlooks[i].Day, got[i], want[i])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestConvectiveOutlookNormalizerMapsProbabilisticOutlookTypes(t *testing.T) {
|
||||||
|
out, err := (ConvectiveOutlookNormalizer{}).Normalize(nil, spcRawEvent(t, spcBundle(t, 38.5, -90.5)))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Normalize() error = %v", err)
|
||||||
|
}
|
||||||
|
run := out.Payload.(model.WeatherOutlookRun)
|
||||||
|
assertAllOutlooksContainLocation(t, run.Outlooks)
|
||||||
|
for _, outlookType := range []string{
|
||||||
|
spcprovider.OutlookTypeTornado,
|
||||||
|
spcprovider.OutlookTypeHail,
|
||||||
|
spcprovider.OutlookTypeWind,
|
||||||
|
} {
|
||||||
|
if findOutlook(run.Outlooks, 1, outlookType) == nil {
|
||||||
|
t.Fatalf("missing day 1 outlook type %q", outlookType)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestConvectiveOutlookNormalizerSkipsEmptyGeometryCollectionPlaceholder(t *testing.T) {
|
||||||
|
bundle := spcBundle(t, 36, -99)
|
||||||
|
replaced := false
|
||||||
|
for i := range bundle.Products {
|
||||||
|
if bundle.Products[i].Day == 2 && bundle.Products[i].OutlookType == spcprovider.OutlookTypeTornado {
|
||||||
|
bundle.Products[i].Body = json.RawMessage(emptyGeometryCollectionGeoJSON())
|
||||||
|
replaced = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !replaced {
|
||||||
|
t.Fatalf("test setup did not find day 2 tornado product")
|
||||||
|
}
|
||||||
|
|
||||||
|
out, err := (ConvectiveOutlookNormalizer{}).Normalize(nil, spcRawEvent(t, bundle))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Normalize() error = %v", err)
|
||||||
|
}
|
||||||
|
run := out.Payload.(model.WeatherOutlookRun)
|
||||||
|
if len(run.Outlooks) != 3 {
|
||||||
|
t.Fatalf("Outlooks length = %d, want 3", len(run.Outlooks))
|
||||||
|
}
|
||||||
|
assertAllOutlooksContainLocation(t, run.Outlooks)
|
||||||
|
assertDiscussionDays(t, run.Discussions, 2)
|
||||||
|
if got := findOutlook(run.Outlooks, 2, spcprovider.OutlookTypeTornado); got != nil {
|
||||||
|
t.Fatalf("day 2 tornado outlook = %+v, want nil placeholder skipped", *got)
|
||||||
|
}
|
||||||
|
wantAsOf := time.Date(2026, 6, 12, 10, 0, 0, 0, time.UTC)
|
||||||
|
if !run.AsOf.Equal(wantAsOf) {
|
||||||
|
t.Fatalf("AsOf = %s, want placeholder ISSUE_ISO %s", run.AsOf, wantAsOf)
|
||||||
|
}
|
||||||
|
if out.EffectiveAt == nil || !out.EffectiveAt.Equal(wantAsOf) {
|
||||||
|
t.Fatalf("EffectiveAt = %v, want placeholder ISSUE_ISO %s", out.EffectiveAt, wantAsOf)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestConvectiveOutlookNormalizerIncludesOnlyDayWithContainingPolygons(t *testing.T) {
|
||||||
|
out, err := (ConvectiveOutlookNormalizer{}).Normalize(nil, spcRawEvent(t, spcBundle(t, 36, -99)))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Normalize() error = %v", err)
|
||||||
|
}
|
||||||
|
run := out.Payload.(model.WeatherOutlookRun)
|
||||||
|
if len(run.Outlooks) == 0 {
|
||||||
|
t.Fatalf("Outlooks length = 0, want retained day 2 outlooks")
|
||||||
|
}
|
||||||
|
assertAllOutlooksContainLocation(t, run.Outlooks)
|
||||||
|
for i, outlook := range run.Outlooks {
|
||||||
|
if outlook.Day != 2 {
|
||||||
|
t.Fatalf("outlook[%d].Day = %d, want 2", i, outlook.Day)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
assertDiscussionDays(t, run.Discussions, 2)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestConvectiveOutlookNormalizerUsesOneDiscussionForMultipleSameDayOutlooks(t *testing.T) {
|
||||||
|
out, err := (ConvectiveOutlookNormalizer{}).Normalize(nil, spcRawEvent(t, spcBundle(t, 38.5, -90.5)))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Normalize() error = %v", err)
|
||||||
|
}
|
||||||
|
run := out.Payload.(model.WeatherOutlookRun)
|
||||||
|
if got := countOutlooksByDay(run.Outlooks, 1); got < 2 {
|
||||||
|
t.Fatalf("day 1 outlook count = %d, want multiple", got)
|
||||||
|
}
|
||||||
|
assertAllOutlooksContainLocation(t, run.Outlooks)
|
||||||
|
assertDiscussionDays(t, run.Discussions, 1)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestConvectiveOutlookNormalizerPreservesCorrectionMarker(t *testing.T) {
|
||||||
|
bundle := spcBundle(t, 36, -99)
|
||||||
|
out, err := (ConvectiveOutlookNormalizer{}).Normalize(nil, spcRawEvent(t, bundle))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Normalize() error = %v", err)
|
||||||
|
}
|
||||||
|
run := out.Payload.(model.WeatherOutlookRun)
|
||||||
|
got := findOutlook(run.Outlooks, 2, spcprovider.OutlookTypeTornado)
|
||||||
|
if got == nil {
|
||||||
|
t.Fatalf("missing day 2 tornado outlook")
|
||||||
|
}
|
||||||
|
assertDiscussionDays(t, run.Discussions, 2)
|
||||||
|
if !strings.Contains(run.Discussions[0].Headline, "CORR 1") {
|
||||||
|
t.Fatalf("day 2 headline = %q, want correction marker", run.Discussions[0].Headline)
|
||||||
|
}
|
||||||
|
if !strings.Contains(run.Discussions[0].Discussion, "CORR 1") {
|
||||||
|
t.Fatalf("day 2 discussion = %q, want correction marker", run.Discussions[0].Discussion)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestConvectiveOutlookNormalizerMissingRSSNormalizes(t *testing.T) {
|
||||||
|
bundle := spcBundle(t, 0, 0)
|
||||||
|
bundle.RSS = nil
|
||||||
|
if _, err := (ConvectiveOutlookNormalizer{}).Normalize(nil, spcRawEvent(t, bundle)); err != nil {
|
||||||
|
t.Fatalf("Normalize() error = %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestConvectiveOutlookNormalizerInvalidRequiredTimestampFailsWithContext(t *testing.T) {
|
||||||
|
bundle := spcBundle(t, 0, 0)
|
||||||
|
bundle.Products[0].Body = json.RawMessage(strings.Replace(
|
||||||
|
string(bundle.Products[0].Body),
|
||||||
|
`"ISSUE_ISO": "2026-06-11T12:34:56Z"`,
|
||||||
|
`"ISSUE_ISO": "bad"`,
|
||||||
|
1,
|
||||||
|
))
|
||||||
|
|
||||||
|
_, err := (ConvectiveOutlookNormalizer{}).Normalize(nil, spcRawEvent(t, bundle))
|
||||||
|
if err == nil {
|
||||||
|
t.Fatalf("Normalize() error = nil, want error")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "product day1_categorical feature 0.ISSUE_ISO") {
|
||||||
|
t.Fatalf("error = %q, want product and feature context", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestConvectiveOutlookNormalizerInvalidGeometryFailsWithContext(t *testing.T) {
|
||||||
|
bundle := spcBundle(t, 0, 0)
|
||||||
|
bundle.Products[0].Body = json.RawMessage(strings.Replace(
|
||||||
|
string(bundle.Products[0].Body),
|
||||||
|
`"geometry": {`,
|
||||||
|
`"geometry": {"type":"LineString","coordinates":[[-91,38],[-90,39]]}, "oldGeometry": {`,
|
||||||
|
1,
|
||||||
|
))
|
||||||
|
|
||||||
|
_, err := (ConvectiveOutlookNormalizer{}).Normalize(nil, spcRawEvent(t, bundle))
|
||||||
|
if err == nil {
|
||||||
|
t.Fatalf("Normalize() error = nil, want error")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "product day1_categorical feature 0.geometry") {
|
||||||
|
t.Fatalf("error = %q, want product and feature context", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestConvectiveOutlookNormalizerRejectsMissingLabel(t *testing.T) {
|
||||||
|
bundle := spcBundle(t, 0, 0)
|
||||||
|
bundle.Products[0].Body = json.RawMessage(strings.Replace(
|
||||||
|
string(bundle.Products[0].Body),
|
||||||
|
`"LABEL": "SLGT"`,
|
||||||
|
`"LABEL": ""`,
|
||||||
|
1,
|
||||||
|
))
|
||||||
|
|
||||||
|
_, err := (ConvectiveOutlookNormalizer{}).Normalize(nil, spcRawEvent(t, bundle))
|
||||||
|
if err == nil {
|
||||||
|
t.Fatalf("Normalize() error = nil, want error")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "product day1_categorical feature 0.LABEL") {
|
||||||
|
t.Fatalf("error = %q, want label context", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestConvectiveOutlookNormalizerRejectsMissingDiscussion(t *testing.T) {
|
||||||
|
bundle := spcBundle(t, 38.5, -90.5)
|
||||||
|
bundle.Discussions = bundle.Discussions[1:]
|
||||||
|
|
||||||
|
_, err := (ConvectiveOutlookNormalizer{}).Normalize(nil, spcRawEvent(t, bundle))
|
||||||
|
if err == nil {
|
||||||
|
t.Fatalf("Normalize() error = nil, want error")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "product day1_categorical: discussion for day 1 is required") {
|
||||||
|
t.Fatalf("error = %q, want missing discussion context", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestConvectiveOutlookNormalizerOutputJSONShape(t *testing.T) {
|
||||||
|
out, err := (ConvectiveOutlookNormalizer{}).Normalize(nil, spcRawEvent(t, spcBundle(t, 38.5, -90.5)))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Normalize() error = %v", err)
|
||||||
|
}
|
||||||
|
raw, err := json.Marshal(out.Payload)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Marshal(payload) error = %v", err)
|
||||||
|
}
|
||||||
|
got := string(raw)
|
||||||
|
for _, want := range []string{`"asOf"`, `"outlooks"`, `"discussions"`, `"containsLocation"`, `"geometry"`} {
|
||||||
|
if !strings.Contains(got, want) {
|
||||||
|
t.Fatalf("payload JSON missing %s: %s", want, got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
outlookStart := strings.Index(got, `"outlooks"`)
|
||||||
|
discussionStart := strings.Index(got, `"discussions"`)
|
||||||
|
if outlookStart == -1 || discussionStart == -1 || discussionStart <= outlookStart {
|
||||||
|
t.Fatalf("payload JSON has unexpected outlook/discussion order: %s", got)
|
||||||
|
}
|
||||||
|
outlookJSON := got[outlookStart:discussionStart]
|
||||||
|
for _, unwanted := range []string{`"headline"`, `"summary"`, `"discussion"`} {
|
||||||
|
if strings.Contains(outlookJSON, unwanted) {
|
||||||
|
t.Fatalf("outlook JSON exposed polygon-level prose key %s: %s", unwanted, got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, unwanted := range []string{`"products"`, `"fetchedAt"`, `"body"`} {
|
||||||
|
if strings.Contains(got, unwanted) {
|
||||||
|
t.Fatalf("payload JSON exposed raw key %s: %s", unwanted, got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func emptyGeometryCollectionGeoJSON() []byte {
|
||||||
|
return []byte(`{
|
||||||
|
"type": "FeatureCollection",
|
||||||
|
"features": [
|
||||||
|
{
|
||||||
|
"type": "Feature",
|
||||||
|
"properties": {
|
||||||
|
"VALID_ISO": "2026-06-12T12:00:00Z",
|
||||||
|
"EXPIRE_ISO": "2026-06-13T12:00:00Z",
|
||||||
|
"ISSUE_ISO": "2026-06-12T10:00:00Z",
|
||||||
|
"FORECASTER": "DOE",
|
||||||
|
"LABEL": "Less Than 2% All Areas",
|
||||||
|
"LABEL2": "",
|
||||||
|
"DN": 0
|
||||||
|
},
|
||||||
|
"geometry": {
|
||||||
|
"type": "GeometryCollection",
|
||||||
|
"geometries": []
|
||||||
|
}
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}`)
|
||||||
|
}
|
||||||
|
|
||||||
|
func spcRawEvent(t *testing.T, bundle spcprovider.RawConvectiveOutlookBundle) event.Event {
|
||||||
|
t.Helper()
|
||||||
|
raw, err := json.Marshal(bundle)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Marshal(bundle) error = %v", err)
|
||||||
|
}
|
||||||
|
effectiveAt := time.Date(2026, 6, 11, 19, 45, 0, 0, time.UTC)
|
||||||
|
return event.Event{
|
||||||
|
ID: "evt-spc-outlook-1",
|
||||||
|
Kind: event.Kind(standards.KindOutlook),
|
||||||
|
Source: "spc-test",
|
||||||
|
EmittedAt: time.Date(2026, 6, 11, 20, 5, 0, 0, time.UTC),
|
||||||
|
EffectiveAt: &effectiveAt,
|
||||||
|
Schema: standards.SchemaRawSPCConvectiveOutlookV1,
|
||||||
|
Payload: json.RawMessage(raw),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func spcBundle(t *testing.T, latitude, longitude float64) spcprovider.RawConvectiveOutlookBundle {
|
||||||
|
t.Helper()
|
||||||
|
fetchedAt := time.Date(2026, 6, 11, 20, 0, 0, 0, time.UTC)
|
||||||
|
products := make([]spcprovider.RawOutlookProduct, 0, len(spcprovider.GeoJSONProducts()))
|
||||||
|
for _, product := range spcprovider.GeoJSONProducts() {
|
||||||
|
products = append(products, spcprovider.RawOutlookProduct{
|
||||||
|
Key: product.Key,
|
||||||
|
Day: product.Day,
|
||||||
|
OutlookType: product.OutlookType,
|
||||||
|
URL: "https://example.invalid/" + product.Key + ".geojson",
|
||||||
|
FetchedAt: fetchedAt,
|
||||||
|
Body: json.RawMessage(geoJSONFixtureForProduct(t, product.Key)),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
return spcprovider.RawConvectiveOutlookBundle{
|
||||||
|
LocationID: "stl",
|
||||||
|
LocationName: "St. Louis, MO",
|
||||||
|
Latitude: latitude,
|
||||||
|
Longitude: longitude,
|
||||||
|
FetchedAt: fetchedAt,
|
||||||
|
Products: products,
|
||||||
|
Discussions: []spcprovider.RawDiscussionPage{
|
||||||
|
{Key: "day1", Day: 1, URL: "https://example.invalid/day1.html", FetchedAt: fetchedAt, Body: string(readSPCTestFixture(t, "day1_prt.html"))},
|
||||||
|
{Key: "day2", Day: 2, URL: "https://example.invalid/day2.html", FetchedAt: fetchedAt, Body: string(readSPCTestFixture(t, "day2_prt_corr.html"))},
|
||||||
|
{Key: "day3", Day: 3, URL: "https://example.invalid/day3.html", FetchedAt: fetchedAt, Body: string(readSPCTestFixture(t, "day3_prt.html"))},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func geoJSONFixtureForProduct(t *testing.T, key string) []byte {
|
||||||
|
t.Helper()
|
||||||
|
switch {
|
||||||
|
case strings.HasPrefix(key, "day1_"):
|
||||||
|
return readSPCTestFixture(t, "day1_cat.geojson")
|
||||||
|
case strings.HasPrefix(key, "day2_"):
|
||||||
|
return readSPCTestFixture(t, "day2_torn.geojson")
|
||||||
|
case strings.HasPrefix(key, "day3_"):
|
||||||
|
return readSPCTestFixture(t, "day3_cat.geojson")
|
||||||
|
default:
|
||||||
|
t.Fatalf("unknown product key %q", key)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func findOutlook(outlooks []model.WeatherOutlook, day int, outlookType string) *model.WeatherOutlook {
|
||||||
|
for i := range outlooks {
|
||||||
|
if outlooks[i].Day == day && outlooks[i].OutlookType == outlookType {
|
||||||
|
return &outlooks[i]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func countOutlooksByDay(outlooks []model.WeatherOutlook, day int) int {
|
||||||
|
count := 0
|
||||||
|
for _, outlook := range outlooks {
|
||||||
|
if outlook.Day == day {
|
||||||
|
count++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return count
|
||||||
|
}
|
||||||
|
|
||||||
|
func assertAllOutlooksContainLocation(t *testing.T, outlooks []model.WeatherOutlook) {
|
||||||
|
t.Helper()
|
||||||
|
for i, outlook := range outlooks {
|
||||||
|
if !outlook.ContainsLocation {
|
||||||
|
t.Fatalf("outlook[%d].ContainsLocation = false, want true", i)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func assertDiscussionDays(t *testing.T, discussions []model.WeatherOutlookDiscussion, want ...int) {
|
||||||
|
t.Helper()
|
||||||
|
if len(discussions) != len(want) {
|
||||||
|
t.Fatalf("Discussions length = %d, want %d", len(discussions), len(want))
|
||||||
|
}
|
||||||
|
for i, day := range want {
|
||||||
|
if discussions[i].Day != day {
|
||||||
|
t.Fatalf("Discussions[%d].Day = %d, want %d", i, discussions[i].Day, day)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func assertTime(t *testing.T, name string, got time.Time, year int, month time.Month, day int, hour int, minute int, second int) {
|
||||||
|
t.Helper()
|
||||||
|
want := time.Date(year, month, day, hour, minute, second, 0, time.UTC)
|
||||||
|
if !got.Equal(want) {
|
||||||
|
t.Fatalf("%s = %s, want %s", name, got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
17
internal/normalizers/spc/fixture_test.go
Normal file
17
internal/normalizers/spc/fixture_test.go
Normal file
@@ -0,0 +1,17 @@
|
|||||||
|
package spc
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func readSPCTestFixture(t *testing.T, name string) []byte {
|
||||||
|
t.Helper()
|
||||||
|
path := filepath.Join("..", "..", "providers", "spc", "testdata", name)
|
||||||
|
raw, err := os.ReadFile(path)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("read fixture %s: %v", path, err)
|
||||||
|
}
|
||||||
|
return raw
|
||||||
|
}
|
||||||
14
internal/normalizers/spc/register.go
Normal file
14
internal/normalizers/spc/register.go
Normal file
@@ -0,0 +1,14 @@
|
|||||||
|
package spc
|
||||||
|
|
||||||
|
import (
|
||||||
|
fknormalize "gitea.maximumdirect.net/ejr/feedkit/processors/normalize"
|
||||||
|
)
|
||||||
|
|
||||||
|
var builtins = []fknormalize.Normalizer{
|
||||||
|
ConvectiveOutlookNormalizer{},
|
||||||
|
}
|
||||||
|
|
||||||
|
// Register appends SPC normalizers in stable order.
|
||||||
|
func Register(in []fknormalize.Normalizer) []fknormalize.Normalizer {
|
||||||
|
return append(in, builtins...)
|
||||||
|
}
|
||||||
@@ -27,11 +27,34 @@ type ForecastDiscussionSection struct {
|
|||||||
Text string
|
Text string
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type forecastDiscussionSectionRole uint8
|
||||||
|
|
||||||
|
const (
|
||||||
|
forecastDiscussionSectionRoleKeyMessages forecastDiscussionSectionRole = iota
|
||||||
|
forecastDiscussionSectionRoleShortTerm
|
||||||
|
forecastDiscussionSectionRoleLongTerm
|
||||||
|
)
|
||||||
|
|
||||||
|
type forecastDiscussionSectionHeading struct {
|
||||||
|
section string
|
||||||
|
qualifier string
|
||||||
|
}
|
||||||
|
|
||||||
|
type forecastDiscussionSectionBlock struct {
|
||||||
|
heading forecastDiscussionSectionHeading
|
||||||
|
body []string
|
||||||
|
}
|
||||||
|
|
||||||
var (
|
var (
|
||||||
forecastDiscussionHeaderRE = regexp.MustCompile(`^\.(KEY MESSAGES|SHORT TERM|LONG TERM|AVIATION)\.\.\.(.*)$`)
|
forecastDiscussionSectionRoles = map[string]forecastDiscussionSectionRole{
|
||||||
forecastDiscussionAFDRE = regexp.MustCompile(`^AFD([A-Z]{3})$`)
|
"KEY MESSAGES": forecastDiscussionSectionRoleKeyMessages,
|
||||||
forecastDiscussionWMORE = regexp.MustCompile(`\bK([A-Z]{3})\b`)
|
"KEY POINTS": forecastDiscussionSectionRoleKeyMessages,
|
||||||
forecastDiscussionSigRE = regexp.MustCompile(`^[A-Z]{2,6}$`)
|
"SHORT TERM": forecastDiscussionSectionRoleShortTerm,
|
||||||
|
"LONG TERM": forecastDiscussionSectionRoleLongTerm,
|
||||||
|
}
|
||||||
|
forecastDiscussionAFDRE = regexp.MustCompile(`^AFD([A-Z]{3})$`)
|
||||||
|
forecastDiscussionWMORE = regexp.MustCompile(`\bK([A-Z]{3})\b`)
|
||||||
|
forecastDiscussionSigRE = regexp.MustCompile(`^[A-Z]{2,6}$`)
|
||||||
)
|
)
|
||||||
|
|
||||||
func ParseForecastDiscussionHTML(raw string) (ForecastDiscussion, error) {
|
func ParseForecastDiscussionHTML(raw string) (ForecastDiscussion, error) {
|
||||||
@@ -99,22 +122,30 @@ func ParseForecastDiscussionText(text string) (ForecastDiscussion, error) {
|
|||||||
IssuedAt: issuedAt.UTC(),
|
IssuedAt: issuedAt.UTC(),
|
||||||
}
|
}
|
||||||
|
|
||||||
if block, ok := extractForecastDiscussionSection(lines, "KEY MESSAGES"); ok {
|
seenRoles := make(map[forecastDiscussionSectionRole]bool, len(forecastDiscussionSectionRoles))
|
||||||
out.KeyMessages = parseForecastDiscussionKeyMessages(block)
|
for _, block := range parseForecastDiscussionSectionBlocks(lines) {
|
||||||
}
|
role, ok := forecastDiscussionSectionRoles[block.heading.section]
|
||||||
if block, ok := extractForecastDiscussionSection(lines, "SHORT TERM"); ok {
|
if !ok || seenRoles[role] {
|
||||||
section, err := parseForecastDiscussionTextSection(block)
|
continue
|
||||||
if err != nil {
|
|
||||||
return ForecastDiscussion{}, fmt.Errorf("parse SHORT TERM: %w", err)
|
|
||||||
}
|
}
|
||||||
out.ShortTerm = §ion
|
seenRoles[role] = true
|
||||||
}
|
|
||||||
if block, ok := extractForecastDiscussionSection(lines, "LONG TERM"); ok {
|
switch role {
|
||||||
section, err := parseForecastDiscussionTextSection(block)
|
case forecastDiscussionSectionRoleKeyMessages:
|
||||||
if err != nil {
|
out.KeyMessages = parseForecastDiscussionKeyMessages(block.body)
|
||||||
return ForecastDiscussion{}, fmt.Errorf("parse LONG TERM: %w", err)
|
case forecastDiscussionSectionRoleShortTerm:
|
||||||
|
section, err := parseForecastDiscussionTextSection(block)
|
||||||
|
if err != nil {
|
||||||
|
return ForecastDiscussion{}, fmt.Errorf("parse %s: %w", block.heading.section, err)
|
||||||
|
}
|
||||||
|
out.ShortTerm = §ion
|
||||||
|
case forecastDiscussionSectionRoleLongTerm:
|
||||||
|
section, err := parseForecastDiscussionTextSection(block)
|
||||||
|
if err != nil {
|
||||||
|
return ForecastDiscussion{}, fmt.Errorf("parse %s: %w", block.heading.section, err)
|
||||||
|
}
|
||||||
|
out.LongTerm = §ion
|
||||||
}
|
}
|
||||||
out.LongTerm = §ion
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return out, nil
|
return out, nil
|
||||||
@@ -285,8 +316,9 @@ func parseForecastDiscussionHeader(lines []string) (string, time.Time, error) {
|
|||||||
|
|
||||||
func parseForecastDiscussionIssueTime(line string) (time.Time, error) {
|
func parseForecastDiscussionIssueTime(line string) (time.Time, error) {
|
||||||
line = strings.TrimSpace(line)
|
line = strings.TrimSpace(line)
|
||||||
line = strings.TrimPrefix(line, "Issued at ")
|
if isForecastDiscussionIssuedAtLine(line) {
|
||||||
line = strings.TrimSpace(line)
|
line = strings.TrimSpace(line[len("Issued at"):])
|
||||||
|
}
|
||||||
|
|
||||||
parts := strings.Fields(line)
|
parts := strings.Fields(line)
|
||||||
if len(parts) != 7 {
|
if len(parts) != 7 {
|
||||||
@@ -386,40 +418,217 @@ func forecastDiscussionLocation(abbrev string) (*time.Location, error) {
|
|||||||
return time.FixedZone(abbr, offset), nil
|
return time.FixedZone(abbr, offset), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func extractForecastDiscussionSection(lines []string, section string) ([]string, bool) {
|
func parseForecastDiscussionSectionHeading(line string) (forecastDiscussionSectionHeading, bool) {
|
||||||
target := "." + section + "..."
|
line = strings.TrimSpace(line)
|
||||||
for i, raw := range lines {
|
if len(line) < 2 || line[0] != '.' {
|
||||||
|
return forecastDiscussionSectionHeading{}, false
|
||||||
|
}
|
||||||
|
|
||||||
|
if strings.HasSuffix(line, "/...") {
|
||||||
|
return parseForecastDiscussionSlashQualifiedHeading(line)
|
||||||
|
}
|
||||||
|
if strings.HasSuffix(line, "...") {
|
||||||
|
if heading, ok := parseForecastDiscussionParenthesizedTerminalHeading(line); ok {
|
||||||
|
return heading, true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return parseForecastDiscussionEllipsisHeading(line)
|
||||||
|
}
|
||||||
|
|
||||||
|
func parseForecastDiscussionSlashQualifiedHeading(line string) (forecastDiscussionSectionHeading, bool) {
|
||||||
|
content := strings.TrimSuffix(line[1:], "/...")
|
||||||
|
separator := -1
|
||||||
|
for i := 1; i < len(content); i++ {
|
||||||
|
if content[i] == '/' && isForecastDiscussionHorizontalWhitespace(content[i-1]) {
|
||||||
|
separator = i
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if separator < 0 {
|
||||||
|
return forecastDiscussionSectionHeading{}, false
|
||||||
|
}
|
||||||
|
|
||||||
|
section, ok := normalizeForecastDiscussionSectionIdentity(content[:separator])
|
||||||
|
if !ok {
|
||||||
|
return forecastDiscussionSectionHeading{}, false
|
||||||
|
}
|
||||||
|
qualifier := strings.TrimSpace(content[separator+1:])
|
||||||
|
if qualifier == "" {
|
||||||
|
return forecastDiscussionSectionHeading{}, false
|
||||||
|
}
|
||||||
|
return forecastDiscussionSectionHeading{section: section, qualifier: qualifier}, true
|
||||||
|
}
|
||||||
|
|
||||||
|
func parseForecastDiscussionParenthesizedTerminalHeading(line string) (forecastDiscussionSectionHeading, bool) {
|
||||||
|
if len(line) < 4 || line[0] != '.' || !strings.HasSuffix(line, "...") {
|
||||||
|
return forecastDiscussionSectionHeading{}, false
|
||||||
|
}
|
||||||
|
|
||||||
|
content := strings.TrimRight(line[1:len(line)-3], " \t")
|
||||||
|
if !strings.HasSuffix(content, ")") {
|
||||||
|
return forecastDiscussionSectionHeading{}, false
|
||||||
|
}
|
||||||
|
|
||||||
|
separator := -1
|
||||||
|
for i := 1; i < len(content); i++ {
|
||||||
|
if content[i] == '(' && isForecastDiscussionHorizontalWhitespace(content[i-1]) {
|
||||||
|
separator = i
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if separator < 0 {
|
||||||
|
return forecastDiscussionSectionHeading{}, false
|
||||||
|
}
|
||||||
|
|
||||||
|
section, ok := normalizeForecastDiscussionSectionIdentity(content[:separator])
|
||||||
|
if !ok {
|
||||||
|
return forecastDiscussionSectionHeading{}, false
|
||||||
|
}
|
||||||
|
qualifier := content[separator:]
|
||||||
|
if len(qualifier) <= 2 || strings.TrimSpace(qualifier[1:len(qualifier)-1]) == "" {
|
||||||
|
return forecastDiscussionSectionHeading{}, false
|
||||||
|
}
|
||||||
|
return forecastDiscussionSectionHeading{section: section, qualifier: qualifier}, true
|
||||||
|
}
|
||||||
|
|
||||||
|
func parseForecastDiscussionEllipsisHeading(line string) (forecastDiscussionSectionHeading, bool) {
|
||||||
|
content := line[1:]
|
||||||
|
delimiter := strings.Index(content, "...")
|
||||||
|
if delimiter < 0 {
|
||||||
|
return forecastDiscussionSectionHeading{}, false
|
||||||
|
}
|
||||||
|
|
||||||
|
section, ok := normalizeForecastDiscussionSectionIdentity(content[:delimiter])
|
||||||
|
if !ok {
|
||||||
|
return forecastDiscussionSectionHeading{}, false
|
||||||
|
}
|
||||||
|
return forecastDiscussionSectionHeading{
|
||||||
|
section: section,
|
||||||
|
qualifier: strings.TrimSpace(content[delimiter+3:]),
|
||||||
|
}, true
|
||||||
|
}
|
||||||
|
|
||||||
|
func normalizeForecastDiscussionSectionIdentity(raw string) (string, bool) {
|
||||||
|
var normalized strings.Builder
|
||||||
|
pendingSpace := false
|
||||||
|
hasLetterOrDigit := false
|
||||||
|
|
||||||
|
for i := 0; i < len(raw); i++ {
|
||||||
|
b := raw[i]
|
||||||
|
switch {
|
||||||
|
case isForecastDiscussionIdentityLetterOrDigit(b):
|
||||||
|
hasLetterOrDigit = true
|
||||||
|
case b == ' ' || b == '\t':
|
||||||
|
pendingSpace = normalized.Len() > 0
|
||||||
|
continue
|
||||||
|
case b == '/' && i > 0 && isForecastDiscussionHorizontalWhitespace(raw[i-1]):
|
||||||
|
return "", false
|
||||||
|
case b != '/' && b != '&' && b != '\'' && b != '-':
|
||||||
|
return "", false
|
||||||
|
}
|
||||||
|
|
||||||
|
if pendingSpace {
|
||||||
|
normalized.WriteByte(' ')
|
||||||
|
pendingSpace = false
|
||||||
|
}
|
||||||
|
normalized.WriteByte(b)
|
||||||
|
}
|
||||||
|
if !hasLetterOrDigit {
|
||||||
|
return "", false
|
||||||
|
}
|
||||||
|
return normalized.String(), true
|
||||||
|
}
|
||||||
|
|
||||||
|
func isForecastDiscussionIdentityLetterOrDigit(b byte) bool {
|
||||||
|
return b >= 'A' && b <= 'Z' || b >= '0' && b <= '9'
|
||||||
|
}
|
||||||
|
|
||||||
|
func isForecastDiscussionHorizontalWhitespace(b byte) bool {
|
||||||
|
return b == ' ' || b == '\t'
|
||||||
|
}
|
||||||
|
|
||||||
|
func parseForecastDiscussionSectionBlocks(lines []string) []forecastDiscussionSectionBlock {
|
||||||
|
var blocks []forecastDiscussionSectionBlock
|
||||||
|
var active *forecastDiscussionSectionBlock
|
||||||
|
embeddedHeadings := false
|
||||||
|
|
||||||
|
finish := func() {
|
||||||
|
if active == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
blocks = append(blocks, *active)
|
||||||
|
active = nil
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, raw := range lines {
|
||||||
line := strings.TrimSpace(raw)
|
line := strings.TrimSpace(raw)
|
||||||
if !strings.HasPrefix(line, target) {
|
if line == "$$" {
|
||||||
|
finish()
|
||||||
|
break
|
||||||
|
}
|
||||||
|
if line == "&&" || strings.Contains(line, "WATCHES/WARNINGS/ADVISORIES") {
|
||||||
|
finish()
|
||||||
|
embeddedHeadings = false
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
out := []string{line}
|
heading, ok := parseForecastDiscussionSectionHeading(raw)
|
||||||
for j := i + 1; j < len(lines); j++ {
|
if ok {
|
||||||
next := strings.TrimSpace(lines[j])
|
finish()
|
||||||
if next == "&&" || next == "$$" || strings.Contains(next, "WATCHES/WARNINGS/ADVISORIES") {
|
active = &forecastDiscussionSectionBlock{heading: heading}
|
||||||
break
|
embeddedHeadings = isForecastDiscussionEmbeddedSectionWrapper(heading.section)
|
||||||
}
|
continue
|
||||||
if j > i+1 && isForecastDiscussionSectionHeader(next) {
|
}
|
||||||
break
|
if embeddedHeadings {
|
||||||
}
|
heading, ok = parseForecastDiscussionEmbeddedSectionHeading(raw)
|
||||||
out = append(out, lines[j])
|
if ok {
|
||||||
|
finish()
|
||||||
|
active = &forecastDiscussionSectionBlock{heading: heading}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if active != nil {
|
||||||
|
active.body = append(active.body, raw)
|
||||||
}
|
}
|
||||||
return out, true
|
|
||||||
}
|
}
|
||||||
return nil, false
|
finish()
|
||||||
|
|
||||||
|
return blocks
|
||||||
}
|
}
|
||||||
|
|
||||||
func isForecastDiscussionSectionHeader(line string) bool {
|
func isForecastDiscussionEmbeddedSectionWrapper(section string) bool {
|
||||||
return forecastDiscussionHeaderRE.MatchString(strings.TrimSpace(line))
|
return section == "PREV DISCUSSION"
|
||||||
}
|
}
|
||||||
|
|
||||||
func parseForecastDiscussionKeyMessages(block []string) []string {
|
func parseForecastDiscussionEmbeddedSectionHeading(line string) (forecastDiscussionSectionHeading, bool) {
|
||||||
if len(block) <= 1 {
|
line = strings.TrimSpace(line)
|
||||||
|
if line == "" || line[0] == '.' {
|
||||||
|
return forecastDiscussionSectionHeading{}, false
|
||||||
|
}
|
||||||
|
return parseForecastDiscussionSectionHeading("." + line)
|
||||||
|
}
|
||||||
|
|
||||||
|
func parseForecastDiscussionKeyMessages(body []string) []string {
|
||||||
|
body = removeForecastDiscussionPresentationMarkers(body)
|
||||||
|
body = trimBlankLines(body)
|
||||||
|
if len(body) > 0 && isForecastDiscussionKeyMessageMetadataLine(body[0]) {
|
||||||
|
body = trimBlankLines(body[1:])
|
||||||
|
}
|
||||||
|
if len(body) == 0 {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
hasMarkers := false
|
||||||
|
for _, raw := range body {
|
||||||
|
line := strings.TrimSpace(raw)
|
||||||
|
if line == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if _, ok := stripForecastDiscussionKeyMessageMarker(line); ok {
|
||||||
|
hasMarkers = true
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
body := trimBlankLines(block[1:])
|
|
||||||
var messages []string
|
var messages []string
|
||||||
var current strings.Builder
|
var current strings.Builder
|
||||||
|
|
||||||
@@ -431,15 +640,21 @@ func parseForecastDiscussionKeyMessages(block []string) []string {
|
|||||||
current.Reset()
|
current.Reset()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
seenMarker := false
|
||||||
for _, raw := range body {
|
for _, raw := range body {
|
||||||
line := strings.TrimSpace(raw)
|
line := strings.TrimSpace(raw)
|
||||||
if line == "" {
|
if line == "" {
|
||||||
|
if !hasMarkers || !seenMarker {
|
||||||
|
flush()
|
||||||
|
}
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
if strings.HasPrefix(line, "-") {
|
if stripped, ok := stripForecastDiscussionKeyMessageMarker(line); ok {
|
||||||
flush()
|
flush()
|
||||||
line = strings.TrimSpace(strings.TrimPrefix(line, "-"))
|
seenMarker = true
|
||||||
current.WriteString(line)
|
line = stripped
|
||||||
|
}
|
||||||
|
if line == "" {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
if current.Len() > 0 {
|
if current.Len() > 0 {
|
||||||
@@ -452,25 +667,24 @@ func parseForecastDiscussionKeyMessages(block []string) []string {
|
|||||||
return messages
|
return messages
|
||||||
}
|
}
|
||||||
|
|
||||||
func parseForecastDiscussionTextSection(block []string) (ForecastDiscussionSection, error) {
|
func parseForecastDiscussionTextSection(block forecastDiscussionSectionBlock) (ForecastDiscussionSection, error) {
|
||||||
if len(block) == 0 {
|
|
||||||
return ForecastDiscussionSection{}, fmt.Errorf("empty section")
|
|
||||||
}
|
|
||||||
|
|
||||||
section := ForecastDiscussionSection{
|
section := ForecastDiscussionSection{
|
||||||
Qualifier: parseForecastDiscussionQualifier(strings.TrimSpace(block[0])),
|
Qualifier: block.heading.qualifier,
|
||||||
}
|
}
|
||||||
|
|
||||||
body := trimBlankLines(block[1:])
|
body := trimBlankLines(removeForecastDiscussionPresentationMarkers(block.body))
|
||||||
|
if section.Qualifier == "" && len(body) > 0 && isForecastDiscussionStandaloneParenthetical(body[0]) {
|
||||||
|
section.Qualifier = strings.TrimSpace(body[0])
|
||||||
|
body = trimBlankLines(body[1:])
|
||||||
|
}
|
||||||
if len(body) == 0 {
|
if len(body) == 0 {
|
||||||
return section, nil
|
return section, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
first := strings.TrimSpace(body[0])
|
if isForecastDiscussionIssuedAtLine(body[0]) {
|
||||||
if strings.HasPrefix(first, "Issued at ") {
|
issuedAt, err := parseForecastDiscussionIssueTime(body[0])
|
||||||
issuedAt, err := parseForecastDiscussionIssueTime(first)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return ForecastDiscussionSection{}, fmt.Errorf("parse section issuedAt %q: %w", first, err)
|
return ForecastDiscussionSection{}, fmt.Errorf("parse section issuedAt %q: %w", strings.TrimSpace(body[0]), err)
|
||||||
}
|
}
|
||||||
tt := issuedAt.UTC()
|
tt := issuedAt.UTC()
|
||||||
section.IssuedAt = &tt
|
section.IssuedAt = &tt
|
||||||
@@ -482,12 +696,147 @@ func parseForecastDiscussionTextSection(block []string) (ForecastDiscussionSecti
|
|||||||
return section, nil
|
return section, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func parseForecastDiscussionQualifier(header string) string {
|
func isForecastDiscussionPresentationMarker(line string) bool {
|
||||||
m := forecastDiscussionHeaderRE.FindStringSubmatch(header)
|
switch {
|
||||||
if len(m) != 3 {
|
case strings.EqualFold(strings.TrimSpace(line), "-- Changed Discussion --"):
|
||||||
return ""
|
return true
|
||||||
|
case strings.EqualFold(strings.TrimSpace(line), "-- End Changed Discussion --"):
|
||||||
|
return true
|
||||||
|
default:
|
||||||
|
return false
|
||||||
}
|
}
|
||||||
return strings.TrimSpace(m[2])
|
}
|
||||||
|
|
||||||
|
func removeForecastDiscussionPresentationMarkers(lines []string) []string {
|
||||||
|
body := make([]string, 0, len(lines))
|
||||||
|
for _, line := range lines {
|
||||||
|
if !isForecastDiscussionPresentationMarker(line) {
|
||||||
|
body = append(body, line)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return body
|
||||||
|
}
|
||||||
|
|
||||||
|
func isForecastDiscussionStandaloneParenthetical(line string) bool {
|
||||||
|
line = strings.TrimSpace(line)
|
||||||
|
return len(line) > 2 && line[0] == '(' && line[len(line)-1] == ')' && strings.TrimSpace(line[1:len(line)-1]) != ""
|
||||||
|
}
|
||||||
|
|
||||||
|
func isForecastDiscussionIssuedAtLine(line string) bool {
|
||||||
|
line = strings.TrimSpace(line)
|
||||||
|
return len(line) > len("Issued at") &&
|
||||||
|
strings.EqualFold(line[:len("Issued at")], "Issued at") &&
|
||||||
|
isForecastDiscussionHorizontalWhitespace(line[len("Issued at")])
|
||||||
|
}
|
||||||
|
|
||||||
|
func isForecastDiscussionKeyMessageMetadataLine(line string) bool {
|
||||||
|
line = strings.TrimSpace(line)
|
||||||
|
for _, label := range []string{"Issued at", "Updated at"} {
|
||||||
|
if !hasForecastDiscussionASCIIPrefix(line, label) || len(line) == len(label) || !isForecastDiscussionHorizontalWhitespace(line[len(label)]) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if _, err := parseForecastDiscussionIssueTime(strings.TrimSpace(line[len(label):])); err == nil {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return isForecastDiscussionKeyMessageAsOfLine(line)
|
||||||
|
}
|
||||||
|
|
||||||
|
func isForecastDiscussionKeyMessageAsOfLine(line string) bool {
|
||||||
|
const label = "As of"
|
||||||
|
|
||||||
|
if !hasForecastDiscussionASCIIPrefix(line, label) || len(line) == len(label) || !isForecastDiscussionHorizontalWhitespace(line[len(label)]) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
remainder := strings.TrimSpace(line[len(label):])
|
||||||
|
if !strings.HasSuffix(remainder, "...") {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
fields := strings.Fields(strings.TrimSpace(strings.TrimSuffix(remainder, "...")))
|
||||||
|
if len(fields) != 3 {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
if _, _, err := parseForecastDiscussionClock(fields[0], fields[1]); err != nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
switch strings.ToLower(fields[2]) {
|
||||||
|
case "monday", "tuesday", "wednesday", "thursday", "friday", "saturday", "sunday":
|
||||||
|
return true
|
||||||
|
default:
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func stripForecastDiscussionKeyMessageMarker(line string) (string, bool) {
|
||||||
|
if line == "" {
|
||||||
|
return "", false
|
||||||
|
}
|
||||||
|
|
||||||
|
if line[0] == '-' || line[0] == '*' {
|
||||||
|
content := line[1:]
|
||||||
|
hadWhitespace := len(content) > 0 && isForecastDiscussionHorizontalWhitespace(content[0])
|
||||||
|
content = strings.TrimLeft(content, " \t")
|
||||||
|
if hadWhitespace {
|
||||||
|
if stripped, ok := stripForecastDiscussionKeyMessageNumericMarker(content); ok {
|
||||||
|
content = stripped
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return content, true
|
||||||
|
}
|
||||||
|
|
||||||
|
return stripForecastDiscussionKeyMessageNumericMarker(line)
|
||||||
|
}
|
||||||
|
|
||||||
|
func stripForecastDiscussionKeyMessageNumericMarker(line string) (string, bool) {
|
||||||
|
digitStart := 0
|
||||||
|
digitEnd := 0
|
||||||
|
parenthesized := len(line) > 0 && line[0] == '('
|
||||||
|
if parenthesized {
|
||||||
|
digitStart = 1
|
||||||
|
digitEnd = 1
|
||||||
|
}
|
||||||
|
for digitEnd < len(line) && line[digitEnd] >= '0' && line[digitEnd] <= '9' {
|
||||||
|
digitEnd++
|
||||||
|
}
|
||||||
|
if digitEnd == digitStart || digitEnd == len(line) {
|
||||||
|
return "", false
|
||||||
|
}
|
||||||
|
if parenthesized && line[digitEnd] != ')' {
|
||||||
|
return "", false
|
||||||
|
}
|
||||||
|
if !parenthesized && line[digitEnd] != ')' && line[digitEnd] != '.' {
|
||||||
|
return "", false
|
||||||
|
}
|
||||||
|
|
||||||
|
markerEnd := digitEnd + 1
|
||||||
|
if markerEnd < len(line) && !isForecastDiscussionHorizontalWhitespace(line[markerEnd]) {
|
||||||
|
return "", false
|
||||||
|
}
|
||||||
|
value, err := strconv.ParseUint(line[digitStart:digitEnd], 10, 0)
|
||||||
|
if err != nil || value == 0 {
|
||||||
|
return "", false
|
||||||
|
}
|
||||||
|
return strings.TrimLeft(line[markerEnd:], " \t"), true
|
||||||
|
}
|
||||||
|
|
||||||
|
func hasForecastDiscussionASCIIPrefix(line, prefix string) bool {
|
||||||
|
if len(line) < len(prefix) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
for i := range prefix {
|
||||||
|
actual := line[i]
|
||||||
|
if actual >= 'A' && actual <= 'Z' {
|
||||||
|
actual += 'a' - 'A'
|
||||||
|
}
|
||||||
|
expected := prefix[i]
|
||||||
|
if expected >= 'A' && expected <= 'Z' {
|
||||||
|
expected += 'a' - 'A'
|
||||||
|
}
|
||||||
|
if actual != expected {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true
|
||||||
}
|
}
|
||||||
|
|
||||||
func trimBlankLines(lines []string) []string {
|
func trimBlankLines(lines []string) []string {
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
26
internal/providers/nws/testdata/forecast_discussion_bgm_numbered_sample.html
vendored
Normal file
26
internal/providers/nws/testdata/forecast_discussion_bgm_numbered_sample.html
vendored
Normal file
@@ -0,0 +1,26 @@
|
|||||||
|
<!DOCTYPE html>
|
||||||
|
<!-- Representative BGM/CTP-style layout; prose is concise edited test data, not an archived product. -->
|
||||||
|
<html>
|
||||||
|
<body>
|
||||||
|
<pre class="glossaryProduct">
|
||||||
|
FXUS61 KBGM 101730
|
||||||
|
AFDBGM
|
||||||
|
|
||||||
|
Area Forecast Discussion
|
||||||
|
National Weather Service Binghamton NY
|
||||||
|
130 PM EDT Fri Apr 10 2026
|
||||||
|
|
||||||
|
.KEY MESSAGES...
|
||||||
|
1) Periods of rain are expected through Saturday,
|
||||||
|
with locally heavier amounts possible.
|
||||||
|
2. Cooler temperatures return late this weekend.
|
||||||
|
|
||||||
|
.DISCUSSION...
|
||||||
|
Discussion details remain boundary-only content.
|
||||||
|
|
||||||
|
$$
|
||||||
|
|
||||||
|
WFO BGM
|
||||||
|
</pre>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
48
internal/providers/nws/testdata/forecast_discussion_bou_sample.html
vendored
Normal file
48
internal/providers/nws/testdata/forecast_discussion_bou_sample.html
vendored
Normal file
@@ -0,0 +1,48 @@
|
|||||||
|
<!DOCTYPE html>
|
||||||
|
<html>
|
||||||
|
<body>
|
||||||
|
<pre class="glossaryProduct">
|
||||||
|
FXUS65 KBOU 071900
|
||||||
|
AFDBOU
|
||||||
|
|
||||||
|
Area Forecast Discussion
|
||||||
|
National Weather Service Denver CO
|
||||||
|
100 PM MDT Tue Apr 7 2026
|
||||||
|
|
||||||
|
.KEY MESSAGES...
|
||||||
|
-- Changed Discussion --
|
||||||
|
Updated at 100 PM MDT Tue Apr 7 2026
|
||||||
|
- Strong winds are expected along the Front Range this evening.
|
||||||
|
- Cooler temperatures arrive on Wednesday.
|
||||||
|
-- End Changed Discussion --
|
||||||
|
|
||||||
|
&&
|
||||||
|
|
||||||
|
.SHORT TERM...
|
||||||
|
(Tonight through Wednesday)
|
||||||
|
Issued at 100 PM MDT Tue Apr 7 2026
|
||||||
|
|
||||||
|
Gusty west winds will continue through the evening before decreasing overnight.
|
||||||
|
|
||||||
|
&&
|
||||||
|
|
||||||
|
.LONG TERM...
|
||||||
|
(Thursday through Saturday)
|
||||||
|
ISSUED AT 100 PM MDT Tue Apr 7 2026
|
||||||
|
|
||||||
|
Warmer and drier conditions return Thursday, followed by a chance of showers Friday.
|
||||||
|
|
||||||
|
&&
|
||||||
|
|
||||||
|
.AVIATION...
|
||||||
|
|
||||||
|
VFR conditions are expected at Denver-area terminals through Wednesday morning.
|
||||||
|
|
||||||
|
&&
|
||||||
|
|
||||||
|
$$
|
||||||
|
|
||||||
|
WFO BOU
|
||||||
|
</pre>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
27
internal/providers/nws/testdata/forecast_discussion_lwx_parenthesized_number_sample.html
vendored
Normal file
27
internal/providers/nws/testdata/forecast_discussion_lwx_parenthesized_number_sample.html
vendored
Normal file
@@ -0,0 +1,27 @@
|
|||||||
|
<!DOCTYPE html>
|
||||||
|
<!-- Representative LWX-style layout; prose is concise edited test data, not an archived product. -->
|
||||||
|
<html>
|
||||||
|
<body>
|
||||||
|
<pre class="glossaryProduct">
|
||||||
|
FXUS61 KLWX 021800
|
||||||
|
AFDLWX
|
||||||
|
|
||||||
|
Area Forecast Discussion
|
||||||
|
National Weather Service Baltimore MD/Washington DC
|
||||||
|
200 PM EDT Sun Aug 2 2026
|
||||||
|
|
||||||
|
.KEY MESSAGES...
|
||||||
|
- (1) Thunderstorms remain possible near the Blue Ridge this evening.
|
||||||
|
- (2) Seasonably warm conditions continue Monday.
|
||||||
|
|
||||||
|
&&
|
||||||
|
|
||||||
|
.AVIATION...
|
||||||
|
Aviation details remain boundary-only content.
|
||||||
|
|
||||||
|
$$
|
||||||
|
|
||||||
|
WFO LWX
|
||||||
|
</pre>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
25
internal/providers/nws/testdata/forecast_discussion_mfr_key_points_sample.html
vendored
Normal file
25
internal/providers/nws/testdata/forecast_discussion_mfr_key_points_sample.html
vendored
Normal file
@@ -0,0 +1,25 @@
|
|||||||
|
<!DOCTYPE html>
|
||||||
|
<!-- Representative MFR-style layout; prose is concise edited test data, not an archived product. -->
|
||||||
|
<html>
|
||||||
|
<body>
|
||||||
|
<pre class="glossaryProduct">
|
||||||
|
FXUS66 KMFR 101945
|
||||||
|
AFDMFR
|
||||||
|
|
||||||
|
Area Forecast Discussion
|
||||||
|
National Weather Service Medford OR
|
||||||
|
1245 PM PDT Fri Apr 10 2026
|
||||||
|
|
||||||
|
.KEY POINTS...
|
||||||
|
* Gusty winds will develop over exposed ridges,
|
||||||
|
especially during the afternoon.
|
||||||
|
* Inland valleys remain dry through Saturday.
|
||||||
|
.DISCUSSION (Today through Thursday)...
|
||||||
|
Discussion details must not be included with key points.
|
||||||
|
|
||||||
|
$$
|
||||||
|
|
||||||
|
WFO MFR
|
||||||
|
</pre>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
33
internal/providers/nws/testdata/forecast_discussion_mfr_prev_discussion_sample.html
vendored
Normal file
33
internal/providers/nws/testdata/forecast_discussion_mfr_prev_discussion_sample.html
vendored
Normal file
@@ -0,0 +1,33 @@
|
|||||||
|
<!DOCTYPE html>
|
||||||
|
<!-- Representative current MFR previous-discussion wrapper layout; prose is concise edited test data, not an archived product. -->
|
||||||
|
<html>
|
||||||
|
<body>
|
||||||
|
<pre class="glossaryProduct">
|
||||||
|
FXUS66 KMFR 022219
|
||||||
|
AFDMFR
|
||||||
|
|
||||||
|
Area Forecast Discussion
|
||||||
|
National Weather Service Medford OR
|
||||||
|
319 PM PDT Sun Aug 2 2026
|
||||||
|
|
||||||
|
.PREV DISCUSSION... /Issued 319 PM PDT Sun Aug 2 2026/
|
||||||
|
|
||||||
|
KEY MESSAGES...
|
||||||
|
|
||||||
|
* Heat returns to inland valleys Monday.
|
||||||
|
* Gusty afternoon winds develop east of the Cascades.
|
||||||
|
|
||||||
|
DISCUSSION...
|
||||||
|
Discussion details must not be included with key messages.
|
||||||
|
|
||||||
|
&&
|
||||||
|
|
||||||
|
.MFR WATCHES/WARNINGS/ADVISORIES...
|
||||||
|
None.
|
||||||
|
|
||||||
|
$$
|
||||||
|
|
||||||
|
WFO MFR
|
||||||
|
</pre>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
29
internal/providers/nws/testdata/forecast_discussion_rah_as_of_sample.html
vendored
Normal file
29
internal/providers/nws/testdata/forecast_discussion_rah_as_of_sample.html
vendored
Normal file
@@ -0,0 +1,29 @@
|
|||||||
|
<!DOCTYPE html>
|
||||||
|
<!-- Representative RAH-style layout; prose is concise edited test data, not an archived product. -->
|
||||||
|
<html>
|
||||||
|
<body>
|
||||||
|
<pre class="glossaryProduct">
|
||||||
|
FXUS62 KRAH 021635
|
||||||
|
AFDRAH
|
||||||
|
|
||||||
|
Area Forecast Discussion
|
||||||
|
National Weather Service Raleigh NC
|
||||||
|
1235 PM EDT Sun Aug 2 2026
|
||||||
|
|
||||||
|
.KEY MESSAGES...
|
||||||
|
As of 1235 PM Sunday...
|
||||||
|
|
||||||
|
1) Scattered storms may produce locally heavy rain this afternoon.
|
||||||
|
2) Drier weather arrives Monday.
|
||||||
|
|
||||||
|
&&
|
||||||
|
|
||||||
|
.DISCUSSION...
|
||||||
|
Discussion details remain boundary-only content.
|
||||||
|
|
||||||
|
$$
|
||||||
|
|
||||||
|
WFO RAH
|
||||||
|
</pre>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
188
internal/providers/spc/discussion.go
Normal file
188
internal/providers/spc/discussion.go
Normal file
@@ -0,0 +1,188 @@
|
|||||||
|
package spc
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"html"
|
||||||
|
"regexp"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
var (
|
||||||
|
scriptBlockRE = regexp.MustCompile(`(?is)<script\b[^>]*>.*?</script>`)
|
||||||
|
preBlockRE = regexp.MustCompile(`(?is)<pre\b[^>]*>(.*?)</pre>`)
|
||||||
|
tagRE = regexp.MustCompile(`(?is)<[^>]+>`)
|
||||||
|
updatedRE = regexp.MustCompile(`(?im)^\s*Updated:\s*(.+?)\s*$`)
|
||||||
|
pageUpdatedRE = regexp.MustCompile(`(?i)\bUpdated:\s*((?:\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z)|(?:[A-Z][a-z]{2}\s+[A-Z][a-z]{2}\s+\d{1,2}\s+\d{2}:\d{2}:\d{2}\s+UTC\s+\d{4})|(?:\d{4}\s+UTC\s+[A-Z][a-z]{2}\s+[A-Z][a-z]{2}\s+\d{1,2}\s+\d{4})|(?:\d{4}Z\s+[A-Z][a-z]{2}\s+[A-Z][a-z]{2}\s+\d{1,2}\s+\d{4}))`)
|
||||||
|
productCodeRE = regexp.MustCompile(`(?i)^SPC\s+AC\s+\d+\s*$`)
|
||||||
|
sectionRE = regexp.MustCompile(`^\s*\.\.\.[A-Z0-9 /-]+\.{3}\s*$`)
|
||||||
|
)
|
||||||
|
|
||||||
|
// DiscussionText contains parsed text from an SPC print page.
|
||||||
|
type DiscussionText struct {
|
||||||
|
ProductTitle string
|
||||||
|
Headline string
|
||||||
|
Summary string
|
||||||
|
Discussion string
|
||||||
|
UpdatedAt *time.Time
|
||||||
|
}
|
||||||
|
|
||||||
|
// ExtractProductText extracts and cleans the first useful preformatted SPC
|
||||||
|
// product text block from a print page.
|
||||||
|
func ExtractProductText(rawHTML string) (string, error) {
|
||||||
|
matches := preBlockRE.FindAllStringSubmatch(rawHTML, -1)
|
||||||
|
for _, match := range matches {
|
||||||
|
if len(match) < 2 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
text := cleanHTMLText(match[1])
|
||||||
|
if strings.TrimSpace(text) != "" {
|
||||||
|
return text, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return "", fmt.Errorf("no useful pre block found")
|
||||||
|
}
|
||||||
|
|
||||||
|
// ParseDiscussionHTML extracts SPC product text and page-level metadata from a
|
||||||
|
// print-page HTML document.
|
||||||
|
func ParseDiscussionHTML(rawHTML string) (DiscussionText, error) {
|
||||||
|
text, err := ExtractProductText(rawHTML)
|
||||||
|
if err != nil {
|
||||||
|
return DiscussionText{}, err
|
||||||
|
}
|
||||||
|
parsed := ParseDiscussionText(text)
|
||||||
|
if updatedAt := ParsePageUpdatedTimestamp(rawHTML); updatedAt != nil {
|
||||||
|
parsed.UpdatedAt = updatedAt
|
||||||
|
}
|
||||||
|
return parsed, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// ParseDiscussionText extracts common SPC narrative metadata from cleaned
|
||||||
|
// product text.
|
||||||
|
func ParseDiscussionText(text string) DiscussionText {
|
||||||
|
text = trimBlankLines(normalizeNewlines(text))
|
||||||
|
title := ParseProductTitle(text)
|
||||||
|
return DiscussionText{
|
||||||
|
ProductTitle: title,
|
||||||
|
Headline: title,
|
||||||
|
Summary: ExtractSummary(text),
|
||||||
|
Discussion: text,
|
||||||
|
UpdatedAt: ParseUpdatedTimestamp(text),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ParsePageUpdatedTimestamp parses the page-level Updated row from an SPC print
|
||||||
|
// page. SPC currently places this outside the product <pre> block.
|
||||||
|
func ParsePageUpdatedTimestamp(rawHTML string) *time.Time {
|
||||||
|
text := cleanHTMLText(rawHTML)
|
||||||
|
text = strings.ReplaceAll(text, "\u00a0", " ")
|
||||||
|
text = strings.Join(strings.Fields(text), " ")
|
||||||
|
match := pageUpdatedRE.FindStringSubmatch(text)
|
||||||
|
if len(match) != 2 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return parseUpdatedValue(match[1])
|
||||||
|
}
|
||||||
|
|
||||||
|
// ParseUpdatedTimestamp parses an SPC print-page Updated line when present.
|
||||||
|
func ParseUpdatedTimestamp(text string) *time.Time {
|
||||||
|
match := updatedRE.FindStringSubmatch(normalizeNewlines(text))
|
||||||
|
if len(match) != 2 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return parseUpdatedValue(match[1])
|
||||||
|
}
|
||||||
|
|
||||||
|
// ParseProductTitle returns the first non-empty product line from cleaned text.
|
||||||
|
func ParseProductTitle(text string) string {
|
||||||
|
for _, line := range strings.Split(normalizeNewlines(text), "\n") {
|
||||||
|
line = strings.TrimSpace(line)
|
||||||
|
if line == "" || strings.HasPrefix(line, "Updated:") || productCodeRE.MatchString(line) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
return line
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
// ParseHeadline returns the human-facing headline from cleaned text.
|
||||||
|
func ParseHeadline(text string) string {
|
||||||
|
return ParseProductTitle(text)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ExtractSummary returns text under the ...SUMMARY... section through the next
|
||||||
|
// SPC section heading.
|
||||||
|
func ExtractSummary(text string) string {
|
||||||
|
lines := strings.Split(normalizeNewlines(text), "\n")
|
||||||
|
start := -1
|
||||||
|
for i, line := range lines {
|
||||||
|
if strings.EqualFold(strings.TrimSpace(line), "...SUMMARY...") {
|
||||||
|
start = i + 1
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if start < 0 {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
var out []string
|
||||||
|
for _, line := range lines[start:] {
|
||||||
|
if sectionRE.MatchString(line) {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
out = append(out, line)
|
||||||
|
}
|
||||||
|
return trimBlankLines(strings.Join(out, "\n"))
|
||||||
|
}
|
||||||
|
|
||||||
|
func cleanHTMLText(raw string) string {
|
||||||
|
raw = scriptBlockRE.ReplaceAllString(raw, "")
|
||||||
|
raw = tagRE.ReplaceAllString(raw, "")
|
||||||
|
raw = html.UnescapeString(raw)
|
||||||
|
raw = normalizeNewlines(raw)
|
||||||
|
return trimBlankLines(raw)
|
||||||
|
}
|
||||||
|
|
||||||
|
func normalizeNewlines(text string) string {
|
||||||
|
text = strings.ReplaceAll(text, "\r\n", "\n")
|
||||||
|
text = strings.ReplaceAll(text, "\r", "\n")
|
||||||
|
return text
|
||||||
|
}
|
||||||
|
|
||||||
|
func trimBlankLines(text string) string {
|
||||||
|
lines := strings.Split(normalizeNewlines(text), "\n")
|
||||||
|
start := 0
|
||||||
|
for start < len(lines) && strings.TrimSpace(lines[start]) == "" {
|
||||||
|
start++
|
||||||
|
}
|
||||||
|
end := len(lines)
|
||||||
|
for end > start && strings.TrimSpace(lines[end-1]) == "" {
|
||||||
|
end--
|
||||||
|
}
|
||||||
|
return strings.Join(lines[start:end], "\n")
|
||||||
|
}
|
||||||
|
|
||||||
|
func parseUpdatedValue(value string) *time.Time {
|
||||||
|
value = strings.TrimSpace(value)
|
||||||
|
if value == "" {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if t := parseOptionalISOTimestamp(value); t != nil {
|
||||||
|
return t
|
||||||
|
}
|
||||||
|
for _, layout := range []string{
|
||||||
|
"Mon Jan 2 15:04:05 UTC 2006",
|
||||||
|
"1504 UTC Mon Jan 2 2006",
|
||||||
|
"1504Z Mon Jan 2 2006",
|
||||||
|
"3:04 PM UTC Mon Jan 2 2006",
|
||||||
|
time.RFC1123,
|
||||||
|
time.RFC1123Z,
|
||||||
|
} {
|
||||||
|
t, err := time.Parse(layout, value)
|
||||||
|
if err == nil {
|
||||||
|
tt := t.UTC()
|
||||||
|
return &tt
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
109
internal/providers/spc/discussion_test.go
Normal file
109
internal/providers/spc/discussion_test.go
Normal file
@@ -0,0 +1,109 @@
|
|||||||
|
package spc
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestExtractProductTextCleansPreBlock(t *testing.T) {
|
||||||
|
raw := string(readTestFile(t, "day1_prt.html"))
|
||||||
|
|
||||||
|
got, err := ExtractProductText(raw)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ExtractProductText() error = %v", err)
|
||||||
|
}
|
||||||
|
if strings.Contains(got, "<script") || strings.Contains(got, "<pre") {
|
||||||
|
t.Fatalf("ExtractProductText() retained HTML: %q", got)
|
||||||
|
}
|
||||||
|
if strings.Contains(got, "ignore me") {
|
||||||
|
t.Fatalf("ExtractProductText() retained script content: %q", got)
|
||||||
|
}
|
||||||
|
if !strings.Contains(got, "Day 1 Convective Outlook") {
|
||||||
|
t.Fatalf("ExtractProductText() missing headline: %q", got)
|
||||||
|
}
|
||||||
|
if !strings.HasPrefix(got, "SPC AC 111234") {
|
||||||
|
t.Fatalf("ExtractProductText() = %q, want product code prefix", got)
|
||||||
|
}
|
||||||
|
if strings.HasPrefix(got, "\n") || strings.HasSuffix(got, "\n") {
|
||||||
|
t.Fatalf("ExtractProductText() retained surrounding blank lines: %q", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseDiscussionHTMLExtractsHeadlineSummaryAndUpdated(t *testing.T) {
|
||||||
|
got, err := ParseDiscussionHTML(string(readTestFile(t, "day1_prt.html")))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ParseDiscussionHTML() error = %v", err)
|
||||||
|
}
|
||||||
|
if got.ProductTitle != "Day 1 Convective Outlook" {
|
||||||
|
t.Fatalf("ProductTitle = %q", got.ProductTitle)
|
||||||
|
}
|
||||||
|
if got.Headline != "Day 1 Convective Outlook" {
|
||||||
|
t.Fatalf("Headline = %q", got.Headline)
|
||||||
|
}
|
||||||
|
wantSummary := "Severe thunderstorms are possible across parts of the central Plains\nand mid Mississippi Valley this afternoon and evening."
|
||||||
|
if got.Summary != wantSummary {
|
||||||
|
t.Fatalf("Summary = %q, want %q", got.Summary, wantSummary)
|
||||||
|
}
|
||||||
|
if !strings.Contains(got.Discussion, "...DISCUSSION...") {
|
||||||
|
t.Fatalf("Discussion missing full text: %q", got.Discussion)
|
||||||
|
}
|
||||||
|
if !strings.HasPrefix(got.Discussion, "SPC AC 111234") {
|
||||||
|
t.Fatalf("Discussion = %q, want product code prefix", got.Discussion)
|
||||||
|
}
|
||||||
|
wantUpdated := time.Date(2026, 6, 11, 12, 45, 0, 0, time.UTC)
|
||||||
|
if got.UpdatedAt == nil || !got.UpdatedAt.Equal(wantUpdated) {
|
||||||
|
t.Fatalf("UpdatedAt = %v, want %s", got.UpdatedAt, wantUpdated)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseProductTitleSkipsSPCProductCode(t *testing.T) {
|
||||||
|
got := ParseProductTitle("SPC AC 101959\nDay 1 Convective Outlook\nNWS Storm Prediction Center Norman OK")
|
||||||
|
if got != "Day 1 Convective Outlook" {
|
||||||
|
t.Fatalf("ParseProductTitle() = %q, want Day 1 Convective Outlook", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseDiscussionTextPreservesCorrectionMarker(t *testing.T) {
|
||||||
|
got, err := ParseDiscussionHTML(string(readTestFile(t, "day2_prt_corr.html")))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ParseDiscussionHTML() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if !strings.Contains(got.Headline, "CORR 1") {
|
||||||
|
t.Fatalf("Headline = %q, want correction marker", got.Headline)
|
||||||
|
}
|
||||||
|
if !strings.Contains(got.Discussion, "CORR 1") {
|
||||||
|
t.Fatalf("Discussion = %q, want correction marker", got.Discussion)
|
||||||
|
}
|
||||||
|
wantUpdated := time.Date(2026, 6, 11, 17, 30, 0, 0, time.UTC)
|
||||||
|
if got.UpdatedAt == nil || !got.UpdatedAt.Equal(wantUpdated) {
|
||||||
|
t.Fatalf("UpdatedAt = %v, want %s", got.UpdatedAt, wantUpdated)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseUpdatedTimestampReturnsNilWhenAbsent(t *testing.T) {
|
||||||
|
text, err := ExtractProductText(string(readTestFile(t, "day2_prt_corr.html")))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ExtractProductText() error = %v", err)
|
||||||
|
}
|
||||||
|
if got := ParseUpdatedTimestamp(text); got != nil {
|
||||||
|
t.Fatalf("ParseUpdatedTimestamp() = %v, want nil", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseUpdatedTimestampAcceptsSPCUTCFormat(t *testing.T) {
|
||||||
|
got := ParseUpdatedTimestamp("Updated: 1945 UTC Thu Jun 11 2026")
|
||||||
|
want := time.Date(2026, 6, 11, 19, 45, 0, 0, time.UTC)
|
||||||
|
if got == nil || !got.Equal(want) {
|
||||||
|
t.Fatalf("ParseUpdatedTimestamp() = %v, want %s", got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParsePageUpdatedTimestampAcceptsLiveSPCShape(t *testing.T) {
|
||||||
|
got := ParsePageUpdatedTimestamp(string(readTestFile(t, "day3_prt.html")))
|
||||||
|
want := time.Date(2026, 6, 11, 20, 0, 0, 0, time.UTC)
|
||||||
|
if got == nil || !got.Equal(want) {
|
||||||
|
t.Fatalf("ParsePageUpdatedTimestamp() = %v, want %s", got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
8
internal/providers/spc/doc.go
Normal file
8
internal/providers/spc/doc.go
Normal file
@@ -0,0 +1,8 @@
|
|||||||
|
// Package spc contains deterministic helper code for Storm Prediction Center
|
||||||
|
// products used by sources and normalizers.
|
||||||
|
//
|
||||||
|
// Rules:
|
||||||
|
// - No network I/O here.
|
||||||
|
// - Keep helpers deterministic and easy to unit test.
|
||||||
|
// - Preserve upstream payload fragments needed for canonical mapping.
|
||||||
|
package spc
|
||||||
17
internal/providers/spc/fixture_test.go
Normal file
17
internal/providers/spc/fixture_test.go
Normal file
@@ -0,0 +1,17 @@
|
|||||||
|
package spc
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func readTestFile(t *testing.T, name string) []byte {
|
||||||
|
t.Helper()
|
||||||
|
path := filepath.Join("testdata", name)
|
||||||
|
raw, err := os.ReadFile(path)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("read %s: %v", path, err)
|
||||||
|
}
|
||||||
|
return raw
|
||||||
|
}
|
||||||
120
internal/providers/spc/geojson.go
Normal file
120
internal/providers/spc/geojson.go
Normal file
@@ -0,0 +1,120 @@
|
|||||||
|
package spc
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
// GeoJSONFeatureCollection is the minimal SPC outlook FeatureCollection shape
|
||||||
|
// needed by weatherfeeder.
|
||||||
|
type GeoJSONFeatureCollection struct {
|
||||||
|
Type string `json:"type"`
|
||||||
|
Features []GeoJSONFeature `json:"features"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// GeoJSONFeature preserves typed SPC properties and compact raw geometry.
|
||||||
|
type GeoJSONFeature struct {
|
||||||
|
Type string `json:"type"`
|
||||||
|
Properties GeoJSONProperties `json:"properties"`
|
||||||
|
Geometry json.RawMessage `json:"geometry"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// GeoJSONProperties contains the SPC fields used by canonical mapping.
|
||||||
|
type GeoJSONProperties struct {
|
||||||
|
ValidISO string `json:"VALID_ISO"`
|
||||||
|
ExpireISO string `json:"EXPIRE_ISO"`
|
||||||
|
IssueISO string `json:"ISSUE_ISO"`
|
||||||
|
Forecaster string `json:"FORECASTER"`
|
||||||
|
Label string `json:"LABEL"`
|
||||||
|
Label2 string `json:"LABEL2"`
|
||||||
|
DN *int `json:"DN"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type geometryMetadata struct {
|
||||||
|
Type string `json:"type"`
|
||||||
|
Geometries []json.RawMessage `json:"geometries"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// DecodeGeoJSON decodes an SPC GeoJSON outlook product and compacts feature
|
||||||
|
// geometry JSON for stable downstream storage.
|
||||||
|
func DecodeGeoJSON(raw []byte) (GeoJSONFeatureCollection, error) {
|
||||||
|
var collection GeoJSONFeatureCollection
|
||||||
|
if err := json.Unmarshal(raw, &collection); err != nil {
|
||||||
|
return GeoJSONFeatureCollection{}, fmt.Errorf("decode geojson: %w", err)
|
||||||
|
}
|
||||||
|
for i := range collection.Features {
|
||||||
|
geom, err := compactJSON(collection.Features[i].Geometry)
|
||||||
|
if err != nil {
|
||||||
|
return GeoJSONFeatureCollection{}, fmt.Errorf("features[%d].geometry: %w", i, err)
|
||||||
|
}
|
||||||
|
collection.Features[i].Geometry = geom
|
||||||
|
}
|
||||||
|
return collection, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// IsEmptyGeometryCollection reports whether raw is SPC's no-polygon placeholder
|
||||||
|
// geometry shape: a GeometryCollection with no child geometries.
|
||||||
|
func IsEmptyGeometryCollection(raw json.RawMessage) bool {
|
||||||
|
var meta geometryMetadata
|
||||||
|
if err := json.Unmarshal(raw, &meta); err != nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
return meta.Type == "GeometryCollection" && len(meta.Geometries) == 0
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *GeoJSONProperties) UnmarshalJSON(raw []byte) error {
|
||||||
|
type alias GeoJSONProperties
|
||||||
|
var aux struct {
|
||||||
|
alias
|
||||||
|
DN any `json:"DN"`
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(raw, &aux); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
*p = GeoJSONProperties(aux.alias)
|
||||||
|
dn, err := parseSeverityRank(aux.DN)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
p.DN = dn
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func parseSeverityRank(value any) (*int, error) {
|
||||||
|
switch v := value.(type) {
|
||||||
|
case nil:
|
||||||
|
return nil, nil
|
||||||
|
case float64:
|
||||||
|
rank := int(v)
|
||||||
|
if float64(rank) != v {
|
||||||
|
return nil, fmt.Errorf("DN must be an integer, got %v", v)
|
||||||
|
}
|
||||||
|
return &rank, nil
|
||||||
|
case string:
|
||||||
|
v = strings.TrimSpace(v)
|
||||||
|
if v == "" {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
rank, err := strconv.Atoi(v)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("DN must be an integer, got %q", v)
|
||||||
|
}
|
||||||
|
return &rank, nil
|
||||||
|
default:
|
||||||
|
return nil, fmt.Errorf("DN must be an integer or string, got %T", value)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func compactJSON(raw json.RawMessage) (json.RawMessage, error) {
|
||||||
|
if len(raw) == 0 {
|
||||||
|
return nil, fmt.Errorf("missing")
|
||||||
|
}
|
||||||
|
var buf bytes.Buffer
|
||||||
|
if err := json.Compact(&buf, raw); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return json.RawMessage(buf.Bytes()), nil
|
||||||
|
}
|
||||||
115
internal/providers/spc/geojson_test.go
Normal file
115
internal/providers/spc/geojson_test.go
Normal file
@@ -0,0 +1,115 @@
|
|||||||
|
package spc
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestDecodeGeoJSONExposesSPCPropertiesAndCompactGeometry(t *testing.T) {
|
||||||
|
raw := readTestFile(t, "day1_cat.geojson")
|
||||||
|
|
||||||
|
got, err := DecodeGeoJSON(raw)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("DecodeGeoJSON() error = %v", err)
|
||||||
|
}
|
||||||
|
if got.Type != "FeatureCollection" {
|
||||||
|
t.Fatalf("Type = %q, want FeatureCollection", got.Type)
|
||||||
|
}
|
||||||
|
if len(got.Features) != 1 {
|
||||||
|
t.Fatalf("Features length = %d, want 1", len(got.Features))
|
||||||
|
}
|
||||||
|
|
||||||
|
feature := got.Features[0]
|
||||||
|
props := feature.Properties
|
||||||
|
if props.ValidISO != "2026-06-11T13:00:00Z" {
|
||||||
|
t.Fatalf("VALID_ISO = %q", props.ValidISO)
|
||||||
|
}
|
||||||
|
if props.ExpireISO != "2026-06-12T12:00:00Z" {
|
||||||
|
t.Fatalf("EXPIRE_ISO = %q", props.ExpireISO)
|
||||||
|
}
|
||||||
|
if props.IssueISO != "2026-06-11T12:34:56Z" {
|
||||||
|
t.Fatalf("ISSUE_ISO = %q", props.IssueISO)
|
||||||
|
}
|
||||||
|
if props.Forecaster != "SMITH" {
|
||||||
|
t.Fatalf("FORECASTER = %q", props.Forecaster)
|
||||||
|
}
|
||||||
|
if props.Label != "SLGT" {
|
||||||
|
t.Fatalf("LABEL = %q", props.Label)
|
||||||
|
}
|
||||||
|
if props.Label2 != "Slight Risk" {
|
||||||
|
t.Fatalf("LABEL2 = %q", props.Label2)
|
||||||
|
}
|
||||||
|
if props.DN == nil || *props.DN != 3 {
|
||||||
|
t.Fatalf("DN = %v, want 3", props.DN)
|
||||||
|
}
|
||||||
|
|
||||||
|
wantGeometry := `{"type":"Polygon","coordinates":[[[-91.0,38.0],[-90.0,38.0],[-90.0,39.0],[-91.0,39.0],[-91.0,38.0]]]}`
|
||||||
|
if string(feature.Geometry) != wantGeometry {
|
||||||
|
t.Fatalf("Geometry = %s, want %s", feature.Geometry, wantGeometry)
|
||||||
|
}
|
||||||
|
if strings.Contains(string(feature.Geometry), "\n") || strings.Contains(string(feature.Geometry), " ") {
|
||||||
|
t.Fatalf("Geometry is not compact: %q", feature.Geometry)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDecodeGeoJSONParsesSeverityRankString(t *testing.T) {
|
||||||
|
raw := readTestFile(t, "day2_torn.geojson")
|
||||||
|
|
||||||
|
got, err := DecodeGeoJSON(raw)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("DecodeGeoJSON() error = %v", err)
|
||||||
|
}
|
||||||
|
props := got.Features[0].Properties
|
||||||
|
if props.DN == nil || *props.DN != 5 {
|
||||||
|
t.Fatalf("DN = %v, want 5", props.DN)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestIsEmptyGeometryCollection(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
raw string
|
||||||
|
want bool
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "empty geometry collection",
|
||||||
|
raw: `{"type":"GeometryCollection","geometries":[]}`,
|
||||||
|
want: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "non-empty geometry collection",
|
||||||
|
raw: `{"type":"GeometryCollection","geometries":[{"type":"Polygon","coordinates":[]} ]}`,
|
||||||
|
want: false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "polygon",
|
||||||
|
raw: `{"type":"Polygon","coordinates":[]}`,
|
||||||
|
want: false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "invalid json",
|
||||||
|
raw: `{`,
|
||||||
|
want: false,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
if got := IsEmptyGeometryCollection([]byte(tt.raw)); got != tt.want {
|
||||||
|
t.Fatalf("IsEmptyGeometryCollection() = %v, want %v", got, tt.want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseISOTimestampTrimsAndReturnsUTC(t *testing.T) {
|
||||||
|
got, err := ParseISOTimestamp(" 2026-06-11T12:34:56Z ")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ParseISOTimestamp() error = %v", err)
|
||||||
|
}
|
||||||
|
want := time.Date(2026, 6, 11, 12, 34, 56, 0, time.UTC)
|
||||||
|
if !got.Equal(want) {
|
||||||
|
t.Fatalf("ParseISOTimestamp() = %s, want %s", got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
86
internal/providers/spc/product.go
Normal file
86
internal/providers/spc/product.go
Normal file
@@ -0,0 +1,86 @@
|
|||||||
|
package spc
|
||||||
|
|
||||||
|
import "fmt"
|
||||||
|
|
||||||
|
const (
|
||||||
|
OutlookTypeCategorical = "categorical"
|
||||||
|
OutlookTypeTornado = "tornado"
|
||||||
|
OutlookTypeHail = "hail"
|
||||||
|
OutlookTypeWind = "wind"
|
||||||
|
)
|
||||||
|
|
||||||
|
// GeoJSONProduct describes one required SPC convective outlook GeoJSON product.
|
||||||
|
type GeoJSONProduct struct {
|
||||||
|
Key string
|
||||||
|
Day int
|
||||||
|
OutlookType string
|
||||||
|
URL string
|
||||||
|
}
|
||||||
|
|
||||||
|
// DiscussionProduct describes one required SPC convective outlook print page.
|
||||||
|
type DiscussionProduct struct {
|
||||||
|
Key string
|
||||||
|
Day int
|
||||||
|
URL string
|
||||||
|
}
|
||||||
|
|
||||||
|
var geoJSONProducts = []GeoJSONProduct{
|
||||||
|
{Key: "day1_categorical", Day: 1, OutlookType: OutlookTypeCategorical, URL: "https://www.spc.noaa.gov/products/outlook/day1otlk_cat.nolyr.geojson"},
|
||||||
|
{Key: "day1_tornado", Day: 1, OutlookType: OutlookTypeTornado, URL: "https://www.spc.noaa.gov/products/outlook/day1otlk_torn.nolyr.geojson"},
|
||||||
|
{Key: "day1_hail", Day: 1, OutlookType: OutlookTypeHail, URL: "https://www.spc.noaa.gov/products/outlook/day1otlk_hail.nolyr.geojson"},
|
||||||
|
{Key: "day1_wind", Day: 1, OutlookType: OutlookTypeWind, URL: "https://www.spc.noaa.gov/products/outlook/day1otlk_wind.nolyr.geojson"},
|
||||||
|
{Key: "day2_categorical", Day: 2, OutlookType: OutlookTypeCategorical, URL: "https://www.spc.noaa.gov/products/outlook/day2otlk_cat.nolyr.geojson"},
|
||||||
|
{Key: "day2_tornado", Day: 2, OutlookType: OutlookTypeTornado, URL: "https://www.spc.noaa.gov/products/outlook/day2otlk_torn.nolyr.geojson"},
|
||||||
|
{Key: "day2_hail", Day: 2, OutlookType: OutlookTypeHail, URL: "https://www.spc.noaa.gov/products/outlook/day2otlk_hail.nolyr.geojson"},
|
||||||
|
{Key: "day2_wind", Day: 2, OutlookType: OutlookTypeWind, URL: "https://www.spc.noaa.gov/products/outlook/day2otlk_wind.nolyr.geojson"},
|
||||||
|
{Key: "day3_categorical", Day: 3, OutlookType: OutlookTypeCategorical, URL: "https://www.spc.noaa.gov/products/outlook/day3otlk_cat.nolyr.geojson"},
|
||||||
|
}
|
||||||
|
|
||||||
|
var discussionProducts = []DiscussionProduct{
|
||||||
|
{Key: "day1", Day: 1, URL: "https://www.spc.noaa.gov/products/outlook/day1otlk_prt.html"},
|
||||||
|
{Key: "day2", Day: 2, URL: "https://www.spc.noaa.gov/products/outlook/day2otlk_prt.html"},
|
||||||
|
{Key: "day3", Day: 3, URL: "https://www.spc.noaa.gov/products/outlook/day3otlk_prt.html"},
|
||||||
|
}
|
||||||
|
|
||||||
|
// GeoJSONProducts returns the required SPC convective outlook GeoJSON products
|
||||||
|
// in stable day/type order.
|
||||||
|
func GeoJSONProducts() []GeoJSONProduct {
|
||||||
|
out := make([]GeoJSONProduct, len(geoJSONProducts))
|
||||||
|
copy(out, geoJSONProducts)
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// DiscussionProducts returns the required SPC convective outlook print pages in
|
||||||
|
// stable day order.
|
||||||
|
func DiscussionProducts() []DiscussionProduct {
|
||||||
|
out := make([]DiscussionProduct, len(discussionProducts))
|
||||||
|
copy(out, discussionProducts)
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// GeoJSONProductByKey returns product metadata for a configured product key.
|
||||||
|
func GeoJSONProductByKey(key string) (GeoJSONProduct, bool) {
|
||||||
|
for _, product := range geoJSONProducts {
|
||||||
|
if product.Key == key {
|
||||||
|
return product, true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return GeoJSONProduct{}, false
|
||||||
|
}
|
||||||
|
|
||||||
|
// DiscussionProductByKey returns discussion metadata for a configured day key.
|
||||||
|
func DiscussionProductByKey(key string) (DiscussionProduct, bool) {
|
||||||
|
for _, product := range discussionProducts {
|
||||||
|
if product.Key == key {
|
||||||
|
return product, true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return DiscussionProduct{}, false
|
||||||
|
}
|
||||||
|
|
||||||
|
func validateProductDay(day int) error {
|
||||||
|
if day < 1 || day > 3 {
|
||||||
|
return fmt.Errorf("day must be 1, 2, or 3, got %d", day)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
53
internal/providers/spc/product_test.go
Normal file
53
internal/providers/spc/product_test.go
Normal file
@@ -0,0 +1,53 @@
|
|||||||
|
package spc
|
||||||
|
|
||||||
|
import "testing"
|
||||||
|
|
||||||
|
func TestGeoJSONProductsStableOrder(t *testing.T) {
|
||||||
|
got := GeoJSONProducts()
|
||||||
|
if len(got) != 9 {
|
||||||
|
t.Fatalf("GeoJSONProducts() length = %d, want 9", len(got))
|
||||||
|
}
|
||||||
|
|
||||||
|
wantKeys := []string{
|
||||||
|
"day1_categorical",
|
||||||
|
"day1_tornado",
|
||||||
|
"day1_hail",
|
||||||
|
"day1_wind",
|
||||||
|
"day2_categorical",
|
||||||
|
"day2_tornado",
|
||||||
|
"day2_hail",
|
||||||
|
"day2_wind",
|
||||||
|
"day3_categorical",
|
||||||
|
}
|
||||||
|
for i, want := range wantKeys {
|
||||||
|
if got[i].Key != want {
|
||||||
|
t.Fatalf("GeoJSONProducts()[%d].Key = %q, want %q", i, got[i].Key, want)
|
||||||
|
}
|
||||||
|
if err := validateProductDay(got[i].Day); err != nil {
|
||||||
|
t.Fatalf("GeoJSONProducts()[%d].Day invalid: %v", i, err)
|
||||||
|
}
|
||||||
|
if got[i].URL == "" {
|
||||||
|
t.Fatalf("GeoJSONProducts()[%d].URL is empty", i)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDiscussionProductsStableOrder(t *testing.T) {
|
||||||
|
got := DiscussionProducts()
|
||||||
|
if len(got) != 3 {
|
||||||
|
t.Fatalf("DiscussionProducts() length = %d, want 3", len(got))
|
||||||
|
}
|
||||||
|
|
||||||
|
wantKeys := []string{"day1", "day2", "day3"}
|
||||||
|
for i, want := range wantKeys {
|
||||||
|
if got[i].Key != want {
|
||||||
|
t.Fatalf("DiscussionProducts()[%d].Key = %q, want %q", i, got[i].Key, want)
|
||||||
|
}
|
||||||
|
if got[i].Day != i+1 {
|
||||||
|
t.Fatalf("DiscussionProducts()[%d].Day = %d, want %d", i, got[i].Day, i+1)
|
||||||
|
}
|
||||||
|
if got[i].URL == "" {
|
||||||
|
t.Fatalf("DiscussionProducts()[%d].URL is empty", i)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
45
internal/providers/spc/raw.go
Normal file
45
internal/providers/spc/raw.go
Normal file
@@ -0,0 +1,45 @@
|
|||||||
|
package spc
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// RawConvectiveOutlookBundle is the provider payload shape for SPC convective
|
||||||
|
// outlook fetch bundles.
|
||||||
|
type RawConvectiveOutlookBundle struct {
|
||||||
|
LocationID string `json:"locationId,omitempty"`
|
||||||
|
LocationName string `json:"locationName,omitempty"`
|
||||||
|
Latitude float64 `json:"latitude"`
|
||||||
|
Longitude float64 `json:"longitude"`
|
||||||
|
FetchedAt time.Time `json:"fetchedAt"`
|
||||||
|
Products []RawOutlookProduct `json:"products"`
|
||||||
|
Discussions []RawDiscussionPage `json:"discussions"`
|
||||||
|
RSS *RawRSSFeed `json:"rss,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// RawOutlookProduct contains one fetched SPC GeoJSON product.
|
||||||
|
type RawOutlookProduct struct {
|
||||||
|
Key string `json:"key"`
|
||||||
|
Day int `json:"day"`
|
||||||
|
OutlookType string `json:"outlookType"`
|
||||||
|
URL string `json:"url"`
|
||||||
|
FetchedAt time.Time `json:"fetchedAt"`
|
||||||
|
Body json.RawMessage `json:"body"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// RawDiscussionPage contains one fetched SPC print page.
|
||||||
|
type RawDiscussionPage struct {
|
||||||
|
Key string `json:"key"`
|
||||||
|
Day int `json:"day"`
|
||||||
|
URL string `json:"url"`
|
||||||
|
FetchedAt time.Time `json:"fetchedAt"`
|
||||||
|
Body string `json:"body"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// RawRSSFeed contains optional fetched SPC RSS metadata.
|
||||||
|
type RawRSSFeed struct {
|
||||||
|
URL string `json:"url"`
|
||||||
|
FetchedAt time.Time `json:"fetchedAt"`
|
||||||
|
Body string `json:"body"`
|
||||||
|
}
|
||||||
51
internal/providers/spc/raw_test.go
Normal file
51
internal/providers/spc/raw_test.go
Normal file
@@ -0,0 +1,51 @@
|
|||||||
|
package spc
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestRawConvectiveOutlookBundleJSONShape(t *testing.T) {
|
||||||
|
fetchedAt := time.Date(2026, 6, 11, 20, 0, 0, 0, time.UTC)
|
||||||
|
bundle := RawConvectiveOutlookBundle{
|
||||||
|
LocationID: "stl",
|
||||||
|
LocationName: "St. Louis, MO",
|
||||||
|
Latitude: 38.6239,
|
||||||
|
Longitude: -90.3571,
|
||||||
|
FetchedAt: fetchedAt,
|
||||||
|
Products: []RawOutlookProduct{{
|
||||||
|
Key: "day1_categorical",
|
||||||
|
Day: 1,
|
||||||
|
OutlookType: OutlookTypeCategorical,
|
||||||
|
URL: "https://example.invalid/day1.geojson",
|
||||||
|
FetchedAt: fetchedAt,
|
||||||
|
Body: json.RawMessage(`{"type":"FeatureCollection","features":[]}`),
|
||||||
|
}},
|
||||||
|
Discussions: []RawDiscussionPage{{
|
||||||
|
Key: "day1",
|
||||||
|
Day: 1,
|
||||||
|
URL: "https://example.invalid/day1.html",
|
||||||
|
FetchedAt: fetchedAt,
|
||||||
|
Body: "Day 1 Convective Outlook",
|
||||||
|
}},
|
||||||
|
}
|
||||||
|
|
||||||
|
raw, err := json.Marshal(bundle)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Marshal() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var got map[string]any
|
||||||
|
if err := json.Unmarshal(raw, &got); err != nil {
|
||||||
|
t.Fatalf("Unmarshal() error = %v", err)
|
||||||
|
}
|
||||||
|
for _, key := range []string{"locationId", "locationName", "latitude", "longitude", "fetchedAt", "products", "discussions"} {
|
||||||
|
if _, ok := got[key]; !ok {
|
||||||
|
t.Fatalf("marshaled bundle missing key %q in %s", key, raw)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if _, ok := got["rss"]; ok {
|
||||||
|
t.Fatalf("marshaled bundle included empty rss: %s", raw)
|
||||||
|
}
|
||||||
|
}
|
||||||
81
internal/providers/spc/rss.go
Normal file
81
internal/providers/spc/rss.go
Normal file
@@ -0,0 +1,81 @@
|
|||||||
|
package spc
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/xml"
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// RSSFeed is a minimal view of the optional SPC RSS feed.
|
||||||
|
type RSSFeed struct {
|
||||||
|
Title string
|
||||||
|
Link string
|
||||||
|
Description string
|
||||||
|
LastBuildDate *time.Time
|
||||||
|
Items []RSSItem
|
||||||
|
}
|
||||||
|
|
||||||
|
// RSSItem is a minimal view of one optional SPC RSS item.
|
||||||
|
type RSSItem struct {
|
||||||
|
Title string
|
||||||
|
Link string
|
||||||
|
Description string
|
||||||
|
PubDate string
|
||||||
|
GUID string
|
||||||
|
}
|
||||||
|
|
||||||
|
// ParseRSSFeed decodes supplemental SPC RSS metadata.
|
||||||
|
func ParseRSSFeed(raw string) (RSSFeed, error) {
|
||||||
|
var doc struct {
|
||||||
|
Channel struct {
|
||||||
|
Title string `xml:"title"`
|
||||||
|
Link string `xml:"link"`
|
||||||
|
Description string `xml:"description"`
|
||||||
|
LastBuildDate string `xml:"lastBuildDate"`
|
||||||
|
Items []struct {
|
||||||
|
Title string `xml:"title"`
|
||||||
|
Link string `xml:"link"`
|
||||||
|
Description string `xml:"description"`
|
||||||
|
PubDate string `xml:"pubDate"`
|
||||||
|
GUID string `xml:"guid"`
|
||||||
|
} `xml:"item"`
|
||||||
|
} `xml:"channel"`
|
||||||
|
}
|
||||||
|
if err := xml.Unmarshal([]byte(raw), &doc); err != nil {
|
||||||
|
return RSSFeed{}, fmt.Errorf("decode rss: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
feed := RSSFeed{
|
||||||
|
Title: strings.TrimSpace(doc.Channel.Title),
|
||||||
|
Link: strings.TrimSpace(doc.Channel.Link),
|
||||||
|
Description: strings.TrimSpace(doc.Channel.Description),
|
||||||
|
LastBuildDate: parseRSSDate(doc.Channel.LastBuildDate),
|
||||||
|
Items: make([]RSSItem, 0, len(doc.Channel.Items)),
|
||||||
|
}
|
||||||
|
for _, item := range doc.Channel.Items {
|
||||||
|
feed.Items = append(feed.Items, RSSItem{
|
||||||
|
Title: strings.TrimSpace(item.Title),
|
||||||
|
Link: strings.TrimSpace(item.Link),
|
||||||
|
Description: strings.TrimSpace(item.Description),
|
||||||
|
PubDate: strings.TrimSpace(item.PubDate),
|
||||||
|
GUID: strings.TrimSpace(item.GUID),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
return feed, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func parseRSSDate(value string) *time.Time {
|
||||||
|
value = strings.TrimSpace(value)
|
||||||
|
if value == "" {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
for _, layout := range []string{time.RFC1123Z, time.RFC1123} {
|
||||||
|
t, err := time.Parse(layout, value)
|
||||||
|
if err == nil {
|
||||||
|
tt := t.UTC()
|
||||||
|
return &tt
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
43
internal/providers/spc/rss_test.go
Normal file
43
internal/providers/spc/rss_test.go
Normal file
@@ -0,0 +1,43 @@
|
|||||||
|
package spc
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestParseRSSFeed(t *testing.T) {
|
||||||
|
const raw = `<?xml version="1.0"?>
|
||||||
|
<rss version="2.0">
|
||||||
|
<channel>
|
||||||
|
<title>SPC AC RSS</title>
|
||||||
|
<link>https://www.spc.noaa.gov/products/</link>
|
||||||
|
<description>SPC products</description>
|
||||||
|
<lastBuildDate>Thu, 11 Jun 2026 19:00:00 +0000</lastBuildDate>
|
||||||
|
<item>
|
||||||
|
<title>Day 1 Convective Outlook</title>
|
||||||
|
<link>https://www.spc.noaa.gov/products/outlook/day1otlk.html</link>
|
||||||
|
<description>Outlook text</description>
|
||||||
|
<pubDate>Thu, 11 Jun 2026 18:55:00 +0000</pubDate>
|
||||||
|
<guid>day1</guid>
|
||||||
|
</item>
|
||||||
|
</channel>
|
||||||
|
</rss>`
|
||||||
|
|
||||||
|
got, err := ParseRSSFeed(raw)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ParseRSSFeed() error = %v", err)
|
||||||
|
}
|
||||||
|
if got.Title != "SPC AC RSS" {
|
||||||
|
t.Fatalf("Title = %q", got.Title)
|
||||||
|
}
|
||||||
|
wantBuild := time.Date(2026, 6, 11, 19, 0, 0, 0, time.UTC)
|
||||||
|
if got.LastBuildDate == nil || !got.LastBuildDate.Equal(wantBuild) {
|
||||||
|
t.Fatalf("LastBuildDate = %v, want %s", got.LastBuildDate, wantBuild)
|
||||||
|
}
|
||||||
|
if len(got.Items) != 1 {
|
||||||
|
t.Fatalf("Items length = %d, want 1", len(got.Items))
|
||||||
|
}
|
||||||
|
if got.Items[0].GUID != "day1" {
|
||||||
|
t.Fatalf("Item GUID = %q", got.Items[0].GUID)
|
||||||
|
}
|
||||||
|
}
|
||||||
29
internal/providers/spc/testdata/day1_cat.geojson
vendored
Normal file
29
internal/providers/spc/testdata/day1_cat.geojson
vendored
Normal file
@@ -0,0 +1,29 @@
|
|||||||
|
{
|
||||||
|
"type": "FeatureCollection",
|
||||||
|
"features": [
|
||||||
|
{
|
||||||
|
"type": "Feature",
|
||||||
|
"properties": {
|
||||||
|
"VALID_ISO": "2026-06-11T13:00:00Z",
|
||||||
|
"EXPIRE_ISO": "2026-06-12T12:00:00Z",
|
||||||
|
"ISSUE_ISO": "2026-06-11T12:34:56Z",
|
||||||
|
"FORECASTER": "SMITH",
|
||||||
|
"LABEL": "SLGT",
|
||||||
|
"LABEL2": "Slight Risk",
|
||||||
|
"DN": 3
|
||||||
|
},
|
||||||
|
"geometry": {
|
||||||
|
"type": "Polygon",
|
||||||
|
"coordinates": [
|
||||||
|
[
|
||||||
|
[-91.0, 38.0],
|
||||||
|
[-90.0, 38.0],
|
||||||
|
[-90.0, 39.0],
|
||||||
|
[-91.0, 39.0],
|
||||||
|
[-91.0, 38.0]
|
||||||
|
]
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
22
internal/providers/spc/testdata/day1_prt.html
vendored
Normal file
22
internal/providers/spc/testdata/day1_prt.html
vendored
Normal file
@@ -0,0 +1,22 @@
|
|||||||
|
<!doctype html>
|
||||||
|
<html>
|
||||||
|
<head><title>Day 1 Convective Outlook</title></head>
|
||||||
|
<body>
|
||||||
|
<table>
|
||||||
|
<tr><td align="center" class="rpttext" nowrap>Updated: Thu Jun 11 12:45:00 UTC 2026 (<a href="archive/day1-geojson.zip">geojson</a>)</td></tr>
|
||||||
|
</table>
|
||||||
|
<pre>
|
||||||
|
<script>window.bad = "<b>ignore me</b>";</script>
|
||||||
|
SPC AC 111234
|
||||||
|
Day 1 Convective Outlook
|
||||||
|
NWS Storm Prediction Center Norman OK
|
||||||
|
|
||||||
|
...SUMMARY...
|
||||||
|
Severe thunderstorms are possible across parts of the central Plains
|
||||||
|
and mid Mississippi Valley this afternoon and evening.
|
||||||
|
|
||||||
|
...DISCUSSION...
|
||||||
|
The primary threats will be damaging wind and large hail.
|
||||||
|
</pre>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
19
internal/providers/spc/testdata/day2_prt_corr.html
vendored
Normal file
19
internal/providers/spc/testdata/day2_prt_corr.html
vendored
Normal file
@@ -0,0 +1,19 @@
|
|||||||
|
<!doctype html>
|
||||||
|
<html>
|
||||||
|
<body>
|
||||||
|
<table>
|
||||||
|
<tr><td class="rpttext">Updated: Thu Jun 11 17:30:00 UTC 2026 </td></tr>
|
||||||
|
</table>
|
||||||
|
<pre>
|
||||||
|
SPC AC 111730
|
||||||
|
Day 2 Convective Outlook CORR 1
|
||||||
|
NWS Storm Prediction Center Norman OK
|
||||||
|
|
||||||
|
...SUMMARY...
|
||||||
|
Scattered severe thunderstorms remain possible across the southern Plains.
|
||||||
|
|
||||||
|
...DISCUSSION...
|
||||||
|
Corrected outlook text remains otherwise unchanged.
|
||||||
|
</pre>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
29
internal/providers/spc/testdata/day2_torn.geojson
vendored
Normal file
29
internal/providers/spc/testdata/day2_torn.geojson
vendored
Normal file
@@ -0,0 +1,29 @@
|
|||||||
|
{
|
||||||
|
"type": "FeatureCollection",
|
||||||
|
"features": [
|
||||||
|
{
|
||||||
|
"type": "Feature",
|
||||||
|
"properties": {
|
||||||
|
"VALID_ISO": "2026-06-12T12:00:00Z",
|
||||||
|
"EXPIRE_ISO": "2026-06-13T12:00:00Z",
|
||||||
|
"ISSUE_ISO": "2026-06-11T17:30:00Z",
|
||||||
|
"FORECASTER": "DOE",
|
||||||
|
"LABEL": "5",
|
||||||
|
"LABEL2": "5% Tornado",
|
||||||
|
"DN": "5"
|
||||||
|
},
|
||||||
|
"geometry": {
|
||||||
|
"type": "Polygon",
|
||||||
|
"coordinates": [
|
||||||
|
[
|
||||||
|
[-100.0, 35.0],
|
||||||
|
[-98.0, 35.0],
|
||||||
|
[-98.0, 37.0],
|
||||||
|
[-100.0, 37.0],
|
||||||
|
[-100.0, 35.0]
|
||||||
|
]
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
31
internal/providers/spc/testdata/day3_cat.geojson
vendored
Normal file
31
internal/providers/spc/testdata/day3_cat.geojson
vendored
Normal file
@@ -0,0 +1,31 @@
|
|||||||
|
{
|
||||||
|
"type": "FeatureCollection",
|
||||||
|
"features": [
|
||||||
|
{
|
||||||
|
"type": "Feature",
|
||||||
|
"properties": {
|
||||||
|
"VALID_ISO": "2026-06-13T12:00:00Z",
|
||||||
|
"EXPIRE_ISO": "2026-06-14T12:00:00Z",
|
||||||
|
"ISSUE_ISO": "2026-06-11T19:45:00Z",
|
||||||
|
"FORECASTER": "LEE",
|
||||||
|
"LABEL": "MRGL",
|
||||||
|
"LABEL2": "Marginal Risk",
|
||||||
|
"DN": 2
|
||||||
|
},
|
||||||
|
"geometry": {
|
||||||
|
"type": "MultiPolygon",
|
||||||
|
"coordinates": [
|
||||||
|
[
|
||||||
|
[
|
||||||
|
[-103.0, 34.0],
|
||||||
|
[-101.0, 34.0],
|
||||||
|
[-101.0, 36.0],
|
||||||
|
[-103.0, 36.0],
|
||||||
|
[-103.0, 34.0]
|
||||||
|
]
|
||||||
|
]
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
19
internal/providers/spc/testdata/day3_prt.html
vendored
Normal file
19
internal/providers/spc/testdata/day3_prt.html
vendored
Normal file
@@ -0,0 +1,19 @@
|
|||||||
|
<!doctype html>
|
||||||
|
<html>
|
||||||
|
<body>
|
||||||
|
<table>
|
||||||
|
<tr><td class="rpttext">Updated: Thu Jun 11 20:00:00 UTC 2026 </td></tr>
|
||||||
|
</table>
|
||||||
|
<pre>
|
||||||
|
SPC AC 112000
|
||||||
|
Day 3 Convective Outlook
|
||||||
|
NWS Storm Prediction Center Norman OK
|
||||||
|
|
||||||
|
...SUMMARY...
|
||||||
|
A corridor of strong to severe storms may develop near a frontal zone.
|
||||||
|
|
||||||
|
...DISCUSSION...
|
||||||
|
Confidence remains moderate for organized storms.
|
||||||
|
</pre>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
24
internal/providers/spc/time.go
Normal file
24
internal/providers/spc/time.go
Normal file
@@ -0,0 +1,24 @@
|
|||||||
|
package spc
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// ParseISOTimestamp parses SPC ISO timestamps from GeoJSON properties.
|
||||||
|
func ParseISOTimestamp(value string) (time.Time, error) {
|
||||||
|
return time.Parse(time.RFC3339, strings.TrimSpace(value))
|
||||||
|
}
|
||||||
|
|
||||||
|
func parseOptionalISOTimestamp(value string) *time.Time {
|
||||||
|
value = strings.TrimSpace(value)
|
||||||
|
if value == "" {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
t, err := ParseISOTimestamp(value)
|
||||||
|
if err != nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
tt := t.UTC()
|
||||||
|
return &tt
|
||||||
|
}
|
||||||
@@ -11,6 +11,7 @@
|
|||||||
// - weather.forecast_discussion.v1 -> model.WeatherForecastDiscussion
|
// - weather.forecast_discussion.v1 -> model.WeatherForecastDiscussion
|
||||||
// - weather.weather_story.v1 -> model.WeatherStoryRun
|
// - weather.weather_story.v1 -> model.WeatherStoryRun
|
||||||
// - weather.alert.v1 -> model.WeatherAlertRun
|
// - weather.alert.v1 -> model.WeatherAlertRun
|
||||||
|
// - weather.outlook.v2 -> model.WeatherOutlookRun
|
||||||
//
|
//
|
||||||
// Parent/child relationships:
|
// Parent/child relationships:
|
||||||
// - observations.event_id -> observation_present_weather.event_id
|
// - observations.event_id -> observation_present_weather.event_id
|
||||||
@@ -19,9 +20,11 @@
|
|||||||
// - weather_story_runs.event_id -> weather_stories.run_event_id
|
// - weather_story_runs.event_id -> weather_stories.run_event_id
|
||||||
// - alert_runs.event_id -> alerts.run_event_id
|
// - alert_runs.event_id -> alerts.run_event_id
|
||||||
// - alerts.(run_event_id, alert_index) -> alert_references.(run_event_id, alert_index)
|
// - alerts.(run_event_id, alert_index) -> alert_references.(run_event_id, alert_index)
|
||||||
|
// - outlook_runs.event_id -> outlooks.run_event_id
|
||||||
|
// - outlook_runs.event_id -> outlook_discussions.run_event_id
|
||||||
//
|
//
|
||||||
// Dedupe and retention behavior:
|
// Dedupe and retention behavior:
|
||||||
// - Parent primary keys (event_id): observations, forecasts, alert_runs.
|
// - Parent primary keys (event_id): observations, forecasts, alert_runs, outlook_runs.
|
||||||
// - Child primary keys use positional indexes to preserve payload order.
|
// - Child primary keys use positional indexes to preserve payload order.
|
||||||
// - Prune columns:
|
// - Prune columns:
|
||||||
// - observations.observed_at
|
// - observations.observed_at
|
||||||
@@ -35,11 +38,14 @@
|
|||||||
// - alert_runs.as_of
|
// - alert_runs.as_of
|
||||||
// - alerts.as_of
|
// - alerts.as_of
|
||||||
// - alert_references.as_of
|
// - alert_references.as_of
|
||||||
|
// - outlook_runs.as_of
|
||||||
|
// - outlooks.as_of
|
||||||
|
// - outlook_discussions.as_of
|
||||||
//
|
//
|
||||||
// Envelope field mapping (shared parent columns)
|
// Envelope field mapping (shared parent columns)
|
||||||
//
|
//
|
||||||
// These columns exist on parent tables such as observations, forecasts,
|
// These columns exist on parent tables such as observations, forecasts,
|
||||||
// forecast_discussions, weather_story_runs, and alert_runs:
|
// forecast_discussions, weather_story_runs, alert_runs, and outlook_runs:
|
||||||
// - event_id TEXT -> event.id
|
// - event_id TEXT -> event.id
|
||||||
// - event_kind TEXT -> event.kind
|
// - event_kind TEXT -> event.kind
|
||||||
// - event_source TEXT -> event.source
|
// - event_source TEXT -> event.source
|
||||||
@@ -192,6 +198,7 @@
|
|||||||
// - sent TIMESTAMPTZ NULL -> payload.alerts[i].sent
|
// - sent TIMESTAMPTZ NULL -> payload.alerts[i].sent
|
||||||
// - effective TIMESTAMPTZ NULL -> payload.alerts[i].effective
|
// - effective TIMESTAMPTZ NULL -> payload.alerts[i].effective
|
||||||
// - onset TIMESTAMPTZ NULL -> payload.alerts[i].onset
|
// - onset TIMESTAMPTZ NULL -> payload.alerts[i].onset
|
||||||
|
// - ends TIMESTAMPTZ NULL -> payload.alerts[i].ends
|
||||||
// - expires TIMESTAMPTZ NULL -> payload.alerts[i].expires
|
// - expires TIMESTAMPTZ NULL -> payload.alerts[i].expires
|
||||||
// - area_description TEXT NULL -> payload.alerts[i].areaDescription
|
// - area_description TEXT NULL -> payload.alerts[i].areaDescription
|
||||||
// - sender_name TEXT NULL -> payload.alerts[i].senderName
|
// - sender_name TEXT NULL -> payload.alerts[i].senderName
|
||||||
@@ -208,6 +215,57 @@
|
|||||||
// - sender TEXT NULL -> payload.alerts[i].references[j].sender
|
// - sender TEXT NULL -> payload.alerts[i].references[j].sender
|
||||||
// - sent TIMESTAMPTZ NULL -> payload.alerts[i].references[j].sent
|
// - sent TIMESTAMPTZ NULL -> payload.alerts[i].references[j].sent
|
||||||
//
|
//
|
||||||
|
// 10. outlook_runs (PK: event_id)
|
||||||
|
//
|
||||||
|
// - event_id TEXT -> event.id
|
||||||
|
// - event_kind TEXT -> event.kind
|
||||||
|
// - event_source TEXT -> event.source
|
||||||
|
// - event_schema TEXT -> event.schema
|
||||||
|
// - event_emitted_at TIMESTAMPTZ -> event.emitted_at
|
||||||
|
// - event_effective_at TIMESTAMPTZ NULL -> event.effective_at
|
||||||
|
// - location_id TEXT NULL -> payload.locationId
|
||||||
|
// - location_name TEXT NULL -> payload.locationName
|
||||||
|
// - latitude DOUBLE PRECISION NULL -> payload.latitude
|
||||||
|
// - longitude DOUBLE PRECISION NULL -> payload.longitude
|
||||||
|
// - as_of TIMESTAMPTZ -> payload.asOf
|
||||||
|
// - issued_at TIMESTAMPTZ NULL -> payload.issuedAt
|
||||||
|
// - outlook_count INTEGER -> len(payload.outlooks)
|
||||||
|
// - discussion_count INTEGER -> len(payload.discussions)
|
||||||
|
//
|
||||||
|
// 11. outlooks (PK: run_event_id, outlook_index)
|
||||||
|
//
|
||||||
|
// - run_event_id TEXT -> outlook_runs.event_id / payload.outlooks[i]
|
||||||
|
// - outlook_index INTEGER -> i (array position in payload.outlooks)
|
||||||
|
// - as_of TIMESTAMPTZ -> payload.asOf (copied from parent)
|
||||||
|
// - outlook_id TEXT -> payload.outlooks[i].id
|
||||||
|
// - provider TEXT -> payload.outlooks[i].provider
|
||||||
|
// - product TEXT -> payload.outlooks[i].product
|
||||||
|
// - day INTEGER -> payload.outlooks[i].day
|
||||||
|
// - outlook_type TEXT -> payload.outlooks[i].outlookType
|
||||||
|
// - label TEXT -> payload.outlooks[i].label
|
||||||
|
// - label_text TEXT NULL -> payload.outlooks[i].labelText
|
||||||
|
// - severity_rank INTEGER NULL -> payload.outlooks[i].severityRank
|
||||||
|
// - valid_from TIMESTAMPTZ -> payload.outlooks[i].validFrom
|
||||||
|
// - valid_to TIMESTAMPTZ -> payload.outlooks[i].validTo
|
||||||
|
// - issued_at TIMESTAMPTZ -> payload.outlooks[i].issuedAt
|
||||||
|
// - expires_at TIMESTAMPTZ -> payload.outlooks[i].expiresAt
|
||||||
|
// - forecaster TEXT NULL -> payload.outlooks[i].forecaster
|
||||||
|
// - source_url TEXT NULL -> payload.outlooks[i].sourceUrl
|
||||||
|
// - image_url TEXT NULL -> payload.outlooks[i].imageUrl
|
||||||
|
// - contains_location BOOLEAN -> payload.outlooks[i].containsLocation
|
||||||
|
// - geometry_json TEXT -> compact JSON payload.outlooks[i].geometry
|
||||||
|
//
|
||||||
|
// 12. outlook_discussions (PK: run_event_id, discussion_index)
|
||||||
|
//
|
||||||
|
// - run_event_id TEXT -> outlook_runs.event_id / payload.discussions[i]
|
||||||
|
// - discussion_index INTEGER -> i (array position in payload.discussions)
|
||||||
|
// - as_of TIMESTAMPTZ -> payload.asOf (copied from parent)
|
||||||
|
// - day INTEGER -> payload.discussions[i].day
|
||||||
|
// - headline TEXT NULL -> payload.discussions[i].headline
|
||||||
|
// - summary TEXT NULL -> payload.discussions[i].summary
|
||||||
|
// - discussion TEXT NULL -> payload.discussions[i].discussion
|
||||||
|
// - updated_at TIMESTAMPTZ NULL -> payload.discussions[i].updatedAt
|
||||||
|
//
|
||||||
// Reconstructing canonical JSON payloads
|
// Reconstructing canonical JSON payloads
|
||||||
//
|
//
|
||||||
// - WeatherObservation:
|
// - WeatherObservation:
|
||||||
@@ -226,4 +284,9 @@
|
|||||||
// read one row from alert_runs, join alerts by run_event_id ordered by
|
// read one row from alert_runs, join alerts by run_event_id ordered by
|
||||||
// alert_index, then join alert_references by (run_event_id, alert_index)
|
// alert_index, then join alert_references by (run_event_id, alert_index)
|
||||||
// ordered by reference_index to rebuild references per alert.
|
// ordered by reference_index to rebuild references per alert.
|
||||||
|
//
|
||||||
|
// - WeatherOutlookRun:
|
||||||
|
// read one row from outlook_runs, join outlooks by run_event_id ordered by
|
||||||
|
// outlook_index to rebuild outlooks, then join outlook_discussions by
|
||||||
|
// run_event_id ordered by discussion_index to rebuild discussions.
|
||||||
package postgres
|
package postgres
|
||||||
|
|||||||
33
internal/sinks/postgres/docs_test.go
Normal file
33
internal/sinks/postgres/docs_test.go
Normal file
@@ -0,0 +1,33 @@
|
|||||||
|
package postgres
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestDocumentedOutlookDiscussionStorage(t *testing.T) {
|
||||||
|
for _, path := range []string{
|
||||||
|
"../../../docs/integrations/postgres.md",
|
||||||
|
"../../../docs/internal/postgres-sink.md",
|
||||||
|
} {
|
||||||
|
t.Run(path, func(t *testing.T) {
|
||||||
|
raw, err := os.ReadFile(path)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ReadFile(%s) error = %v", path, err)
|
||||||
|
}
|
||||||
|
doc := string(raw)
|
||||||
|
|
||||||
|
for _, want := range []string{
|
||||||
|
tableOutlookDiscussions,
|
||||||
|
"discussion_count",
|
||||||
|
"discussion_index",
|
||||||
|
"weather.outlook.v2",
|
||||||
|
} {
|
||||||
|
if !strings.Contains(doc, want) {
|
||||||
|
t.Fatalf("%s missing %q", path, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,6 +1,7 @@
|
|||||||
package postgres
|
package postgres
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"bytes"
|
||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
@@ -26,6 +27,8 @@ func mapPostgresEvent(_ context.Context, e fkevent.Event) ([]fksinks.PostgresWri
|
|||||||
return mapWeatherStoryEvent(e)
|
return mapWeatherStoryEvent(e)
|
||||||
case standards.SchemaWeatherAlertV1:
|
case standards.SchemaWeatherAlertV1:
|
||||||
return mapAlertEvent(e)
|
return mapAlertEvent(e)
|
||||||
|
case standards.SchemaWeatherOutlookV2:
|
||||||
|
return mapOutlookEvent(e)
|
||||||
default:
|
default:
|
||||||
return nil, nil
|
return nil, nil
|
||||||
}
|
}
|
||||||
@@ -45,13 +48,7 @@ func mapObservationEvent(e fkevent.Event) ([]fksinks.PostgresWrite, error) {
|
|||||||
|
|
||||||
writes = append(writes, fksinks.PostgresWrite{
|
writes = append(writes, fksinks.PostgresWrite{
|
||||||
Table: tableObservations,
|
Table: tableObservations,
|
||||||
Values: map[string]any{
|
Values: parentEventValues(e, map[string]any{
|
||||||
"event_id": e.ID,
|
|
||||||
"event_kind": string(e.Kind),
|
|
||||||
"event_source": e.Source,
|
|
||||||
"event_schema": e.Schema,
|
|
||||||
"event_emitted_at": e.EmittedAt.UTC(),
|
|
||||||
"event_effective_at": nullableTime(e.EffectiveAt),
|
|
||||||
"station_id": nullableString(obs.StationID),
|
"station_id": nullableString(obs.StationID),
|
||||||
"station_name": nullableString(obs.StationName),
|
"station_name": nullableString(obs.StationName),
|
||||||
"observed_at": observedAt,
|
"observed_at": observedAt,
|
||||||
@@ -67,7 +64,7 @@ func mapObservationEvent(e fkevent.Event) ([]fksinks.PostgresWrite, error) {
|
|||||||
"visibility_meters": nullableFloat64(obs.VisibilityMeters),
|
"visibility_meters": nullableFloat64(obs.VisibilityMeters),
|
||||||
"relative_humidity_percent": nullableFloat64(obs.RelativeHumidityPercent),
|
"relative_humidity_percent": nullableFloat64(obs.RelativeHumidityPercent),
|
||||||
"apparent_temperature_c": nullableFloat64(obs.ApparentTemperatureC),
|
"apparent_temperature_c": nullableFloat64(obs.ApparentTemperatureC),
|
||||||
},
|
}),
|
||||||
})
|
})
|
||||||
|
|
||||||
for i, pw := range obs.PresentWeather {
|
for i, pw := range obs.PresentWeather {
|
||||||
@@ -106,23 +103,17 @@ func mapForecastEvent(e fkevent.Event) ([]fksinks.PostgresWrite, error) {
|
|||||||
|
|
||||||
writes = append(writes, fksinks.PostgresWrite{
|
writes = append(writes, fksinks.PostgresWrite{
|
||||||
Table: tableForecasts,
|
Table: tableForecasts,
|
||||||
Values: map[string]any{
|
Values: parentEventValues(e, map[string]any{
|
||||||
"event_id": e.ID,
|
"location_id": nullableString(run.LocationID),
|
||||||
"event_kind": string(e.Kind),
|
"location_name": nullableString(run.LocationName),
|
||||||
"event_source": e.Source,
|
"issued_at": issuedAt,
|
||||||
"event_schema": e.Schema,
|
"updated_at": nullableTime(run.UpdatedAt),
|
||||||
"event_emitted_at": e.EmittedAt.UTC(),
|
"product": string(run.Product),
|
||||||
"event_effective_at": nullableTime(e.EffectiveAt),
|
"latitude": nullableFloat64(run.Latitude),
|
||||||
"location_id": nullableString(run.LocationID),
|
"longitude": nullableFloat64(run.Longitude),
|
||||||
"location_name": nullableString(run.LocationName),
|
"elevation_meters": nullableFloat64(run.ElevationMeters),
|
||||||
"issued_at": issuedAt,
|
"period_count": len(run.Periods),
|
||||||
"updated_at": nullableTime(run.UpdatedAt),
|
}),
|
||||||
"product": string(run.Product),
|
|
||||||
"latitude": nullableFloat64(run.Latitude),
|
|
||||||
"longitude": nullableFloat64(run.Longitude),
|
|
||||||
"elevation_meters": nullableFloat64(run.ElevationMeters),
|
|
||||||
"period_count": len(run.Periods),
|
|
||||||
},
|
|
||||||
})
|
})
|
||||||
|
|
||||||
for i, p := range run.Periods {
|
for i, p := range run.Periods {
|
||||||
@@ -183,13 +174,7 @@ func mapForecastDiscussionEvent(e fkevent.Event) ([]fksinks.PostgresWrite, error
|
|||||||
writes := make([]fksinks.PostgresWrite, 0, 1+len(run.KeyMessages))
|
writes := make([]fksinks.PostgresWrite, 0, 1+len(run.KeyMessages))
|
||||||
writes = append(writes, fksinks.PostgresWrite{
|
writes = append(writes, fksinks.PostgresWrite{
|
||||||
Table: tableForecastDiscussions,
|
Table: tableForecastDiscussions,
|
||||||
Values: map[string]any{
|
Values: parentEventValues(e, map[string]any{
|
||||||
"event_id": e.ID,
|
|
||||||
"event_kind": string(e.Kind),
|
|
||||||
"event_source": e.Source,
|
|
||||||
"event_schema": e.Schema,
|
|
||||||
"event_emitted_at": e.EmittedAt.UTC(),
|
|
||||||
"event_effective_at": nullableTime(e.EffectiveAt),
|
|
||||||
"office_id": nullableString(run.OfficeID),
|
"office_id": nullableString(run.OfficeID),
|
||||||
"office_name": nullableString(run.OfficeName),
|
"office_name": nullableString(run.OfficeName),
|
||||||
"issued_at": issuedAt,
|
"issued_at": issuedAt,
|
||||||
@@ -202,7 +187,7 @@ func mapForecastDiscussionEvent(e fkevent.Event) ([]fksinks.PostgresWrite, error
|
|||||||
"long_term_issued_at": longTermIssuedAt,
|
"long_term_issued_at": longTermIssuedAt,
|
||||||
"long_term_text": longTermText,
|
"long_term_text": longTermText,
|
||||||
"key_message_count": len(run.KeyMessages),
|
"key_message_count": len(run.KeyMessages),
|
||||||
},
|
}),
|
||||||
})
|
})
|
||||||
|
|
||||||
for i, msg := range run.KeyMessages {
|
for i, msg := range run.KeyMessages {
|
||||||
@@ -233,17 +218,11 @@ func mapWeatherStoryEvent(e fkevent.Event) ([]fksinks.PostgresWrite, error) {
|
|||||||
writes := make([]fksinks.PostgresWrite, 0, 1+len(run.Stories))
|
writes := make([]fksinks.PostgresWrite, 0, 1+len(run.Stories))
|
||||||
writes = append(writes, fksinks.PostgresWrite{
|
writes = append(writes, fksinks.PostgresWrite{
|
||||||
Table: tableWeatherStoryRuns,
|
Table: tableWeatherStoryRuns,
|
||||||
Values: map[string]any{
|
Values: parentEventValues(e, map[string]any{
|
||||||
"event_id": e.ID,
|
"office_id": nullableString(run.OfficeID),
|
||||||
"event_kind": string(e.Kind),
|
"as_of": asOf,
|
||||||
"event_source": e.Source,
|
"story_count": len(run.Stories),
|
||||||
"event_schema": e.Schema,
|
}),
|
||||||
"event_emitted_at": e.EmittedAt.UTC(),
|
|
||||||
"event_effective_at": nullableTime(e.EffectiveAt),
|
|
||||||
"office_id": nullableString(run.OfficeID),
|
|
||||||
"as_of": asOf,
|
|
||||||
"story_count": len(run.Stories),
|
|
||||||
},
|
|
||||||
})
|
})
|
||||||
|
|
||||||
for i, story := range run.Stories {
|
for i, story := range run.Stories {
|
||||||
@@ -287,20 +266,14 @@ func mapAlertEvent(e fkevent.Event) ([]fksinks.PostgresWrite, error) {
|
|||||||
|
|
||||||
writes = append(writes, fksinks.PostgresWrite{
|
writes = append(writes, fksinks.PostgresWrite{
|
||||||
Table: tableAlertRuns,
|
Table: tableAlertRuns,
|
||||||
Values: map[string]any{
|
Values: parentEventValues(e, map[string]any{
|
||||||
"event_id": e.ID,
|
"location_id": nullableString(run.LocationID),
|
||||||
"event_kind": string(e.Kind),
|
"location_name": nullableString(run.LocationName),
|
||||||
"event_source": e.Source,
|
"as_of": asOf,
|
||||||
"event_schema": e.Schema,
|
"latitude": nullableFloat64(run.Latitude),
|
||||||
"event_emitted_at": e.EmittedAt.UTC(),
|
"longitude": nullableFloat64(run.Longitude),
|
||||||
"event_effective_at": nullableTime(e.EffectiveAt),
|
"alert_count": len(run.Alerts),
|
||||||
"location_id": nullableString(run.LocationID),
|
}),
|
||||||
"location_name": nullableString(run.LocationName),
|
|
||||||
"as_of": asOf,
|
|
||||||
"latitude": nullableFloat64(run.Latitude),
|
|
||||||
"longitude": nullableFloat64(run.Longitude),
|
|
||||||
"alert_count": len(run.Alerts),
|
|
||||||
},
|
|
||||||
})
|
})
|
||||||
|
|
||||||
for i, a := range run.Alerts {
|
for i, a := range run.Alerts {
|
||||||
@@ -329,6 +302,7 @@ func mapAlertEvent(e fkevent.Event) ([]fksinks.PostgresWrite, error) {
|
|||||||
"sent": nullableTime(a.Sent),
|
"sent": nullableTime(a.Sent),
|
||||||
"effective": nullableTime(a.Effective),
|
"effective": nullableTime(a.Effective),
|
||||||
"onset": nullableTime(a.Onset),
|
"onset": nullableTime(a.Onset),
|
||||||
|
"ends": nullableTime(a.Ends),
|
||||||
"expires": nullableTime(a.Expires),
|
"expires": nullableTime(a.Expires),
|
||||||
"area_description": nullableString(a.AreaDescription),
|
"area_description": nullableString(a.AreaDescription),
|
||||||
"sender_name": nullableString(a.SenderName),
|
"sender_name": nullableString(a.SenderName),
|
||||||
@@ -356,6 +330,143 @@ func mapAlertEvent(e fkevent.Event) ([]fksinks.PostgresWrite, error) {
|
|||||||
return writes, nil
|
return writes, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func mapOutlookEvent(e fkevent.Event) ([]fksinks.PostgresWrite, error) {
|
||||||
|
run, err := decodePayload[model.WeatherOutlookRun](e.Payload)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("decode outlook payload: %w", err)
|
||||||
|
}
|
||||||
|
if run.AsOf.IsZero() {
|
||||||
|
return nil, fmt.Errorf("decode outlook payload: asOf is required")
|
||||||
|
}
|
||||||
|
|
||||||
|
asOf := run.AsOf.UTC()
|
||||||
|
if err := validateOutlookDiscussions(run.Discussions); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
writes := make([]fksinks.PostgresWrite, 0, 1+len(run.Outlooks)+len(run.Discussions))
|
||||||
|
writes = append(writes, fksinks.PostgresWrite{
|
||||||
|
Table: tableOutlookRuns,
|
||||||
|
Values: parentEventValues(e, map[string]any{
|
||||||
|
"location_id": nullableString(run.LocationID),
|
||||||
|
"location_name": nullableString(run.LocationName),
|
||||||
|
"latitude": nullableFloat64(run.Latitude),
|
||||||
|
"longitude": nullableFloat64(run.Longitude),
|
||||||
|
"as_of": asOf,
|
||||||
|
"issued_at": nullableTime(run.IssuedAt),
|
||||||
|
"outlook_count": len(run.Outlooks),
|
||||||
|
"discussion_count": len(run.Discussions),
|
||||||
|
}),
|
||||||
|
})
|
||||||
|
|
||||||
|
for i, outlook := range run.Outlooks {
|
||||||
|
if err := validateOutlook(outlook, i); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
geometryJSON, err := requiredCompactJSONText(outlook.Geometry)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("decode outlook payload: outlooks[%d].geometry: %w", i, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
writes = append(writes, fksinks.PostgresWrite{
|
||||||
|
Table: tableOutlooks,
|
||||||
|
Values: map[string]any{
|
||||||
|
"run_event_id": e.ID,
|
||||||
|
"outlook_index": i,
|
||||||
|
"as_of": asOf,
|
||||||
|
"outlook_id": outlook.ID,
|
||||||
|
"provider": outlook.Provider,
|
||||||
|
"product": outlook.Product,
|
||||||
|
"day": outlook.Day,
|
||||||
|
"outlook_type": outlook.OutlookType,
|
||||||
|
"label": outlook.Label,
|
||||||
|
"label_text": nullableString(outlook.LabelText),
|
||||||
|
"severity_rank": nullableInt(outlook.SeverityRank),
|
||||||
|
"valid_from": outlook.ValidFrom.UTC(),
|
||||||
|
"valid_to": outlook.ValidTo.UTC(),
|
||||||
|
"issued_at": outlook.IssuedAt.UTC(),
|
||||||
|
"expires_at": outlook.ExpiresAt.UTC(),
|
||||||
|
"forecaster": nullableString(outlook.Forecaster),
|
||||||
|
"source_url": nullableString(outlook.SourceURL),
|
||||||
|
"image_url": nullableString(outlook.ImageURL),
|
||||||
|
"contains_location": outlook.ContainsLocation,
|
||||||
|
"geometry_json": geometryJSON,
|
||||||
|
},
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
for i, discussion := range run.Discussions {
|
||||||
|
writes = append(writes, fksinks.PostgresWrite{
|
||||||
|
Table: tableOutlookDiscussions,
|
||||||
|
Values: map[string]any{
|
||||||
|
"run_event_id": e.ID,
|
||||||
|
"discussion_index": i,
|
||||||
|
"as_of": asOf,
|
||||||
|
"day": discussion.Day,
|
||||||
|
"headline": nullableString(discussion.Headline),
|
||||||
|
"summary": nullableString(discussion.Summary),
|
||||||
|
"discussion": nullableString(discussion.Discussion),
|
||||||
|
"updated_at": nullableTime(discussion.UpdatedAt),
|
||||||
|
},
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
return writes, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func validateOutlook(outlook model.WeatherOutlook, index int) error {
|
||||||
|
if strings.TrimSpace(outlook.ID) == "" {
|
||||||
|
return fmt.Errorf("decode outlook payload: outlooks[%d].id is required", index)
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(outlook.Provider) == "" {
|
||||||
|
return fmt.Errorf("decode outlook payload: outlooks[%d].provider is required", index)
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(outlook.Product) == "" {
|
||||||
|
return fmt.Errorf("decode outlook payload: outlooks[%d].product is required", index)
|
||||||
|
}
|
||||||
|
if outlook.Day == 0 {
|
||||||
|
return fmt.Errorf("decode outlook payload: outlooks[%d].day is required", index)
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(outlook.OutlookType) == "" {
|
||||||
|
return fmt.Errorf("decode outlook payload: outlooks[%d].outlookType is required", index)
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(outlook.Label) == "" {
|
||||||
|
return fmt.Errorf("decode outlook payload: outlooks[%d].label is required", index)
|
||||||
|
}
|
||||||
|
if outlook.ValidFrom.IsZero() || outlook.ValidTo.IsZero() {
|
||||||
|
return fmt.Errorf("decode outlook payload: outlooks[%d] validFrom/validTo are required", index)
|
||||||
|
}
|
||||||
|
if outlook.IssuedAt.IsZero() || outlook.ExpiresAt.IsZero() {
|
||||||
|
return fmt.Errorf("decode outlook payload: outlooks[%d] issuedAt/expiresAt are required", index)
|
||||||
|
}
|
||||||
|
if len(outlook.Geometry) == 0 {
|
||||||
|
return fmt.Errorf("decode outlook payload: outlooks[%d].geometry is required", index)
|
||||||
|
}
|
||||||
|
if !outlook.ContainsLocation {
|
||||||
|
return fmt.Errorf("decode outlook payload: outlooks[%d].containsLocation must be true", index)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func validateOutlookDiscussions(discussions []model.WeatherOutlookDiscussion) error {
|
||||||
|
seenDays := map[int]int{}
|
||||||
|
for i, discussion := range discussions {
|
||||||
|
if discussion.Day < 1 || discussion.Day > 3 {
|
||||||
|
return fmt.Errorf("decode outlook payload: discussions[%d].day must be 1, 2, or 3", i)
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(discussion.Headline) == "" &&
|
||||||
|
strings.TrimSpace(discussion.Summary) == "" &&
|
||||||
|
strings.TrimSpace(discussion.Discussion) == "" {
|
||||||
|
return fmt.Errorf("decode outlook payload: discussions[%d] headline, summary, or discussion is required", i)
|
||||||
|
}
|
||||||
|
if first, ok := seenDays[discussion.Day]; ok {
|
||||||
|
return fmt.Errorf("decode outlook payload: discussions[%d].day duplicates discussions[%d].day %d", i, first, discussion.Day)
|
||||||
|
}
|
||||||
|
seenDays[discussion.Day] = i
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
func decodePayload[T any](payload any) (T, error) {
|
func decodePayload[T any](payload any) (T, error) {
|
||||||
var out T
|
var out T
|
||||||
if payload == nil {
|
if payload == nil {
|
||||||
@@ -382,6 +493,21 @@ func decodePayload[T any](payload any) (T, error) {
|
|||||||
return out, nil
|
return out, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func parentEventValues(e fkevent.Event, values map[string]any) map[string]any {
|
||||||
|
out := map[string]any{
|
||||||
|
"event_id": e.ID,
|
||||||
|
"event_kind": string(e.Kind),
|
||||||
|
"event_source": e.Source,
|
||||||
|
"event_schema": e.Schema,
|
||||||
|
"event_emitted_at": e.EmittedAt.UTC(),
|
||||||
|
"event_effective_at": nullableTime(e.EffectiveAt),
|
||||||
|
}
|
||||||
|
for k, v := range values {
|
||||||
|
out[k] = v
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
func nullableDiscussionSection(section *model.WeatherForecastDiscussionSection) (any, any, any) {
|
func nullableDiscussionSection(section *model.WeatherForecastDiscussionSection) (any, any, any) {
|
||||||
if section == nil {
|
if section == nil {
|
||||||
return nil, nil, nil
|
return nil, nil, nil
|
||||||
@@ -418,6 +544,13 @@ func nullableBool(v *bool) any {
|
|||||||
return *v
|
return *v
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func nullableInt(v *int) any {
|
||||||
|
if v == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return *v
|
||||||
|
}
|
||||||
|
|
||||||
func nullableTime(v *time.Time) any {
|
func nullableTime(v *time.Time) any {
|
||||||
if v == nil || v.IsZero() {
|
if v == nil || v.IsZero() {
|
||||||
return nil
|
return nil
|
||||||
@@ -445,3 +578,19 @@ func compactJSONText(v any) (any, error) {
|
|||||||
}
|
}
|
||||||
return string(b), nil
|
return string(b), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func requiredCompactJSONText(v any) (string, error) {
|
||||||
|
compact, err := compactJSONText(v)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
s, ok := compact.(string)
|
||||||
|
if !ok || strings.TrimSpace(s) == "" || strings.TrimSpace(s) == "null" {
|
||||||
|
return "", fmt.Errorf("is required")
|
||||||
|
}
|
||||||
|
var buf bytes.Buffer
|
||||||
|
if err := json.Compact(&buf, []byte(s)); err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
return buf.String(), nil
|
||||||
|
}
|
||||||
|
|||||||
@@ -27,7 +27,7 @@ func TestMapPostgresEventObservationStructPayload(t *testing.T) {
|
|||||||
PresentWeather: []model.PresentWeather{{Raw: map[string]any{"a": 1, "b": "x"}}},
|
PresentWeather: []model.PresentWeather{{Raw: map[string]any{"a": 1, "b": "x"}}},
|
||||||
}
|
}
|
||||||
|
|
||||||
writes, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherObservationV1, "observation", obs))
|
writes, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherObservationV1, standards.KindObservation, obs))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("mapPostgresEvent() error = %v", err)
|
t.Fatalf("mapPostgresEvent() error = %v", err)
|
||||||
}
|
}
|
||||||
@@ -74,7 +74,7 @@ func TestMapPostgresEventForecastStructPayload(t *testing.T) {
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
writes, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherForecastV1, "forecast", run))
|
writes, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherForecastV1, standards.KindForecast, run))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("mapPostgresEvent() error = %v", err)
|
t.Fatalf("mapPostgresEvent() error = %v", err)
|
||||||
}
|
}
|
||||||
@@ -103,6 +103,8 @@ func TestMapPostgresEventForecastStructPayload(t *testing.T) {
|
|||||||
|
|
||||||
func TestMapPostgresEventAlertStructPayload(t *testing.T) {
|
func TestMapPostgresEventAlertStructPayload(t *testing.T) {
|
||||||
sent := time.Date(2026, 3, 16, 17, 0, 0, 0, time.UTC)
|
sent := time.Date(2026, 3, 16, 17, 0, 0, 0, time.UTC)
|
||||||
|
ends := time.Date(2026, 3, 16, 20, 0, 0, 0, time.UTC)
|
||||||
|
expires := time.Date(2026, 3, 16, 18, 30, 0, 0, time.UTC)
|
||||||
run := model.WeatherAlertRun{
|
run := model.WeatherAlertRun{
|
||||||
AsOf: time.Date(2026, 3, 16, 18, 0, 0, 0, time.UTC),
|
AsOf: time.Date(2026, 3, 16, 18, 0, 0, 0, time.UTC),
|
||||||
Alerts: []model.WeatherAlert{
|
Alerts: []model.WeatherAlert{
|
||||||
@@ -110,6 +112,8 @@ func TestMapPostgresEventAlertStructPayload(t *testing.T) {
|
|||||||
ID: "urn:alert:1",
|
ID: "urn:alert:1",
|
||||||
Headline: "Winter Weather Advisory",
|
Headline: "Winter Weather Advisory",
|
||||||
Severity: "Moderate",
|
Severity: "Moderate",
|
||||||
|
Ends: &ends,
|
||||||
|
Expires: &expires,
|
||||||
References: []model.AlertReference{
|
References: []model.AlertReference{
|
||||||
{ID: "urn:ref:1", Sent: &sent},
|
{ID: "urn:ref:1", Sent: &sent},
|
||||||
{Identifier: "ref-two"},
|
{Identifier: "ref-two"},
|
||||||
@@ -122,7 +126,7 @@ func TestMapPostgresEventAlertStructPayload(t *testing.T) {
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
writes, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherAlertV1, "alert", run))
|
writes, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherAlertV1, standards.KindAlert, run))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("mapPostgresEvent() error = %v", err)
|
t.Fatalf("mapPostgresEvent() error = %v", err)
|
||||||
}
|
}
|
||||||
@@ -145,6 +149,20 @@ func TestMapPostgresEventAlertStructPayload(t *testing.T) {
|
|||||||
if got := firstAlert.Values["reference_count"]; got != 2 {
|
if got := firstAlert.Values["reference_count"]; got != 2 {
|
||||||
t.Fatalf("alerts reference_count = %#v, want 2", got)
|
t.Fatalf("alerts reference_count = %#v, want 2", got)
|
||||||
}
|
}
|
||||||
|
if got := firstAlert.Values["ends"]; got != ends {
|
||||||
|
t.Fatalf("alerts ends = %#v, want %#v", got, ends)
|
||||||
|
}
|
||||||
|
if got := firstAlert.Values["expires"]; got != expires {
|
||||||
|
t.Fatalf("alerts expires = %#v, want %#v", got, expires)
|
||||||
|
}
|
||||||
|
|
||||||
|
alertWrites := writesForTable(writes, tableAlerts)
|
||||||
|
if len(alertWrites) != 2 {
|
||||||
|
t.Fatalf("alert writes len = %d, want 2", len(alertWrites))
|
||||||
|
}
|
||||||
|
if got := alertWrites[1].Values["ends"]; got != nil {
|
||||||
|
t.Fatalf("second alert ends = %#v, want nil", got)
|
||||||
|
}
|
||||||
|
|
||||||
assertAllWritesIncludeAllColumns(t, writes)
|
assertAllWritesIncludeAllColumns(t, writes)
|
||||||
}
|
}
|
||||||
@@ -163,7 +181,7 @@ func TestMapPostgresEventForecastDiscussionStructPayload(t *testing.T) {
|
|||||||
LongTerm: &model.WeatherForecastDiscussionSection{Text: "Long term text"},
|
LongTerm: &model.WeatherForecastDiscussionSection{Text: "Long term text"},
|
||||||
}
|
}
|
||||||
|
|
||||||
writes, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherForecastDiscussionV1, "forecast_discussion", run))
|
writes, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherForecastDiscussionV1, standards.KindForecastDiscussion, run))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("mapPostgresEvent() error = %v", err)
|
t.Fatalf("mapPostgresEvent() error = %v", err)
|
||||||
}
|
}
|
||||||
@@ -213,7 +231,7 @@ func TestMapPostgresEventWeatherStoryStructPayload(t *testing.T) {
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
writes, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherStoryV1, "weather_story", run))
|
writes, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherStoryV1, standards.KindWeatherStory, run))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("mapPostgresEvent() error = %v", err)
|
t.Fatalf("mapPostgresEvent() error = %v", err)
|
||||||
}
|
}
|
||||||
@@ -239,8 +257,331 @@ func TestMapPostgresEventWeatherStoryStructPayload(t *testing.T) {
|
|||||||
assertAllWritesIncludeAllColumns(t, writes)
|
assertAllWritesIncludeAllColumns(t, writes)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestMapPostgresEventOutlookStructPayload(t *testing.T) {
|
||||||
|
lat := 38.6239
|
||||||
|
lon := -90.3571
|
||||||
|
issuedAt := time.Date(2026, 6, 11, 19, 45, 0, 0, time.FixedZone("UTC-5", -5*60*60))
|
||||||
|
updatedAt := time.Date(2026, 6, 11, 21, 15, 0, 0, time.FixedZone("UTC-5", -5*60*60))
|
||||||
|
severity := 3
|
||||||
|
run := model.WeatherOutlookRun{
|
||||||
|
LocationID: "stl",
|
||||||
|
LocationName: "St. Louis, MO",
|
||||||
|
Latitude: &lat,
|
||||||
|
Longitude: &lon,
|
||||||
|
AsOf: time.Date(2026, 6, 12, 0, 45, 0, 0, time.UTC),
|
||||||
|
IssuedAt: &issuedAt,
|
||||||
|
Outlooks: []model.WeatherOutlook{
|
||||||
|
{
|
||||||
|
ID: "outlook-1",
|
||||||
|
Provider: "spc",
|
||||||
|
Product: "convective",
|
||||||
|
Day: 1,
|
||||||
|
OutlookType: "categorical",
|
||||||
|
Label: "SLGT",
|
||||||
|
LabelText: "Slight Risk",
|
||||||
|
SeverityRank: &severity,
|
||||||
|
ValidFrom: time.Date(2026, 6, 11, 13, 0, 0, 0, time.FixedZone("UTC-5", -5*60*60)),
|
||||||
|
ValidTo: time.Date(2026, 6, 12, 12, 0, 0, 0, time.UTC),
|
||||||
|
IssuedAt: issuedAt,
|
||||||
|
ExpiresAt: time.Date(2026, 6, 12, 12, 0, 0, 0, time.UTC),
|
||||||
|
Forecaster: "SMITH",
|
||||||
|
SourceURL: "https://example.invalid/day1.geojson",
|
||||||
|
ContainsLocation: true,
|
||||||
|
Geometry: json.RawMessage(`{ "type" : "Polygon", "coordinates" : [ [ [ -91.0, 38.0 ], [ -90.0, 38.0 ], [ -90.0, 39.0 ], [ -91.0, 39.0 ], [ -91.0, 38.0 ] ] ] }`),
|
||||||
|
},
|
||||||
|
{
|
||||||
|
ID: "outlook-2",
|
||||||
|
Provider: "spc",
|
||||||
|
Product: "convective",
|
||||||
|
Day: 1,
|
||||||
|
OutlookType: "wind",
|
||||||
|
Label: "15",
|
||||||
|
ValidFrom: time.Date(2026, 6, 11, 13, 0, 0, 0, time.UTC),
|
||||||
|
ValidTo: time.Date(2026, 6, 12, 12, 0, 0, 0, time.UTC),
|
||||||
|
IssuedAt: time.Date(2026, 6, 11, 19, 45, 0, 0, time.UTC),
|
||||||
|
ExpiresAt: time.Date(2026, 6, 12, 12, 0, 0, 0, time.UTC),
|
||||||
|
ContainsLocation: true,
|
||||||
|
Geometry: json.RawMessage(`{"type":"Polygon","coordinates":[[[-100,35],[-98,35],[-98,37],[-100,37],[-100,35]]]}`),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
Discussions: []model.WeatherOutlookDiscussion{
|
||||||
|
{
|
||||||
|
Day: 1,
|
||||||
|
Headline: "Day 1 Convective Outlook",
|
||||||
|
Summary: "Severe thunderstorms are possible.",
|
||||||
|
Discussion: "Full discussion text.",
|
||||||
|
UpdatedAt: &updatedAt,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
writes, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherOutlookV2, standards.KindOutlook, run))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("mapPostgresEvent() error = %v", err)
|
||||||
|
}
|
||||||
|
if len(writes) != 4 {
|
||||||
|
t.Fatalf("mapPostgresEvent() writes len = %d, want 4", len(writes))
|
||||||
|
}
|
||||||
|
if writes[0].Table != tableOutlookRuns {
|
||||||
|
t.Fatalf("writes[0].Table = %q, want %q", writes[0].Table, tableOutlookRuns)
|
||||||
|
}
|
||||||
|
if got := writes[0].Values["outlook_count"]; got != 2 {
|
||||||
|
t.Fatalf("outlook_runs outlook_count = %#v, want 2", got)
|
||||||
|
}
|
||||||
|
if got := writes[0].Values["discussion_count"]; got != 1 {
|
||||||
|
t.Fatalf("outlook_runs discussion_count = %#v, want 1", got)
|
||||||
|
}
|
||||||
|
if got := writes[0].Values["issued_at"]; got != issuedAt.UTC() {
|
||||||
|
t.Fatalf("outlook_runs issued_at = %#v, want UTC %s", got, issuedAt.UTC())
|
||||||
|
}
|
||||||
|
if writes[1].Table != tableOutlooks || writes[2].Table != tableOutlooks {
|
||||||
|
t.Fatalf("outlook writes not in expected order")
|
||||||
|
}
|
||||||
|
if got := writes[1].Values["outlook_index"]; got != 0 {
|
||||||
|
t.Fatalf("first outlook index = %#v, want 0", got)
|
||||||
|
}
|
||||||
|
if got := writes[1].Values["outlook_id"]; got != "outlook-1" {
|
||||||
|
t.Fatalf("first outlook_id = %#v, want outlook-1", got)
|
||||||
|
}
|
||||||
|
if got := writes[1].Values["provider"]; got != "spc" {
|
||||||
|
t.Fatalf("first provider = %#v, want spc", got)
|
||||||
|
}
|
||||||
|
if got := writes[1].Values["valid_from"]; got != run.Outlooks[0].ValidFrom.UTC() {
|
||||||
|
t.Fatalf("first valid_from = %#v, want UTC %s", got, run.Outlooks[0].ValidFrom.UTC())
|
||||||
|
}
|
||||||
|
if got := writes[1].Values["geometry_json"]; got != `{"type":"Polygon","coordinates":[[[-91.0,38.0],[-90.0,38.0],[-90.0,39.0],[-91.0,39.0],[-91.0,38.0]]]}` {
|
||||||
|
t.Fatalf("first geometry_json = %#v", got)
|
||||||
|
}
|
||||||
|
if got := writes[2].Values["contains_location"]; got != true {
|
||||||
|
t.Fatalf("second contains_location = %#v, want true", got)
|
||||||
|
}
|
||||||
|
if writes[3].Table != tableOutlookDiscussions {
|
||||||
|
t.Fatalf("writes[3].Table = %q, want %q", writes[3].Table, tableOutlookDiscussions)
|
||||||
|
}
|
||||||
|
if got := writes[3].Values["discussion_index"]; got != 0 {
|
||||||
|
t.Fatalf("discussion_index = %#v, want 0", got)
|
||||||
|
}
|
||||||
|
if got := writes[3].Values["as_of"]; got != run.AsOf.UTC() {
|
||||||
|
t.Fatalf("discussion as_of = %#v, want %s", got, run.AsOf.UTC())
|
||||||
|
}
|
||||||
|
if got := writes[3].Values["day"]; got != 1 {
|
||||||
|
t.Fatalf("discussion day = %#v, want 1", got)
|
||||||
|
}
|
||||||
|
if got := writes[3].Values["headline"]; got != "Day 1 Convective Outlook" {
|
||||||
|
t.Fatalf("discussion headline = %#v", got)
|
||||||
|
}
|
||||||
|
if got := writes[3].Values["summary"]; got != "Severe thunderstorms are possible." {
|
||||||
|
t.Fatalf("discussion summary = %#v", got)
|
||||||
|
}
|
||||||
|
if got := writes[3].Values["discussion"]; got != "Full discussion text." {
|
||||||
|
t.Fatalf("discussion text = %#v", got)
|
||||||
|
}
|
||||||
|
if got := writes[3].Values["updated_at"]; got != updatedAt.UTC() {
|
||||||
|
t.Fatalf("discussion updated_at = %#v, want UTC %s", got, updatedAt.UTC())
|
||||||
|
}
|
||||||
|
|
||||||
|
assertAllWritesIncludeAllColumns(t, writes)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMapPostgresEventOutlookEmptyLocalRun(t *testing.T) {
|
||||||
|
run := model.WeatherOutlookRun{
|
||||||
|
AsOf: time.Date(2026, 6, 11, 19, 45, 0, 0, time.UTC),
|
||||||
|
}
|
||||||
|
|
||||||
|
writes, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherOutlookV2, standards.KindOutlook, run))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("mapPostgresEvent() error = %v", err)
|
||||||
|
}
|
||||||
|
if len(writes) != 1 {
|
||||||
|
t.Fatalf("mapPostgresEvent() writes len = %d, want 1", len(writes))
|
||||||
|
}
|
||||||
|
if writes[0].Table != tableOutlookRuns {
|
||||||
|
t.Fatalf("writes[0].Table = %q, want %q", writes[0].Table, tableOutlookRuns)
|
||||||
|
}
|
||||||
|
if got := writes[0].Values["outlook_count"]; got != 0 {
|
||||||
|
t.Fatalf("outlook_runs outlook_count = %#v, want 0", got)
|
||||||
|
}
|
||||||
|
if got := writes[0].Values["discussion_count"]; got != 0 {
|
||||||
|
t.Fatalf("outlook_runs discussion_count = %#v, want 0", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
assertAllWritesIncludeAllColumns(t, writes)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMapPostgresEventOutlookRejectsMissingAsOf(t *testing.T) {
|
||||||
|
_, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherOutlookV2, standards.KindOutlook, model.WeatherOutlookRun{}))
|
||||||
|
if err == nil {
|
||||||
|
t.Fatalf("mapPostgresEvent() error = nil, want missing asOf error")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "asOf is required") {
|
||||||
|
t.Fatalf("error = %q, want asOf context", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMapPostgresEventOutlookRejectsMissingIDAndProvider(t *testing.T) {
|
||||||
|
base := model.WeatherOutlook{
|
||||||
|
ID: "outlook-1",
|
||||||
|
Provider: "spc",
|
||||||
|
Product: "convective",
|
||||||
|
Day: 1,
|
||||||
|
OutlookType: "categorical",
|
||||||
|
Label: "SLGT",
|
||||||
|
ValidFrom: time.Date(2026, 6, 11, 13, 0, 0, 0, time.UTC),
|
||||||
|
ValidTo: time.Date(2026, 6, 12, 12, 0, 0, 0, time.UTC),
|
||||||
|
IssuedAt: time.Date(2026, 6, 11, 19, 45, 0, 0, time.UTC),
|
||||||
|
ExpiresAt: time.Date(2026, 6, 12, 12, 0, 0, 0, time.UTC),
|
||||||
|
ContainsLocation: true,
|
||||||
|
Geometry: json.RawMessage(`{"type":"Polygon","coordinates":[[[-91,38],[-90,38],[-90,39],[-91,39],[-91,38]]]}`),
|
||||||
|
}
|
||||||
|
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
mutate func(*model.WeatherOutlook)
|
||||||
|
wantErr string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "missing id",
|
||||||
|
mutate: func(outlook *model.WeatherOutlook) { outlook.ID = "" },
|
||||||
|
wantErr: "outlooks[0].id is required",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "missing provider",
|
||||||
|
mutate: func(outlook *model.WeatherOutlook) { outlook.Provider = "" },
|
||||||
|
wantErr: "outlooks[0].provider is required",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
outlook := base
|
||||||
|
tt.mutate(&outlook)
|
||||||
|
run := model.WeatherOutlookRun{
|
||||||
|
AsOf: time.Date(2026, 6, 11, 19, 45, 0, 0, time.UTC),
|
||||||
|
Outlooks: []model.WeatherOutlook{outlook},
|
||||||
|
}
|
||||||
|
_, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherOutlookV2, standards.KindOutlook, run))
|
||||||
|
if err == nil {
|
||||||
|
t.Fatalf("mapPostgresEvent() error = nil, want %q", tt.wantErr)
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), tt.wantErr) {
|
||||||
|
t.Fatalf("error = %q, want %q", err, tt.wantErr)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMapPostgresEventOutlookRejectsMissingRequiredTimes(t *testing.T) {
|
||||||
|
run := model.WeatherOutlookRun{
|
||||||
|
AsOf: time.Date(2026, 6, 11, 19, 45, 0, 0, time.UTC),
|
||||||
|
Outlooks: []model.WeatherOutlook{{
|
||||||
|
ID: "outlook-1",
|
||||||
|
Provider: "spc",
|
||||||
|
Product: "convective",
|
||||||
|
Day: 1,
|
||||||
|
OutlookType: "categorical",
|
||||||
|
Label: "SLGT",
|
||||||
|
ContainsLocation: true,
|
||||||
|
Geometry: json.RawMessage(`{"type":"Polygon","coordinates":[[[-91,38],[-90,38],[-90,39],[-91,39],[-91,38]]]}`),
|
||||||
|
}},
|
||||||
|
}
|
||||||
|
_, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherOutlookV2, standards.KindOutlook, run))
|
||||||
|
if err == nil {
|
||||||
|
t.Fatalf("mapPostgresEvent() error = nil, want missing time error")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "outlooks[0] validFrom/validTo are required") {
|
||||||
|
t.Fatalf("error = %q, want outlook time context", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMapPostgresEventOutlookRejectsEmptyGeometry(t *testing.T) {
|
||||||
|
run := model.WeatherOutlookRun{
|
||||||
|
AsOf: time.Date(2026, 6, 11, 19, 45, 0, 0, time.UTC),
|
||||||
|
Outlooks: []model.WeatherOutlook{{
|
||||||
|
ID: "outlook-1",
|
||||||
|
Provider: "spc",
|
||||||
|
Product: "convective",
|
||||||
|
Day: 1,
|
||||||
|
OutlookType: "categorical",
|
||||||
|
Label: "SLGT",
|
||||||
|
ValidFrom: time.Date(2026, 6, 11, 13, 0, 0, 0, time.UTC),
|
||||||
|
ValidTo: time.Date(2026, 6, 12, 12, 0, 0, 0, time.UTC),
|
||||||
|
IssuedAt: time.Date(2026, 6, 11, 19, 45, 0, 0, time.UTC),
|
||||||
|
ExpiresAt: time.Date(2026, 6, 12, 12, 0, 0, 0, time.UTC),
|
||||||
|
ContainsLocation: true,
|
||||||
|
}},
|
||||||
|
}
|
||||||
|
_, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherOutlookV2, standards.KindOutlook, run))
|
||||||
|
if err == nil {
|
||||||
|
t.Fatalf("mapPostgresEvent() error = nil, want geometry error")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "outlooks[0].geometry is required") {
|
||||||
|
t.Fatalf("error = %q, want geometry context", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMapPostgresEventOutlookRejectsDuplicateDiscussionDay(t *testing.T) {
|
||||||
|
run := model.WeatherOutlookRun{
|
||||||
|
AsOf: time.Date(2026, 6, 11, 19, 45, 0, 0, time.UTC),
|
||||||
|
Discussions: []model.WeatherOutlookDiscussion{
|
||||||
|
{Day: 1, Discussion: "First day one discussion."},
|
||||||
|
{Day: 1, Discussion: "Duplicate day one discussion."},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
_, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherOutlookV2, standards.KindOutlook, run))
|
||||||
|
if err == nil {
|
||||||
|
t.Fatalf("mapPostgresEvent() error = nil, want duplicate discussion day error")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "discussions[1].day duplicates discussions[0].day 1") {
|
||||||
|
t.Fatalf("error = %q, want duplicate discussion day context", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMapPostgresEventOutlookRejectsInvalidDiscussionDay(t *testing.T) {
|
||||||
|
run := model.WeatherOutlookRun{
|
||||||
|
AsOf: time.Date(2026, 6, 11, 19, 45, 0, 0, time.UTC),
|
||||||
|
Discussions: []model.WeatherOutlookDiscussion{{Day: 4, Discussion: "Invalid day."}},
|
||||||
|
}
|
||||||
|
_, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherOutlookV2, standards.KindOutlook, run))
|
||||||
|
if err == nil {
|
||||||
|
t.Fatalf("mapPostgresEvent() error = nil, want invalid discussion day error")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "discussions[0].day must be 1, 2, or 3") {
|
||||||
|
t.Fatalf("error = %q, want invalid discussion day context", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMapPostgresEventOutlookRejectsEmptyDiscussionContent(t *testing.T) {
|
||||||
|
run := model.WeatherOutlookRun{
|
||||||
|
AsOf: time.Date(2026, 6, 11, 19, 45, 0, 0, time.UTC),
|
||||||
|
Discussions: []model.WeatherOutlookDiscussion{{Day: 1}},
|
||||||
|
}
|
||||||
|
_, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherOutlookV2, standards.KindOutlook, run))
|
||||||
|
if err == nil {
|
||||||
|
t.Fatalf("mapPostgresEvent() error = nil, want empty discussion content error")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "discussions[0] headline, summary, or discussion is required") {
|
||||||
|
t.Fatalf("error = %q, want empty discussion content context", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMapPostgresEventOutlookRejectsContainsLocationFalse(t *testing.T) {
|
||||||
|
run := model.WeatherOutlookRun{
|
||||||
|
AsOf: time.Date(2026, 6, 11, 19, 45, 0, 0, time.UTC),
|
||||||
|
Outlooks: []model.WeatherOutlook{validTestOutlook()},
|
||||||
|
}
|
||||||
|
run.Outlooks[0].ContainsLocation = false
|
||||||
|
|
||||||
|
_, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherOutlookV2, standards.KindOutlook, run))
|
||||||
|
if err == nil {
|
||||||
|
t.Fatalf("mapPostgresEvent() error = nil, want containsLocation error")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "outlooks[0].containsLocation must be true") {
|
||||||
|
t.Fatalf("error = %q, want containsLocation context", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestMapPostgresEventWeatherStoryRejectsMissingAsOf(t *testing.T) {
|
func TestMapPostgresEventWeatherStoryRejectsMissingAsOf(t *testing.T) {
|
||||||
_, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherStoryV1, "weather_story", model.WeatherStoryRun{}))
|
_, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherStoryV1, standards.KindWeatherStory, model.WeatherStoryRun{}))
|
||||||
if err == nil {
|
if err == nil {
|
||||||
t.Fatalf("mapPostgresEvent() error = nil, want missing asOf error")
|
t.Fatalf("mapPostgresEvent() error = nil, want missing asOf error")
|
||||||
}
|
}
|
||||||
@@ -254,7 +595,7 @@ func TestMapPostgresEventWeatherStoryRejectsMissingStoryTimes(t *testing.T) {
|
|||||||
AsOf: time.Date(2026, 5, 30, 9, 0, 34, 0, time.UTC),
|
AsOf: time.Date(2026, 5, 30, 9, 0, 34, 0, time.UTC),
|
||||||
Stories: []model.WeatherStory{{Title: "missing times"}},
|
Stories: []model.WeatherStory{{Title: "missing times"}},
|
||||||
}
|
}
|
||||||
_, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherStoryV1, "weather_story", run))
|
_, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherStoryV1, standards.KindWeatherStory, run))
|
||||||
if err == nil {
|
if err == nil {
|
||||||
t.Fatalf("mapPostgresEvent() error = nil, want missing story times error")
|
t.Fatalf("mapPostgresEvent() error = nil, want missing story times error")
|
||||||
}
|
}
|
||||||
@@ -284,7 +625,7 @@ func TestMapPostgresEventMapPayload(t *testing.T) {
|
|||||||
t.Fatalf("json.Unmarshal() error = %v", err)
|
t.Fatalf("json.Unmarshal() error = %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
writes, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherForecastV1, "forecast", payload))
|
writes, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherForecastV1, standards.KindForecast, payload))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("mapPostgresEvent() error = %v", err)
|
t.Fatalf("mapPostgresEvent() error = %v", err)
|
||||||
}
|
}
|
||||||
@@ -299,7 +640,18 @@ func TestMapPostgresEventMapPayload(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestMapPostgresEventUnknownSchemaNoOp(t *testing.T) {
|
func TestMapPostgresEventUnknownSchemaNoOp(t *testing.T) {
|
||||||
writes, err := mapPostgresEvent(context.Background(), testEvent("weather.unknown.v1", "observation", map[string]any{"x": 1}))
|
writes, err := mapPostgresEvent(context.Background(), testEvent("weather.unknown.v1", standards.KindObservation, map[string]any{"x": 1}))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("mapPostgresEvent() error = %v", err)
|
||||||
|
}
|
||||||
|
if len(writes) != 0 {
|
||||||
|
t.Fatalf("mapPostgresEvent() writes len = %d, want 0", len(writes))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMapPostgresEventLegacyOutlookSchemaNoOp(t *testing.T) {
|
||||||
|
run := model.WeatherOutlookRun{AsOf: time.Date(2026, 6, 11, 19, 45, 0, 0, time.UTC)}
|
||||||
|
writes, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherOutlookV1, standards.KindOutlook, run))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("mapPostgresEvent() error = %v", err)
|
t.Fatalf("mapPostgresEvent() error = %v", err)
|
||||||
}
|
}
|
||||||
@@ -309,7 +661,7 @@ func TestMapPostgresEventUnknownSchemaNoOp(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestMapPostgresEventMalformedPayload(t *testing.T) {
|
func TestMapPostgresEventMalformedPayload(t *testing.T) {
|
||||||
_, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherForecastV1, "forecast", "bad"))
|
_, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherForecastV1, standards.KindForecast, "bad"))
|
||||||
if err == nil {
|
if err == nil {
|
||||||
t.Fatalf("mapPostgresEvent() expected error for malformed payload")
|
t.Fatalf("mapPostgresEvent() expected error for malformed payload")
|
||||||
}
|
}
|
||||||
@@ -319,7 +671,7 @@ func TestMapPostgresEventMalformedPayload(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestMapPostgresEventForecastDiscussionMalformedPayload(t *testing.T) {
|
func TestMapPostgresEventForecastDiscussionMalformedPayload(t *testing.T) {
|
||||||
_, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherForecastDiscussionV1, "forecast_discussion", "bad"))
|
_, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherForecastDiscussionV1, standards.KindForecastDiscussion, "bad"))
|
||||||
if err == nil {
|
if err == nil {
|
||||||
t.Fatalf("mapPostgresEvent() expected error for malformed payload")
|
t.Fatalf("mapPostgresEvent() expected error for malformed payload")
|
||||||
}
|
}
|
||||||
@@ -328,6 +680,56 @@ func TestMapPostgresEventForecastDiscussionMalformedPayload(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestParentEventValuesAddsEnvelopeAndPreservesProductValues(t *testing.T) {
|
||||||
|
emittedAt := time.Date(2026, 3, 16, 13, 31, 0, 0, time.FixedZone("CDT", -5*60*60))
|
||||||
|
effectiveAt := time.Date(2026, 3, 16, 13, 30, 0, 0, time.FixedZone("CDT", -5*60*60))
|
||||||
|
event := fkevent.Event{
|
||||||
|
ID: "evt-envelope",
|
||||||
|
Kind: fkevent.Kind(standards.KindForecast),
|
||||||
|
Source: "test-source",
|
||||||
|
Schema: standards.SchemaWeatherForecastV1,
|
||||||
|
EmittedAt: emittedAt,
|
||||||
|
EffectiveAt: &effectiveAt,
|
||||||
|
}
|
||||||
|
|
||||||
|
got := parentEventValues(event, map[string]any{"product_col": "product-value"})
|
||||||
|
|
||||||
|
assertParentEnvelopeValues(t, got, event)
|
||||||
|
if got["product_col"] != "product-value" {
|
||||||
|
t.Fatalf("product_col = %#v, want product-value", got["product_col"])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParentEventValuesNullEffectiveAt(t *testing.T) {
|
||||||
|
base := fkevent.Event{
|
||||||
|
ID: "evt-envelope",
|
||||||
|
Kind: fkevent.Kind(standards.KindObservation),
|
||||||
|
Source: "test-source",
|
||||||
|
Schema: standards.SchemaWeatherObservationV1,
|
||||||
|
EmittedAt: time.Date(2026, 3, 16, 18, 31, 0, 0, time.UTC),
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range []struct {
|
||||||
|
name string
|
||||||
|
mut func(*fkevent.Event)
|
||||||
|
}{
|
||||||
|
{name: "nil", mut: func(*fkevent.Event) {}},
|
||||||
|
{name: "zero", mut: func(event *fkevent.Event) {
|
||||||
|
zero := time.Time{}
|
||||||
|
event.EffectiveAt = &zero
|
||||||
|
}},
|
||||||
|
} {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
event := base
|
||||||
|
tt.mut(&event)
|
||||||
|
got := parentEventValues(event, nil)
|
||||||
|
if got["event_effective_at"] != nil {
|
||||||
|
t.Fatalf("event_effective_at = %#v, want nil", got["event_effective_at"])
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func testEvent(schema string, kind fkevent.Kind, payload any) fkevent.Event {
|
func testEvent(schema string, kind fkevent.Kind, payload any) fkevent.Event {
|
||||||
effectiveAt := time.Date(2026, 3, 16, 18, 30, 0, 0, time.UTC)
|
effectiveAt := time.Date(2026, 3, 16, 18, 30, 0, 0, time.UTC)
|
||||||
return fkevent.Event{
|
return fkevent.Event{
|
||||||
@@ -341,6 +743,30 @@ func testEvent(schema string, kind fkevent.Kind, payload any) fkevent.Event {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func assertParentEnvelopeValues(t *testing.T, values map[string]any, event fkevent.Event) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
if got := values["event_id"]; got != event.ID {
|
||||||
|
t.Fatalf("event_id = %#v, want %q", got, event.ID)
|
||||||
|
}
|
||||||
|
if got := values["event_kind"]; got != string(event.Kind) {
|
||||||
|
t.Fatalf("event_kind = %#v, want %q", got, event.Kind)
|
||||||
|
}
|
||||||
|
if got := values["event_source"]; got != event.Source {
|
||||||
|
t.Fatalf("event_source = %#v, want %q", got, event.Source)
|
||||||
|
}
|
||||||
|
if got := values["event_schema"]; got != event.Schema {
|
||||||
|
t.Fatalf("event_schema = %#v, want %q", got, event.Schema)
|
||||||
|
}
|
||||||
|
if got := values["event_emitted_at"]; got != event.EmittedAt.UTC() {
|
||||||
|
t.Fatalf("event_emitted_at = %#v, want %s", got, event.EmittedAt.UTC())
|
||||||
|
}
|
||||||
|
wantEffective := nullableTime(event.EffectiveAt)
|
||||||
|
if got := values["event_effective_at"]; got != wantEffective {
|
||||||
|
t.Fatalf("event_effective_at = %#v, want %#v", got, wantEffective)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func firstWriteForTable(writes []fksinks.PostgresWrite, table string) (fksinks.PostgresWrite, bool) {
|
func firstWriteForTable(writes []fksinks.PostgresWrite, table string) (fksinks.PostgresWrite, bool) {
|
||||||
for _, w := range writes {
|
for _, w := range writes {
|
||||||
if w.Table == table {
|
if w.Table == table {
|
||||||
@@ -350,6 +776,16 @@ func firstWriteForTable(writes []fksinks.PostgresWrite, table string) (fksinks.P
|
|||||||
return fksinks.PostgresWrite{}, false
|
return fksinks.PostgresWrite{}, false
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func writesForTable(writes []fksinks.PostgresWrite, table string) []fksinks.PostgresWrite {
|
||||||
|
out := make([]fksinks.PostgresWrite, 0)
|
||||||
|
for _, w := range writes {
|
||||||
|
if w.Table == table {
|
||||||
|
out = append(out, w)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
func assertAllWritesIncludeAllColumns(t *testing.T, writes []fksinks.PostgresWrite) {
|
func assertAllWritesIncludeAllColumns(t *testing.T, writes []fksinks.PostgresWrite) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
colCounts := tableColumnCounts()
|
colCounts := tableColumnCounts()
|
||||||
@@ -373,6 +809,23 @@ func tableColumnCounts() map[string]int {
|
|||||||
return m
|
return m
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func validTestOutlook() model.WeatherOutlook {
|
||||||
|
return model.WeatherOutlook{
|
||||||
|
ID: "outlook-1",
|
||||||
|
Provider: "spc",
|
||||||
|
Product: "convective",
|
||||||
|
Day: 1,
|
||||||
|
OutlookType: "categorical",
|
||||||
|
Label: "SLGT",
|
||||||
|
ValidFrom: time.Date(2026, 6, 11, 13, 0, 0, 0, time.UTC),
|
||||||
|
ValidTo: time.Date(2026, 6, 12, 12, 0, 0, 0, time.UTC),
|
||||||
|
IssuedAt: time.Date(2026, 6, 11, 19, 45, 0, 0, time.UTC),
|
||||||
|
ExpiresAt: time.Date(2026, 6, 12, 12, 0, 0, 0, time.UTC),
|
||||||
|
ContainsLocation: true,
|
||||||
|
Geometry: json.RawMessage(`{"type":"Polygon","coordinates":[[[-91,38],[-90,38],[-90,39],[-91,39],[-91,38]]]}`),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func wmoCodePtr(v model.WMOCode) *model.WMOCode {
|
func wmoCodePtr(v model.WMOCode) *model.WMOCode {
|
||||||
out := v
|
out := v
|
||||||
return &out
|
return &out
|
||||||
|
|||||||
@@ -16,6 +16,9 @@ const (
|
|||||||
tableAlertRuns = "alert_runs"
|
tableAlertRuns = "alert_runs"
|
||||||
tableAlerts = "alerts"
|
tableAlerts = "alerts"
|
||||||
tableAlertReferences = "alert_references"
|
tableAlertReferences = "alert_references"
|
||||||
|
tableOutlookRuns = "outlook_runs"
|
||||||
|
tableOutlooks = "outlooks"
|
||||||
|
tableOutlookDiscussions = "outlook_discussions"
|
||||||
)
|
)
|
||||||
|
|
||||||
// PostgresSchema returns weatherfeeder's Postgres schema definition.
|
// PostgresSchema returns weatherfeeder's Postgres schema definition.
|
||||||
@@ -24,13 +27,7 @@ func PostgresSchema() fksinks.PostgresSchema {
|
|||||||
Tables: []fksinks.PostgresTable{
|
Tables: []fksinks.PostgresTable{
|
||||||
{
|
{
|
||||||
Name: tableObservations,
|
Name: tableObservations,
|
||||||
Columns: []fksinks.PostgresColumn{
|
Columns: parentEnvelopeColumns([]fksinks.PostgresColumn{
|
||||||
{Name: "event_id", Type: "TEXT", Nullable: false},
|
|
||||||
{Name: "event_kind", Type: "TEXT", Nullable: false},
|
|
||||||
{Name: "event_source", Type: "TEXT", Nullable: false},
|
|
||||||
{Name: "event_schema", Type: "TEXT", Nullable: false},
|
|
||||||
{Name: "event_emitted_at", Type: "TIMESTAMPTZ", Nullable: false},
|
|
||||||
{Name: "event_effective_at", Type: "TIMESTAMPTZ", Nullable: true},
|
|
||||||
{Name: "station_id", Type: "TEXT", Nullable: true},
|
{Name: "station_id", Type: "TEXT", Nullable: true},
|
||||||
{Name: "station_name", Type: "TEXT", Nullable: true},
|
{Name: "station_name", Type: "TEXT", Nullable: true},
|
||||||
{Name: "observed_at", Type: "TIMESTAMPTZ", Nullable: false},
|
{Name: "observed_at", Type: "TIMESTAMPTZ", Nullable: false},
|
||||||
@@ -46,7 +43,7 @@ func PostgresSchema() fksinks.PostgresSchema {
|
|||||||
{Name: "visibility_meters", Type: "DOUBLE PRECISION", Nullable: true},
|
{Name: "visibility_meters", Type: "DOUBLE PRECISION", Nullable: true},
|
||||||
{Name: "relative_humidity_percent", Type: "DOUBLE PRECISION", Nullable: true},
|
{Name: "relative_humidity_percent", Type: "DOUBLE PRECISION", Nullable: true},
|
||||||
{Name: "apparent_temperature_c", Type: "DOUBLE PRECISION", Nullable: true},
|
{Name: "apparent_temperature_c", Type: "DOUBLE PRECISION", Nullable: true},
|
||||||
},
|
}...),
|
||||||
PrimaryKey: []string{"event_id"},
|
PrimaryKey: []string{"event_id"},
|
||||||
PruneColumn: "observed_at",
|
PruneColumn: "observed_at",
|
||||||
Indexes: []fksinks.PostgresIndex{
|
Indexes: []fksinks.PostgresIndex{
|
||||||
@@ -71,13 +68,7 @@ func PostgresSchema() fksinks.PostgresSchema {
|
|||||||
},
|
},
|
||||||
{
|
{
|
||||||
Name: tableForecasts,
|
Name: tableForecasts,
|
||||||
Columns: []fksinks.PostgresColumn{
|
Columns: parentEnvelopeColumns([]fksinks.PostgresColumn{
|
||||||
{Name: "event_id", Type: "TEXT", Nullable: false},
|
|
||||||
{Name: "event_kind", Type: "TEXT", Nullable: false},
|
|
||||||
{Name: "event_source", Type: "TEXT", Nullable: false},
|
|
||||||
{Name: "event_schema", Type: "TEXT", Nullable: false},
|
|
||||||
{Name: "event_emitted_at", Type: "TIMESTAMPTZ", Nullable: false},
|
|
||||||
{Name: "event_effective_at", Type: "TIMESTAMPTZ", Nullable: true},
|
|
||||||
{Name: "location_id", Type: "TEXT", Nullable: true},
|
{Name: "location_id", Type: "TEXT", Nullable: true},
|
||||||
{Name: "location_name", Type: "TEXT", Nullable: true},
|
{Name: "location_name", Type: "TEXT", Nullable: true},
|
||||||
{Name: "issued_at", Type: "TIMESTAMPTZ", Nullable: false},
|
{Name: "issued_at", Type: "TIMESTAMPTZ", Nullable: false},
|
||||||
@@ -87,7 +78,7 @@ func PostgresSchema() fksinks.PostgresSchema {
|
|||||||
{Name: "longitude", Type: "DOUBLE PRECISION", Nullable: true},
|
{Name: "longitude", Type: "DOUBLE PRECISION", Nullable: true},
|
||||||
{Name: "elevation_meters", Type: "DOUBLE PRECISION", Nullable: true},
|
{Name: "elevation_meters", Type: "DOUBLE PRECISION", Nullable: true},
|
||||||
{Name: "period_count", Type: "INTEGER", Nullable: false},
|
{Name: "period_count", Type: "INTEGER", Nullable: false},
|
||||||
},
|
}...),
|
||||||
PrimaryKey: []string{"event_id"},
|
PrimaryKey: []string{"event_id"},
|
||||||
PruneColumn: "issued_at",
|
PruneColumn: "issued_at",
|
||||||
Indexes: []fksinks.PostgresIndex{
|
Indexes: []fksinks.PostgresIndex{
|
||||||
@@ -135,13 +126,7 @@ func PostgresSchema() fksinks.PostgresSchema {
|
|||||||
},
|
},
|
||||||
{
|
{
|
||||||
Name: tableForecastDiscussions,
|
Name: tableForecastDiscussions,
|
||||||
Columns: []fksinks.PostgresColumn{
|
Columns: parentEnvelopeColumns([]fksinks.PostgresColumn{
|
||||||
{Name: "event_id", Type: "TEXT", Nullable: false},
|
|
||||||
{Name: "event_kind", Type: "TEXT", Nullable: false},
|
|
||||||
{Name: "event_source", Type: "TEXT", Nullable: false},
|
|
||||||
{Name: "event_schema", Type: "TEXT", Nullable: false},
|
|
||||||
{Name: "event_emitted_at", Type: "TIMESTAMPTZ", Nullable: false},
|
|
||||||
{Name: "event_effective_at", Type: "TIMESTAMPTZ", Nullable: true},
|
|
||||||
{Name: "office_id", Type: "TEXT", Nullable: true},
|
{Name: "office_id", Type: "TEXT", Nullable: true},
|
||||||
{Name: "office_name", Type: "TEXT", Nullable: true},
|
{Name: "office_name", Type: "TEXT", Nullable: true},
|
||||||
{Name: "issued_at", Type: "TIMESTAMPTZ", Nullable: false},
|
{Name: "issued_at", Type: "TIMESTAMPTZ", Nullable: false},
|
||||||
@@ -154,7 +139,7 @@ func PostgresSchema() fksinks.PostgresSchema {
|
|||||||
{Name: "long_term_issued_at", Type: "TIMESTAMPTZ", Nullable: true},
|
{Name: "long_term_issued_at", Type: "TIMESTAMPTZ", Nullable: true},
|
||||||
{Name: "long_term_text", Type: "TEXT", Nullable: true},
|
{Name: "long_term_text", Type: "TEXT", Nullable: true},
|
||||||
{Name: "key_message_count", Type: "INTEGER", Nullable: false},
|
{Name: "key_message_count", Type: "INTEGER", Nullable: false},
|
||||||
},
|
}...),
|
||||||
PrimaryKey: []string{"event_id"},
|
PrimaryKey: []string{"event_id"},
|
||||||
PruneColumn: "issued_at",
|
PruneColumn: "issued_at",
|
||||||
Indexes: []fksinks.PostgresIndex{
|
Indexes: []fksinks.PostgresIndex{
|
||||||
@@ -178,17 +163,11 @@ func PostgresSchema() fksinks.PostgresSchema {
|
|||||||
},
|
},
|
||||||
{
|
{
|
||||||
Name: tableWeatherStoryRuns,
|
Name: tableWeatherStoryRuns,
|
||||||
Columns: []fksinks.PostgresColumn{
|
Columns: parentEnvelopeColumns([]fksinks.PostgresColumn{
|
||||||
{Name: "event_id", Type: "TEXT", Nullable: false},
|
|
||||||
{Name: "event_kind", Type: "TEXT", Nullable: false},
|
|
||||||
{Name: "event_source", Type: "TEXT", Nullable: false},
|
|
||||||
{Name: "event_schema", Type: "TEXT", Nullable: false},
|
|
||||||
{Name: "event_emitted_at", Type: "TIMESTAMPTZ", Nullable: false},
|
|
||||||
{Name: "event_effective_at", Type: "TIMESTAMPTZ", Nullable: true},
|
|
||||||
{Name: "office_id", Type: "TEXT", Nullable: true},
|
{Name: "office_id", Type: "TEXT", Nullable: true},
|
||||||
{Name: "as_of", Type: "TIMESTAMPTZ", Nullable: false},
|
{Name: "as_of", Type: "TIMESTAMPTZ", Nullable: false},
|
||||||
{Name: "story_count", Type: "INTEGER", Nullable: false},
|
{Name: "story_count", Type: "INTEGER", Nullable: false},
|
||||||
},
|
}...),
|
||||||
PrimaryKey: []string{"event_id"},
|
PrimaryKey: []string{"event_id"},
|
||||||
PruneColumn: "as_of",
|
PruneColumn: "as_of",
|
||||||
Indexes: []fksinks.PostgresIndex{
|
Indexes: []fksinks.PostgresIndex{
|
||||||
@@ -223,20 +202,14 @@ func PostgresSchema() fksinks.PostgresSchema {
|
|||||||
},
|
},
|
||||||
{
|
{
|
||||||
Name: tableAlertRuns,
|
Name: tableAlertRuns,
|
||||||
Columns: []fksinks.PostgresColumn{
|
Columns: parentEnvelopeColumns([]fksinks.PostgresColumn{
|
||||||
{Name: "event_id", Type: "TEXT", Nullable: false},
|
|
||||||
{Name: "event_kind", Type: "TEXT", Nullable: false},
|
|
||||||
{Name: "event_source", Type: "TEXT", Nullable: false},
|
|
||||||
{Name: "event_schema", Type: "TEXT", Nullable: false},
|
|
||||||
{Name: "event_emitted_at", Type: "TIMESTAMPTZ", Nullable: false},
|
|
||||||
{Name: "event_effective_at", Type: "TIMESTAMPTZ", Nullable: true},
|
|
||||||
{Name: "location_id", Type: "TEXT", Nullable: true},
|
{Name: "location_id", Type: "TEXT", Nullable: true},
|
||||||
{Name: "location_name", Type: "TEXT", Nullable: true},
|
{Name: "location_name", Type: "TEXT", Nullable: true},
|
||||||
{Name: "as_of", Type: "TIMESTAMPTZ", Nullable: false},
|
{Name: "as_of", Type: "TIMESTAMPTZ", Nullable: false},
|
||||||
{Name: "latitude", Type: "DOUBLE PRECISION", Nullable: true},
|
{Name: "latitude", Type: "DOUBLE PRECISION", Nullable: true},
|
||||||
{Name: "longitude", Type: "DOUBLE PRECISION", Nullable: true},
|
{Name: "longitude", Type: "DOUBLE PRECISION", Nullable: true},
|
||||||
{Name: "alert_count", Type: "INTEGER", Nullable: false},
|
{Name: "alert_count", Type: "INTEGER", Nullable: false},
|
||||||
},
|
}...),
|
||||||
PrimaryKey: []string{"event_id"},
|
PrimaryKey: []string{"event_id"},
|
||||||
PruneColumn: "as_of",
|
PruneColumn: "as_of",
|
||||||
Indexes: []fksinks.PostgresIndex{
|
Indexes: []fksinks.PostgresIndex{
|
||||||
@@ -265,6 +238,7 @@ func PostgresSchema() fksinks.PostgresSchema {
|
|||||||
{Name: "sent", Type: "TIMESTAMPTZ", Nullable: true},
|
{Name: "sent", Type: "TIMESTAMPTZ", Nullable: true},
|
||||||
{Name: "effective", Type: "TIMESTAMPTZ", Nullable: true},
|
{Name: "effective", Type: "TIMESTAMPTZ", Nullable: true},
|
||||||
{Name: "onset", Type: "TIMESTAMPTZ", Nullable: true},
|
{Name: "onset", Type: "TIMESTAMPTZ", Nullable: true},
|
||||||
|
{Name: "ends", Type: "TIMESTAMPTZ", Nullable: true},
|
||||||
{Name: "expires", Type: "TIMESTAMPTZ", Nullable: true},
|
{Name: "expires", Type: "TIMESTAMPTZ", Nullable: true},
|
||||||
{Name: "area_description", Type: "TEXT", Nullable: true},
|
{Name: "area_description", Type: "TEXT", Nullable: true},
|
||||||
{Name: "sender_name", Type: "TEXT", Nullable: true},
|
{Name: "sender_name", Type: "TEXT", Nullable: true},
|
||||||
@@ -297,7 +271,89 @@ func PostgresSchema() fksinks.PostgresSchema {
|
|||||||
{Name: "idx_wf_alert_refs_sent", Columns: []string{"sent"}},
|
{Name: "idx_wf_alert_refs_sent", Columns: []string{"sent"}},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
Name: tableOutlookRuns,
|
||||||
|
Columns: parentEnvelopeColumns([]fksinks.PostgresColumn{
|
||||||
|
{Name: "location_id", Type: "TEXT", Nullable: true},
|
||||||
|
{Name: "location_name", Type: "TEXT", Nullable: true},
|
||||||
|
{Name: "latitude", Type: "DOUBLE PRECISION", Nullable: true},
|
||||||
|
{Name: "longitude", Type: "DOUBLE PRECISION", Nullable: true},
|
||||||
|
{Name: "as_of", Type: "TIMESTAMPTZ", Nullable: false},
|
||||||
|
{Name: "issued_at", Type: "TIMESTAMPTZ", Nullable: true},
|
||||||
|
{Name: "outlook_count", Type: "INTEGER", Nullable: false},
|
||||||
|
{Name: "discussion_count", Type: "INTEGER", Nullable: false},
|
||||||
|
}...),
|
||||||
|
PrimaryKey: []string{"event_id"},
|
||||||
|
PruneColumn: "as_of",
|
||||||
|
Indexes: []fksinks.PostgresIndex{
|
||||||
|
{Name: "idx_wf_outlook_run_location_as_of", Columns: []string{"location_id", "as_of"}},
|
||||||
|
{Name: "idx_wf_outlook_run_as_of", Columns: []string{"as_of"}},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Name: tableOutlooks,
|
||||||
|
Columns: []fksinks.PostgresColumn{
|
||||||
|
{Name: "run_event_id", Type: "TEXT REFERENCES outlook_runs(event_id) ON DELETE CASCADE", Nullable: false},
|
||||||
|
{Name: "outlook_index", Type: "INTEGER", Nullable: false},
|
||||||
|
{Name: "as_of", Type: "TIMESTAMPTZ", Nullable: false},
|
||||||
|
{Name: "outlook_id", Type: "TEXT", Nullable: false},
|
||||||
|
{Name: "provider", Type: "TEXT", Nullable: false},
|
||||||
|
{Name: "product", Type: "TEXT", Nullable: false},
|
||||||
|
{Name: "day", Type: "INTEGER", Nullable: false},
|
||||||
|
{Name: "outlook_type", Type: "TEXT", Nullable: false},
|
||||||
|
{Name: "label", Type: "TEXT", Nullable: false},
|
||||||
|
{Name: "label_text", Type: "TEXT", Nullable: true},
|
||||||
|
{Name: "severity_rank", Type: "INTEGER", Nullable: true},
|
||||||
|
{Name: "valid_from", Type: "TIMESTAMPTZ", Nullable: false},
|
||||||
|
{Name: "valid_to", Type: "TIMESTAMPTZ", Nullable: false},
|
||||||
|
{Name: "issued_at", Type: "TIMESTAMPTZ", Nullable: false},
|
||||||
|
{Name: "expires_at", Type: "TIMESTAMPTZ", Nullable: false},
|
||||||
|
{Name: "forecaster", Type: "TEXT", Nullable: true},
|
||||||
|
{Name: "source_url", Type: "TEXT", Nullable: true},
|
||||||
|
{Name: "image_url", Type: "TEXT", Nullable: true},
|
||||||
|
{Name: "contains_location", Type: "BOOLEAN", Nullable: false},
|
||||||
|
{Name: "geometry_json", Type: "TEXT", Nullable: false},
|
||||||
|
},
|
||||||
|
PrimaryKey: []string{"run_event_id", "outlook_index"},
|
||||||
|
PruneColumn: "as_of",
|
||||||
|
Indexes: []fksinks.PostgresIndex{
|
||||||
|
{Name: "idx_wf_outlooks_contains_valid", Columns: []string{"contains_location", "valid_from", "valid_to"}},
|
||||||
|
{Name: "idx_wf_outlooks_day_type_label", Columns: []string{"day", "outlook_type", "label"}},
|
||||||
|
{Name: "idx_wf_outlooks_valid", Columns: []string{"valid_from", "valid_to"}},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Name: tableOutlookDiscussions,
|
||||||
|
Columns: []fksinks.PostgresColumn{
|
||||||
|
{Name: "run_event_id", Type: "TEXT REFERENCES outlook_runs(event_id) ON DELETE CASCADE", Nullable: false},
|
||||||
|
{Name: "discussion_index", Type: "INTEGER", Nullable: false},
|
||||||
|
{Name: "as_of", Type: "TIMESTAMPTZ", Nullable: false},
|
||||||
|
{Name: "day", Type: "INTEGER", Nullable: false},
|
||||||
|
{Name: "headline", Type: "TEXT", Nullable: true},
|
||||||
|
{Name: "summary", Type: "TEXT", Nullable: true},
|
||||||
|
{Name: "discussion", Type: "TEXT", Nullable: true},
|
||||||
|
{Name: "updated_at", Type: "TIMESTAMPTZ", Nullable: true},
|
||||||
|
},
|
||||||
|
PrimaryKey: []string{"run_event_id", "discussion_index"},
|
||||||
|
PruneColumn: "as_of",
|
||||||
|
Indexes: []fksinks.PostgresIndex{
|
||||||
|
{Name: "idx_wf_outlook_discussions_day_as_of", Columns: []string{"day", "as_of"}},
|
||||||
|
{Name: "idx_wf_outlook_discussions_run_day", Columns: []string{"run_event_id", "day"}, Unique: true},
|
||||||
|
},
|
||||||
|
},
|
||||||
},
|
},
|
||||||
MapEvent: mapPostgresEvent,
|
MapEvent: mapPostgresEvent,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func parentEnvelopeColumns(extra ...fksinks.PostgresColumn) []fksinks.PostgresColumn {
|
||||||
|
columns := []fksinks.PostgresColumn{
|
||||||
|
{Name: "event_id", Type: "TEXT", Nullable: false},
|
||||||
|
{Name: "event_kind", Type: "TEXT", Nullable: false},
|
||||||
|
{Name: "event_source", Type: "TEXT", Nullable: false},
|
||||||
|
{Name: "event_schema", Type: "TEXT", Nullable: false},
|
||||||
|
{Name: "event_emitted_at", Type: "TIMESTAMPTZ", Nullable: false},
|
||||||
|
{Name: "event_effective_at", Type: "TIMESTAMPTZ", Nullable: true},
|
||||||
|
}
|
||||||
|
return append(columns, extra...)
|
||||||
|
}
|
||||||
|
|||||||
@@ -1,6 +1,12 @@
|
|||||||
package postgres
|
package postgres
|
||||||
|
|
||||||
import "testing"
|
import (
|
||||||
|
"reflect"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
fksinks "gitea.maximumdirect.net/ejr/feedkit/sinks"
|
||||||
|
)
|
||||||
|
|
||||||
func TestWeatherPostgresSchemaShape(t *testing.T) {
|
func TestWeatherPostgresSchemaShape(t *testing.T) {
|
||||||
s := PostgresSchema()
|
s := PostgresSchema()
|
||||||
@@ -20,6 +26,9 @@ func TestWeatherPostgresSchemaShape(t *testing.T) {
|
|||||||
tableAlertRuns: true,
|
tableAlertRuns: true,
|
||||||
tableAlerts: true,
|
tableAlerts: true,
|
||||||
tableAlertReferences: true,
|
tableAlertReferences: true,
|
||||||
|
tableOutlookRuns: true,
|
||||||
|
tableOutlooks: true,
|
||||||
|
tableOutlookDiscussions: true,
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(s.Tables) != len(wantTables) {
|
if len(s.Tables) != len(wantTables) {
|
||||||
@@ -43,6 +52,54 @@ func TestWeatherPostgresSchemaShape(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestWeatherPostgresSchemaIncludesOutlookTables(t *testing.T) {
|
||||||
|
runColumns := columnsForTable(t, tableOutlookRuns)
|
||||||
|
for _, col := range []string{"event_id", "event_kind", "event_source", "event_schema", "event_emitted_at", "event_effective_at", "location_id", "location_name", "latitude", "longitude", "as_of", "issued_at", "outlook_count", "discussion_count"} {
|
||||||
|
if !runColumns[col] {
|
||||||
|
t.Fatalf("%s missing %s column", tableOutlookRuns, col)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
assertTablePrimaryKey(t, tableOutlookRuns, []string{"event_id"})
|
||||||
|
assertTableIndex(t, tableOutlookRuns, "idx_wf_outlook_run_location_as_of", []string{"location_id", "as_of"})
|
||||||
|
assertTableIndex(t, tableOutlookRuns, "idx_wf_outlook_run_as_of", []string{"as_of"})
|
||||||
|
|
||||||
|
outlookColumns := columnsForTable(t, tableOutlooks)
|
||||||
|
for _, col := range []string{"run_event_id", "outlook_index", "as_of", "outlook_id", "provider", "product", "day", "outlook_type", "label", "label_text", "severity_rank", "valid_from", "valid_to", "issued_at", "expires_at", "forecaster", "source_url", "image_url", "contains_location", "geometry_json"} {
|
||||||
|
if !outlookColumns[col] {
|
||||||
|
t.Fatalf("%s missing %s column", tableOutlooks, col)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, col := range []string{"headline", "summary", "discussion"} {
|
||||||
|
if outlookColumns[col] {
|
||||||
|
t.Fatalf("%s still includes legacy %s column", tableOutlooks, col)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
assertTablePrimaryKey(t, tableOutlooks, []string{"run_event_id", "outlook_index"})
|
||||||
|
assertTableIndex(t, tableOutlooks, "idx_wf_outlooks_contains_valid", []string{"contains_location", "valid_from", "valid_to"})
|
||||||
|
assertTableIndex(t, tableOutlooks, "idx_wf_outlooks_day_type_label", []string{"day", "outlook_type", "label"})
|
||||||
|
assertTableIndex(t, tableOutlooks, "idx_wf_outlooks_valid", []string{"valid_from", "valid_to"})
|
||||||
|
|
||||||
|
discussionColumns := columnsForTable(t, tableOutlookDiscussions)
|
||||||
|
for _, col := range []string{"run_event_id", "discussion_index", "as_of", "day", "headline", "summary", "discussion", "updated_at"} {
|
||||||
|
if !discussionColumns[col] {
|
||||||
|
t.Fatalf("%s missing %s column", tableOutlookDiscussions, col)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
assertTablePrimaryKey(t, tableOutlookDiscussions, []string{"run_event_id", "discussion_index"})
|
||||||
|
assertTablePruneColumn(t, tableOutlookDiscussions, "as_of")
|
||||||
|
assertTableIndex(t, tableOutlookDiscussions, "idx_wf_outlook_discussions_day_as_of", []string{"day", "as_of"})
|
||||||
|
assertTableUniqueIndex(t, tableOutlookDiscussions, "idx_wf_outlook_discussions_run_day", []string{"run_event_id", "day"})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestWeatherPostgresSchemaIncludesAlertEndsColumn(t *testing.T) {
|
||||||
|
alertColumns := columnsForTable(t, tableAlerts)
|
||||||
|
for _, col := range []string{"run_event_id", "alert_index", "as_of", "alert_id", "onset", "ends", "expires"} {
|
||||||
|
if !alertColumns[col] {
|
||||||
|
t.Fatalf("%s missing %s column", tableAlerts, col)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestWeatherPostgresSchemaIncludesWeatherStoryColumns(t *testing.T) {
|
func TestWeatherPostgresSchemaIncludesWeatherStoryColumns(t *testing.T) {
|
||||||
runColumns := columnsForTable(t, tableWeatherStoryRuns)
|
runColumns := columnsForTable(t, tableWeatherStoryRuns)
|
||||||
if !runColumns["as_of"] {
|
if !runColumns["as_of"] {
|
||||||
@@ -60,20 +117,102 @@ func TestWeatherPostgresSchemaIncludesWeatherStoryColumns(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func columnsForTable(t *testing.T, table string) map[string]bool {
|
func TestWeatherPostgresSchemaParentTablesStartWithEnvelopeColumns(t *testing.T) {
|
||||||
|
for _, table := range []string{
|
||||||
|
tableObservations,
|
||||||
|
tableForecasts,
|
||||||
|
tableForecastDiscussions,
|
||||||
|
tableWeatherStoryRuns,
|
||||||
|
tableAlertRuns,
|
||||||
|
tableOutlookRuns,
|
||||||
|
} {
|
||||||
|
t.Run(table, func(t *testing.T) {
|
||||||
|
columns := orderedColumnsForTable(t, table)
|
||||||
|
want := parentEnvelopeColumns()
|
||||||
|
if len(columns) < len(want) {
|
||||||
|
t.Fatalf("%s has %d columns, want at least %d", table, len(columns), len(want))
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(columns[:len(want)], want) {
|
||||||
|
t.Fatalf("%s envelope prefix = %#v, want %#v", table, columns[:len(want)], want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func assertTablePrimaryKey(t *testing.T, table string, want []string) {
|
||||||
|
t.Helper()
|
||||||
|
tbl := tableByName(t, table)
|
||||||
|
if strings.Join(tbl.PrimaryKey, ",") != strings.Join(want, ",") {
|
||||||
|
t.Fatalf("%s primary key = %#v, want %#v", table, tbl.PrimaryKey, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func assertTablePruneColumn(t *testing.T, table string, want string) {
|
||||||
|
t.Helper()
|
||||||
|
tbl := tableByName(t, table)
|
||||||
|
if tbl.PruneColumn != want {
|
||||||
|
t.Fatalf("%s prune column = %q, want %q", table, tbl.PruneColumn, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func assertTableIndex(t *testing.T, table string, name string, want []string) {
|
||||||
|
t.Helper()
|
||||||
|
assertTableIndexWithUnique(t, table, name, want, false)
|
||||||
|
}
|
||||||
|
|
||||||
|
func assertTableUniqueIndex(t *testing.T, table string, name string, want []string) {
|
||||||
|
t.Helper()
|
||||||
|
assertTableIndexWithUnique(t, table, name, want, true)
|
||||||
|
}
|
||||||
|
|
||||||
|
func assertTableIndexWithUnique(t *testing.T, table string, name string, want []string, unique bool) {
|
||||||
|
t.Helper()
|
||||||
|
tbl := tableByName(t, table)
|
||||||
|
for _, idx := range tbl.Indexes {
|
||||||
|
if idx.Name == name {
|
||||||
|
if strings.Join(idx.Columns, ",") != strings.Join(want, ",") {
|
||||||
|
t.Fatalf("%s index %s columns = %#v, want %#v", table, name, idx.Columns, want)
|
||||||
|
}
|
||||||
|
if idx.Unique != unique {
|
||||||
|
t.Fatalf("%s index %s unique = %v, want %v", table, name, idx.Unique, unique)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
t.Fatalf("%s missing index %s", table, name)
|
||||||
|
}
|
||||||
|
|
||||||
|
func tableByName(t *testing.T, table string) fksinks.PostgresTable {
|
||||||
|
t.Helper()
|
||||||
|
for _, tbl := range PostgresSchema().Tables {
|
||||||
|
if tbl.Name == table {
|
||||||
|
return tbl
|
||||||
|
}
|
||||||
|
}
|
||||||
|
t.Fatalf("missing table %q", table)
|
||||||
|
return fksinks.PostgresTable{}
|
||||||
|
}
|
||||||
|
|
||||||
|
func orderedColumnsForTable(t *testing.T, table string) []fksinks.PostgresColumn {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
schema := PostgresSchema()
|
schema := PostgresSchema()
|
||||||
for _, tbl := range schema.Tables {
|
for _, tbl := range schema.Tables {
|
||||||
if tbl.Name != table {
|
if tbl.Name == table {
|
||||||
continue
|
return tbl.Columns
|
||||||
}
|
}
|
||||||
cols := make(map[string]bool, len(tbl.Columns))
|
|
||||||
for _, col := range tbl.Columns {
|
|
||||||
cols[col.Name] = true
|
|
||||||
}
|
|
||||||
return cols
|
|
||||||
}
|
}
|
||||||
t.Fatalf("missing table %q", table)
|
t.Fatalf("missing table %q", table)
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func columnsForTable(t *testing.T, table string) map[string]bool {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
ordered := orderedColumnsForTable(t, table)
|
||||||
|
cols := make(map[string]bool, len(ordered))
|
||||||
|
for _, col := range ordered {
|
||||||
|
cols[col.Name] = true
|
||||||
|
}
|
||||||
|
return cols
|
||||||
|
}
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ import (
|
|||||||
"gitea.maximumdirect.net/ejr/weatherfeeder/internal/sources/nws"
|
"gitea.maximumdirect.net/ejr/weatherfeeder/internal/sources/nws"
|
||||||
"gitea.maximumdirect.net/ejr/weatherfeeder/internal/sources/openmeteo"
|
"gitea.maximumdirect.net/ejr/weatherfeeder/internal/sources/openmeteo"
|
||||||
"gitea.maximumdirect.net/ejr/weatherfeeder/internal/sources/openweather"
|
"gitea.maximumdirect.net/ejr/weatherfeeder/internal/sources/openweather"
|
||||||
|
"gitea.maximumdirect.net/ejr/weatherfeeder/internal/sources/spc"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/ejr/feedkit/config"
|
"gitea.maximumdirect.net/ejr/feedkit/config"
|
||||||
fksource "gitea.maximumdirect.net/ejr/feedkit/sources"
|
fksource "gitea.maximumdirect.net/ejr/feedkit/sources"
|
||||||
@@ -15,19 +16,22 @@ type pollDriverRegistration struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
var pollDriverRegistrations = []pollDriverRegistration{
|
var pollDriverRegistrations = []pollDriverRegistration{
|
||||||
{driver: "nws_observation", factory: func(cfg config.SourceConfig) (fksource.PollSource, error) { return nws.NewObservationSource(cfg) }},
|
{driver: nws.DriverObservation, factory: func(cfg config.SourceConfig) (fksource.PollSource, error) { return nws.NewObservationSource(cfg) }},
|
||||||
{driver: "nws_alerts", factory: func(cfg config.SourceConfig) (fksource.PollSource, error) { return nws.NewAlertsSource(cfg) }},
|
{driver: nws.DriverAlerts, factory: func(cfg config.SourceConfig) (fksource.PollSource, error) { return nws.NewAlertsSource(cfg) }},
|
||||||
{driver: "nws_forecast_hourly", factory: func(cfg config.SourceConfig) (fksource.PollSource, error) { return nws.NewHourlyForecastSource(cfg) }},
|
{driver: nws.DriverForecastHourly, factory: func(cfg config.SourceConfig) (fksource.PollSource, error) { return nws.NewHourlyForecastSource(cfg) }},
|
||||||
{driver: "nws_forecast_narrative", factory: func(cfg config.SourceConfig) (fksource.PollSource, error) { return nws.NewNarrativeForecastSource(cfg) }},
|
{driver: nws.DriverForecastNarrative, factory: func(cfg config.SourceConfig) (fksource.PollSource, error) { return nws.NewNarrativeForecastSource(cfg) }},
|
||||||
{driver: "nws_forecast_discussion", factory: func(cfg config.SourceConfig) (fksource.PollSource, error) {
|
{driver: nws.DriverForecastDiscussion, factory: func(cfg config.SourceConfig) (fksource.PollSource, error) {
|
||||||
return nws.NewForecastDiscussionSource(cfg)
|
return nws.NewForecastDiscussionSource(cfg)
|
||||||
}},
|
}},
|
||||||
{driver: "nws_weatherstories", factory: func(cfg config.SourceConfig) (fksource.PollSource, error) { return nws.NewWeatherStoriesSource(cfg) }},
|
{driver: nws.DriverWeatherStories, factory: func(cfg config.SourceConfig) (fksource.PollSource, error) { return nws.NewWeatherStoriesSource(cfg) }},
|
||||||
{driver: "openmeteo_observation", factory: func(cfg config.SourceConfig) (fksource.PollSource, error) { return openmeteo.NewObservationSource(cfg) }},
|
{driver: openmeteo.DriverObservation, factory: func(cfg config.SourceConfig) (fksource.PollSource, error) { return openmeteo.NewObservationSource(cfg) }},
|
||||||
{driver: "openmeteo_forecast", factory: func(cfg config.SourceConfig) (fksource.PollSource, error) { return openmeteo.NewForecastSource(cfg) }},
|
{driver: openmeteo.DriverForecast, factory: func(cfg config.SourceConfig) (fksource.PollSource, error) { return openmeteo.NewForecastSource(cfg) }},
|
||||||
{driver: "openweather_observation", factory: func(cfg config.SourceConfig) (fksource.PollSource, error) {
|
{driver: openweather.DriverObservation, factory: func(cfg config.SourceConfig) (fksource.PollSource, error) {
|
||||||
return openweather.NewObservationSource(cfg)
|
return openweather.NewObservationSource(cfg)
|
||||||
}},
|
}},
|
||||||
|
{driver: spc.DriverConvectiveOutlook, factory: func(cfg config.SourceConfig) (fksource.PollSource, error) {
|
||||||
|
return spc.NewConvectiveOutlookSource(cfg)
|
||||||
|
}},
|
||||||
}
|
}
|
||||||
|
|
||||||
// RegisterBuiltins registers the source drivers that ship with this binary.
|
// RegisterBuiltins registers the source drivers that ship with this binary.
|
||||||
|
|||||||
@@ -6,57 +6,29 @@ import (
|
|||||||
|
|
||||||
"gitea.maximumdirect.net/ejr/feedkit/config"
|
"gitea.maximumdirect.net/ejr/feedkit/config"
|
||||||
fksource "gitea.maximumdirect.net/ejr/feedkit/sources"
|
fksource "gitea.maximumdirect.net/ejr/feedkit/sources"
|
||||||
|
"gitea.maximumdirect.net/ejr/weatherfeeder/internal/sources/openweather"
|
||||||
|
"gitea.maximumdirect.net/ejr/weatherfeeder/internal/sources/spc"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestRegisterBuiltinsRegistersNWSHourlyForecastDriver(t *testing.T) {
|
func TestRegisterBuiltinsRegistersCurrentPollDrivers(t *testing.T) {
|
||||||
reg := fksource.NewRegistry()
|
reg := fksource.NewRegistry()
|
||||||
RegisterBuiltins(reg)
|
RegisterBuiltins(reg)
|
||||||
|
|
||||||
in, err := reg.BuildInput(sourceConfigForDriver("nws_forecast_hourly"))
|
if len(pollDriverRegistrations) == 0 {
|
||||||
if err != nil {
|
t.Fatalf("pollDriverRegistrations is empty")
|
||||||
t.Fatalf("BuildInput(nws_forecast_hourly) error = %v", err)
|
|
||||||
}
|
}
|
||||||
if _, ok := in.(fksource.PollSource); !ok {
|
|
||||||
t.Fatalf("BuildInput(nws_forecast_hourly) type = %T, want PollSource", in)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRegisterBuiltinsRegistersNWSNarrativeForecastDriver(t *testing.T) {
|
for _, tt := range pollDriverRegistrations {
|
||||||
reg := fksource.NewRegistry()
|
tt := tt
|
||||||
RegisterBuiltins(reg)
|
t.Run(tt.driver, func(t *testing.T) {
|
||||||
|
in, err := reg.BuildInput(sourceConfigForDriver(tt.driver))
|
||||||
in, err := reg.BuildInput(sourceConfigForDriver("nws_forecast_narrative"))
|
if err != nil {
|
||||||
if err != nil {
|
t.Fatalf("BuildInput(%q) error = %v", tt.driver, err)
|
||||||
t.Fatalf("BuildInput(nws_forecast_narrative) error = %v", err)
|
}
|
||||||
}
|
if _, ok := in.(fksource.PollSource); !ok {
|
||||||
if _, ok := in.(fksource.PollSource); !ok {
|
t.Fatalf("BuildInput(%q) type = %T, want PollSource", tt.driver, in)
|
||||||
t.Fatalf("BuildInput(nws_forecast_narrative) type = %T, want PollSource", in)
|
}
|
||||||
}
|
})
|
||||||
}
|
|
||||||
|
|
||||||
func TestRegisterBuiltinsRegistersNWSForecastDiscussionDriver(t *testing.T) {
|
|
||||||
reg := fksource.NewRegistry()
|
|
||||||
RegisterBuiltins(reg)
|
|
||||||
|
|
||||||
in, err := reg.BuildInput(sourceConfigForDriver("nws_forecast_discussion"))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("BuildInput(nws_forecast_discussion) error = %v", err)
|
|
||||||
}
|
|
||||||
if _, ok := in.(fksource.PollSource); !ok {
|
|
||||||
t.Fatalf("BuildInput(nws_forecast_discussion) type = %T, want PollSource", in)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRegisterBuiltinsRegistersNWSWeatherStoriesDriver(t *testing.T) {
|
|
||||||
reg := fksource.NewRegistry()
|
|
||||||
RegisterBuiltins(reg)
|
|
||||||
|
|
||||||
in, err := reg.BuildInput(sourceConfigForDriver("nws_weatherstories"))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("BuildInput(nws_weatherstories) error = %v", err)
|
|
||||||
}
|
|
||||||
if _, ok := in.(fksource.PollSource); !ok {
|
|
||||||
t.Fatalf("BuildInput(nws_weatherstories) type = %T, want PollSource", in)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -73,45 +45,23 @@ func TestRegisterBuiltinsDoesNotRegisterLegacyNWSForecastDriver(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRegisterBuiltinsRegistersAllCurrentDrivers(t *testing.T) {
|
|
||||||
reg := fksource.NewRegistry()
|
|
||||||
RegisterBuiltins(reg)
|
|
||||||
|
|
||||||
drivers := []string{
|
|
||||||
"nws_observation",
|
|
||||||
"nws_alerts",
|
|
||||||
"nws_forecast_hourly",
|
|
||||||
"nws_forecast_narrative",
|
|
||||||
"nws_forecast_discussion",
|
|
||||||
"nws_weatherstories",
|
|
||||||
"openmeteo_observation",
|
|
||||||
"openmeteo_forecast",
|
|
||||||
"openweather_observation",
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, driver := range drivers {
|
|
||||||
in, err := reg.BuildInput(sourceConfigForDriver(driver))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("BuildInput(%s) error = %v", driver, err)
|
|
||||||
}
|
|
||||||
if _, ok := in.(fksource.PollSource); !ok {
|
|
||||||
t.Fatalf("BuildInput(%s) type = %T, want PollSource", driver, in)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func sourceConfigForDriver(driver string) config.SourceConfig {
|
func sourceConfigForDriver(driver string) config.SourceConfig {
|
||||||
url := "https://example.invalid"
|
url := "https://example.invalid"
|
||||||
if driver == "openweather_observation" {
|
if driver == openweather.DriverObservation {
|
||||||
url = "https://example.invalid?units=metric"
|
url = "https://example.invalid?units=metric"
|
||||||
}
|
}
|
||||||
|
params := map[string]any{
|
||||||
|
"url": url,
|
||||||
|
"user_agent": "test-agent",
|
||||||
|
}
|
||||||
|
if driver == spc.DriverConvectiveOutlook {
|
||||||
|
params["latitude"] = 38.6239
|
||||||
|
params["longitude"] = -90.3571
|
||||||
|
}
|
||||||
return config.SourceConfig{
|
return config.SourceConfig{
|
||||||
Name: "test-source",
|
Name: "test-source",
|
||||||
Driver: driver,
|
Driver: driver,
|
||||||
Mode: config.SourceModePoll,
|
Mode: config.SourceModePoll,
|
||||||
Params: map[string]any{
|
Params: params,
|
||||||
"url": url,
|
|
||||||
"user_agent": "test-agent",
|
|
||||||
},
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
32
internal/sources/docs_test.go
Normal file
32
internal/sources/docs_test.go
Normal file
@@ -0,0 +1,32 @@
|
|||||||
|
package sources
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestDocumentedRegisteredSourceDrivers(t *testing.T) {
|
||||||
|
docs := map[string]string{
|
||||||
|
"docs/config.md": readDoc(t, "../../docs/config.md"),
|
||||||
|
"docs/internal/sources.md": readDoc(t, "../../docs/internal/sources.md"),
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, reg := range pollDriverRegistrations {
|
||||||
|
for path, doc := range docs {
|
||||||
|
if !strings.Contains(doc, reg.driver) {
|
||||||
|
t.Fatalf("%s missing source driver %q", path, reg.driver)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func readDoc(t *testing.T, path string) string {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
raw, err := os.ReadFile(path)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ReadFile(%s) error = %v", path, err)
|
||||||
|
}
|
||||||
|
return string(raw)
|
||||||
|
}
|
||||||
58
internal/sources/internal/httpconfig/config.go
Normal file
58
internal/sources/internal/httpconfig/config.go
Normal file
@@ -0,0 +1,58 @@
|
|||||||
|
package httpconfig
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/ejr/feedkit/config"
|
||||||
|
"gitea.maximumdirect.net/ejr/feedkit/transport"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Settings contains common HTTP client config for sources that fetch multiple documents.
|
||||||
|
type Settings struct {
|
||||||
|
Name string
|
||||||
|
UserAgent string
|
||||||
|
Timeout time.Duration
|
||||||
|
BodyLimitBytes int64
|
||||||
|
}
|
||||||
|
|
||||||
|
func Parse(driver string, cfg config.SourceConfig) (Settings, error) {
|
||||||
|
name := strings.TrimSpace(cfg.Name)
|
||||||
|
if name == "" {
|
||||||
|
return Settings{}, fmt.Errorf("%s: name is required", driver)
|
||||||
|
}
|
||||||
|
if cfg.Params == nil {
|
||||||
|
return Settings{}, fmt.Errorf("%s %q: params are required", driver, name)
|
||||||
|
}
|
||||||
|
|
||||||
|
userAgent, ok := cfg.ParamString("user_agent", "userAgent")
|
||||||
|
if !ok {
|
||||||
|
return Settings{}, fmt.Errorf("%s %q: params.user_agent is required", driver, name)
|
||||||
|
}
|
||||||
|
|
||||||
|
timeout := transport.DefaultHTTPTimeout
|
||||||
|
if _, exists := cfg.Params["http_timeout"]; exists {
|
||||||
|
var ok bool
|
||||||
|
timeout, ok = cfg.ParamDuration("http_timeout")
|
||||||
|
if !ok || timeout <= 0 {
|
||||||
|
return Settings{}, fmt.Errorf("source %q: params.http_timeout must be a positive duration", name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
bodyLimit := transport.DefaultHTTPResponseBodyLimitBytes
|
||||||
|
if _, exists := cfg.Params["http_response_body_limit_bytes"]; exists {
|
||||||
|
rawLimit, ok := cfg.ParamInt("http_response_body_limit_bytes")
|
||||||
|
if !ok || rawLimit <= 0 {
|
||||||
|
return Settings{}, fmt.Errorf("source %q: params.http_response_body_limit_bytes must be a positive integer", name)
|
||||||
|
}
|
||||||
|
bodyLimit = int64(rawLimit)
|
||||||
|
}
|
||||||
|
|
||||||
|
return Settings{
|
||||||
|
Name: name,
|
||||||
|
UserAgent: userAgent,
|
||||||
|
Timeout: timeout,
|
||||||
|
BodyLimitBytes: bodyLimit,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
115
internal/sources/internal/httpconfig/config_test.go
Normal file
115
internal/sources/internal/httpconfig/config_test.go
Normal file
@@ -0,0 +1,115 @@
|
|||||||
|
package httpconfig
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/ejr/feedkit/config"
|
||||||
|
"gitea.maximumdirect.net/ejr/feedkit/transport"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestParseUsesRequiredValuesAndDefaults(t *testing.T) {
|
||||||
|
got, err := Parse("test_driver", config.SourceConfig{
|
||||||
|
Name: " test-source ",
|
||||||
|
Params: map[string]any{
|
||||||
|
"user_agent": "test-agent",
|
||||||
|
},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Parse() error = %v", err)
|
||||||
|
}
|
||||||
|
if got.Name != "test-source" {
|
||||||
|
t.Fatalf("Name = %q, want test-source", got.Name)
|
||||||
|
}
|
||||||
|
if got.UserAgent != "test-agent" {
|
||||||
|
t.Fatalf("UserAgent = %q, want test-agent", got.UserAgent)
|
||||||
|
}
|
||||||
|
if got.Timeout != transport.DefaultHTTPTimeout {
|
||||||
|
t.Fatalf("Timeout = %s, want %s", got.Timeout, transport.DefaultHTTPTimeout)
|
||||||
|
}
|
||||||
|
if got.BodyLimitBytes != transport.DefaultHTTPResponseBodyLimitBytes {
|
||||||
|
t.Fatalf("BodyLimitBytes = %d, want %d", got.BodyLimitBytes, transport.DefaultHTTPResponseBodyLimitBytes)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseUsesAliasesAndOverrides(t *testing.T) {
|
||||||
|
got, err := Parse("test_driver", config.SourceConfig{
|
||||||
|
Name: "test-source",
|
||||||
|
Params: map[string]any{
|
||||||
|
"userAgent": "test-agent",
|
||||||
|
"http_timeout": "2s",
|
||||||
|
"http_response_body_limit_bytes": 2048,
|
||||||
|
},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Parse() error = %v", err)
|
||||||
|
}
|
||||||
|
if got.UserAgent != "test-agent" {
|
||||||
|
t.Fatalf("UserAgent = %q, want test-agent", got.UserAgent)
|
||||||
|
}
|
||||||
|
if got.Timeout != 2*time.Second {
|
||||||
|
t.Fatalf("Timeout = %s, want 2s", got.Timeout)
|
||||||
|
}
|
||||||
|
if got.BodyLimitBytes != 2048 {
|
||||||
|
t.Fatalf("BodyLimitBytes = %d, want 2048", got.BodyLimitBytes)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseRejectsInvalidConfig(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
cfg config.SourceConfig
|
||||||
|
wantErr string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "missing name",
|
||||||
|
cfg: config.SourceConfig{Params: map[string]any{"user_agent": "test-agent"}},
|
||||||
|
wantErr: "test_driver: name is required",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "missing params",
|
||||||
|
cfg: config.SourceConfig{Name: "test-source"},
|
||||||
|
wantErr: `test_driver "test-source": params are required`,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "missing user agent",
|
||||||
|
cfg: config.SourceConfig{Name: "test-source", Params: map[string]any{}},
|
||||||
|
wantErr: `test_driver "test-source": params.user_agent is required`,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "invalid timeout",
|
||||||
|
cfg: config.SourceConfig{
|
||||||
|
Name: "test-source",
|
||||||
|
Params: map[string]any{
|
||||||
|
"user_agent": "test-agent",
|
||||||
|
"http_timeout": "0s",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
wantErr: `source "test-source": params.http_timeout must be a positive duration`,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "invalid body limit",
|
||||||
|
cfg: config.SourceConfig{
|
||||||
|
Name: "test-source",
|
||||||
|
Params: map[string]any{
|
||||||
|
"user_agent": "test-agent",
|
||||||
|
"http_response_body_limit_bytes": 0,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
wantErr: `source "test-source": params.http_response_body_limit_bytes must be a positive integer`,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
_, err := Parse("test_driver", tt.cfg)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatalf("Parse() error = nil, want %q", tt.wantErr)
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), tt.wantErr) {
|
||||||
|
t.Fatalf("Parse() error = %q, want %q", err, tt.wantErr)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -26,10 +26,8 @@ type AlertsSource struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func NewAlertsSource(cfg config.SourceConfig) (*AlertsSource, error) {
|
func NewAlertsSource(cfg config.SourceConfig) (*AlertsSource, error) {
|
||||||
const driver = "nws_alerts"
|
|
||||||
|
|
||||||
// NWS alerts responses are GeoJSON-ish; allow fallback to plain JSON as well.
|
// NWS alerts responses are GeoJSON-ish; allow fallback to plain JSON as well.
|
||||||
hs, err := fksources.NewHTTPSource(driver, cfg, "application/geo+json, application/json")
|
hs, err := fksources.NewHTTPSource(DriverAlerts, cfg, "application/geo+json, application/json")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@@ -40,7 +38,7 @@ func NewAlertsSource(cfg config.SourceConfig) (*AlertsSource, error) {
|
|||||||
func (s *AlertsSource) Name() string { return s.http.Name }
|
func (s *AlertsSource) Name() string { return s.http.Name }
|
||||||
|
|
||||||
// Kinds is used for routing/policy.
|
// Kinds is used for routing/policy.
|
||||||
func (s *AlertsSource) Kinds() []event.Kind { return []event.Kind{event.Kind("alert")} }
|
func (s *AlertsSource) Kinds() []event.Kind { return []event.Kind{event.Kind(standards.KindAlert)} }
|
||||||
|
|
||||||
func (s *AlertsSource) Poll(ctx context.Context) ([]event.Event, error) {
|
func (s *AlertsSource) Poll(ctx context.Context) ([]event.Event, error) {
|
||||||
raw, meta, changed, err := s.fetchRaw(ctx)
|
raw, meta, changed, err := s.fetchRaw(ctx)
|
||||||
@@ -71,7 +69,7 @@ func (s *AlertsSource) Poll(ctx context.Context) ([]event.Event, error) {
|
|||||||
eventID := fksources.DefaultEventID("", s.http.Name, effectiveAt, emittedAt)
|
eventID := fksources.DefaultEventID("", s.http.Name, effectiveAt, emittedAt)
|
||||||
|
|
||||||
return fksources.SingleEvent(
|
return fksources.SingleEvent(
|
||||||
event.Kind("alert"),
|
event.Kind(standards.KindAlert),
|
||||||
s.http.Name,
|
s.http.Name,
|
||||||
standards.SchemaRawNWSAlertsV1,
|
standards.SchemaRawNWSAlertsV1,
|
||||||
eventID,
|
eventID,
|
||||||
|
|||||||
11
internal/sources/nws/driver.go
Normal file
11
internal/sources/nws/driver.go
Normal file
@@ -0,0 +1,11 @@
|
|||||||
|
package nws
|
||||||
|
|
||||||
|
// Source driver strings registered by weatherfeeder for NWS sources.
|
||||||
|
const (
|
||||||
|
DriverObservation = "nws_observation"
|
||||||
|
DriverAlerts = "nws_alerts"
|
||||||
|
DriverForecastHourly = "nws_forecast_hourly"
|
||||||
|
DriverForecastNarrative = "nws_forecast_narrative"
|
||||||
|
DriverForecastDiscussion = "nws_forecast_discussion"
|
||||||
|
DriverWeatherStories = "nws_weatherstories"
|
||||||
|
)
|
||||||
@@ -10,6 +10,7 @@ import (
|
|||||||
"gitea.maximumdirect.net/ejr/feedkit/event"
|
"gitea.maximumdirect.net/ejr/feedkit/event"
|
||||||
fksources "gitea.maximumdirect.net/ejr/feedkit/sources"
|
fksources "gitea.maximumdirect.net/ejr/feedkit/sources"
|
||||||
nwscommon "gitea.maximumdirect.net/ejr/weatherfeeder/internal/providers/nws"
|
nwscommon "gitea.maximumdirect.net/ejr/weatherfeeder/internal/providers/nws"
|
||||||
|
"gitea.maximumdirect.net/ejr/weatherfeeder/standards"
|
||||||
)
|
)
|
||||||
|
|
||||||
const nwsForecastAccept = "application/geo+json, application/json"
|
const nwsForecastAccept = "application/geo+json, application/json"
|
||||||
@@ -44,7 +45,9 @@ func newForecastSource(cfg config.SourceConfig, driver, rawSchema string) (*fore
|
|||||||
|
|
||||||
func (s *forecastSource) Name() string { return s.http.Name }
|
func (s *forecastSource) Name() string { return s.http.Name }
|
||||||
|
|
||||||
func (s *forecastSource) Kinds() []event.Kind { return []event.Kind{event.Kind("forecast")} }
|
func (s *forecastSource) Kinds() []event.Kind {
|
||||||
|
return []event.Kind{event.Kind(standards.KindForecast)}
|
||||||
|
}
|
||||||
|
|
||||||
func (s *forecastSource) Poll(ctx context.Context) ([]event.Event, error) {
|
func (s *forecastSource) Poll(ctx context.Context) ([]event.Event, error) {
|
||||||
raw, meta, changed, err := s.fetchRaw(ctx)
|
raw, meta, changed, err := s.fetchRaw(ctx)
|
||||||
@@ -69,7 +72,7 @@ func (s *forecastSource) Poll(ctx context.Context) ([]event.Event, error) {
|
|||||||
eventID := fksources.DefaultEventID("", s.http.Name, effectiveAt, emittedAt)
|
eventID := fksources.DefaultEventID("", s.http.Name, effectiveAt, emittedAt)
|
||||||
|
|
||||||
return fksources.SingleEvent(
|
return fksources.SingleEvent(
|
||||||
event.Kind("forecast"),
|
event.Kind(standards.KindForecast),
|
||||||
s.http.Name,
|
s.http.Name,
|
||||||
s.rawSchema,
|
s.rawSchema,
|
||||||
eventID,
|
eventID,
|
||||||
|
|||||||
@@ -20,9 +20,7 @@ type ForecastDiscussionSource struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func NewForecastDiscussionSource(cfg config.SourceConfig) (*ForecastDiscussionSource, error) {
|
func NewForecastDiscussionSource(cfg config.SourceConfig) (*ForecastDiscussionSource, error) {
|
||||||
const driver = "nws_forecast_discussion"
|
hs, err := fksources.NewHTTPSource(DriverForecastDiscussion, cfg, "text/html, application/xhtml+xml")
|
||||||
|
|
||||||
hs, err := fksources.NewHTTPSource(driver, cfg, "text/html, application/xhtml+xml")
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@@ -33,7 +31,7 @@ func NewForecastDiscussionSource(cfg config.SourceConfig) (*ForecastDiscussionSo
|
|||||||
func (s *ForecastDiscussionSource) Name() string { return s.http.Name }
|
func (s *ForecastDiscussionSource) Name() string { return s.http.Name }
|
||||||
|
|
||||||
func (s *ForecastDiscussionSource) Kinds() []event.Kind {
|
func (s *ForecastDiscussionSource) Kinds() []event.Kind {
|
||||||
return []event.Kind{event.Kind("forecast_discussion")}
|
return []event.Kind{event.Kind(standards.KindForecastDiscussion)}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *ForecastDiscussionSource) Poll(ctx context.Context) ([]event.Event, error) {
|
func (s *ForecastDiscussionSource) Poll(ctx context.Context) ([]event.Event, error) {
|
||||||
@@ -57,7 +55,7 @@ func (s *ForecastDiscussionSource) Poll(ctx context.Context) ([]event.Event, err
|
|||||||
eventID := fksources.DefaultEventID("", s.http.Name, effectiveAt, emittedAt)
|
eventID := fksources.DefaultEventID("", s.http.Name, effectiveAt, emittedAt)
|
||||||
|
|
||||||
return fksources.SingleEvent(
|
return fksources.SingleEvent(
|
||||||
event.Kind("forecast_discussion"),
|
event.Kind(standards.KindForecastDiscussion),
|
||||||
s.http.Name,
|
s.http.Name,
|
||||||
standards.SchemaRawNWSForecastDiscussionV1,
|
standards.SchemaRawNWSForecastDiscussionV1,
|
||||||
eventID,
|
eventID,
|
||||||
|
|||||||
@@ -27,7 +27,7 @@ func TestForecastDiscussionSourcePollEmitsExpectedEvent(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("NewForecastDiscussionSource() error = %v", err)
|
t.Fatalf("NewForecastDiscussionSource() error = %v", err)
|
||||||
}
|
}
|
||||||
if got := src.Kinds(); len(got) != 1 || got[0] != event.Kind("forecast_discussion") {
|
if got := src.Kinds(); len(got) != 1 || got[0] != event.Kind(standards.KindForecastDiscussion) {
|
||||||
t.Fatalf("Kinds() = %#v, want [forecast_discussion]", got)
|
t.Fatalf("Kinds() = %#v, want [forecast_discussion]", got)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -40,7 +40,7 @@ func TestForecastDiscussionSourcePollEmitsExpectedEvent(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
got := events[0]
|
got := events[0]
|
||||||
if got.Kind != event.Kind("forecast_discussion") {
|
if got.Kind != event.Kind(standards.KindForecastDiscussion) {
|
||||||
t.Fatalf("Kind = %q, want forecast_discussion", got.Kind)
|
t.Fatalf("Kind = %q, want forecast_discussion", got.Kind)
|
||||||
}
|
}
|
||||||
if got.Schema != standards.SchemaRawNWSForecastDiscussionV1 {
|
if got.Schema != standards.SchemaRawNWSForecastDiscussionV1 {
|
||||||
@@ -117,7 +117,7 @@ func TestForecastDiscussionSourcePollRejectsInvalidHTML(t *testing.T) {
|
|||||||
func forecastDiscussionSourceConfig(url string) config.SourceConfig {
|
func forecastDiscussionSourceConfig(url string) config.SourceConfig {
|
||||||
return config.SourceConfig{
|
return config.SourceConfig{
|
||||||
Name: "test-forecast-discussion-source",
|
Name: "test-forecast-discussion-source",
|
||||||
Driver: "nws_forecast_discussion",
|
Driver: DriverForecastDiscussion,
|
||||||
Mode: config.SourceModePoll,
|
Mode: config.SourceModePoll,
|
||||||
Params: map[string]any{
|
Params: map[string]any{
|
||||||
"url": url,
|
"url": url,
|
||||||
|
|||||||
@@ -18,8 +18,7 @@ type HourlyForecastSource struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func NewHourlyForecastSource(cfg config.SourceConfig) (*HourlyForecastSource, error) {
|
func NewHourlyForecastSource(cfg config.SourceConfig) (*HourlyForecastSource, error) {
|
||||||
const driver = "nws_forecast_hourly"
|
src, err := newForecastSource(cfg, DriverForecastHourly, standards.SchemaRawNWSHourlyForecastV1)
|
||||||
src, err := newForecastSource(cfg, driver, standards.SchemaRawNWSHourlyForecastV1)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -18,8 +18,7 @@ type NarrativeForecastSource struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func NewNarrativeForecastSource(cfg config.SourceConfig) (*NarrativeForecastSource, error) {
|
func NewNarrativeForecastSource(cfg config.SourceConfig) (*NarrativeForecastSource, error) {
|
||||||
const driver = "nws_forecast_narrative"
|
src, err := newForecastSource(cfg, DriverForecastNarrative, standards.SchemaRawNWSNarrativeForecastV1)
|
||||||
src, err := newForecastSource(cfg, driver, standards.SchemaRawNWSNarrativeForecastV1)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -26,7 +26,7 @@ func TestForecastSourcesEmitExpectedSchemaAndPreferGeneratedAt(t *testing.T) {
|
|||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
name: "hourly",
|
name: "hourly",
|
||||||
driver: "nws_forecast_hourly",
|
driver: DriverForecastHourly,
|
||||||
wantSchema: standards.SchemaRawNWSHourlyForecastV1,
|
wantSchema: standards.SchemaRawNWSHourlyForecastV1,
|
||||||
newSource: func(cfg config.SourceConfig) (forecastPoller, error) {
|
newSource: func(cfg config.SourceConfig) (forecastPoller, error) {
|
||||||
return NewHourlyForecastSource(cfg)
|
return NewHourlyForecastSource(cfg)
|
||||||
@@ -34,7 +34,7 @@ func TestForecastSourcesEmitExpectedSchemaAndPreferGeneratedAt(t *testing.T) {
|
|||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "narrative",
|
name: "narrative",
|
||||||
driver: "nws_forecast_narrative",
|
driver: DriverForecastNarrative,
|
||||||
wantSchema: standards.SchemaRawNWSNarrativeForecastV1,
|
wantSchema: standards.SchemaRawNWSNarrativeForecastV1,
|
||||||
newSource: func(cfg config.SourceConfig) (forecastPoller, error) {
|
newSource: func(cfg config.SourceConfig) (forecastPoller, error) {
|
||||||
return NewNarrativeForecastSource(cfg)
|
return NewNarrativeForecastSource(cfg)
|
||||||
@@ -55,7 +55,7 @@ func TestForecastSourcesEmitExpectedSchemaAndPreferGeneratedAt(t *testing.T) {
|
|||||||
}
|
}
|
||||||
if ks, ok := src.(interface{ Kinds() []event.Kind }); !ok {
|
if ks, ok := src.(interface{ Kinds() []event.Kind }); !ok {
|
||||||
t.Fatalf("source does not implement Kinds()")
|
t.Fatalf("source does not implement Kinds()")
|
||||||
} else if gotKinds := ks.Kinds(); len(gotKinds) != 1 || gotKinds[0] != event.Kind("forecast") {
|
} else if gotKinds := ks.Kinds(); len(gotKinds) != 1 || gotKinds[0] != event.Kind(standards.KindForecast) {
|
||||||
t.Fatalf("Kinds() = %#v, want [forecast]", gotKinds)
|
t.Fatalf("Kinds() = %#v, want [forecast]", gotKinds)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -69,7 +69,7 @@ func TestForecastSourcesEmitExpectedSchemaAndPreferGeneratedAt(t *testing.T) {
|
|||||||
if got[0].Schema != tt.wantSchema {
|
if got[0].Schema != tt.wantSchema {
|
||||||
t.Fatalf("Poll() schema = %q, want %q", got[0].Schema, tt.wantSchema)
|
t.Fatalf("Poll() schema = %q, want %q", got[0].Schema, tt.wantSchema)
|
||||||
}
|
}
|
||||||
if got[0].Kind != event.Kind("forecast") {
|
if got[0].Kind != event.Kind(standards.KindForecast) {
|
||||||
t.Fatalf("Poll() kind = %q, want forecast", got[0].Kind)
|
t.Fatalf("Poll() kind = %q, want forecast", got[0].Kind)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -117,7 +117,7 @@ func TestForecastSourcePollEffectiveAtFallbackOrder(t *testing.T) {
|
|||||||
}))
|
}))
|
||||||
defer srv.Close()
|
defer srv.Close()
|
||||||
|
|
||||||
src, err := NewHourlyForecastSource(forecastSourceConfig("nws_forecast_hourly", srv.URL))
|
src, err := NewHourlyForecastSource(forecastSourceConfig(DriverForecastHourly, srv.URL))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("NewHourlyForecastSource() error = %v", err)
|
t.Fatalf("NewHourlyForecastSource() error = %v", err)
|
||||||
}
|
}
|
||||||
@@ -148,7 +148,7 @@ func TestForecastSourcePollMetadataDecodeFailureStillEmitsRawEvent(t *testing.T)
|
|||||||
}))
|
}))
|
||||||
defer srv.Close()
|
defer srv.Close()
|
||||||
|
|
||||||
src, err := NewNarrativeForecastSource(forecastSourceConfig("nws_forecast_narrative", srv.URL))
|
src, err := NewNarrativeForecastSource(forecastSourceConfig(DriverForecastNarrative, srv.URL))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("NewNarrativeForecastSource() error = %v", err)
|
t.Fatalf("NewNarrativeForecastSource() error = %v", err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -20,9 +20,7 @@ type ObservationSource struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func NewObservationSource(cfg config.SourceConfig) (*ObservationSource, error) {
|
func NewObservationSource(cfg config.SourceConfig) (*ObservationSource, error) {
|
||||||
const driver = "nws_observation"
|
hs, err := fksources.NewHTTPSource(DriverObservation, cfg, "application/geo+json, application/json")
|
||||||
|
|
||||||
hs, err := fksources.NewHTTPSource(driver, cfg, "application/geo+json, application/json")
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@@ -32,7 +30,9 @@ func NewObservationSource(cfg config.SourceConfig) (*ObservationSource, error) {
|
|||||||
|
|
||||||
func (s *ObservationSource) Name() string { return s.http.Name }
|
func (s *ObservationSource) Name() string { return s.http.Name }
|
||||||
|
|
||||||
func (s *ObservationSource) Kinds() []event.Kind { return []event.Kind{event.Kind("observation")} }
|
func (s *ObservationSource) Kinds() []event.Kind {
|
||||||
|
return []event.Kind{event.Kind(standards.KindObservation)}
|
||||||
|
}
|
||||||
|
|
||||||
func (s *ObservationSource) Poll(ctx context.Context) ([]event.Event, error) {
|
func (s *ObservationSource) Poll(ctx context.Context) ([]event.Event, error) {
|
||||||
raw, meta, changed, err := s.fetchRaw(ctx)
|
raw, meta, changed, err := s.fetchRaw(ctx)
|
||||||
@@ -54,7 +54,7 @@ func (s *ObservationSource) Poll(ctx context.Context) ([]event.Event, error) {
|
|||||||
eventID := fksources.DefaultEventID(meta.ID, s.http.Name, effectiveAt, emittedAt)
|
eventID := fksources.DefaultEventID(meta.ID, s.http.Name, effectiveAt, emittedAt)
|
||||||
|
|
||||||
return fksources.SingleEvent(
|
return fksources.SingleEvent(
|
||||||
event.Kind("observation"),
|
event.Kind(standards.KindObservation),
|
||||||
s.http.Name,
|
s.http.Name,
|
||||||
standards.SchemaRawNWSObservationV1,
|
standards.SchemaRawNWSObservationV1,
|
||||||
eventID,
|
eventID,
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ import (
|
|||||||
|
|
||||||
"gitea.maximumdirect.net/ejr/feedkit/config"
|
"gitea.maximumdirect.net/ejr/feedkit/config"
|
||||||
"gitea.maximumdirect.net/ejr/feedkit/event"
|
"gitea.maximumdirect.net/ejr/feedkit/event"
|
||||||
|
"gitea.maximumdirect.net/ejr/weatherfeeder/standards"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestObservationSourcePollReturnsNoEventsOn304(t *testing.T) {
|
func TestObservationSourcePollReturnsNoEventsOn304(t *testing.T) {
|
||||||
@@ -31,7 +32,7 @@ func TestObservationSourcePollReturnsNoEventsOn304(t *testing.T) {
|
|||||||
|
|
||||||
src, err := NewObservationSource(config.SourceConfig{
|
src, err := NewObservationSource(config.SourceConfig{
|
||||||
Name: "NWSObservationTest",
|
Name: "NWSObservationTest",
|
||||||
Driver: "nws_observation",
|
Driver: DriverObservation,
|
||||||
Mode: config.SourceModePoll,
|
Mode: config.SourceModePoll,
|
||||||
Params: map[string]any{
|
Params: map[string]any{
|
||||||
"url": srv.URL,
|
"url": srv.URL,
|
||||||
@@ -41,7 +42,7 @@ func TestObservationSourcePollReturnsNoEventsOn304(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("NewObservationSource() error = %v", err)
|
t.Fatalf("NewObservationSource() error = %v", err)
|
||||||
}
|
}
|
||||||
if got := src.Kinds(); len(got) != 1 || got[0] != event.Kind("observation") {
|
if got := src.Kinds(); len(got) != 1 || got[0] != event.Kind(standards.KindObservation) {
|
||||||
t.Fatalf("Kinds() = %#v, want [observation]", got)
|
t.Fatalf("Kinds() = %#v, want [observation]", got)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -52,7 +53,7 @@ func TestObservationSourcePollReturnsNoEventsOn304(t *testing.T) {
|
|||||||
if len(first) != 1 {
|
if len(first) != 1 {
|
||||||
t.Fatalf("first Poll() len = %d, want 1", len(first))
|
t.Fatalf("first Poll() len = %d, want 1", len(first))
|
||||||
}
|
}
|
||||||
if first[0].Kind != event.Kind("observation") {
|
if first[0].Kind != event.Kind(standards.KindObservation) {
|
||||||
t.Fatalf("first Poll() kind = %q", first[0].Kind)
|
t.Fatalf("first Poll() kind = %q", first[0].Kind)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -22,9 +22,7 @@ type WeatherStoriesSource struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func NewWeatherStoriesSource(cfg config.SourceConfig) (*WeatherStoriesSource, error) {
|
func NewWeatherStoriesSource(cfg config.SourceConfig) (*WeatherStoriesSource, error) {
|
||||||
const driver = "nws_weatherstories"
|
hs, err := fksources.NewHTTPSource(DriverWeatherStories, cfg, "application/geo+json, application/json")
|
||||||
|
|
||||||
hs, err := fksources.NewHTTPSource(driver, cfg, "application/geo+json, application/json")
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@@ -35,7 +33,7 @@ func NewWeatherStoriesSource(cfg config.SourceConfig) (*WeatherStoriesSource, er
|
|||||||
func (s *WeatherStoriesSource) Name() string { return s.http.Name }
|
func (s *WeatherStoriesSource) Name() string { return s.http.Name }
|
||||||
|
|
||||||
func (s *WeatherStoriesSource) Kinds() []event.Kind {
|
func (s *WeatherStoriesSource) Kinds() []event.Kind {
|
||||||
return []event.Kind{event.Kind("weather_story")}
|
return []event.Kind{event.Kind(standards.KindWeatherStory)}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *WeatherStoriesSource) Poll(ctx context.Context) ([]event.Event, error) {
|
func (s *WeatherStoriesSource) Poll(ctx context.Context) ([]event.Event, error) {
|
||||||
@@ -61,7 +59,7 @@ func (s *WeatherStoriesSource) Poll(ctx context.Context) ([]event.Event, error)
|
|||||||
eventID := fksources.DefaultEventID("", s.http.Name, effectiveAt, emittedAt)
|
eventID := fksources.DefaultEventID("", s.http.Name, effectiveAt, emittedAt)
|
||||||
|
|
||||||
return fksources.SingleEvent(
|
return fksources.SingleEvent(
|
||||||
event.Kind("weather_story"),
|
event.Kind(standards.KindWeatherStory),
|
||||||
s.http.Name,
|
s.http.Name,
|
||||||
standards.SchemaRawNWSWeatherStoriesV1,
|
standards.SchemaRawNWSWeatherStoriesV1,
|
||||||
eventID,
|
eventID,
|
||||||
|
|||||||
@@ -28,7 +28,7 @@ func TestWeatherStoriesSourcePollEmitsExpectedEventAndPrefersLatestUpdateTime(t
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("NewWeatherStoriesSource() error = %v", err)
|
t.Fatalf("NewWeatherStoriesSource() error = %v", err)
|
||||||
}
|
}
|
||||||
if got := src.Kinds(); len(got) != 1 || got[0] != event.Kind("weather_story") {
|
if got := src.Kinds(); len(got) != 1 || got[0] != event.Kind(standards.KindWeatherStory) {
|
||||||
t.Fatalf("Kinds() = %#v, want [weather_story]", got)
|
t.Fatalf("Kinds() = %#v, want [weather_story]", got)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -41,7 +41,7 @@ func TestWeatherStoriesSourcePollEmitsExpectedEventAndPrefersLatestUpdateTime(t
|
|||||||
}
|
}
|
||||||
|
|
||||||
got := events[0]
|
got := events[0]
|
||||||
if got.Kind != event.Kind("weather_story") {
|
if got.Kind != event.Kind(standards.KindWeatherStory) {
|
||||||
t.Fatalf("Kind = %q, want weather_story", got.Kind)
|
t.Fatalf("Kind = %q, want weather_story", got.Kind)
|
||||||
}
|
}
|
||||||
if got.Schema != standards.SchemaRawNWSWeatherStoriesV1 {
|
if got.Schema != standards.SchemaRawNWSWeatherStoriesV1 {
|
||||||
@@ -148,7 +148,7 @@ func TestWeatherStoriesSourcePollMetadataDecodeFailureStillEmitsRawEvent(t *test
|
|||||||
func weatherStoriesSourceConfig(url string) config.SourceConfig {
|
func weatherStoriesSourceConfig(url string) config.SourceConfig {
|
||||||
return config.SourceConfig{
|
return config.SourceConfig{
|
||||||
Name: "test-weatherstories-source",
|
Name: "test-weatherstories-source",
|
||||||
Driver: "nws_weatherstories",
|
Driver: DriverWeatherStories,
|
||||||
Mode: config.SourceModePoll,
|
Mode: config.SourceModePoll,
|
||||||
Params: map[string]any{
|
Params: map[string]any{
|
||||||
"url": url,
|
"url": url,
|
||||||
|
|||||||
7
internal/sources/openmeteo/driver.go
Normal file
7
internal/sources/openmeteo/driver.go
Normal file
@@ -0,0 +1,7 @@
|
|||||||
|
package openmeteo
|
||||||
|
|
||||||
|
// Source driver strings registered by weatherfeeder for Open-Meteo sources.
|
||||||
|
const (
|
||||||
|
DriverObservation = "openmeteo_observation"
|
||||||
|
DriverForecast = "openmeteo_forecast"
|
||||||
|
)
|
||||||
@@ -19,9 +19,7 @@ type ForecastSource struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func NewForecastSource(cfg config.SourceConfig) (*ForecastSource, error) {
|
func NewForecastSource(cfg config.SourceConfig) (*ForecastSource, error) {
|
||||||
const driver = "openmeteo_forecast"
|
hs, err := fksources.NewHTTPSource(DriverForecast, cfg, "application/json")
|
||||||
|
|
||||||
hs, err := fksources.NewHTTPSource(driver, cfg, "application/json")
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@@ -31,7 +29,9 @@ func NewForecastSource(cfg config.SourceConfig) (*ForecastSource, error) {
|
|||||||
|
|
||||||
func (s *ForecastSource) Name() string { return s.http.Name }
|
func (s *ForecastSource) Name() string { return s.http.Name }
|
||||||
|
|
||||||
func (s *ForecastSource) Kinds() []event.Kind { return []event.Kind{event.Kind("forecast")} }
|
func (s *ForecastSource) Kinds() []event.Kind {
|
||||||
|
return []event.Kind{event.Kind(standards.KindForecast)}
|
||||||
|
}
|
||||||
|
|
||||||
func (s *ForecastSource) Poll(ctx context.Context) ([]event.Event, error) {
|
func (s *ForecastSource) Poll(ctx context.Context) ([]event.Event, error) {
|
||||||
raw, meta, changed, err := s.fetchRaw(ctx)
|
raw, meta, changed, err := s.fetchRaw(ctx)
|
||||||
@@ -55,7 +55,7 @@ func (s *ForecastSource) Poll(ctx context.Context) ([]event.Event, error) {
|
|||||||
eventID := fksources.DefaultEventID("", s.http.Name, effectiveAt, emittedAt)
|
eventID := fksources.DefaultEventID("", s.http.Name, effectiveAt, emittedAt)
|
||||||
|
|
||||||
return fksources.SingleEvent(
|
return fksources.SingleEvent(
|
||||||
event.Kind("forecast"),
|
event.Kind(standards.KindForecast),
|
||||||
s.http.Name,
|
s.http.Name,
|
||||||
standards.SchemaRawOpenMeteoHourlyForecastV1,
|
standards.SchemaRawOpenMeteoHourlyForecastV1,
|
||||||
eventID,
|
eventID,
|
||||||
|
|||||||
@@ -19,9 +19,7 @@ type ObservationSource struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func NewObservationSource(cfg config.SourceConfig) (*ObservationSource, error) {
|
func NewObservationSource(cfg config.SourceConfig) (*ObservationSource, error) {
|
||||||
const driver = "openmeteo_observation"
|
hs, err := fksources.NewHTTPSource(DriverObservation, cfg, "application/json")
|
||||||
|
|
||||||
hs, err := fksources.NewHTTPSource(driver, cfg, "application/json")
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@@ -31,7 +29,9 @@ func NewObservationSource(cfg config.SourceConfig) (*ObservationSource, error) {
|
|||||||
|
|
||||||
func (s *ObservationSource) Name() string { return s.http.Name }
|
func (s *ObservationSource) Name() string { return s.http.Name }
|
||||||
|
|
||||||
func (s *ObservationSource) Kinds() []event.Kind { return []event.Kind{event.Kind("observation")} }
|
func (s *ObservationSource) Kinds() []event.Kind {
|
||||||
|
return []event.Kind{event.Kind(standards.KindObservation)}
|
||||||
|
}
|
||||||
|
|
||||||
func (s *ObservationSource) Poll(ctx context.Context) ([]event.Event, error) {
|
func (s *ObservationSource) Poll(ctx context.Context) ([]event.Event, error) {
|
||||||
raw, meta, changed, err := s.fetchRaw(ctx)
|
raw, meta, changed, err := s.fetchRaw(ctx)
|
||||||
@@ -52,7 +52,7 @@ func (s *ObservationSource) Poll(ctx context.Context) ([]event.Event, error) {
|
|||||||
eventID := fksources.DefaultEventID("", s.http.Name, effectiveAt, emittedAt)
|
eventID := fksources.DefaultEventID("", s.http.Name, effectiveAt, emittedAt)
|
||||||
|
|
||||||
return fksources.SingleEvent(
|
return fksources.SingleEvent(
|
||||||
event.Kind("observation"),
|
event.Kind(standards.KindObservation),
|
||||||
s.http.Name,
|
s.http.Name,
|
||||||
standards.SchemaRawOpenMeteoCurrentV1,
|
standards.SchemaRawOpenMeteoCurrentV1,
|
||||||
eventID,
|
eventID,
|
||||||
|
|||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user