Compare commits
52 Commits
979d754d18
...
main
| Author | SHA1 | Date | |
|---|---|---|---|
| 9b88be4dd2 | |||
| 456a46e01b | |||
| 3740c779eb | |||
| 6943a5ebc9 | |||
| 9a09454621 | |||
| b8c6708439 | |||
| a62cb87b78 | |||
| 0b5eaf46f4 | |||
| 882059014c | |||
| 0a3e52d0e5 | |||
| 2b19a121fa | |||
| 29c65971eb | |||
| 3ecf4c5b7f | |||
| f402e27542 | |||
| f720b6cdc0 | |||
| 50215d2105 | |||
| 74411e3f54 | |||
| 4358a7cdce | |||
| dcea5261ab | |||
| 97141c7a9b | |||
| 2e2d36024e | |||
| 4d2cddf801 | |||
| 21a35a5205 | |||
| 435d1ade07 | |||
| 819ac24aed | |||
| 2c472449e8 | |||
| 5d7f604a2c | |||
| 8041f99782 | |||
| c417c892d9 | |||
| 481215c5db | |||
| 5d94d3f32d | |||
| f8f1b8d4a5 | |||
| 06d5973746 | |||
| 8045b27173 | |||
| 985468c1b9 | |||
| 6a0b30b7c7 | |||
| 86ce4eb68c | |||
| 33541a71fc | |||
| b7277e0c02 | |||
| dec05821bf | |||
| 1a9f462fbf | |||
| a990da957b | |||
| a4cd63ca4e | |||
| fba519cab0 | |||
| da8ff81692 | |||
| f91a185f9d | |||
| e966276c40 | |||
| 1e2db468ea | |||
| cefd4dfc7c | |||
| 42c646c328 | |||
| b2c429983c | |||
| 0f20d1e4cb |
@@ -1,7 +1,8 @@
|
||||
.git
|
||||
.gitignore
|
||||
**/*.md
|
||||
!docs/*.md
|
||||
!docs/**/*.md
|
||||
dist/
|
||||
tmp/
|
||||
.DS_Store
|
||||
|
||||
|
||||
4
AGENTS.md
Normal file
4
AGENTS.md
Normal file
@@ -0,0 +1,4 @@
|
||||
Please carefully review the documents in `docs/policy` before making any changes to this repository.
|
||||
- `architecture.md` provides the canonical high-level architecture policy for this repository.
|
||||
- `development.md` provides more granular development policy for this repository.
|
||||
- `documentation.md` provides the canonical documentation policy for this repository.
|
||||
@@ -5,9 +5,9 @@ provider-specific responses into canonical weather events, and routes those
|
||||
events to configured sinks.
|
||||
|
||||
It currently supports NWS observations, alerts, hourly forecasts, narrative
|
||||
forecasts, forecast discussions, and weather stories; Open-Meteo observations
|
||||
and hourly forecasts; and OpenWeather observations. Implemented sinks are
|
||||
stdout, NATS, and Postgres.
|
||||
forecasts, forecast discussions, and weather stories; SPC Day 1-3 convective
|
||||
outlooks; Open-Meteo observations and hourly forecasts; and OpenWeather
|
||||
observations. Implemented sinks are stdout, NATS, and Postgres.
|
||||
|
||||
## Quickstart
|
||||
|
||||
@@ -29,9 +29,12 @@ current working directory.
|
||||
- [Operations guide](docs/operations.md)
|
||||
- [Troubleshooting guide](docs/troubleshooting.md)
|
||||
- [Example configs](examples/)
|
||||
- [Go consumer guide](docs/consumers/api.md)
|
||||
- [Event wire contract](docs/integrations/events.md)
|
||||
- [Postgres table contract](docs/integrations/postgres.md)
|
||||
- [Feedkit integration notes](docs/integrations/feedkit.md)
|
||||
- [NWS integration notes](docs/integrations/nws.md)
|
||||
- [SPC integration notes](docs/integrations/spc.md)
|
||||
- [Open-Meteo integration notes](docs/integrations/openmeteo.md)
|
||||
- [OpenWeather integration notes](docs/integrations/openweather.md)
|
||||
- [Architecture policy](docs/policy/architecture.md)
|
||||
|
||||
@@ -99,6 +99,18 @@ sources:
|
||||
url: "https://api.weather.gov/alerts?point=38.6239,-90.3571&limit=20"
|
||||
user_agent: "HomeOps (eric@maximumdirect.net)"
|
||||
|
||||
- name: SPCConvectiveOutlookSTL
|
||||
mode: poll
|
||||
kinds: ["outlook"]
|
||||
driver: spc_convective_outlook
|
||||
every: 30m
|
||||
params:
|
||||
latitude: 38.6239
|
||||
longitude: -90.3571
|
||||
location_id: "stl"
|
||||
location_name: "St. Louis, MO"
|
||||
user_agent: "HomeOps (eric@maximumdirect.net)"
|
||||
|
||||
sinks:
|
||||
- name: stdout
|
||||
driver: stdout
|
||||
@@ -126,13 +138,13 @@ sinks:
|
||||
|
||||
routes:
|
||||
- sink: stdout
|
||||
kinds: ["observation", "forecast", "forecast_discussion", "weather_story", "alert"]
|
||||
kinds: ["observation", "forecast", "forecast_discussion", "weather_story", "alert", "outlook"]
|
||||
|
||||
- sink: nats_weatherfeeder
|
||||
kinds: ["observation", "forecast", "forecast_discussion", "weather_story", "alert"]
|
||||
kinds: ["observation", "forecast", "forecast_discussion", "weather_story", "alert", "outlook"]
|
||||
|
||||
# - sink: pg_weatherfeeder
|
||||
# kinds: ["observation", "forecast", "forecast_discussion", "weather_story", "alert"]
|
||||
# kinds: ["observation", "forecast", "forecast_discussion", "weather_story", "alert", "outlook"]
|
||||
|
||||
# - sink: logfile
|
||||
# kinds: ["observation", "alert", "forecast", "forecast_discussion", "weather_story"]
|
||||
# kinds: ["observation", "alert", "forecast", "forecast_discussion", "weather_story", "outlook"]
|
||||
|
||||
@@ -20,6 +20,7 @@ import (
|
||||
|
||||
wfnormalizers "gitea.maximumdirect.net/ejr/weatherfeeder/internal/normalizers"
|
||||
wfsources "gitea.maximumdirect.net/ejr/weatherfeeder/internal/sources"
|
||||
"gitea.maximumdirect.net/ejr/weatherfeeder/standards"
|
||||
)
|
||||
|
||||
type testInput struct {
|
||||
@@ -36,10 +37,10 @@ type testKindsSource struct {
|
||||
func (s testKindsSource) Kinds() []fkevent.Kind { return s.kinds }
|
||||
|
||||
func TestValidateSourceExpectedKindsSubsetAllowed(t *testing.T) {
|
||||
sc := config.SourceConfig{Kinds: []string{"observation"}}
|
||||
sc := config.SourceConfig{Kinds: []string{standards.KindObservation}}
|
||||
in := testKindsSource{
|
||||
testInput: testInput{name: "test"},
|
||||
kinds: []fkevent.Kind{"observation", "forecast"},
|
||||
kinds: []fkevent.Kind{fkevent.Kind(standards.KindObservation), fkevent.Kind(standards.KindForecast)},
|
||||
}
|
||||
|
||||
if err := fksources.ValidateExpectedKinds(sc, in); err != nil {
|
||||
@@ -48,10 +49,10 @@ func TestValidateSourceExpectedKindsSubsetAllowed(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestValidateSourceExpectedKindsMismatchFails(t *testing.T) {
|
||||
sc := config.SourceConfig{Kinds: []string{"alert"}}
|
||||
sc := config.SourceConfig{Kinds: []string{standards.KindAlert}}
|
||||
in := testKindsSource{
|
||||
testInput: testInput{name: "test"},
|
||||
kinds: []fkevent.Kind{"observation", "forecast"},
|
||||
kinds: []fkevent.Kind{fkevent.Kind(standards.KindObservation), fkevent.Kind(standards.KindForecast)},
|
||||
}
|
||||
|
||||
err := fksources.ValidateExpectedKinds(sc, in)
|
||||
@@ -64,7 +65,7 @@ func TestValidateSourceExpectedKindsMismatchFails(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestValidateSourceExpectedKindsNoMetadataSkipsCheck(t *testing.T) {
|
||||
sc := config.SourceConfig{Kinds: []string{"alert"}}
|
||||
sc := config.SourceConfig{Kinds: []string{standards.KindAlert}}
|
||||
in := testInput{name: "test"}
|
||||
|
||||
if err := fksources.ValidateExpectedKinds(sc, in); err != nil {
|
||||
@@ -111,6 +112,10 @@ func TestMaintainedConfigExamplesLoad(t *testing.T) {
|
||||
func assertConfigSourcesBuildSchedulerJobs(t *testing.T, cfg *config.Config) {
|
||||
t.Helper()
|
||||
|
||||
if len(cfg.Sources) == 0 {
|
||||
t.Fatalf("config has no sources")
|
||||
}
|
||||
|
||||
reg := fksources.NewRegistry()
|
||||
wfsources.RegisterBuiltins(reg)
|
||||
|
||||
@@ -158,7 +163,7 @@ func TestNormalizeNoMatchPassThrough(t *testing.T) {
|
||||
pl := &fkpipeline.Pipeline{Processors: chain}
|
||||
in := fkevent.Event{
|
||||
ID: "evt-no-match",
|
||||
Kind: fkevent.Kind("observation"),
|
||||
Kind: fkevent.Kind(standards.KindObservation),
|
||||
Source: "test",
|
||||
EmittedAt: time.Now().UTC(),
|
||||
Schema: "raw.weatherfeeder.unknown.v1",
|
||||
@@ -188,7 +193,7 @@ func TestDedupeDropsSecondEventWithSameID(t *testing.T) {
|
||||
pl := &fkpipeline.Pipeline{Processors: chain}
|
||||
in := fkevent.Event{
|
||||
ID: "evt-dedupe-1",
|
||||
Kind: fkevent.Kind("observation"),
|
||||
Kind: fkevent.Kind(standards.KindObservation),
|
||||
Source: "test",
|
||||
EmittedAt: time.Now().UTC(),
|
||||
Schema: "raw.weatherfeeder.unknown.v1",
|
||||
|
||||
@@ -52,6 +52,18 @@ sources:
|
||||
url: "https://api.weather.gov/alerts?point=38.6239,-90.3571&limit=20"
|
||||
user_agent: "Example weatherfeeder operator (ops@example.com)"
|
||||
|
||||
- name: SPCConvectiveOutlookLocal
|
||||
mode: poll
|
||||
driver: spc_convective_outlook
|
||||
every: 30m
|
||||
kinds: ["outlook"]
|
||||
params:
|
||||
latitude: 38.6239
|
||||
longitude: -90.3571
|
||||
location_id: "local"
|
||||
location_name: "Configured point"
|
||||
user_agent: "Example weatherfeeder operator (ops@example.com)"
|
||||
|
||||
sinks:
|
||||
- name: nats_weather
|
||||
driver: nats
|
||||
@@ -69,10 +81,10 @@ sinks:
|
||||
|
||||
routes:
|
||||
- sink: nats_weather
|
||||
kinds: ["observation", "forecast", "forecast_discussion", "weather_story", "alert"]
|
||||
kinds: ["observation", "forecast", "forecast_discussion", "weather_story", "alert", "outlook"]
|
||||
|
||||
- sink: pg_weather
|
||||
kinds: ["observation", "forecast", "forecast_discussion", "weather_story", "alert"]
|
||||
kinds: ["observation", "forecast", "forecast_discussion", "weather_story", "alert", "outlook"]
|
||||
```
|
||||
|
||||
Do not commit real API keys, database passwords, or personal contact addresses in
|
||||
@@ -95,10 +107,10 @@ copyable configs.
|
||||
| `mode` | no | `poll`, `stream`, or omitted for auto. Current weatherfeeder drivers are polling drivers. |
|
||||
| `every` | yes | Poll interval for current weatherfeeder source drivers. |
|
||||
| `kinds` | no | Expected event kinds. If present, startup verifies they match the source driver. |
|
||||
| `params` | driver-specific | Driver parameters. Current source drivers require HTTP params. |
|
||||
| `params` | driver-specific | Driver parameters. See the source-specific sections below. |
|
||||
|
||||
Current event kinds are `observation`, `forecast`, `forecast_discussion`,
|
||||
`weather_story`, and `alert`.
|
||||
`weather_story`, `alert`, and `outlook`.
|
||||
|
||||
## Source Drivers
|
||||
|
||||
@@ -113,10 +125,11 @@ Current event kinds are `observation`, `forecast`, `forecast_discussion`,
|
||||
| `openmeteo_observation` | `observation` | Open-Meteo current conditions. |
|
||||
| `openmeteo_forecast` | `forecast` | Open-Meteo hourly forecast. |
|
||||
| `openweather_observation` | `observation` | OpenWeather current weather. |
|
||||
| `spc_convective_outlook` | `outlook` | SPC Day 1-3 convective outlooks. |
|
||||
|
||||
## HTTP Source Params
|
||||
|
||||
All current source drivers use the shared HTTP polling helper.
|
||||
Most source drivers use the shared HTTP polling helper.
|
||||
|
||||
| Param | Required | Description |
|
||||
|---|:---:|---|
|
||||
@@ -132,6 +145,44 @@ source emits no events for that poll.
|
||||
OpenWeather observation URLs must include `units=metric`. Startup fails if the
|
||||
URL omits it or sets another unit system.
|
||||
|
||||
## SPC Convective Outlook Params
|
||||
|
||||
`spc_convective_outlook` fetches the nine required Day 1-3 GeoJSON outlook
|
||||
products and the three required Day 1-3 print pages as one atomic bundle.
|
||||
|
||||
| Param | Required | Description |
|
||||
|---|:---:|---|
|
||||
| `latitude` | yes | Location latitude in decimal degrees. |
|
||||
| `longitude` | yes | Location longitude in decimal degrees. |
|
||||
| `user_agent` | yes | User-Agent sent to SPC. `userAgent` is also accepted. |
|
||||
| `location_id` | no | Operator-defined location identifier copied into canonical outlook runs. |
|
||||
| `location_name` | no | Human location label copied into canonical outlook runs. |
|
||||
| `geojson_urls` | no | Map of product key to override URL. Used for tests and upstream URL changes. |
|
||||
| `discussion_urls` | no | Map of day key to override print-page URL. Used for tests and upstream URL changes. |
|
||||
| `rss_url` | no | Optional RSS URL. RSS is not fetched unless this is configured. |
|
||||
| `http_timeout` | no | Positive duration for the HTTP client timeout. |
|
||||
| `http_response_body_limit_bytes` | no | Positive integer response body limit in bytes. |
|
||||
|
||||
GeoJSON product keys are `day1_categorical`, `day1_tornado`, `day1_hail`,
|
||||
`day1_wind`, `day2_categorical`, `day2_tornado`, `day2_hail`, `day2_wind`,
|
||||
and `day3_categorical`. SPC does not provide Day 3 tornado, hail, or wind
|
||||
GeoJSON products. Discussion keys are `day1`, `day2`, and `day3`.
|
||||
|
||||
```yaml
|
||||
sources:
|
||||
- name: SPCConvectiveOutlookSTL
|
||||
mode: poll
|
||||
kinds: ["outlook"]
|
||||
driver: spc_convective_outlook
|
||||
every: 30m
|
||||
params:
|
||||
latitude: 38.6239
|
||||
longitude: -90.3571
|
||||
location_id: "stl"
|
||||
location_name: "St. Louis, MO"
|
||||
user_agent: "Example weatherfeeder operator (ops@example.com)"
|
||||
```
|
||||
|
||||
## Sink Fields
|
||||
|
||||
| Field | Required | Description |
|
||||
|
||||
92
docs/consumers/api.md
Normal file
92
docs/consumers/api.md
Normal file
@@ -0,0 +1,92 @@
|
||||
# Consumer API Guide
|
||||
|
||||
## Purpose
|
||||
|
||||
This guide is for developers and LLM coding agents integrating `weatherfeeder`
|
||||
from another Go codebase.
|
||||
|
||||
`weatherfeeder` is primarily a daemon, not an SDK. Its public integration
|
||||
surface is intentionally narrow:
|
||||
|
||||
- `model`: canonical weather payload structs.
|
||||
- `standards`: schema strings, event kind strings, and shared WMO constants.
|
||||
- JSON event output from stdout and NATS sinks.
|
||||
- Postgres tables written by the optional Postgres sink.
|
||||
|
||||
Packages under `internal/` are implementation details and are not public
|
||||
integration surfaces.
|
||||
|
||||
## Recommended Workflow
|
||||
|
||||
Consumers should switch on the event `schema` value and decode `payload` into
|
||||
the matching `model` type.
|
||||
|
||||
Minimal example:
|
||||
|
||||
```go
|
||||
package consumer
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
|
||||
"gitea.maximumdirect.net/ejr/weatherfeeder/model"
|
||||
"gitea.maximumdirect.net/ejr/weatherfeeder/standards"
|
||||
)
|
||||
|
||||
type Event struct {
|
||||
ID string `json:"id"`
|
||||
Kind string `json:"kind"`
|
||||
Schema string `json:"schema"`
|
||||
Payload json.RawMessage `json:"payload"`
|
||||
}
|
||||
|
||||
func Decode(payload []byte) (any, error) {
|
||||
var evt Event
|
||||
if err := json.Unmarshal(payload, &evt); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
switch evt.Schema {
|
||||
case standards.SchemaWeatherObservationV1:
|
||||
var out model.WeatherObservation
|
||||
return &out, json.Unmarshal(evt.Payload, &out)
|
||||
case standards.SchemaWeatherForecastV1:
|
||||
var out model.WeatherForecastRun
|
||||
return &out, json.Unmarshal(evt.Payload, &out)
|
||||
case standards.SchemaWeatherForecastDiscussionV1:
|
||||
var out model.WeatherForecastDiscussion
|
||||
return &out, json.Unmarshal(evt.Payload, &out)
|
||||
case standards.SchemaWeatherStoryV1:
|
||||
var out model.WeatherStoryRun
|
||||
return &out, json.Unmarshal(evt.Payload, &out)
|
||||
case standards.SchemaWeatherAlertV1:
|
||||
var out model.WeatherAlertRun
|
||||
return &out, json.Unmarshal(evt.Payload, &out)
|
||||
case standards.SchemaWeatherOutlookV2:
|
||||
var out model.WeatherOutlookRun
|
||||
return &out, json.Unmarshal(evt.Payload, &out)
|
||||
default:
|
||||
return nil, fmt.Errorf("unsupported weatherfeeder schema %q", evt.Schema)
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Consumer Responsibilities
|
||||
|
||||
- Treat event IDs as opaque.
|
||||
- Treat absent `omitempty` fields as unknown, not zero.
|
||||
- Prefer schema constants from `standards` over string literals in Go code.
|
||||
- Expect canonical numeric measurements to use metric units.
|
||||
- Expect canonical timestamps from normalizers to be UTC unless a field-specific
|
||||
contract says otherwise.
|
||||
- Handle additive fields within the same schema version.
|
||||
- Do not import `internal/...` packages.
|
||||
|
||||
## Canonical References
|
||||
|
||||
- Public payload package: [`pkg-model.md`](pkg-model.md).
|
||||
- Public constants package: [`pkg-standards.md`](pkg-standards.md).
|
||||
- JSON event wire contract: [`../integrations/events.md`](../integrations/events.md).
|
||||
- Postgres table contract: [`../integrations/postgres.md`](../integrations/postgres.md).
|
||||
- Runtime and adapter architecture: [`../policy/architecture.md`](../policy/architecture.md).
|
||||
64
docs/consumers/pkg-model.md
Normal file
64
docs/consumers/pkg-model.md
Normal file
@@ -0,0 +1,64 @@
|
||||
# Package `model`
|
||||
|
||||
## Import Path
|
||||
|
||||
```go
|
||||
import "gitea.maximumdirect.net/ejr/weatherfeeder/model"
|
||||
```
|
||||
|
||||
## Purpose
|
||||
|
||||
Package `model` defines `weatherfeeder`'s canonical weather payload structs.
|
||||
These structs are emitted as the `payload` of canonical `weather.*` events and
|
||||
are also the domain types consumed by downstream applications such as
|
||||
`weatherapi`.
|
||||
|
||||
The JSON field tags on these structs are part of the wire contract. For the full
|
||||
field-by-field JSON contract, use the [event wire contract](../integrations/events.md).
|
||||
|
||||
## Payload Types
|
||||
|
||||
Current canonical schema families map to these public types:
|
||||
|
||||
| Schema | Primary type |
|
||||
|---|---|
|
||||
| `weather.observation.v1` | `WeatherObservation` |
|
||||
| `weather.forecast.v1` | `WeatherForecastRun` |
|
||||
| `weather.forecast_discussion.v1` | `WeatherForecastDiscussion` |
|
||||
| `weather.weather_story.v1` | `WeatherStoryRun` |
|
||||
| `weather.alert.v1` | `WeatherAlertRun` |
|
||||
| `weather.outlook.v2` | `WeatherOutlookRun` |
|
||||
|
||||
Related child types include:
|
||||
|
||||
- `WeatherObservationPresentWeather`
|
||||
- `WeatherForecastPeriod`
|
||||
- `WeatherForecastDiscussionSection`
|
||||
- `WeatherStory`
|
||||
- `WeatherAlert`
|
||||
- `WeatherAlertReference`
|
||||
- `WeatherOutlook`
|
||||
- `WeatherOutlookDiscussion`
|
||||
- `WMOCode`
|
||||
|
||||
## Wire And Compatibility Rules
|
||||
|
||||
- JSON tags define canonical payload field names.
|
||||
- Pointer fields and fields tagged `omitempty` are optional on the wire.
|
||||
- Missing optional fields mean unknown or not applicable.
|
||||
- Canonical measurements use metric units.
|
||||
- Canonical timestamps are `time.Time` values encoded by Go's JSON encoder.
|
||||
- Normalized canonical timestamps are UTC unless a field-specific contract says
|
||||
otherwise.
|
||||
- Additive fields are compatible within a schema version.
|
||||
- Removing, renaming, or changing the meaning of a field requires a new schema
|
||||
identifier.
|
||||
|
||||
## Boundaries
|
||||
|
||||
`model` should not depend on source adapters, sinks, SQL column names, provider
|
||||
HTTP shapes, or runtime configuration.
|
||||
|
||||
Consumers should not rely on packages under `internal/...`. Use `model` with
|
||||
schema constants from [`standards`](pkg-standards.md) and the JSON contract in
|
||||
[`docs/integrations/events.md`](../integrations/events.md).
|
||||
96
docs/consumers/pkg-standards.md
Normal file
96
docs/consumers/pkg-standards.md
Normal file
@@ -0,0 +1,96 @@
|
||||
# Package `standards`
|
||||
|
||||
## Import Path
|
||||
|
||||
```go
|
||||
import "gitea.maximumdirect.net/ejr/weatherfeeder/standards"
|
||||
```
|
||||
|
||||
## Purpose
|
||||
|
||||
Package `standards` defines stable identifiers and shared weather constants used
|
||||
by `weatherfeeder` producers and Go consumers.
|
||||
|
||||
Use this package when switching on event schemas, comparing event kinds, or
|
||||
working with canonical WMO condition codes.
|
||||
|
||||
## Event Kind Constants
|
||||
|
||||
Current event kind constants are:
|
||||
|
||||
| Constant | Value |
|
||||
|---|---|
|
||||
| `KindObservation` | `observation` |
|
||||
| `KindForecast` | `forecast` |
|
||||
| `KindForecastDiscussion` | `forecast_discussion` |
|
||||
| `KindWeatherStory` | `weather_story` |
|
||||
| `KindAlert` | `alert` |
|
||||
| `KindOutlook` | `outlook` |
|
||||
|
||||
These are plain string constants. Convert them at adapter boundaries when using
|
||||
feedkit's `event.Kind` type.
|
||||
|
||||
## Canonical Schema Constants
|
||||
|
||||
Canonical schemas emitted after normalization:
|
||||
|
||||
| Constant | Value |
|
||||
|---|---|
|
||||
| `SchemaWeatherObservationV1` | `weather.observation.v1` |
|
||||
| `SchemaWeatherForecastV1` | `weather.forecast.v1` |
|
||||
| `SchemaWeatherForecastDiscussionV1` | `weather.forecast_discussion.v1` |
|
||||
| `SchemaWeatherStoryV1` | `weather.weather_story.v1` |
|
||||
| `SchemaWeatherAlertV1` | `weather.alert.v1` |
|
||||
| `SchemaWeatherOutlookV2` | `weather.outlook.v2` |
|
||||
|
||||
Historical canonical schema constant:
|
||||
|
||||
| Constant | Value |
|
||||
|---|---|
|
||||
| `SchemaWeatherOutlookV1` | `weather.outlook.v1` |
|
||||
|
||||
## Raw Schema Constants
|
||||
|
||||
Raw source schemas emitted by current registered sources:
|
||||
|
||||
| Constant | Value |
|
||||
|---|---|
|
||||
| `SchemaRawNWSObservationV1` | `raw.nws.observation.v1` |
|
||||
| `SchemaRawOpenMeteoCurrentV1` | `raw.openmeteo.current.v1` |
|
||||
| `SchemaRawOpenWeatherCurrentV1` | `raw.openweather.current.v1` |
|
||||
| `SchemaRawNWSHourlyForecastV1` | `raw.nws.hourly.forecast.v1` |
|
||||
| `SchemaRawNWSNarrativeForecastV1` | `raw.nws.narrative.forecast.v1` |
|
||||
| `SchemaRawNWSForecastDiscussionV1` | `raw.nws.forecast_discussion.v1` |
|
||||
| `SchemaRawNWSWeatherStoriesV1` | `raw.nws.weatherstories.v1` |
|
||||
| `SchemaRawOpenMeteoHourlyForecastV1` | `raw.openmeteo.hourly.forecast.v1` |
|
||||
| `SchemaRawNWSAlertsV1` | `raw.nws.alerts.v1` |
|
||||
| `SchemaRawSPCConvectiveOutlookV1` | `raw.spc.convective_outlook.v1` |
|
||||
|
||||
Additional raw schema constant:
|
||||
|
||||
| Constant | Value |
|
||||
|---|---|
|
||||
| `SchemaRawOpenWeatherHourlyForecastV1` | `raw.openweather.hourly.forecast.v1` |
|
||||
|
||||
`SchemaRawOpenWeatherHourlyForecastV1` exists in code, but no current registered
|
||||
source emits it. Consumers should not expect that raw schema unless a later
|
||||
registered source documents it as part of the current event contract.
|
||||
|
||||
## WMO Constants And Text
|
||||
|
||||
`standards` also defines the canonical `WMOCode` vocabulary and text helpers
|
||||
used by normalized observations and forecasts.
|
||||
|
||||
Consumer guidance:
|
||||
|
||||
- Treat `WMOUnknown` as unknown condition data.
|
||||
- Observation `conditionCode` is required in the current event contract.
|
||||
- Forecast period `conditionCode` is optional because some forecast products do
|
||||
not provide a meaningful WMO condition.
|
||||
- Prefer WMO constants and helper functions from this package instead of
|
||||
duplicating code tables in consumers.
|
||||
|
||||
## Boundaries
|
||||
|
||||
`standards` is provider-agnostic. Provider-specific parsing belongs in
|
||||
`weatherfeeder` internals, not in this package and not in consumers.
|
||||
@@ -37,6 +37,7 @@ Canonical schemas emitted after normalization:
|
||||
| `forecast_discussion` | `weather.forecast_discussion.v1` | `WeatherForecastDiscussion` |
|
||||
| `weather_story` | `weather.weather_story.v1` | `WeatherStoryRun` |
|
||||
| `alert` | `weather.alert.v1` | `WeatherAlertRun` |
|
||||
| `outlook` | `weather.outlook.v2` | `WeatherOutlookRun` |
|
||||
|
||||
Raw upstream schemas emitted by current sources:
|
||||
|
||||
@@ -51,6 +52,7 @@ Raw upstream schemas emitted by current sources:
|
||||
| `weather_story` | `raw.nws.weatherstories.v1` | NWS weather stories JSON |
|
||||
| `forecast` | `raw.openmeteo.hourly.forecast.v1` | Open-Meteo hourly forecast JSON |
|
||||
| `alert` | `raw.nws.alerts.v1` | NWS alerts JSON |
|
||||
| `outlook` | `raw.spc.convective_outlook.v1` | SPC convective outlook raw bundle |
|
||||
|
||||
`standards.SchemaRawOpenWeatherHourlyForecastV1` exists in code, but no current
|
||||
registered source emits it.
|
||||
@@ -209,8 +211,9 @@ Payload type: `WeatherAlertRun`.
|
||||
| `instruction` | string | no | Alert instruction. |
|
||||
| `sent` | timestamp | no | Provider sent time. |
|
||||
| `effective` | timestamp | no | Effective time. |
|
||||
| `onset` | timestamp | no | Onset time. |
|
||||
| `expires` | timestamp | no | Expiration time. |
|
||||
| `onset` | timestamp | no | Alert period start. |
|
||||
| `ends` | timestamp | no | Alert period end. |
|
||||
| `expires` | timestamp | no | Provider expiration metadata; not necessarily the alert period end. |
|
||||
| `areaDescription` | string | no | Affected area description. |
|
||||
| `senderName` | string | no | Provider sender name. |
|
||||
| `references` | array | no | Related alerts. |
|
||||
@@ -218,6 +221,92 @@ Payload type: `WeatherAlertRun`.
|
||||
`references[]` entries contain optional `id`, `identifier`, `sender`, and
|
||||
`sent` fields.
|
||||
|
||||
## `weather.outlook.v2`
|
||||
|
||||
Payload type: `WeatherOutlookRun`.
|
||||
|
||||
The current producer is the SPC convective outlook normalizer. It emits Day 1-3
|
||||
convective outlook snapshots for categorical, tornado, hail, and wind products
|
||||
that apply to the configured forecast point. Raw SPC bundles remain complete;
|
||||
canonical outlook payloads are filtered to local polygons. All timestamps are
|
||||
UTC.
|
||||
|
||||
| Field | Type | Required | Notes |
|
||||
|---|---|:---:|---|
|
||||
| `locationId` | string | no | Operator-configured location identifier. |
|
||||
| `locationName` | string | no | Operator-configured location label. |
|
||||
| `latitude` | number | no | Configured point latitude in decimal degrees. |
|
||||
| `longitude` | number | no | Configured point longitude in decimal degrees. |
|
||||
| `asOf` | timestamp | yes | Snapshot time. For SPC, this is the latest outlook issue time when available. |
|
||||
| `issuedAt` | timestamp | no | Latest issue time across outlook features when any feature exists. |
|
||||
| `outlooks` | array | yes | Ordered outlook polygons containing the configured point. |
|
||||
| `discussions` | array | yes | Run-level day discussions for retained outlook days. |
|
||||
|
||||
`outlooks[]` entries:
|
||||
|
||||
| Field | Type | Required | Notes |
|
||||
|---|---|:---:|---|
|
||||
| `id` | string | yes | Deterministic weatherfeeder outlook identifier. |
|
||||
| `provider` | string | yes | Current value is `spc`. |
|
||||
| `product` | string | yes | Current value is `convective`. |
|
||||
| `day` | integer | yes | SPC outlook day, currently `1`, `2`, or `3`. |
|
||||
| `outlookType` | string | yes | `categorical`, `tornado`, `hail`, or `wind`. |
|
||||
| `label` | string | yes | SPC outlook label such as `SLGT` or `15`. |
|
||||
| `labelText` | string | no | Human label text from SPC, when present. |
|
||||
| `severityRank` | integer | no | SPC `DN` value, when present. |
|
||||
| `validFrom` | timestamp | yes | Valid period start. |
|
||||
| `validTo` | timestamp | yes | Valid period end. |
|
||||
| `issuedAt` | timestamp | yes | Feature issue time. |
|
||||
| `expiresAt` | timestamp | yes | Expiration time; currently equal to `validTo`. |
|
||||
| `forecaster` | string | no | SPC forecaster text, when present. |
|
||||
| `sourceUrl` | string | no | GeoJSON product URL for this outlook feature. |
|
||||
| `imageUrl` | string | no | Reserved for provider image URLs; currently empty. |
|
||||
| `containsLocation` | boolean | yes | Always `true` for emitted v2 outlooks. |
|
||||
| `geometry` | object | yes | Compact GeoJSON `Polygon` or `MultiPolygon` geometry. |
|
||||
|
||||
`geometry` preserves the SPC feature geometry as compact GeoJSON using
|
||||
`[longitude, latitude]` coordinate order. `containsLocation` is computed with
|
||||
that geometry and the configured source `latitude`/`longitude`; boundary points
|
||||
count as contained. Polygons that do not contain the configured point are not
|
||||
included in canonical v2 payloads.
|
||||
|
||||
`discussions[]` entries:
|
||||
|
||||
| Field | Type | Required | Notes |
|
||||
|---|---|:---:|---|
|
||||
| `day` | integer | yes | SPC outlook day, currently `1`, `2`, or `3`. |
|
||||
| `headline` | string | no | Matching Day 1-3 print-page product title. |
|
||||
| `summary` | string | no | Text from the print-page `...SUMMARY...` section. |
|
||||
| `discussion` | string | no | Cleaned full print-page product text. |
|
||||
| `updatedAt` | timestamp | no | Print-page update time, when present. |
|
||||
|
||||
When no SPC polygons apply locally, the run is still emitted with `outlooks: []`
|
||||
and `discussions: []`. Discussions are included only for days represented by at
|
||||
least one retained outlook, and multiple retained outlook types for the same day
|
||||
share one discussion entry.
|
||||
|
||||
### SPC Outlook Supersession
|
||||
|
||||
Consumers should prefer latest-run semantics for current conditions: read the
|
||||
latest `WeatherOutlookRun` for the configured location and use its `outlooks`
|
||||
and `discussions` arrays together.
|
||||
|
||||
Historical SQL consumers that collapse older rows should identify superseded
|
||||
outlooks by `provider`, `product`, `outlookType`, `validFrom`, and `validTo`,
|
||||
then keep rows with the greatest `issuedAt`. `day` and `label` are not identity
|
||||
fields. When multiple retained polygons share that latest `issuedAt`, preserve
|
||||
the full group.
|
||||
|
||||
## Legacy `weather.outlook.v1`
|
||||
|
||||
`weather.outlook.v1` is a historical canonical schema retained as a standards
|
||||
constant for older data and consumers. Current SPC normalization emits
|
||||
`weather.outlook.v2`.
|
||||
|
||||
The v1 payload used `WeatherOutlookRun` and placed `headline`, `summary`, and
|
||||
`discussion` on each `outlooks[]` polygon. It also represented the pre-v2 SPC
|
||||
canonical behavior, where national polygons were preserved in canonical output.
|
||||
|
||||
## Compact Example
|
||||
|
||||
```json
|
||||
|
||||
106
docs/integrations/feedkit.md
Normal file
106
docs/integrations/feedkit.md
Normal file
@@ -0,0 +1,106 @@
|
||||
# Feedkit Integration
|
||||
|
||||
## Purpose
|
||||
|
||||
This document describes the feedkit runtime behavior that `weatherfeeder`
|
||||
currently depends on. It is for maintainers and LLM coding agents changing
|
||||
runtime wiring, config behavior, source construction, processing, routing, or
|
||||
sink behavior.
|
||||
|
||||
Weather-domain behavior belongs in `weatherfeeder`. Generic daemon mechanics
|
||||
belong to feedkit.
|
||||
|
||||
## Current Dependency
|
||||
|
||||
`weatherfeeder` imports feedkit as its daemon framework dependency. The exact
|
||||
module version is declared in `go.mod`.
|
||||
|
||||
Feedkit provides:
|
||||
|
||||
- YAML config loading and validation.
|
||||
- Source, processor, and sink registries.
|
||||
- HTTP source helper behavior.
|
||||
- Scheduler polling.
|
||||
- Normalize and dedupe processors.
|
||||
- Route compilation and sink dispatch.
|
||||
- Built-in stdout, NATS, and Postgres sink mechanics.
|
||||
|
||||
## Config Contract
|
||||
|
||||
`cmd/weatherfeeder` calls feedkit config loading for `config.yml` in the current
|
||||
working directory.
|
||||
|
||||
Implemented behavior relied on by weatherfeeder docs and tests:
|
||||
|
||||
- Top-level config contains `sources`, `sinks`, and optional `routes`.
|
||||
- Config struct fields are decoded strictly, so misspelled struct fields fail
|
||||
startup.
|
||||
- Driver-specific `params` maps are decoded generically and validated by the
|
||||
source or sink constructor that consumes them.
|
||||
- Source `kinds` can be validated against a source's advertised `Kinds()`.
|
||||
|
||||
## Source And HTTP Contract
|
||||
|
||||
Most weatherfeeder sources use feedkit's single-document HTTP source helper for:
|
||||
|
||||
- request construction;
|
||||
- `User-Agent` and `Accept` headers;
|
||||
- optional conditional GET validators;
|
||||
- response body size limits;
|
||||
- context-aware HTTP work;
|
||||
- unchanged `304 Not Modified` responses that emit no events.
|
||||
|
||||
The SPC convective outlook source fetches multiple documents itself, but it uses
|
||||
feedkit transport helpers for HTTP clients and response body limits.
|
||||
|
||||
## Scheduler And Processing Contract
|
||||
|
||||
Weatherfeeder builds feedkit scheduler jobs from source configs. Current source
|
||||
drivers are polling drivers and use the configured `every` interval.
|
||||
|
||||
Events flow through a feedkit pipeline in this order:
|
||||
|
||||
1. normalize processor;
|
||||
2. dedupe processor.
|
||||
|
||||
The normalize processor is configured with `RequireMatch=false`, so unmatched
|
||||
schemas pass through unchanged. Weatherfeeder registers its built-in normalizers
|
||||
and owns the provider-to-canonical mapping.
|
||||
|
||||
The dedupe processor stores a bounded in-memory set of recent event IDs. The
|
||||
bound is configured in `cmd/weatherfeeder`.
|
||||
|
||||
## Dispatch And Sink Contract
|
||||
|
||||
Feedkit compiles routes from config and dispatches processed events to matching
|
||||
sinks. If `routes` is omitted, every configured sink receives every event kind.
|
||||
|
||||
Feedkit owns sink fanout mechanics, per-sink workers, queueing, context-aware
|
||||
shutdown, and sink error logging. Weatherfeeder owns the event kinds and schemas
|
||||
that make routes meaningful.
|
||||
|
||||
Built-in feedkit sinks used by weatherfeeder:
|
||||
|
||||
- `stdout`: validates and writes JSON events to stdout.
|
||||
- `nats`: publishes JSON events to a configured subject.
|
||||
- generic `postgres` sink factory: opens the database, ensures tables/indexes,
|
||||
runs transactions, inserts mapped rows, and prunes when configured.
|
||||
|
||||
Weatherfeeder supplies its Postgres table schema and event mapper to feedkit's
|
||||
Postgres sink factory. The table contract is documented in
|
||||
[`postgres.md`](postgres.md).
|
||||
|
||||
## Boundaries
|
||||
|
||||
Do not move weather-domain policy into feedkit. Weatherfeeder owns:
|
||||
|
||||
- provider source drivers;
|
||||
- raw and canonical schema constants;
|
||||
- event kind meaning;
|
||||
- canonical payload structs;
|
||||
- normalizers;
|
||||
- Postgres table shape and row mapping.
|
||||
|
||||
Do not duplicate generic feedkit mechanics in weatherfeeder unless there is a
|
||||
narrow weather-specific reason. Runtime composition details are documented in
|
||||
[`../internal/runtime.md`](../internal/runtime.md).
|
||||
@@ -44,6 +44,9 @@ normalizer uses fields under `properties` such as `stationId`, `stationName`,
|
||||
`nws_alerts` expects an alerts FeatureCollection. The normalizer uses the
|
||||
collection `updated` timestamp, `title`, each feature ID, alert classification
|
||||
fields, narrative fields, timing fields, sender fields, and references.
|
||||
`properties.onset` and `properties.ends` map to the canonical alert period
|
||||
start and end. `properties.expires` maps only to canonical `expires` provider
|
||||
metadata and is not treated as the alert period end.
|
||||
|
||||
`nws_forecast_hourly` and `nws_forecast_narrative` expect gridpoint forecast
|
||||
GeoJSON with `properties.generatedAt`, `properties.updateTime`, elevation,
|
||||
@@ -98,8 +101,9 @@ unset. Forecast temperatures are converted to Celsius when NWS supplies
|
||||
Fahrenheit, and wind speed strings are converted to kilometers per hour.
|
||||
|
||||
Alert timing fields are parsed best-effort. Invalid per-alert timestamps are
|
||||
left unset rather than failing the whole alert run. Missing alert IDs are
|
||||
synthesized from the run snapshot time and array position.
|
||||
left unset rather than failing the whole alert run. NWS `ends` is preserved
|
||||
separately from `expires`; `expires` does not fall back to `ends`. Missing alert
|
||||
IDs are synthesized from the run snapshot time and array position.
|
||||
|
||||
Forecast discussion parsing requires an issue time. Weather story entries require
|
||||
start time, end time, and update time.
|
||||
|
||||
@@ -23,6 +23,7 @@ Events are mapped only for canonical weather schemas:
|
||||
- `weather.forecast_discussion.v1`
|
||||
- `weather.weather_story.v1`
|
||||
- `weather.alert.v1`
|
||||
- `weather.outlook.v2`
|
||||
|
||||
Unsupported schemas produce no writes for this sink. Mapped events are inserted
|
||||
transactionally. Inserts use ordinary `INSERT`; duplicate primary keys fail the
|
||||
@@ -56,6 +57,9 @@ Parent tables store the feed event envelope:
|
||||
| `alert_runs` | `event_id` | `as_of` |
|
||||
| `alerts` | `run_event_id`, `alert_index` | `as_of` |
|
||||
| `alert_references` | `run_event_id`, `alert_index`, `reference_index` | `as_of` |
|
||||
| `outlook_runs` | `event_id` | `as_of` |
|
||||
| `outlooks` | `run_event_id`, `outlook_index` | `as_of` |
|
||||
| `outlook_discussions` | `run_event_id`, `discussion_index` | `as_of` |
|
||||
|
||||
## Table Contract
|
||||
|
||||
@@ -349,6 +353,7 @@ Indexes:
|
||||
| `sent` | `TIMESTAMPTZ` | yes | `payload.alerts[].sent` |
|
||||
| `effective` | `TIMESTAMPTZ` | yes | `payload.alerts[].effective` |
|
||||
| `onset` | `TIMESTAMPTZ` | yes | `payload.alerts[].onset` |
|
||||
| `ends` | `TIMESTAMPTZ` | yes | `payload.alerts[].ends` |
|
||||
| `expires` | `TIMESTAMPTZ` | yes | `payload.alerts[].expires` |
|
||||
| `area_description` | `TEXT` | yes | `payload.alerts[].areaDescription` |
|
||||
| `sender_name` | `TEXT` | yes | `payload.alerts[].senderName` |
|
||||
@@ -379,6 +384,97 @@ Indexes:
|
||||
| `sender` | `TEXT` | yes | `payload.alerts[].references[].sender` |
|
||||
| `sent` | `TIMESTAMPTZ` | yes | `payload.alerts[].references[].sent` |
|
||||
|
||||
### `outlook_runs`
|
||||
|
||||
Primary key: `event_id`
|
||||
|
||||
Prune column: `as_of`
|
||||
|
||||
Indexes:
|
||||
|
||||
- `idx_wf_outlook_run_location_as_of` on `location_id`, `as_of`
|
||||
- `idx_wf_outlook_run_as_of` on `as_of`
|
||||
|
||||
| Column | Type | Null | Source |
|
||||
|---|---|:---:|---|
|
||||
| `event_id` | `TEXT` | no | `event.id` |
|
||||
| `event_kind` | `TEXT` | no | `event.kind` |
|
||||
| `event_source` | `TEXT` | no | `event.source` |
|
||||
| `event_schema` | `TEXT` | no | `event.schema` |
|
||||
| `event_emitted_at` | `TIMESTAMPTZ` | no | `event.emitted_at` |
|
||||
| `event_effective_at` | `TIMESTAMPTZ` | yes | `event.effective_at` |
|
||||
| `location_id` | `TEXT` | yes | `payload.locationId` |
|
||||
| `location_name` | `TEXT` | yes | `payload.locationName` |
|
||||
| `latitude` | `DOUBLE PRECISION` | yes | `payload.latitude` |
|
||||
| `longitude` | `DOUBLE PRECISION` | yes | `payload.longitude` |
|
||||
| `as_of` | `TIMESTAMPTZ` | no | `payload.asOf` |
|
||||
| `issued_at` | `TIMESTAMPTZ` | yes | `payload.issuedAt` |
|
||||
| `outlook_count` | `INTEGER` | no | `len(payload.outlooks)` |
|
||||
| `discussion_count` | `INTEGER` | no | `len(payload.discussions)` |
|
||||
|
||||
### `outlooks`
|
||||
|
||||
Primary key: `run_event_id`, `outlook_index`
|
||||
|
||||
Prune column: `as_of`
|
||||
|
||||
Foreign key: `run_event_id` references `outlook_runs(event_id)` with cascade
|
||||
delete.
|
||||
|
||||
Indexes:
|
||||
|
||||
- `idx_wf_outlooks_contains_valid` on `contains_location`, `valid_from`, `valid_to`
|
||||
- `idx_wf_outlooks_day_type_label` on `day`, `outlook_type`, `label`
|
||||
- `idx_wf_outlooks_valid` on `valid_from`, `valid_to`
|
||||
|
||||
| Column | Type | Null | Source |
|
||||
|---|---|:---:|---|
|
||||
| `run_event_id` | `TEXT REFERENCES outlook_runs(event_id) ON DELETE CASCADE` | no | Parent event ID. |
|
||||
| `outlook_index` | `INTEGER` | no | `payload.outlooks[]` index. |
|
||||
| `as_of` | `TIMESTAMPTZ` | no | Parent `payload.asOf` |
|
||||
| `outlook_id` | `TEXT` | no | `payload.outlooks[].id` |
|
||||
| `provider` | `TEXT` | no | `payload.outlooks[].provider` |
|
||||
| `product` | `TEXT` | no | `payload.outlooks[].product` |
|
||||
| `day` | `INTEGER` | no | `payload.outlooks[].day` |
|
||||
| `outlook_type` | `TEXT` | no | `payload.outlooks[].outlookType` |
|
||||
| `label` | `TEXT` | no | `payload.outlooks[].label` |
|
||||
| `label_text` | `TEXT` | yes | `payload.outlooks[].labelText` |
|
||||
| `severity_rank` | `INTEGER` | yes | `payload.outlooks[].severityRank` |
|
||||
| `valid_from` | `TIMESTAMPTZ` | no | `payload.outlooks[].validFrom` |
|
||||
| `valid_to` | `TIMESTAMPTZ` | no | `payload.outlooks[].validTo` |
|
||||
| `issued_at` | `TIMESTAMPTZ` | no | `payload.outlooks[].issuedAt` |
|
||||
| `expires_at` | `TIMESTAMPTZ` | no | `payload.outlooks[].expiresAt` |
|
||||
| `forecaster` | `TEXT` | yes | `payload.outlooks[].forecaster` |
|
||||
| `source_url` | `TEXT` | yes | `payload.outlooks[].sourceUrl` |
|
||||
| `image_url` | `TEXT` | yes | `payload.outlooks[].imageUrl` |
|
||||
| `contains_location` | `BOOLEAN` | no | `payload.outlooks[].containsLocation` |
|
||||
| `geometry_json` | `TEXT` | no | Compact JSON from `payload.outlooks[].geometry` |
|
||||
|
||||
### `outlook_discussions`
|
||||
|
||||
Primary key: `run_event_id`, `discussion_index`
|
||||
|
||||
Prune column: `as_of`
|
||||
|
||||
Foreign key: `run_event_id` references `outlook_runs(event_id)` with cascade
|
||||
delete.
|
||||
|
||||
Indexes:
|
||||
|
||||
- `idx_wf_outlook_discussions_day_as_of` on `day`, `as_of`
|
||||
- unique `idx_wf_outlook_discussions_run_day` on `run_event_id`, `day`
|
||||
|
||||
| Column | Type | Null | Source |
|
||||
|---|---|:---:|---|
|
||||
| `run_event_id` | `TEXT REFERENCES outlook_runs(event_id) ON DELETE CASCADE` | no | Parent event ID. |
|
||||
| `discussion_index` | `INTEGER` | no | `payload.discussions[]` index. |
|
||||
| `as_of` | `TIMESTAMPTZ` | no | Parent `payload.asOf` |
|
||||
| `day` | `INTEGER` | no | `payload.discussions[].day` |
|
||||
| `headline` | `TEXT` | yes | `payload.discussions[].headline` |
|
||||
| `summary` | `TEXT` | yes | `payload.discussions[].summary` |
|
||||
| `discussion` | `TEXT` | yes | `payload.discussions[].discussion` |
|
||||
| `updated_at` | `TIMESTAMPTZ` | yes | `payload.discussions[].updatedAt` |
|
||||
|
||||
## Retention
|
||||
|
||||
When sink param `prune` is set, every successful write transaction deletes rows
|
||||
@@ -401,3 +497,6 @@ binary does not provide CLI commands for them.
|
||||
- `WeatherAlertRun`: read `alert_runs`, join `alerts` by `run_event_id` ordered
|
||||
by `alert_index`, then join `alert_references` by `run_event_id` and
|
||||
`alert_index` ordered by `reference_index`.
|
||||
- `WeatherOutlookRun`: read `outlook_runs`, join `outlooks` by `run_event_id`
|
||||
ordered by `outlook_index`, then join `outlook_discussions` by
|
||||
`run_event_id` ordered by `discussion_index`.
|
||||
|
||||
144
docs/integrations/spc.md
Normal file
144
docs/integrations/spc.md
Normal file
@@ -0,0 +1,144 @@
|
||||
# SPC Integration Notes
|
||||
|
||||
## Purpose
|
||||
|
||||
This document describes the Storm Prediction Center convective outlook usage
|
||||
implemented by `weatherfeeder`. It is for developers and operators maintaining
|
||||
SPC source configuration, provider helpers, fixtures, and tests.
|
||||
|
||||
General config syntax belongs in [configuration](../config.md). Emitted JSON
|
||||
events are documented in [event wire contract](events.md).
|
||||
|
||||
## Implemented Driver
|
||||
|
||||
| Driver | Kind | Raw schema | Canonical schema |
|
||||
| --- | --- | --- | --- |
|
||||
| `spc_convective_outlook` | `outlook` | `raw.spc.convective_outlook.v1` | `weather.outlook.v2` |
|
||||
|
||||
## Config Requirements
|
||||
|
||||
The driver requires:
|
||||
|
||||
- `latitude`
|
||||
- `longitude`
|
||||
- `user_agent`
|
||||
|
||||
Optional params are:
|
||||
|
||||
- `location_id`
|
||||
- `location_name`
|
||||
- `geojson_urls`
|
||||
- `discussion_urls`
|
||||
- `rss_url`
|
||||
- `http_timeout`
|
||||
- `http_response_body_limit_bytes`
|
||||
|
||||
RSS is not fetched unless `rss_url` is configured. URL override maps are intended
|
||||
for tests and upstream URL changes; the default driver configuration owns the
|
||||
current Day 1-3 SPC product URLs.
|
||||
|
||||
## Upstream Products Used
|
||||
|
||||
The source fetches nine required GeoJSON products every poll:
|
||||
|
||||
- Day 1 categorical, tornado, hail, and wind
|
||||
- Day 2 categorical, tornado, hail, and wind
|
||||
- Day 3 categorical
|
||||
|
||||
It also fetches three required print pages:
|
||||
|
||||
- Day 1 convective outlook print page
|
||||
- Day 2 convective outlook print page
|
||||
- Day 3 convective outlook print page
|
||||
|
||||
GeoJSON products are authoritative for outlook polygons, valid windows, issue
|
||||
times, labels, and severity rank. Print pages are authoritative for run-level
|
||||
day discussion headline, summary, and discussion text.
|
||||
|
||||
## Accept Headers
|
||||
|
||||
GeoJSON requests use:
|
||||
|
||||
```text
|
||||
application/geo+json, application/json
|
||||
```
|
||||
|
||||
Print-page requests use:
|
||||
|
||||
```text
|
||||
text/html, application/xhtml+xml
|
||||
```
|
||||
|
||||
RSS requests, when configured, use:
|
||||
|
||||
```text
|
||||
application/rss+xml, application/xml, text/xml
|
||||
```
|
||||
|
||||
## Polling And Raw Events
|
||||
|
||||
The source polls all required GeoJSON and print-page products as one bundle. If
|
||||
any required request fails or returns a non-2xx response, the poll returns an
|
||||
error and emits no partial event.
|
||||
|
||||
The raw payload contains fetched bodies plus configured location metadata and
|
||||
per-product metadata. The source parses only the timestamp metadata needed for
|
||||
event effective time selection; canonical mapping belongs to the normalizer.
|
||||
|
||||
The source emits no event when a complete fetched bundle is unchanged from the
|
||||
previous successful poll. It does this with a source-local hash of the fetched
|
||||
document bodies.
|
||||
|
||||
## Time Handling
|
||||
|
||||
Raw source `effective_at` prefers:
|
||||
|
||||
1. the latest valid GeoJSON `ISSUE_ISO`;
|
||||
2. the latest print-page `Updated:` timestamp;
|
||||
3. RSS `lastBuildDate` when RSS was fetched and parseable;
|
||||
4. fetch time.
|
||||
|
||||
The normalizer sets canonical `asOf` and normalized event `effective_at` from
|
||||
the latest valid GeoJSON issue time across the complete raw bundle, including
|
||||
empty no-risk placeholders, with fallback to print-page update time and then the
|
||||
incoming event envelope.
|
||||
|
||||
## Mapping Notes
|
||||
|
||||
The raw source fetches and envelopes the complete SPC bundle. The normalizer
|
||||
decodes every configured GeoJSON product, skips empty no-risk
|
||||
`GeometryCollection` placeholders, and emits canonical outlooks only when the
|
||||
configured point is inside or on the boundary of a real feature. Products are
|
||||
ordered by day, then categorical, tornado, hail, and wind. Retained feature order
|
||||
is preserved within each product.
|
||||
|
||||
The normalizer computes `containsLocation` with the configured latitude and
|
||||
longitude against compact GeoJSON `Polygon` or `MultiPolygon` geometry.
|
||||
Coordinates use GeoJSON order, `[longitude, latitude]`, and boundary points
|
||||
count as contained.
|
||||
|
||||
Canonical outlook runs are emitted even when no polygons apply locally. In that
|
||||
case the payload contains empty `outlooks` and `discussions` arrays.
|
||||
|
||||
Print-page prose is represented as run-level day discussions. Discussions are
|
||||
included only for days represented by at least one retained outlook. Multiple
|
||||
retained outlook types for the same day share one discussion entry.
|
||||
|
||||
For downstream current-state and historical supersession guidance, see the
|
||||
[event wire contract](events.md#spc-outlook-supersession).
|
||||
|
||||
## Failure Behavior
|
||||
|
||||
Constructor validation failures stop daemon startup. Polling failures are
|
||||
returned to the scheduler and emit no event for that poll.
|
||||
|
||||
Normalization fails when required GeoJSON timestamps, labels, geometry, or
|
||||
configured coordinates are missing or invalid. Print-page extraction failures
|
||||
also fail normalization because print pages are required inputs.
|
||||
|
||||
## Tests To Inspect
|
||||
|
||||
- `internal/providers/spc/*_test.go`
|
||||
- `internal/sources/spc/*_test.go`
|
||||
- `internal/normalizers/spc/*_test.go`
|
||||
- fixtures under `internal/providers/spc/testdata`
|
||||
@@ -27,6 +27,7 @@ Current mappings:
|
||||
| `raw.nws.forecast_discussion.v1` | `weather.forecast_discussion.v1` |
|
||||
| `raw.nws.weatherstories.v1` | `weather.weather_story.v1` |
|
||||
| `raw.nws.alerts.v1` | `weather.alert.v1` |
|
||||
| `raw.spc.convective_outlook.v1` | `weather.outlook.v2` |
|
||||
|
||||
## Boundaries
|
||||
|
||||
@@ -54,9 +55,18 @@ Weatherfeeder registers normalizers in a stable order:
|
||||
1. NWS
|
||||
2. Open-Meteo
|
||||
3. OpenWeather
|
||||
4. SPC
|
||||
|
||||
The current normalizers avoid ambiguous matches by using schema equality.
|
||||
|
||||
The SPC outlook normalizer decodes the raw multi-document bundle, maps
|
||||
location-containing GeoJSON features to canonical outlooks, and adds one
|
||||
run-level print-page discussion per retained outlook day. It preserves compact
|
||||
GeoJSON feature geometry and computes `containsLocation` with
|
||||
`internal/geo.ContainsPoint` using the source-configured point. Boundary points
|
||||
count as contained. Polygons that do not contain the point are omitted from the
|
||||
canonical run.
|
||||
|
||||
## State
|
||||
|
||||
Normalizers should be stateless. Shared helpers should be deterministic and free
|
||||
@@ -78,6 +88,7 @@ to four digits after the decimal point.
|
||||
- provider normalizer tests under `internal/normalizers/nws`
|
||||
- provider normalizer tests under `internal/normalizers/openmeteo`
|
||||
- provider normalizer tests under `internal/normalizers/openweather`
|
||||
- provider normalizer tests under `internal/normalizers/spc`
|
||||
- common helper tests under `internal/normalizers/common`
|
||||
|
||||
## Invariants
|
||||
|
||||
@@ -19,10 +19,16 @@ Inputs are canonical feed events. The mapper currently handles these schemas:
|
||||
- `weather.forecast_discussion.v1`
|
||||
- `weather.weather_story.v1`
|
||||
- `weather.alert.v1`
|
||||
- `weather.outlook.v2`
|
||||
|
||||
Outputs are feedkit `PostgresWrite` values for weatherfeeder-owned tables.
|
||||
Unsupported schemas produce no writes and no error.
|
||||
|
||||
Outlook runs are written to `outlook_runs`, retained local polygons are written
|
||||
to `outlooks`, and run-level day discussions are written to
|
||||
`outlook_discussions`. The parent run row stores `outlook_count` and
|
||||
`discussion_count`.
|
||||
|
||||
## Boundaries
|
||||
|
||||
- Weatherfeeder owns table definitions in `schema.go`.
|
||||
@@ -80,6 +86,8 @@ Child rows use positional indexes to preserve canonical array order:
|
||||
- `story_index`
|
||||
- `alert_index`
|
||||
- `reference_index`
|
||||
- `outlook_index`
|
||||
- `discussion_index`
|
||||
|
||||
Required canonical fields are validated before writes are returned:
|
||||
|
||||
@@ -89,10 +97,17 @@ Required canonical fields are validated before writes are returned:
|
||||
- forecast discussions require `issuedAt` and `product`;
|
||||
- weather story runs require `asOf`, and each story requires `startTime`,
|
||||
`endTime`, and `updatedAt`;
|
||||
- alert runs require `asOf`, and each alert requires `id`.
|
||||
- alert runs require `asOf`, and each alert requires `id`;
|
||||
- outlook runs require `asOf`, and each outlook requires `id`, `provider`,
|
||||
`product`, `day`, `outlookType`, `label`, `validFrom`, `validTo`, `issuedAt`,
|
||||
`expiresAt`, `containsLocation: true`, and `geometry`;
|
||||
- outlook discussions require day `1`, `2`, or `3`; at least one of
|
||||
`headline`, `summary`, or `discussion`; and no duplicate discussion day in the
|
||||
same run.
|
||||
|
||||
Nullable canonical values are converted to SQL nulls by mapper helpers.
|
||||
Observation present-weather raw values are stored as compact JSON text.
|
||||
Observation present-weather raw values and outlook geometry values are stored as
|
||||
compact JSON text.
|
||||
|
||||
## Failure Behavior
|
||||
|
||||
|
||||
@@ -68,13 +68,28 @@ Runtime composition uses feedkit for:
|
||||
|
||||
Weatherfeeder registers its own source drivers and its Postgres schema mapper.
|
||||
|
||||
Responsibility split:
|
||||
|
||||
| Runtime concern | Owner |
|
||||
| --- | --- |
|
||||
| Config loading and generic validation | feedkit |
|
||||
| Source, processor, and sink registries | feedkit mechanics; weatherfeeder registrations |
|
||||
| Source polling and stream supervision | feedkit scheduler |
|
||||
| Raw weather data fetching | weatherfeeder source adapters |
|
||||
| Normalizer execution order and pass-through behavior | feedkit normalize processor |
|
||||
| Weather raw-to-canonical mapping | weatherfeeder normalizers |
|
||||
| Dedupe mechanics | feedkit dedupe processor |
|
||||
| Route compilation and sink fanout | feedkit dispatch |
|
||||
| Weather Postgres table shape and row mapping | weatherfeeder Postgres adapter |
|
||||
| Postgres connection, DDL, inserts, transactions, and pruning | feedkit Postgres sink |
|
||||
|
||||
## State
|
||||
|
||||
Weatherfeeder-owned runtime state is in process:
|
||||
|
||||
- event channel contents;
|
||||
- the bounded dedupe key set;
|
||||
- source instances and their HTTP conditional validators;
|
||||
- source instances and their in-memory unchanged-content state;
|
||||
- scheduler and dispatcher goroutines.
|
||||
|
||||
There is no weatherfeeder-owned durable scheduler state, checkpoint, replay log,
|
||||
|
||||
@@ -28,6 +28,7 @@ Current drivers:
|
||||
| `openmeteo_observation` | `observation` | `raw.openmeteo.current.v1` |
|
||||
| `openmeteo_forecast` | `forecast` | `raw.openmeteo.hourly.forecast.v1` |
|
||||
| `openweather_observation` | `observation` | `raw.openweather.current.v1` |
|
||||
| `spc_convective_outlook` | `outlook` | `raw.spc.convective_outlook.v1` |
|
||||
|
||||
## Boundaries
|
||||
|
||||
@@ -40,7 +41,7 @@ Current drivers:
|
||||
|
||||
## Config Fields Used
|
||||
|
||||
All current source drivers use feedkit `HTTPSource`.
|
||||
Most source drivers use feedkit `HTTPSource`.
|
||||
|
||||
Required params:
|
||||
|
||||
@@ -56,12 +57,18 @@ Optional params:
|
||||
OpenWeather observation sources additionally require the configured URL to use
|
||||
metric units. This is enforced by `internal/providers/openweather`.
|
||||
|
||||
The SPC convective outlook source is a multi-document poll source rather than a
|
||||
single-URL `HTTPSource`. It requires `latitude`, `longitude`, and `user_agent`;
|
||||
accepts optional `location_id`, `location_name`, `geojson_urls`,
|
||||
`discussion_urls`, and `rss_url`; and supports `http_timeout` and
|
||||
`http_response_body_limit_bytes`.
|
||||
|
||||
Source-level `kinds`, when configured, are validated against the source's
|
||||
advertised `Kinds()`.
|
||||
|
||||
## External Adapters Used
|
||||
|
||||
Sources use feedkit's HTTP helper for:
|
||||
Most sources use feedkit's HTTP helper for:
|
||||
|
||||
- request construction;
|
||||
- `User-Agent` and `Accept` headers;
|
||||
@@ -71,11 +78,15 @@ Sources use feedkit's HTTP helper for:
|
||||
|
||||
NWS helpers parse NWS timestamps. Open-Meteo helpers parse provider-local times
|
||||
with timezone or UTC-offset data. OpenWeather helpers enforce metric-unit URLs.
|
||||
SPC helpers define Day 1-3 product metadata, parse GeoJSON timestamps, extract
|
||||
cleaned print-page discussion text, and parse optional RSS metadata.
|
||||
|
||||
## State
|
||||
|
||||
HTTP conditional validators are held in each source instance. They are not
|
||||
persisted across process restarts.
|
||||
HTTP conditional validators are held in each single-document HTTP source
|
||||
instance. They are not persisted across process restarts. The SPC outlook source
|
||||
keeps only a source-local hash of the most recent complete required product
|
||||
bundle and emits no event when a later complete bundle is unchanged.
|
||||
|
||||
## Failure Behavior
|
||||
|
||||
@@ -88,12 +99,18 @@ to default ID/effective-time behavior from feedkit source helpers.
|
||||
|
||||
Unchanged conditional responses return no events and no error.
|
||||
|
||||
The SPC outlook source fetches all configured Day 1-3 GeoJSON products and print
|
||||
pages atomically. If any required GeoJSON or print-page request fails, the poll
|
||||
returns an error and emits no partial bundle. RSS is fetched only when `rss_url`
|
||||
is configured.
|
||||
|
||||
## Tests To Inspect
|
||||
|
||||
- `internal/sources/builtins_test.go`
|
||||
- provider source tests under `internal/sources/nws`
|
||||
- provider source tests under `internal/sources/openmeteo`
|
||||
- provider source tests under `internal/sources/openweather`
|
||||
- provider source tests under `internal/sources/spc`
|
||||
- provider helper tests under `internal/providers`
|
||||
|
||||
## Invariants
|
||||
|
||||
@@ -64,21 +64,25 @@ and before each interval tick. If no jitter is configured in code, feedkit uses
|
||||
Poll failures are logged and do not stop the daemon. A failed poll emits no
|
||||
events for that source until a subsequent poll succeeds.
|
||||
|
||||
## Conditional HTTP Fetches
|
||||
## Unchanged Upstream Content
|
||||
|
||||
All current sources use feedkit's HTTP polling helper. By default,
|
||||
`params.conditional` is `true`, so the helper keeps ETag and Last-Modified
|
||||
validators in memory for each source instance.
|
||||
Most sources use feedkit's single-document HTTP polling helper. By default,
|
||||
`params.conditional` is `true` for those sources, so the helper keeps ETag and
|
||||
Last-Modified validators in memory for each source instance.
|
||||
|
||||
If the upstream returns `304 Not Modified`, the source emits no events for that
|
||||
poll. Validator state is in memory only; restarting the process starts with no
|
||||
cached validators.
|
||||
|
||||
The SPC convective outlook source polls multiple required documents as one
|
||||
bundle. It emits no event when a later complete bundle has the same fetched
|
||||
document bodies as the previous successful poll.
|
||||
|
||||
## Processing And Dedupe
|
||||
|
||||
Every event passes through normalization first and dedupe second.
|
||||
|
||||
Normalizers match raw source schemas and produce canonical `weather.*.v1`
|
||||
Normalizers match raw source schemas and produce versioned canonical `weather.*`
|
||||
payloads. If an event has no matching normalizer, the normalize processor passes
|
||||
it through unchanged.
|
||||
|
||||
@@ -128,6 +132,7 @@ table in that same transaction. See the
|
||||
|
||||
- scheduler goroutines and timers;
|
||||
- HTTP conditional request validators;
|
||||
- source-local unchanged-content state;
|
||||
- event channel buffers;
|
||||
- per-sink fanout queues;
|
||||
- the dedupe ID set.
|
||||
|
||||
@@ -23,10 +23,32 @@ The implemented runtime flow is:
|
||||
|
||||
Canonical payload structs live in `model`. Schema identifiers and cross-provider wire conventions live in `standards`. Source adapters live under `internal/sources`. Normalizers live under `internal/normalizers`. Provider-specific parsing helpers shared by sources and normalizers live under `internal/providers`. Sink-specific persistence mapping lives under `internal/sinks`.
|
||||
|
||||
## Architecture Style
|
||||
|
||||
`weatherfeeder` uses a pragmatic ports-and-adapters architecture rather than a
|
||||
formal framework. Provider APIs, config loading, scheduling, dispatch, and sinks
|
||||
sit outside the weather domain model and normalization rules.
|
||||
|
||||
The implementation style is:
|
||||
|
||||
- Pipeline-oriented: events flow from source polling through normalization,
|
||||
dedupe, routing, and sink fanout.
|
||||
- Schema-routed: normalizers select raw payloads by explicit schema strings, not
|
||||
source names or configured routes.
|
||||
- Provider-isolated: NWS, Open-Meteo, OpenWeather, and SPC quirks stay in
|
||||
provider-specific source, provider-helper, and normalizer packages.
|
||||
- Registry-based: built-in source drivers, normalizers, processors, and sinks
|
||||
are assembled explicitly through registries instead of dynamic plugin loading.
|
||||
- Adapter-clean: persistence and external-system details stay behind source and
|
||||
sink adapters, not in `model` or normalizers.
|
||||
- Direct Go: prefer small package-level constructors and straightforward code
|
||||
over broad abstractions.
|
||||
|
||||
## Core Design Principles
|
||||
|
||||
- Hexagonal boundaries: provider APIs, config loading, scheduling, dispatch, and sinks are external mechanisms around the weather domain model and normalization logic.
|
||||
- Raw-to-canonical flow: sources should fetch and envelope raw provider payloads; normalizers should own provider-to-canonical mapping.
|
||||
- Location-focused canonical data: canonical weather events represent data relevant to the configured forecast location or configured provider object; `weatherfeeder` is not a national provider-data archive.
|
||||
- Schema-based routing: normalizers match on event schema, not source name or event kind.
|
||||
- Composable registries: source drivers, normalizers, processors, and sinks are assembled explicitly through registries.
|
||||
- Bounded concurrency: scheduling and sink fanout are concurrent, but the application should keep queues, goroutine ownership, logging, and cancellation behavior visible.
|
||||
@@ -57,6 +79,23 @@ Tests and examples:
|
||||
- The sample `cmd/weatherfeeder/config.yml` is executable test input and is load-tested.
|
||||
- Tests should keep exercising package contracts directly rather than relying only on full-daemon execution.
|
||||
|
||||
## Feedkit Boundary
|
||||
|
||||
`feedkit` provides reusable daemon infrastructure. `weatherfeeder` provides the
|
||||
weather-domain adapters, models, schemas, and mapping policy.
|
||||
|
||||
| Area | Feedkit owns | Weatherfeeder owns |
|
||||
| --- | --- | --- |
|
||||
| Config | Generic YAML shape: sources, sinks, routes, modes, cadence, and params. | Driver-specific config rules such as NWS `user_agent`, OpenWeather `units=metric`, and SPC coordinates. |
|
||||
| Events | Domain-agnostic event envelope: ID, kind, source, emitted/effective times, schema, and payload. | Event kind meaning, schema strings, and canonical weather payloads. |
|
||||
| Sources | Source interfaces, registry, expected-kind validation, HTTP helper, and default event ID helper. | NWS/Open-Meteo/OpenWeather/SPC source drivers and raw schema emission. |
|
||||
| Processing | Processor registry, normalize processor, dedupe processor, and pipeline execution. | Weather normalizers and schema-specific raw-to-canonical mapping. |
|
||||
| Dispatch | Route compilation and sink fanout mechanics. | Which weather event kinds are configured and meaningful. |
|
||||
| Sinks | Generic stdout, NATS, and Postgres sink mechanics. | Weather-specific Postgres schema and canonical event-to-row mapping. |
|
||||
|
||||
Do not move weather-domain policy into `feedkit`, and do not duplicate generic
|
||||
daemon mechanics in `weatherfeeder` when feedkit already provides the boundary.
|
||||
|
||||
## Modules Or Processing Steps
|
||||
|
||||
The implemented processing steps are source polling, normalization, dedupe, and sink dispatch.
|
||||
|
||||
@@ -24,7 +24,8 @@ docs, not here.
|
||||
normalizers.
|
||||
- `internal/sinks/postgres/`: weatherfeeder-owned Postgres schema and canonical
|
||||
event mapper.
|
||||
- `docs/`: current behavior, policies, integration contracts, and roadmap files.
|
||||
- `docs/`: current behavior, consumer guides, integration contracts, policies,
|
||||
and roadmap files.
|
||||
- `examples/`: maintained, copyable configuration examples.
|
||||
|
||||
## Build And Test
|
||||
@@ -70,6 +71,32 @@ Use fixtures, local test servers, and package-level tests.
|
||||
- Prefer explicit registries and small package-level constructors over hidden
|
||||
global behavior.
|
||||
|
||||
## Architecture-Preserving Changes
|
||||
|
||||
When changing `weatherfeeder`, preserve the split between feedkit
|
||||
infrastructure and weather-domain behavior.
|
||||
|
||||
Do:
|
||||
|
||||
- keep generic scheduling, dispatch, processor, config, and sink mechanics in
|
||||
feedkit;
|
||||
- keep weather provider rules in source adapters, provider helpers, and
|
||||
normalizers;
|
||||
- keep canonical weather payloads in `model` and schema/wire identifiers in
|
||||
`standards`;
|
||||
- keep Postgres table and row mapping under `internal/sinks/postgres`;
|
||||
- use explicit registries for built-in sources and normalizers.
|
||||
|
||||
Do not:
|
||||
|
||||
- move provider parsing, WMO mapping, or canonical weather policy into
|
||||
`cmd/weatherfeeder`;
|
||||
- move weather-specific constants, schemas, or validation rules into feedkit;
|
||||
- put database column metadata or sink-specific tags on canonical model structs;
|
||||
- replace explicit registries with dynamic plugin loading;
|
||||
- introduce broad abstractions when a small provider-specific helper preserves
|
||||
clarity.
|
||||
|
||||
## Dependency Policy
|
||||
|
||||
Prefer the Go standard library unless a dependency materially improves
|
||||
@@ -183,6 +210,7 @@ When behavior changes, update the canonical docs in the same change:
|
||||
- CLI behavior: `docs/cli.md`;
|
||||
- operations and recovery: `docs/operations.md`;
|
||||
- troubleshooting: `docs/troubleshooting.md`;
|
||||
- public Go package consumption: `docs/consumers/`;
|
||||
- external contracts: `docs/integrations/`;
|
||||
- internal component behavior: `docs/internal/`;
|
||||
- copyable configs: `examples/`.
|
||||
|
||||
@@ -2,12 +2,13 @@
|
||||
|
||||
## Purpose
|
||||
|
||||
Project documentation must help four audiences:
|
||||
Project documentation must help five audiences:
|
||||
|
||||
1. users who need to run the application;
|
||||
2. administrators/operators who need to configure and operate it;
|
||||
3. developers who need to understand and change it safely;
|
||||
4. LLM coding agents that need clear scope, boundaries, and invariants.
|
||||
4. LLM coding agents that need clear scope, boundaries, and invariants;
|
||||
5. developers and LLM coding agents integrating this project from another codebase.
|
||||
|
||||
Docs should be accurate, concise, task-oriented, and organized by audience. Prefer links to canonical docs over repetition.
|
||||
|
||||
@@ -42,11 +43,14 @@ Canonical homes:
|
||||
|
||||
- project purpose and quickstart: `README.md`
|
||||
- development principles: `docs/policy/architecture.md`
|
||||
- public HTTP API reference: `docs/api.md`
|
||||
- configuration reference: `docs/config.md`
|
||||
- CLI reference: `docs/cli.md`
|
||||
- operations and recovery: `docs/operations.md`
|
||||
- troubleshooting: `docs/troubleshooting.md`
|
||||
- public API/package consumer guidance: `docs/consumers/`
|
||||
- implemented internals: `docs/internal/`
|
||||
- external protocol, service, and file-format contracts: `docs/integrations/`
|
||||
- future work: `docs/roadmap/`
|
||||
- contributor workflow: `docs/policy/development.md`
|
||||
- copyable examples: `examples/`
|
||||
@@ -106,7 +110,7 @@ Recommended:
|
||||
- `examples/`
|
||||
- `docs/policy/development.md`
|
||||
|
||||
### Modular, staged, service-oriented, or orchestration application
|
||||
### Modular, service-oriented, or orchestration application
|
||||
|
||||
Required:
|
||||
- `docs/cli.md`, if CLI-based
|
||||
@@ -119,6 +123,31 @@ Recommended:
|
||||
- `docs/troubleshooting.md`
|
||||
- validated examples under `examples/`
|
||||
|
||||
### Public HTTP API service
|
||||
|
||||
Required:
|
||||
- `docs/api.md`
|
||||
- `docs/cli.md`, if CLI-based
|
||||
- `docs/config.md`, if config-driven
|
||||
- `docs/operations.md`
|
||||
- `docs/internal/`
|
||||
- `docs/policy/development.md`
|
||||
|
||||
Recommended:
|
||||
- `docs/troubleshooting.md`
|
||||
- `docs/consumers/`, for task-oriented client integration guides
|
||||
- `docs/integrations/`, for upstream/downstream service contracts
|
||||
- validated examples under `examples/`
|
||||
|
||||
### Project with public packages or consumer APIs
|
||||
|
||||
Required:
|
||||
- `docs/consumers/api.md`
|
||||
- one `docs/consumers/pkg-<name>.md` file per public package, if public packages exist
|
||||
|
||||
Recommended:
|
||||
- copyable consumer examples under `examples/`, if practical
|
||||
|
||||
## Required Documents
|
||||
|
||||
### README.md
|
||||
@@ -161,6 +190,32 @@ It should include:
|
||||
|
||||
For small projects, this file may be brief. It may simply state that the project is intentionally narrow, monolithic, and dependency-light.
|
||||
|
||||
### docs/api.md
|
||||
|
||||
**Audience:** external HTTP API consumers, developers, LLM coding agents integrating by HTTP
|
||||
|
||||
Required for projects whose primary public interface is HTTP.
|
||||
|
||||
`docs/api.md` is the canonical public HTTP API contract. It should be normative for external consumers and should not be duplicated by README, operations docs, consumer guides, or integration docs.
|
||||
|
||||
It should include:
|
||||
|
||||
1. base URL conventions;
|
||||
2. authentication and authorization behavior, if implemented;
|
||||
3. response envelope;
|
||||
4. supported media types and content negotiation behavior;
|
||||
5. shared query parameters;
|
||||
6. endpoint reference grouped by route family;
|
||||
7. request parameters and validation rules;
|
||||
8. response fields, units, nullability, and optionality;
|
||||
9. error response shape and status codes;
|
||||
10. pagination, caching, rate-limit, idempotency, and retry behavior, if implemented;
|
||||
11. compact request and response examples.
|
||||
|
||||
It must document only implemented endpoints and behavior. Planned endpoints, proposed fields, future filters, and experimental response shapes belong only under `docs/roadmap/`.
|
||||
|
||||
For HTTP API projects, `docs/consumers/` may provide task-oriented client integration guides, but those guides should link to `docs/api.md` for the authoritative endpoint contract.
|
||||
|
||||
### docs/policy/development.md
|
||||
|
||||
**Audience:** developers, LLM coding agents
|
||||
@@ -175,7 +230,7 @@ It should include:
|
||||
- dependency policy;
|
||||
- how to add config fields;
|
||||
- how to add CLI flags;
|
||||
- how to add stages/modules/adapters, if applicable;
|
||||
- how to add modules or adapters, if applicable;
|
||||
- how to update examples;
|
||||
- documentation update expectations.
|
||||
|
||||
@@ -216,7 +271,7 @@ Explain when commands are useful, not just their syntax.
|
||||
|
||||
**Audience:** administrators, operators
|
||||
|
||||
Required for applications that maintain state, support resume behavior, run multiple stages, write durable artifacts, use remote storage, or require recovery procedures.
|
||||
Required for applications that maintain state, support resume behavior, run multi-step workflows, write durable artifacts, use remote storage, or require recovery procedures.
|
||||
|
||||
It should cover:
|
||||
|
||||
@@ -244,11 +299,40 @@ Each entry should include:
|
||||
- safe fix;
|
||||
- relevant links.
|
||||
|
||||
### docs/consumers/
|
||||
|
||||
**Audience:** developers and LLM coding agents integrating this project from another codebase
|
||||
|
||||
Required for projects with public packages, SDKs, client APIs, plugin APIs, or other application-facing integration surfaces.
|
||||
|
||||
This directory describes how an external codebase should consume the project's public API. It should be task-oriented and copyable where useful. It is not the place for internal implementation details or operator procedures.
|
||||
|
||||
For projects whose public API is HTTP, `docs/consumers/` is not required, and it should not duplicate the endpoint reference in `docs/api.md`. If present, it may provide practical integration workflows, client-specific examples, or migration notes that link back to `docs/api.md`.
|
||||
|
||||
`docs/consumers/api.md` should provide the consumer-facing overview and primary implementation workflow. It should include:
|
||||
|
||||
1. intended consumer audience and use cases;
|
||||
2. required inputs supplied by operators or deployment configuration;
|
||||
3. recommended public package or API workflow;
|
||||
4. minimal copyable example;
|
||||
5. consumer responsibilities and boundaries;
|
||||
6. retry, idempotency, or status behavior, if applicable;
|
||||
7. links to package-specific docs and canonical integration contracts.
|
||||
|
||||
Package-specific docs should be named `pkg-<name>.md` and should include:
|
||||
|
||||
1. import path;
|
||||
2. intended use cases;
|
||||
3. primary types and functions needed by consumers;
|
||||
4. minimal examples;
|
||||
5. validation, error, retry, and boundary behavior;
|
||||
6. links to canonical file-format or wire-protocol contracts.
|
||||
|
||||
### docs/internal/
|
||||
|
||||
**Audience:** developers, LLM coding agents
|
||||
|
||||
Required for modular, staged, service-oriented, or orchestration projects.
|
||||
Required for modular, service-oriented, or orchestration projects.
|
||||
|
||||
This directory describes implemented internal components. It is not the roadmap.
|
||||
|
||||
@@ -289,7 +373,9 @@ Roadmap docs should not be confused with current behavior.
|
||||
|
||||
Required for projects that depend on external CLIs, APIs, services, protocols, or file formats where the integration contract is important to maintain.
|
||||
|
||||
This directory contains concise, versioned reference notes for external integration contracts. It should document only the parts of the external system that this project actually uses.
|
||||
This directory contains concise, versioned reference notes for external integration contracts. It should document only the parts of the external system that this project actually uses or exposes.
|
||||
|
||||
For public HTTP API services, `docs/integrations/` should document upstream, downstream, storage, protocol, or runtime contracts that the service depends on or bridges. It should not become a second copy of the public HTTP endpoint reference; that belongs in `docs/api.md`.
|
||||
|
||||
Use one file per integration where useful.
|
||||
|
||||
@@ -346,8 +432,10 @@ Before merging documentation changes, verify:
|
||||
|
||||
- README is concise and orientation-focused.
|
||||
- `docs/policy/architecture.md` describes development principles.
|
||||
- `docs/api.md` is the canonical HTTP contract for HTTP API services.
|
||||
- Future work appears only under `docs/roadmap/`.
|
||||
- User-facing docs avoid unnecessary internals.
|
||||
- Consumer-facing docs explain public APIs without duplicating HTTP endpoint or integration contracts.
|
||||
- Developer-facing docs preserve boundaries and invariants.
|
||||
- Config examples match the schema.
|
||||
- CLI examples match real commands and flags.
|
||||
|
||||
99
docs/roadmap/future.md
Normal file
99
docs/roadmap/future.md
Normal file
@@ -0,0 +1,99 @@
|
||||
# Future Work
|
||||
|
||||
## Purpose
|
||||
|
||||
This document is the catch-all roadmap for planned, deferred, aspirational, experimental, or unimplemented weatherfeeder work. Current behavior belongs in the canonical docs outside `docs/roadmap/`.
|
||||
|
||||
## NWS AFD Parsing Resilience
|
||||
|
||||
The current parser handles the concrete RAH, LWX, and MFR variants that
|
||||
motivated these ideas. Future work should keep those extension points
|
||||
maintainable as additional evidence appears.
|
||||
|
||||
### Generalize Wrapper-Scoped Embedded Sections
|
||||
|
||||
The scanner currently permits undotted nested headings only inside `PREV
|
||||
DISCUSSION`. If other wrapper identities are observed, replace the single
|
||||
wrapper check with a small explicit provider-local registry and add a fixture
|
||||
for each wrapper family. Do not make the leading dot globally optional: wrapper
|
||||
scope is the safeguard against classifying uppercase prose as a section.
|
||||
|
||||
### Extend Conservative Preamble Classification
|
||||
|
||||
Leading key-message metadata currently supports validated `Issued at`, `Updated
|
||||
at`, and `As of <clock> <weekday>...` forms. Add future wording variants as
|
||||
small, ordered classifiers with strict label boundaries and value grammars.
|
||||
Every addition should include collision tests proving that similar message prose
|
||||
and malformed metadata remain canonical content.
|
||||
|
||||
### Keep List-Marker Recognition Extensible
|
||||
|
||||
The marker parser currently supports hyphens, asterisks, `N)`, `N.`, `(N)`, and
|
||||
composite forms such as `- (N)`. If new decorators appear, evolve the helper
|
||||
toward an explicit marker grammar or typed classification result rather than a
|
||||
broad punctuation heuristic. Preserve positive-number and whitespace-boundary
|
||||
checks so ordinary prose is not stripped.
|
||||
|
||||
### Maintain a Cross-Office Fixture Corpus
|
||||
|
||||
The compact RAH, LWX, and current MFR fixtures seed regression coverage for the
|
||||
observed layouts. Future parser changes should add concise, deterministic HTML
|
||||
fixtures for materially distinct office formats and exercise them through both
|
||||
the provider parser and normalizer. Fixture comments should identify the format
|
||||
family and state that edited prose is not an archived product; tests must remain
|
||||
offline and assert both intended extraction and adjacent-section isolation.
|
||||
|
||||
## SPC Convective Outlook Follow-Ups
|
||||
|
||||
### Weatherapi Outlook Endpoints
|
||||
|
||||
Expose persisted SPC convective outlooks through `weatherapi` after the weatherfeeder storage contract is stable.
|
||||
|
||||
Likely endpoints:
|
||||
|
||||
- `GET /outlooks/convective`
|
||||
- `GET /outlooks/convective/active`
|
||||
- `GET /outlooks/convective/location`
|
||||
|
||||
Recommended behavior:
|
||||
|
||||
- Return the latest outlook run by default.
|
||||
- Support active outlook filtering by current time and `containsLocation=true`.
|
||||
- Consider optional query filters for `day`, `outlookType`, and `containsLocation`.
|
||||
- Preserve canonical outlook geometry for downstream display and audit use.
|
||||
|
||||
### SPC Day 4-8 Outlooks
|
||||
|
||||
Add SPC Day 4-8 convective outlook support as a schema-compatible extension only after Day 1-3 operation is proven.
|
||||
|
||||
Notes:
|
||||
|
||||
- Day 4-8 products have different semantics from Day 1-3 categorical/tornado/hail/wind products.
|
||||
- Avoid forcing Day 4-8 assumptions into the current Day 1-3 model until the source shapes and consumer needs are reviewed.
|
||||
- Prefer reusing `weather.outlook.v2` if the fields remain accurate; otherwise write a separate roadmap before changing the canonical contract.
|
||||
|
||||
### Degraded SPC Bundle Mode
|
||||
|
||||
Evaluate whether the SPC source should support degraded partial bundles when one required upstream product fails.
|
||||
|
||||
Current behavior should remain atomic: if a required GeoJSON or print-page fetch fails, emit no event for that poll.
|
||||
|
||||
Future degraded mode would need a clear contract for:
|
||||
|
||||
- distinguishing "no risk polygon" from "product missing";
|
||||
- exposing per-product fetch errors without leaking raw provider internals into canonical events;
|
||||
- deciding whether downstream sinks and APIs should store or serve partial snapshots.
|
||||
|
||||
### Richer SPC Page Assets And Tables
|
||||
|
||||
Evaluate whether to parse additional SPC print-page metadata beyond the current discussion text.
|
||||
|
||||
Possible additions:
|
||||
|
||||
- archive GeoJSON/shapefile/KML links;
|
||||
- image URLs;
|
||||
- page risk tables;
|
||||
- city tables;
|
||||
- richer discussion section metadata.
|
||||
|
||||
Keep GeoJSON products authoritative for polygons, validity windows, and point matching unless a future roadmap explicitly changes that contract.
|
||||
@@ -1,697 +0,0 @@
|
||||
# SPC Convective Outlook Implementation Plan
|
||||
|
||||
## Purpose
|
||||
|
||||
Implement `weatherfeeder` support for Storm Prediction Center Day 1-3 convective outlooks described in [`docs/roadmap/spc.md`](spc.md). This plan is written for an LLM coding agent and should be followed stage by stage.
|
||||
|
||||
This is a planning document only. The implementation must preserve the existing weatherfeeder architecture: sources emit raw provider events, normalizers map raw payloads into canonical model types, and sinks persist canonical schemas.
|
||||
|
||||
## Decisions
|
||||
|
||||
The following choices are fixed for this implementation:
|
||||
|
||||
- Scope is Day 1-3 SPC convective outlooks only.
|
||||
- Day 4-8 outlooks are out of scope.
|
||||
- GeoJSON files are authoritative for polygons, validity windows, issue times, outlook labels, and severity ranking.
|
||||
- Day 1-3 print pages are authoritative for discussion text.
|
||||
- RSS is optional supplemental metadata only and must not be required for correctness.
|
||||
- Do not fetch RSS by default. Include RSS only when an optional `rss_url` source param is configured.
|
||||
- A poll is atomic for required products. If any configured GeoJSON or print-page URL fails or returns a non-2xx response, return an error and emit no event.
|
||||
- Use compact GeoJSON geometry in the canonical payload for auditability and downstream display.
|
||||
- Use standard-library-first parsing. Do not add an HTML parsing dependency unless string-based extraction proves unmaintainable during implementation.
|
||||
- Keep all new planned behavior inside `weatherfeeder`; do not make `weatherapi` changes in this pass.
|
||||
|
||||
## Public Contract
|
||||
|
||||
Add schema constants in `standards/schema.go`:
|
||||
|
||||
- `SchemaRawSPCConvectiveOutlookV1 = "raw.spc.convective_outlook.v1"`
|
||||
- `SchemaWeatherOutlookV1 = "weather.outlook.v1"`
|
||||
|
||||
Add source driver:
|
||||
|
||||
- `spc_convective_outlook`
|
||||
|
||||
Add event kind:
|
||||
|
||||
- `outlook`
|
||||
|
||||
Add canonical model types:
|
||||
|
||||
- `model.WeatherOutlookRun`
|
||||
- `model.WeatherOutlook`
|
||||
|
||||
Canonical run fields:
|
||||
|
||||
```go
|
||||
type WeatherOutlookRun struct {
|
||||
LocationID string `json:"locationId,omitempty"`
|
||||
LocationName string `json:"locationName,omitempty"`
|
||||
Latitude *float64 `json:"latitude,omitempty"`
|
||||
Longitude *float64 `json:"longitude,omitempty"`
|
||||
AsOf time.Time `json:"asOf"`
|
||||
IssuedAt *time.Time `json:"issuedAt,omitempty"`
|
||||
Outlooks []WeatherOutlook `json:"outlooks"`
|
||||
}
|
||||
```
|
||||
|
||||
Canonical outlook fields:
|
||||
|
||||
```go
|
||||
type WeatherOutlook struct {
|
||||
ID string `json:"id"`
|
||||
Provider string `json:"provider"`
|
||||
Product string `json:"product"`
|
||||
Day int `json:"day"`
|
||||
OutlookType string `json:"outlookType"`
|
||||
Label string `json:"label"`
|
||||
LabelText string `json:"labelText,omitempty"`
|
||||
SeverityRank *int `json:"severityRank,omitempty"`
|
||||
ValidFrom time.Time `json:"validFrom"`
|
||||
ValidTo time.Time `json:"validTo"`
|
||||
IssuedAt time.Time `json:"issuedAt"`
|
||||
ExpiresAt time.Time `json:"expiresAt"`
|
||||
Forecaster string `json:"forecaster,omitempty"`
|
||||
Headline string `json:"headline,omitempty"`
|
||||
Summary string `json:"summary,omitempty"`
|
||||
Discussion string `json:"discussion,omitempty"`
|
||||
SourceURL string `json:"sourceUrl,omitempty"`
|
||||
ImageURL string `json:"imageUrl,omitempty"`
|
||||
ContainsLocation bool `json:"containsLocation"`
|
||||
Geometry json.RawMessage `json:"geometry"`
|
||||
}
|
||||
```
|
||||
|
||||
Required canonical fields:
|
||||
|
||||
- Run: `asOf`, `outlooks`.
|
||||
- Outlook: `id`, `provider`, `product`, `day`, `outlookType`, `label`, `validFrom`, `validTo`, `issuedAt`, `expiresAt`, `containsLocation`, `geometry`.
|
||||
|
||||
Canonical values:
|
||||
|
||||
- `provider` is `spc`.
|
||||
- `product` is `convective`.
|
||||
- `outlookType` is one of `categorical`, `tornado`, `hail`, `wind`.
|
||||
- `day` is one of `1`, `2`, `3`.
|
||||
|
||||
## Source Inputs
|
||||
|
||||
Default required GeoJSON products:
|
||||
|
||||
- `https://www.spc.noaa.gov/products/outlook/day1otlk_cat.nolyr.geojson`
|
||||
- `https://www.spc.noaa.gov/products/outlook/day1otlk_torn.nolyr.geojson`
|
||||
- `https://www.spc.noaa.gov/products/outlook/day1otlk_hail.nolyr.geojson`
|
||||
- `https://www.spc.noaa.gov/products/outlook/day1otlk_wind.nolyr.geojson`
|
||||
- `https://www.spc.noaa.gov/products/outlook/day2otlk_cat.nolyr.geojson`
|
||||
- `https://www.spc.noaa.gov/products/outlook/day2otlk_torn.nolyr.geojson`
|
||||
- `https://www.spc.noaa.gov/products/outlook/day2otlk_hail.nolyr.geojson`
|
||||
- `https://www.spc.noaa.gov/products/outlook/day2otlk_wind.nolyr.geojson`
|
||||
- `https://www.spc.noaa.gov/products/outlook/day3otlk_cat.nolyr.geojson`
|
||||
- `https://www.spc.noaa.gov/products/outlook/day3otlk_torn.nolyr.geojson`
|
||||
- `https://www.spc.noaa.gov/products/outlook/day3otlk_hail.nolyr.geojson`
|
||||
- `https://www.spc.noaa.gov/products/outlook/day3otlk_wind.nolyr.geojson`
|
||||
|
||||
Default required print-page products:
|
||||
|
||||
- `https://www.spc.noaa.gov/products/outlook/day1otlk_prt.html`
|
||||
- `https://www.spc.noaa.gov/products/outlook/day2otlk_prt.html`
|
||||
- `https://www.spc.noaa.gov/products/outlook/day3otlk_prt.html`
|
||||
|
||||
Optional RSS product:
|
||||
|
||||
- `https://www.spc.noaa.gov/products/spcacrss.xml`
|
||||
|
||||
Recommended config shape:
|
||||
|
||||
```yaml
|
||||
- name: SPCConvectiveOutlookSTL
|
||||
mode: poll
|
||||
kinds: ["outlook"]
|
||||
driver: spc_convective_outlook
|
||||
every: 30m
|
||||
params:
|
||||
latitude: 38.6239
|
||||
longitude: -90.3571
|
||||
location_id: "stl"
|
||||
location_name: "St. Louis, MO"
|
||||
user_agent: "HomeOps (eric@maximumdirect.net)"
|
||||
```
|
||||
|
||||
Optional source params:
|
||||
|
||||
- `geojson_urls`: map from product key to URL, used by tests and future upstream changes.
|
||||
- `discussion_urls`: map from day key to URL, used by tests and future upstream changes.
|
||||
- `rss_url`: string; when non-empty, fetch RSS as supplemental metadata.
|
||||
|
||||
Product keys for `geojson_urls`:
|
||||
|
||||
- `day1_categorical`, `day1_tornado`, `day1_hail`, `day1_wind`
|
||||
- `day2_categorical`, `day2_tornado`, `day2_hail`, `day2_wind`
|
||||
- `day3_categorical`, `day3_tornado`, `day3_hail`, `day3_wind`
|
||||
|
||||
Discussion keys for `discussion_urls`:
|
||||
|
||||
- `day1`, `day2`, `day3`
|
||||
|
||||
## Raw Bundle Shape
|
||||
|
||||
Create a provider raw bundle type under `internal/providers/spc` or `internal/normalizers/spc` and use it consistently between source tests and normalizer tests. Prefer `internal/providers/spc` if source metadata extraction and normalizer parsing share helpers.
|
||||
|
||||
Suggested raw payload shape:
|
||||
|
||||
```go
|
||||
type RawConvectiveOutlookBundle struct {
|
||||
LocationID string `json:"locationId,omitempty"`
|
||||
LocationName string `json:"locationName,omitempty"`
|
||||
Latitude float64 `json:"latitude"`
|
||||
Longitude float64 `json:"longitude"`
|
||||
FetchedAt time.Time `json:"fetchedAt"`
|
||||
Products []RawOutlookProduct `json:"products"`
|
||||
Discussions []RawDiscussionPage `json:"discussions"`
|
||||
RSS *RawRSSFeed `json:"rss,omitempty"`
|
||||
}
|
||||
```
|
||||
|
||||
```go
|
||||
type RawOutlookProduct struct {
|
||||
Key string `json:"key"`
|
||||
Day int `json:"day"`
|
||||
OutlookType string `json:"outlookType"`
|
||||
URL string `json:"url"`
|
||||
FetchedAt time.Time `json:"fetchedAt"`
|
||||
Body json.RawMessage `json:"body"`
|
||||
}
|
||||
```
|
||||
|
||||
```go
|
||||
type RawDiscussionPage struct {
|
||||
Key string `json:"key"`
|
||||
Day int `json:"day"`
|
||||
URL string `json:"url"`
|
||||
FetchedAt time.Time `json:"fetchedAt"`
|
||||
Body string `json:"body"`
|
||||
}
|
||||
```
|
||||
|
||||
```go
|
||||
type RawRSSFeed struct {
|
||||
URL string `json:"url"`
|
||||
FetchedAt time.Time `json:"fetchedAt"`
|
||||
Body string `json:"body"`
|
||||
}
|
||||
```
|
||||
|
||||
Do not put parsed canonical fields into the raw bundle except configured metadata and product keys needed to identify each fetched upstream document. Source-level timestamp parsing is allowed only for event `effectiveAt` selection.
|
||||
|
||||
## Stage 1: Provider Helpers And Fixtures
|
||||
|
||||
Goal: add deterministic SPC parsing primitives and test fixtures before wiring the source or normalizer.
|
||||
|
||||
Files to add:
|
||||
|
||||
- `internal/providers/spc/doc.go`
|
||||
- `internal/providers/spc/time.go`
|
||||
- `internal/providers/spc/product.go`
|
||||
- `internal/providers/spc/geojson.go`
|
||||
- `internal/providers/spc/discussion.go`
|
||||
- `internal/providers/spc/rss.go`, only if optional RSS parsing is implemented
|
||||
- `internal/providers/spc/testdata/day1_cat.geojson`
|
||||
- `internal/providers/spc/testdata/day2_torn.geojson`
|
||||
- `internal/providers/spc/testdata/day3_wind.geojson`
|
||||
- `internal/providers/spc/testdata/day1_prt.html`
|
||||
- `internal/providers/spc/testdata/day2_prt_corr.html`
|
||||
- `internal/providers/spc/testdata/day3_prt.html`
|
||||
|
||||
Provider helper behavior:
|
||||
|
||||
- Define stable product metadata for the 12 required GeoJSON products.
|
||||
- Define stable discussion metadata for the 3 required print-page products.
|
||||
- Parse SPC ISO timestamps from GeoJSON properties using `time.Parse(time.RFC3339, value)` after trimming whitespace.
|
||||
- Decode enough GeoJSON to expose feature properties and raw geometry without owning canonical mapping.
|
||||
- Preserve raw geometry as compact JSON bytes for later canonical use.
|
||||
- Extract print-page product text from the first useful `<pre>` block.
|
||||
- Strip embedded `<script>` blocks and remaining tags from extracted `<pre>` content.
|
||||
- Use `html.UnescapeString`, normalize CRLF to LF, and trim surrounding blank lines.
|
||||
- Parse print-page `Updated:` timestamps when present.
|
||||
- Parse discussion headline/product title from product text.
|
||||
- Parse `...SUMMARY...` content through the next section heading.
|
||||
- Preserve full product text as discussion text after cleanup.
|
||||
- Preserve correction markers such as `CORR 1` in headline or metadata when present.
|
||||
|
||||
Dependency rule:
|
||||
|
||||
- Start with `strings`, `regexp`, `encoding/json`, `encoding/xml`, and `html` from the standard library.
|
||||
- Do not add `golang.org/x/net/html` or another HTML parser unless tests show the string extraction is too brittle.
|
||||
|
||||
Tests:
|
||||
|
||||
- `go test ./internal/providers/spc`
|
||||
- Product metadata contains exactly 12 GeoJSON products in day/type order.
|
||||
- Discussion metadata contains exactly 3 print pages in day order.
|
||||
- GeoJSON fixture decode exposes `VALID_ISO`, `EXPIRE_ISO`, `ISSUE_ISO`, `FORECASTER`, `LABEL`, `LABEL2`, `DN`, and geometry.
|
||||
- Print-page fixture extraction returns product text without scripts or tags.
|
||||
- Day 2 correction fixture preserves `CORR 1`.
|
||||
- Summary extraction returns only the summary paragraph content.
|
||||
- Updated timestamp parser returns UTC time when present and nil when absent.
|
||||
|
||||
Stage completion criteria:
|
||||
|
||||
- Provider helper tests pass.
|
||||
- No source, normalizer, model, standards, sink, config, or docs current-behavior files are changed in this stage unless needed for package compilation.
|
||||
|
||||
## Stage 2: Geometry Helper
|
||||
|
||||
Goal: implement point-in-polygon support independent of SPC parsing.
|
||||
|
||||
Files to add:
|
||||
|
||||
- `internal/geo/geojson.go`
|
||||
- `internal/geo/point.go`
|
||||
- `internal/geo/point_test.go`
|
||||
|
||||
Behavior:
|
||||
|
||||
- Support GeoJSON `Polygon` and `MultiPolygon` only.
|
||||
- Accept geometry as `json.RawMessage` or `[]byte`.
|
||||
- Interpret GeoJSON coordinate order as `[longitude, latitude]`.
|
||||
- Treat the first ring as the exterior ring.
|
||||
- Treat subsequent rings as holes.
|
||||
- Count boundary points as inside.
|
||||
- Return a clear error for unsupported geometry types, malformed coordinates, empty rings, or invalid JSON.
|
||||
- Use planar ray casting. This is sufficient for operational point-in-polygon checks at SPC polygon scale.
|
||||
|
||||
Tests:
|
||||
|
||||
- Point inside simple polygon returns true.
|
||||
- Point outside simple polygon returns false.
|
||||
- Point on polygon boundary returns true.
|
||||
- Point in a hole returns false.
|
||||
- Point inside one `MultiPolygon` member returns true.
|
||||
- Longitude/latitude ordering regression test fails if coordinates are reversed.
|
||||
- Unsupported geometry returns a useful error.
|
||||
|
||||
Verification:
|
||||
|
||||
```sh
|
||||
go test ./internal/geo
|
||||
```
|
||||
|
||||
Stage completion criteria:
|
||||
|
||||
- Geometry helper has no dependency on SPC, source, normalizer, or sink packages.
|
||||
|
||||
## Stage 3: Source Driver And Raw Schema
|
||||
|
||||
Goal: emit raw SPC outlook bundles from a new poll source.
|
||||
|
||||
Files to update:
|
||||
|
||||
- `standards/schema.go`
|
||||
- `internal/sources/builtins.go`
|
||||
- `internal/sources/builtins_test.go`
|
||||
|
||||
Files to add:
|
||||
|
||||
- `internal/sources/spc/convective_outlook.go`
|
||||
- `internal/sources/spc/convective_outlook_test.go`
|
||||
|
||||
Source constructor:
|
||||
|
||||
- Export `NewConvectiveOutlookSource(cfg config.SourceConfig) (*ConvectiveOutlookSource, error)`.
|
||||
- Register driver `spc_convective_outlook` in `internal/sources/builtins.go`.
|
||||
- Validate required params `latitude` and `longitude`.
|
||||
- Accept optional params `location_id`, `location_name`, `geojson_urls`, `discussion_urls`, and `rss_url`.
|
||||
- Require `user_agent` when the existing HTTP source conventions require it.
|
||||
- Advertise `Kinds() []event.Kind{event.Kind("outlook")}`.
|
||||
|
||||
Fetch behavior:
|
||||
|
||||
- Fetch all required GeoJSON URLs and print-page URLs every poll cycle.
|
||||
- Fetch optional RSS only when `rss_url` is configured and non-empty.
|
||||
- Use Accept `application/geo+json, application/json` for GeoJSON requests.
|
||||
- Use Accept `text/html, application/xhtml+xml` for print pages.
|
||||
- Use Accept `application/rss+xml, application/xml, text/xml` for RSS.
|
||||
- Respect `context.Context` on all requests.
|
||||
- Do not emit partial bundles.
|
||||
- If the complete raw bundle is unchanged from the previous successful poll, return no events. Implement source-local change detection by hashing the combined fetched bodies if feedkit HTTP conditional helpers do not directly support multi-document polling.
|
||||
- Build the raw bundle with fetched bodies and metadata.
|
||||
- Emit one raw event with kind `outlook` and schema `standards.SchemaRawSPCConvectiveOutlookV1`.
|
||||
- Use `fksources.DefaultEventID("", sourceName, effectiveAt, emittedAt)`.
|
||||
|
||||
Effective time policy:
|
||||
|
||||
- Prefer latest valid `ISSUE_ISO` across all GeoJSON features.
|
||||
- Fallback to latest valid print-page `Updated:` timestamp.
|
||||
- Fallback to RSS `lastBuildDate` if RSS was fetched and parseable.
|
||||
- Fallback to fetch time if no provider timestamp is parseable.
|
||||
- Keep fetch-time fallback as the last source effective-time option so downstream consumers still receive a stable effective time when provider metadata is absent.
|
||||
|
||||
Tests:
|
||||
|
||||
- Builtin registry builds `spc_convective_outlook` as a `PollSource`.
|
||||
- `Kinds()` returns `outlook`.
|
||||
- Constructor rejects missing `latitude` or `longitude`.
|
||||
- Poll emits one raw event with schema `raw.spc.convective_outlook.v1`.
|
||||
- Raw bundle includes 12 products and 3 discussions when defaults are overridden to test-server URLs.
|
||||
- Poll effectiveAt prefers latest GeoJSON `ISSUE_ISO`.
|
||||
- Poll effectiveAt falls back to print-page `Updated:` when GeoJSON issue times are absent.
|
||||
- Poll includes RSS only when `rss_url` is configured.
|
||||
- Unchanged response emits no events on a second poll.
|
||||
- A configured GeoJSON fetch failure returns error and emits no event.
|
||||
- A configured print-page fetch failure returns error and emits no event.
|
||||
- Tests use `httptest.Server`; do not call live SPC services.
|
||||
|
||||
Verification:
|
||||
|
||||
```sh
|
||||
go test ./internal/providers/spc ./internal/sources
|
||||
```
|
||||
|
||||
Stage completion criteria:
|
||||
|
||||
- Source emits raw bundles only.
|
||||
- No canonical model, normalizer, or sink mapping is added in this stage except schema constants required for compilation.
|
||||
|
||||
## Stage 4: Canonical Model And Normalizer
|
||||
|
||||
Goal: convert raw SPC bundles into `weather.outlook.v1` events.
|
||||
|
||||
Files to update:
|
||||
|
||||
- `standards/schema.go`
|
||||
- `internal/normalizers/builtins_test.go`
|
||||
|
||||
Files to add:
|
||||
|
||||
- `model/outlook.go`
|
||||
- `internal/normalizers/spc/convective_outlook.go`
|
||||
- `internal/normalizers/spc/register.go`
|
||||
- `internal/normalizers/spc/convective_outlook_test.go`
|
||||
|
||||
Files to update for registration:
|
||||
|
||||
- `internal/normalizers/builtins.go`
|
||||
|
||||
Normalizer behavior:
|
||||
|
||||
- Add `SPCConvectiveOutlookNormalizer`.
|
||||
- Match only `standards.SchemaRawSPCConvectiveOutlookV1`.
|
||||
- Decode `RawConvectiveOutlookBundle`.
|
||||
- Build one `model.WeatherOutlookRun` per raw bundle.
|
||||
- Map each GeoJSON feature to one `model.WeatherOutlook`.
|
||||
- Preserve feature order within each product.
|
||||
- Order products by day then outlook type in this order: `categorical`, `tornado`, `hail`, `wind`.
|
||||
- Compute `containsLocation` with `internal/geo` using the configured bundle latitude and longitude.
|
||||
- Store compact feature geometry JSON in `WeatherOutlook.Geometry`.
|
||||
- Set output schema to `standards.SchemaWeatherOutlookV1`.
|
||||
- Set output effectiveAt to run `AsOf`.
|
||||
- Use `internal/normalizers/common.Finalize` or existing normalizer helper patterns so envelope handling remains consistent.
|
||||
|
||||
GeoJSON property mapping:
|
||||
|
||||
- `VALID_ISO` maps to `validFrom` and is required.
|
||||
- `EXPIRE_ISO` maps to `validTo` and `expiresAt` and is required.
|
||||
- `ISSUE_ISO` maps to `issuedAt` and is required.
|
||||
- `FORECASTER` maps to `forecaster` and is optional.
|
||||
- `LABEL` maps to `label` and is required.
|
||||
- `LABEL2` maps to `labelText` and is optional.
|
||||
- `DN` maps to `severityRank` and is optional.
|
||||
|
||||
Derived mapping:
|
||||
|
||||
- `provider` is `spc`.
|
||||
- `product` is `convective`.
|
||||
- `day` and `outlookType` come from raw product metadata, not from brittle URL parsing when product metadata is available.
|
||||
- `id` is deterministic: join day, outlook type, normalized label, issuedAt UTC in RFC3339 format, validFrom UTC in RFC3339 format, and product-local feature index. Use a stable ASCII-safe format.
|
||||
- `sourceUrl` is the GeoJSON product URL for geometry/probability features.
|
||||
- `imageUrl` is empty for v1.
|
||||
|
||||
Discussion enrichment:
|
||||
|
||||
- Parse each print-page discussion with `internal/providers/spc` helpers.
|
||||
- Attach Day 1 print-page headline, summary, and discussion to Day 1 outlooks, Day 2 to Day 2 outlooks, and Day 3 to Day 3 outlooks.
|
||||
- Use the same discussion text for categorical, tornado, hail, and wind outlooks for the same day.
|
||||
- If a print page is fetched but discussion extraction fails, normalization should fail because print pages are required v1 inputs.
|
||||
- RSS metadata must not be required for discussion enrichment.
|
||||
|
||||
Run timestamp policy:
|
||||
|
||||
- `AsOf` is latest valid `issuedAt` across all outlook features.
|
||||
- Fallback to latest print-page `Updated:` timestamp.
|
||||
- Fallback to input event `EffectiveAt`.
|
||||
- Fallback to input event `EmittedAt`.
|
||||
- `IssuedAt` is latest valid `issuedAt` across all outlook features when any feature exists; otherwise nil.
|
||||
|
||||
Error behavior:
|
||||
|
||||
- Fail normalization if required GeoJSON timestamps are missing or unparseable.
|
||||
- Fail normalization if required `LABEL` is empty.
|
||||
- Fail normalization if required geometry is missing or invalid.
|
||||
- Fail normalization if latitude or longitude is missing or invalid in the raw bundle.
|
||||
- Include product key and feature index in errors.
|
||||
|
||||
Tests:
|
||||
|
||||
- Normalizer matches only `raw.spc.convective_outlook.v1`.
|
||||
- Builtin normalizer order includes the SPC normalizer after existing provider normalizers unless a specific order is needed.
|
||||
- Canonical schema is `weather.outlook.v1`.
|
||||
- Categorical fixture maps expected day, type, label, label text, severity rank, valid times, issue time, forecaster, source URL, and geometry.
|
||||
- Probabilistic fixtures for tornado, hail, and wind map expected `outlookType` values.
|
||||
- `containsLocation` is true for a fixture point inside a polygon.
|
||||
- `containsLocation` is false for a fixture point outside a polygon.
|
||||
- Print-page text maps to headline, summary, and discussion.
|
||||
- Day 2 correction marker `CORR 1` is preserved in headline or discussion metadata.
|
||||
- Missing optional RSS still normalizes successfully.
|
||||
- Invalid required timestamp fails with product key and feature index context.
|
||||
- Invalid geometry fails with product key and feature index context.
|
||||
- Output JSON uses the intended field names and does not expose raw bundle internals.
|
||||
|
||||
Verification:
|
||||
|
||||
```sh
|
||||
go test ./model ./internal/geo ./internal/providers/spc ./internal/normalizers ./internal/normalizers/spc
|
||||
```
|
||||
|
||||
Stage completion criteria:
|
||||
|
||||
- A raw SPC bundle normalizes into canonical `model.WeatherOutlookRun`.
|
||||
- No Postgres schema or current-behavior docs are updated in this stage unless the canonical schema documentation is intentionally updated with implementation.
|
||||
|
||||
## Stage 5: Postgres Sink
|
||||
|
||||
Goal: persist canonical outlook events through the weatherfeeder Postgres sink.
|
||||
|
||||
Files to update:
|
||||
|
||||
- `internal/sinks/postgres/schema.go`
|
||||
- `internal/sinks/postgres/map.go`
|
||||
- `internal/sinks/postgres/schema_test.go`
|
||||
- `internal/sinks/postgres/map_test.go`
|
||||
- `internal/sinks/postgres/doc.go`
|
||||
|
||||
Tables:
|
||||
|
||||
- `outlook_runs`
|
||||
- `outlooks`
|
||||
|
||||
Add table constants:
|
||||
|
||||
- `tableOutlookRuns = "outlook_runs"`
|
||||
- `tableOutlooks = "outlooks"`
|
||||
|
||||
`outlook_runs` columns:
|
||||
|
||||
- `event_id TEXT NOT NULL`
|
||||
- `event_kind TEXT NOT NULL`
|
||||
- `event_source TEXT NOT NULL`
|
||||
- `event_schema TEXT NOT NULL`
|
||||
- `event_emitted_at TIMESTAMPTZ NOT NULL`
|
||||
- `event_effective_at TIMESTAMPTZ NULL`
|
||||
- `location_id TEXT NULL`
|
||||
- `location_name TEXT NULL`
|
||||
- `latitude DOUBLE PRECISION NULL`
|
||||
- `longitude DOUBLE PRECISION NULL`
|
||||
- `as_of TIMESTAMPTZ NOT NULL`
|
||||
- `issued_at TIMESTAMPTZ NULL`
|
||||
- `outlook_count INTEGER NOT NULL`
|
||||
|
||||
`outlook_runs` keys and indexes:
|
||||
|
||||
- Primary key: `event_id`
|
||||
- Prune column: `as_of`
|
||||
- Index `idx_wf_outlook_run_location_as_of` on `location_id, as_of`
|
||||
- Index `idx_wf_outlook_run_as_of` on `as_of`
|
||||
|
||||
`outlooks` columns:
|
||||
|
||||
- `run_event_id TEXT REFERENCES outlook_runs(event_id) ON DELETE CASCADE NOT NULL`
|
||||
- `outlook_index INTEGER NOT NULL`
|
||||
- `as_of TIMESTAMPTZ NOT NULL`
|
||||
- `product TEXT NOT NULL`
|
||||
- `day INTEGER NOT NULL`
|
||||
- `outlook_type TEXT NOT NULL`
|
||||
- `label TEXT NOT NULL`
|
||||
- `label_text TEXT NULL`
|
||||
- `severity_rank INTEGER NULL`
|
||||
- `valid_from TIMESTAMPTZ NOT NULL`
|
||||
- `valid_to TIMESTAMPTZ NOT NULL`
|
||||
- `issued_at TIMESTAMPTZ NOT NULL`
|
||||
- `expires_at TIMESTAMPTZ NOT NULL`
|
||||
- `forecaster TEXT NULL`
|
||||
- `headline TEXT NULL`
|
||||
- `summary TEXT NULL`
|
||||
- `discussion TEXT NULL`
|
||||
- `source_url TEXT NULL`
|
||||
- `image_url TEXT NULL`
|
||||
- `contains_location BOOLEAN NOT NULL`
|
||||
- `geometry_json TEXT NOT NULL`
|
||||
|
||||
`outlooks` keys and indexes:
|
||||
|
||||
- Primary key: `run_event_id, outlook_index`
|
||||
- Prune column: `as_of`
|
||||
- Index `idx_wf_outlooks_contains_valid` on `contains_location, valid_from, valid_to`
|
||||
- Index `idx_wf_outlooks_day_type_label` on `day, outlook_type, label`
|
||||
- Index `idx_wf_outlooks_valid` on `valid_from, valid_to`
|
||||
|
||||
Mapper behavior:
|
||||
|
||||
- Extend `mapPostgresEvent` for `standards.SchemaWeatherOutlookV1`.
|
||||
- Decode `model.WeatherOutlookRun`.
|
||||
- Require run `AsOf`.
|
||||
- Map envelope columns exactly like existing parent run tables.
|
||||
- Store all times as UTC.
|
||||
- Write one `outlook_runs` row and one `outlooks` row per outlook.
|
||||
- Use `outlook_index` as the zero-based slice index.
|
||||
- Require outlook `ID`, `Provider`, `Product`, `Day`, `OutlookType`, `Label`, `ValidFrom`, `ValidTo`, `IssuedAt`, `ExpiresAt`, and non-empty `Geometry`.
|
||||
- Store compact geometry JSON text in `geometry_json` using the existing compact JSON helper or a similar helper.
|
||||
- Preserve all outlook polygons, not only those containing the configured point.
|
||||
|
||||
Tests:
|
||||
|
||||
- Schema includes `outlook_runs` and `outlooks`.
|
||||
- Schema includes required columns and indexes.
|
||||
- Mapper writes one run row plus one row per outlook.
|
||||
- Mapper stores UTC times.
|
||||
- Mapper stores compact geometry JSON.
|
||||
- Mapper preserves `contains_location` false as false, not nil.
|
||||
- Mapper rejects missing run `asOf`.
|
||||
- Mapper rejects missing required outlook times.
|
||||
- Mapper rejects empty geometry.
|
||||
- Mapper ignores unrelated schemas unchanged.
|
||||
|
||||
Verification:
|
||||
|
||||
```sh
|
||||
go test ./internal/sinks/postgres
|
||||
```
|
||||
|
||||
Stage completion criteria:
|
||||
|
||||
- Postgres schema and mapper tests pass.
|
||||
- No weatherapi read path is added in this stage.
|
||||
|
||||
## Stage 6: Config, Runtime Wiring, And Current-Behavior Docs
|
||||
|
||||
Goal: expose the completed source/normalizer/sink behavior in maintained configuration and documentation after implementation exists.
|
||||
|
||||
Files to update:
|
||||
|
||||
- `cmd/weatherfeeder/config.yml`
|
||||
- `cmd/weatherfeeder/main_test.go`, if config load expectations need updates
|
||||
- `docs/config.md`
|
||||
- `docs/integrations/events.md`
|
||||
- `docs/integrations/postgres.md`
|
||||
- `docs/internal/sources.md`
|
||||
- `docs/internal/normalizers.md`
|
||||
- `docs/internal/postgres-sink.md`
|
||||
- `README.md`
|
||||
|
||||
Config changes:
|
||||
|
||||
- Add sample source `SPCConvectiveOutlookSTL` with driver `spc_convective_outlook`, kind `outlook`, `every: 30m`, latitude, longitude, location metadata, and `user_agent`.
|
||||
- Add `outlook` route examples for stdout, NATS, and Postgres where the existing config style includes kind lists.
|
||||
- Keep optional RSS disabled in sample config unless there is a specific operator reason to include it.
|
||||
|
||||
Docs changes:
|
||||
|
||||
- Update current-behavior docs only after the code for that behavior exists.
|
||||
- `docs/config.md` should document required and optional source params.
|
||||
- `docs/integrations/events.md` should document `weather.outlook.v1`, including field definitions, required fields, optional fields, geometry semantics, and `containsLocation` semantics.
|
||||
- `docs/integrations/postgres.md` should document `outlook_runs` and `outlooks`.
|
||||
- Internal docs should explain provider boundaries, print-page discussion parsing, and point-in-polygon behavior where useful for maintainers.
|
||||
- `README.md` should only briefly list SPC convective outlook support and link to canonical docs.
|
||||
- Do not document weatherapi endpoints in weatherfeeder current-behavior docs.
|
||||
|
||||
Tests:
|
||||
|
||||
- Existing config load tests pass.
|
||||
- Add config test coverage if the sample config is expected to build scheduler jobs for the new source.
|
||||
- Documentation examples should use the real driver name and event kind.
|
||||
|
||||
Verification:
|
||||
|
||||
```sh
|
||||
go test ./cmd/weatherfeeder ./internal/sources ./internal/normalizers ./internal/sinks/postgres
|
||||
```
|
||||
|
||||
Stage completion criteria:
|
||||
|
||||
- Sample config remains loadable.
|
||||
- Current-behavior docs match implemented code.
|
||||
- No roadmap-only claims leak into non-roadmap docs beyond the implemented behavior.
|
||||
|
||||
## Stage 7: Full Verification And Cleanup
|
||||
|
||||
Goal: validate the complete feature and remove implementation-only rough edges.
|
||||
|
||||
Run focused tests:
|
||||
|
||||
```sh
|
||||
go test ./internal/providers/spc ./internal/geo ./internal/sources ./internal/normalizers/... ./internal/sinks/postgres ./model ./cmd/weatherfeeder
|
||||
```
|
||||
|
||||
Run full tests:
|
||||
|
||||
```sh
|
||||
go test ./...
|
||||
```
|
||||
|
||||
Manual review checklist:
|
||||
|
||||
- `standards/schema.go` contains raw and canonical SPC schema constants.
|
||||
- `model/outlook.go` uses stable JSON tags and no provider-specific names except canonical strings.
|
||||
- Source registry includes `spc_convective_outlook`.
|
||||
- Normalizer registry includes the SPC normalizer.
|
||||
- Source tests do not use live SPC services.
|
||||
- Normalizer tests use fixtures and cover discussion parsing, geometry, and timestamp failures.
|
||||
- Postgres tests cover schema shape and mapper validation.
|
||||
- Docs follow `docs/policy/documentation.md` and use canonical homes.
|
||||
- No weatherapi files are changed.
|
||||
- No broad dependency was added without clear justification.
|
||||
|
||||
Acceptance criteria:
|
||||
|
||||
- A configured SPC source emits one raw bundled `outlook` event per changed complete product snapshot.
|
||||
- Normalization produces `weather.outlook.v1` events.
|
||||
- All Day 1-3 categorical, tornado, hail, and wind products are represented.
|
||||
- Each outlook includes `containsLocation` for the configured latitude and longitude.
|
||||
- Each outlook preserves compact GeoJSON geometry.
|
||||
- Day 1-3 print-page discussion text is preserved in canonical headline, summary, and discussion fields where parseable.
|
||||
- RSS metadata is optional and supplemental only.
|
||||
- Postgres sink persists outlook runs and outlook rows.
|
||||
- Sample config and current-behavior docs describe the implemented kind, driver, schema, and storage contract.
|
||||
|
||||
## Suggested Prompt Boundaries
|
||||
|
||||
This feature is too broad for a single safe implementation prompt. Use these implementation prompts in order:
|
||||
|
||||
1. Implement Stage 1 provider helpers and fixtures only.
|
||||
2. Implement Stage 2 geometry helper only.
|
||||
3. Implement Stage 3 source driver and raw schema only.
|
||||
4. Implement Stage 4 canonical model and normalizer only.
|
||||
5. Implement Stage 5 Postgres sink mapping only.
|
||||
6. Implement Stage 6 config and current-behavior documentation only.
|
||||
7. Run Stage 7 verification and perform targeted fixes only.
|
||||
|
||||
Each prompt should run its stage-specific tests before moving on. Do not proceed to the next stage with failing tests unless the failure is unrelated and explicitly documented.
|
||||
87
docs/roadmap/outlook-schema-transition.md
Normal file
87
docs/roadmap/outlook-schema-transition.md
Normal file
@@ -0,0 +1,87 @@
|
||||
# SPC Outlook Postgres Schema Transition
|
||||
|
||||
## Purpose
|
||||
|
||||
This document describes how to reset existing Postgres outlook tables from the
|
||||
`weather.outlook.v1` storage shape to the `weather.outlook.v2` compatible
|
||||
storage shape.
|
||||
|
||||
Updated `weatherfeeder` versions create outlook tables with run-level
|
||||
discussion storage. Existing databases that already contain the old outlook
|
||||
table family need a manual reset because the Postgres sink creates tables with
|
||||
`CREATE TABLE IF NOT EXISTS`.
|
||||
|
||||
## Scope
|
||||
|
||||
This reset drops only the outlook table family and lets updated `weatherfeeder`
|
||||
recreate it:
|
||||
|
||||
- `outlook_discussions`
|
||||
- `outlooks`
|
||||
- `outlook_runs`
|
||||
|
||||
Other weather tables are not affected.
|
||||
|
||||
## Warning
|
||||
|
||||
These commands delete stored SPC outlook history. Existing `weather.outlook.v1`
|
||||
outlook rows are intentionally removed. Downstream readers should be updated
|
||||
intentionally for the new outlook shape.
|
||||
|
||||
## Deployment Order
|
||||
|
||||
1. Stop `weatherfeeder`.
|
||||
2. Drop the existing outlook tables.
|
||||
3. Deploy updated `weatherfeeder`.
|
||||
4. Start `weatherfeeder` so the Postgres sink recreates the new outlook tables.
|
||||
5. Deploy updated downstream consumers such as `weatherapi`.
|
||||
|
||||
## Reset SQL
|
||||
|
||||
```sql
|
||||
DROP TABLE IF EXISTS outlook_discussions;
|
||||
DROP TABLE IF EXISTS outlooks;
|
||||
DROP TABLE IF EXISTS outlook_runs;
|
||||
```
|
||||
|
||||
## Verification SQL
|
||||
|
||||
Before or after the updated daemon starts, this query shows which outlook tables
|
||||
exist:
|
||||
|
||||
```sql
|
||||
SELECT table_name
|
||||
FROM information_schema.tables
|
||||
WHERE table_name IN ('outlook_runs', 'outlooks', 'outlook_discussions')
|
||||
ORDER BY table_name;
|
||||
```
|
||||
|
||||
After the updated daemon has started and recreated the tables, verify the new
|
||||
run column:
|
||||
|
||||
```sql
|
||||
SELECT column_name, is_nullable, data_type
|
||||
FROM information_schema.columns
|
||||
WHERE table_name = 'outlook_runs'
|
||||
AND column_name = 'discussion_count';
|
||||
```
|
||||
|
||||
Verify the discussion table indexes:
|
||||
|
||||
```sql
|
||||
SELECT indexname
|
||||
FROM pg_indexes
|
||||
WHERE tablename = 'outlook_discussions'
|
||||
ORDER BY indexname;
|
||||
```
|
||||
|
||||
Verify that legacy polygon-level prose columns are gone from `outlooks`:
|
||||
|
||||
```sql
|
||||
SELECT column_name
|
||||
FROM information_schema.columns
|
||||
WHERE table_name = 'outlooks'
|
||||
AND column_name IN ('headline', 'summary', 'discussion');
|
||||
```
|
||||
|
||||
The final query should return zero rows.
|
||||
@@ -1,349 +0,0 @@
|
||||
# SPC Convective Outlook Support Roadmap
|
||||
|
||||
## Summary
|
||||
|
||||
Add `weatherfeeder` support for Storm Prediction Center convective outlooks as a new canonical outlook domain. The feature should poll SPC GeoJSON outlook products, enrich them with Day 1-3 print-page discussion text, compute whether the configured forecast point is inside each active outlook polygon, normalize the result into a provider-independent schema, and persist it through existing sinks.
|
||||
|
||||
This is a new domain, not an extension of `weather.alert.v1`. SPC outlooks describe probabilistic/categorical risk areas over a validity window; alerts describe active actionable hazard messages.
|
||||
|
||||
## Public Contract
|
||||
|
||||
Add schema constants:
|
||||
|
||||
- Raw schema: `raw.spc.convective_outlook.v1`
|
||||
- Canonical schema: `weather.outlook.v1`
|
||||
|
||||
Add source driver:
|
||||
|
||||
- `spc_convective_outlook`
|
||||
|
||||
Add event kind:
|
||||
|
||||
- `outlook`
|
||||
|
||||
Add canonical model types:
|
||||
|
||||
- `model.WeatherOutlookRun`
|
||||
- `model.WeatherOutlook`
|
||||
|
||||
Recommended canonical run fields:
|
||||
|
||||
- `locationId`, `locationName`
|
||||
- `latitude`, `longitude`
|
||||
- `asOf`
|
||||
- `issuedAt`
|
||||
- `outlooks`
|
||||
|
||||
Recommended canonical outlook fields:
|
||||
|
||||
- `id`
|
||||
- `provider`
|
||||
- `product`
|
||||
- `day`
|
||||
- `outlookType`
|
||||
- `label`
|
||||
- `labelText`
|
||||
- `severityRank`
|
||||
- `validFrom`
|
||||
- `validTo`
|
||||
- `issuedAt`
|
||||
- `expiresAt`
|
||||
- `forecaster`
|
||||
- `headline`
|
||||
- `summary`
|
||||
- `discussion`
|
||||
- `sourceUrl`
|
||||
- `imageUrl`
|
||||
- `containsLocation`
|
||||
- `geometry`
|
||||
|
||||
Contract defaults:
|
||||
|
||||
- `product` should be `convective`.
|
||||
- `outlookType` should be one of `categorical`, `tornado`, `hail`, `wind`.
|
||||
- `day` should be `1`, `2`, or `3` for this first implementation.
|
||||
- `containsLocation` is computed against configured forecast coordinates.
|
||||
- `geometry` should preserve compact GeoJSON geometry for auditability and future API use.
|
||||
- Day 4-8 outlooks are out of scope for this first implementation. They have different product semantics and should be added in a later schema-compatible extension only after Day 1-3 support is proven.
|
||||
|
||||
## Source Scope
|
||||
|
||||
The source should fetch a bundle of SPC products in one poll cycle and emit one raw event containing fetched GeoJSON products, fetched Day 1-3 print pages, optional RSS metadata, configured point metadata, and per-product fetch metadata. The source may decode minimal timestamps for event identity/effective time, but provider-to-canonical mapping belongs in the normalizer.
|
||||
|
||||
Poll these GeoJSON URLs:
|
||||
|
||||
- `https://www.spc.noaa.gov/products/outlook/day1otlk_cat.nolyr.geojson`
|
||||
- `https://www.spc.noaa.gov/products/outlook/day1otlk_torn.nolyr.geojson`
|
||||
- `https://www.spc.noaa.gov/products/outlook/day1otlk_hail.nolyr.geojson`
|
||||
- `https://www.spc.noaa.gov/products/outlook/day1otlk_wind.nolyr.geojson`
|
||||
- `https://www.spc.noaa.gov/products/outlook/day2otlk_cat.nolyr.geojson`
|
||||
- `https://www.spc.noaa.gov/products/outlook/day2otlk_torn.nolyr.geojson`
|
||||
- `https://www.spc.noaa.gov/products/outlook/day2otlk_hail.nolyr.geojson`
|
||||
- `https://www.spc.noaa.gov/products/outlook/day2otlk_wind.nolyr.geojson`
|
||||
- `https://www.spc.noaa.gov/products/outlook/day3otlk_cat.nolyr.geojson`
|
||||
- `https://www.spc.noaa.gov/products/outlook/day3otlk_torn.nolyr.geojson`
|
||||
- `https://www.spc.noaa.gov/products/outlook/day3otlk_hail.nolyr.geojson`
|
||||
- `https://www.spc.noaa.gov/products/outlook/day3otlk_wind.nolyr.geojson`
|
||||
|
||||
Implementation note: the initial candidate list duplicated Day 3 wind and omitted Day 2 wind. Use the corrected 12-product matrix above.
|
||||
|
||||
Poll these print-page discussion URLs:
|
||||
|
||||
- `https://www.spc.noaa.gov/products/outlook/day1otlk_prt.html`
|
||||
- `https://www.spc.noaa.gov/products/outlook/day2otlk_prt.html`
|
||||
- `https://www.spc.noaa.gov/products/outlook/day3otlk_prt.html`
|
||||
|
||||
Print-page usage:
|
||||
|
||||
- Use print pages as the authoritative source for Day 1-3 outlook discussion text.
|
||||
- Preserve source URL, page title/header where available, page-level `Updated:` timestamp where parseable, and the raw HTML or extracted `<pre>` product block in the raw bundle.
|
||||
- Extract canonical `headline`, `summary`, `discussion`, and discussion metadata in the normalizer or shared SPC provider helper.
|
||||
- Treat the GeoJSON files, not print pages, as authoritative for polygons and validity windows.
|
||||
- Ignore page risk tables and image assets for v1 unless they are needed to support discussion parsing.
|
||||
|
||||
Optionally poll:
|
||||
|
||||
- `https://www.spc.noaa.gov/products/spcacrss.xml`
|
||||
|
||||
RSS usage:
|
||||
|
||||
- Use RSS as supplemental discovery or health metadata, not as the primary discussion or geometry source.
|
||||
- Preserve item title, link, description text, pubDate, and guid where useful.
|
||||
- Do not depend on RSS completeness for Day 1-3 discussion text. The RSS feed can omit current outlook products, including a current Day 2 discussion.
|
||||
- Do not attempt to parse polygons from RSS HTML.
|
||||
|
||||
Polling cadence:
|
||||
|
||||
- Use a default cadence of `30m`, not daily/twice-daily. SPC current outlook files can update several times per day, and a 30-minute poll with HTTP caching is low cost and less likely to miss updates.
|
||||
- Rely on ETag/Last-Modified handling from the HTTP source layer where available.
|
||||
- Keep the cadence configurable via `every`.
|
||||
|
||||
Recommended config shape:
|
||||
|
||||
```yaml
|
||||
- name: SPCConvectiveOutlookSTL
|
||||
mode: poll
|
||||
kinds: ["outlook"]
|
||||
driver: spc_convective_outlook
|
||||
every: 30m
|
||||
params:
|
||||
latitude: 38.6239
|
||||
longitude: -90.3571
|
||||
location_id: "stl"
|
||||
location_name: "St. Louis, MO"
|
||||
user_agent: "HomeOps (eric@maximumdirect.net)"
|
||||
```
|
||||
|
||||
The source should own default SPC URLs, with optional params to override GeoJSON URLs, print-page URLs, and the RSS URL for tests or future provider changes.
|
||||
|
||||
## Implementation Stages
|
||||
|
||||
### Stage 1: Raw Source and Schema
|
||||
|
||||
- Add schema constants.
|
||||
- Add `internal/sources/spc` package.
|
||||
- Implement `spc_convective_outlook` as a poll source.
|
||||
- Fetch all configured GeoJSON products and Day 1-3 print pages during a poll cycle.
|
||||
- Fetch RSS only when configured or retained as an optional supplemental source.
|
||||
- Emit one raw event with kind `outlook` and schema `raw.spc.convective_outlook.v1`.
|
||||
- Use the latest valid `ISSUE_ISO`, print-page `Updated:` timestamp, RSS `lastBuildDate`, or fetch time for `effectiveAt`, in that order.
|
||||
- Preserve partial fetch errors as source errors for the first implementation; do not emit incomplete outlook bundles unless a later explicit degraded-mode policy is added.
|
||||
- Keep source tests deterministic with local test servers and fixtures; do not call live SPC services from unit tests.
|
||||
|
||||
Tests:
|
||||
|
||||
- source driver builds as a `PollSource`
|
||||
- source advertises kind `outlook`
|
||||
- source emits one raw bundled event
|
||||
- source chooses expected effective time
|
||||
- source uses configured location metadata
|
||||
- source fails clearly on missing latitude/longitude
|
||||
- source fails clearly when a configured GeoJSON or print-page fetch fails
|
||||
|
||||
### Stage 2: GeoJSON and Spatial Matching
|
||||
|
||||
- Add an internal geometry helper package, for example `internal/geo`.
|
||||
- Support GeoJSON `Polygon` and `MultiPolygon`.
|
||||
- Implement point-in-polygon with ring support:
|
||||
- first ring is the exterior
|
||||
- subsequent rings are holes
|
||||
- boundary counts as inside
|
||||
- GeoJSON coordinate order is `[longitude, latitude]`
|
||||
- Use planar ray casting for this operational point-in-polygon check.
|
||||
|
||||
Tests:
|
||||
|
||||
- point inside polygon
|
||||
- point outside polygon
|
||||
- point on polygon boundary
|
||||
- point inside a hole is outside
|
||||
- point inside one multipolygon member is inside
|
||||
- longitude/latitude order regression test
|
||||
|
||||
### Stage 3: Normalizer and Canonical Mapping
|
||||
|
||||
- Add `model.WeatherOutlookRun` and `model.WeatherOutlook`.
|
||||
- Add `SPCConvectiveOutlookNormalizer`.
|
||||
- Match only `raw.spc.convective_outlook.v1`.
|
||||
- Decode the raw source bundle.
|
||||
- Map each GeoJSON feature to one `WeatherOutlook`.
|
||||
- Preserve SPC feature order within each product, then order products by day and type.
|
||||
- Map properties:
|
||||
- `VALID_ISO` -> `validFrom`
|
||||
- `EXPIRE_ISO` -> `validTo` and `expiresAt`
|
||||
- `ISSUE_ISO` -> `issuedAt`
|
||||
- `FORECASTER` -> `forecaster`
|
||||
- `LABEL` -> `label`
|
||||
- `LABEL2` -> `labelText`
|
||||
- `DN` -> `severityRank`
|
||||
- Derive:
|
||||
- `day` from product key or URL
|
||||
- `outlookType` from product key or URL
|
||||
- `id` from day, type, label, issuedAt, validFrom, and feature index
|
||||
- `containsLocation` from configured point and GeoJSON geometry
|
||||
- Enrich narrative fields from the matching Day 1-3 print page:
|
||||
- parse the main product text from the `<pre>` block
|
||||
- strip embedded tags and scripts before text parsing
|
||||
- preserve line breaks in discussion text
|
||||
- parse page/title metadata, product title, issued line, valid line, correction markers such as `CORR 1`, forecaster, summary, and full discussion where practical
|
||||
- use print-page text for canonical `headline`, `summary`, and `discussion`
|
||||
- Use RSS only as optional supplemental metadata when present; missing or incomplete RSS must not prevent GeoJSON plus print-page normalization.
|
||||
- Set run `asOf` to the latest valid `issuedAt` across features, falling back to print-page `Updated:` timestamps, RSS `lastBuildDate`, then input event time.
|
||||
- Set run `issuedAt` to the latest valid `issuedAt` across features.
|
||||
- Set output event `effectiveAt` to run `asOf`.
|
||||
|
||||
Tests:
|
||||
|
||||
- normalizer routes only raw SPC schema
|
||||
- categorical and probabilistic products map expected fields
|
||||
- `containsLocation` is true for a known point inside a fixture polygon
|
||||
- `containsLocation` is false outside
|
||||
- print-page discussion text maps to canonical headline, summary, and discussion
|
||||
- correction markers such as `CORR 1` are preserved where present
|
||||
- missing optional RSS still permits GeoJSON and print-page normalization
|
||||
- malformed required GeoJSON timestamps fail with useful context
|
||||
- canonical JSON shape does not expose raw-provider-only bundle internals
|
||||
|
||||
### Stage 4: Postgres Sink
|
||||
|
||||
Add tables:
|
||||
|
||||
- `outlook_runs`
|
||||
- `outlooks`
|
||||
|
||||
Suggested `outlook_runs` columns:
|
||||
|
||||
- event envelope columns
|
||||
- `location_id`
|
||||
- `location_name`
|
||||
- `latitude`
|
||||
- `longitude`
|
||||
- `as_of`
|
||||
- `issued_at`
|
||||
- `outlook_count`
|
||||
|
||||
Suggested `outlooks` columns:
|
||||
|
||||
- `run_event_id`
|
||||
- `outlook_index`
|
||||
- `as_of`
|
||||
- `product`
|
||||
- `day`
|
||||
- `outlook_type`
|
||||
- `label`
|
||||
- `label_text`
|
||||
- `severity_rank`
|
||||
- `valid_from`
|
||||
- `valid_to`
|
||||
- `issued_at`
|
||||
- `expires_at`
|
||||
- `forecaster`
|
||||
- `headline`
|
||||
- `summary`
|
||||
- `discussion`
|
||||
- `source_url`
|
||||
- `image_url`
|
||||
- `contains_location`
|
||||
- `geometry_json`
|
||||
|
||||
Suggested indexes:
|
||||
|
||||
- `outlook_runs(location_id, as_of)`
|
||||
- `outlooks(contains_location, valid_from, valid_to)`
|
||||
- `outlooks(day, outlook_type, label)`
|
||||
- `outlooks(valid_from, valid_to)`
|
||||
|
||||
Mapping rules:
|
||||
|
||||
- Store `geometry` as compact GeoJSON text in `geometry_json`.
|
||||
- Require run `asOf`.
|
||||
- Require outlook `validFrom`, `validTo`, `issuedAt`, `day`, `outlookType`, and `label`.
|
||||
- Preserve all outlook polygons, not only polygons containing the configured point.
|
||||
|
||||
Tests:
|
||||
|
||||
- schema includes outlook tables and indexes
|
||||
- mapper writes one run row plus one row per outlook
|
||||
- mapper stores compact geometry JSON
|
||||
- mapper rejects missing required run/outlook fields
|
||||
|
||||
### Stage 5: Config and Documentation
|
||||
|
||||
- Update sample config with `SPCConvectiveOutlookSTL`.
|
||||
- Add `outlook` to route examples.
|
||||
- Update current-behavior docs in the implementation change, not before behavior exists.
|
||||
- Update `docs/config.md` with source params.
|
||||
- Update `docs/integrations/events.md` with `weather.outlook.v1`.
|
||||
- Update `docs/integrations/postgres.md` with outlook table contract.
|
||||
- Update relevant internal docs under `docs/internal/` if source, normalizer, or sink behavior needs maintainer context.
|
||||
- Update `README.md` only with brief implemented-provider capability and links to canonical docs.
|
||||
|
||||
Docs should clearly state:
|
||||
|
||||
- Print pages are used for Day 1-3 narrative/discussion text.
|
||||
- RSS, if retained, is supplemental and not required for complete discussion coverage.
|
||||
- GeoJSON is used for polygons, validity windows, and point matching.
|
||||
- `containsLocation` is computed by weatherfeeder at ingestion time.
|
||||
- Geometry is stored for downstream audit/display.
|
||||
- Day 4-8 outlooks are not included in the initial implementation.
|
||||
|
||||
### Stage 6: Weatherapi Follow-Up
|
||||
|
||||
Do not include weatherapi changes in the first weatherfeeder implementation unless explicitly requested.
|
||||
|
||||
Likely future weatherapi endpoints:
|
||||
|
||||
- `GET /outlooks/convective`
|
||||
- `GET /outlooks/convective/active`
|
||||
- `GET /outlooks/convective/location`
|
||||
|
||||
Recommended API behavior:
|
||||
|
||||
- latest run by default
|
||||
- active outlooks filtered by current time and `containsLocation=true`
|
||||
- optional filter query params for `day`, `outlookType`, and `containsLocation`
|
||||
|
||||
## Verification Commands
|
||||
|
||||
Run focused tests:
|
||||
|
||||
```sh
|
||||
go test ./internal/sources ./internal/normalizers/... ./internal/sinks/postgres ./model
|
||||
```
|
||||
|
||||
Run full weatherfeeder tests:
|
||||
|
||||
```sh
|
||||
go test ./...
|
||||
```
|
||||
|
||||
## Acceptance Criteria
|
||||
|
||||
- A configured SPC source emits raw outlook bundles.
|
||||
- Normalization produces `weather.outlook.v1` events.
|
||||
- Each outlook indicates whether the configured forecast point is inside its polygon.
|
||||
- All current Day 1-3 categorical/tornado/hail/wind products are represented.
|
||||
- Day 1-3 print-page discussion text is preserved where available.
|
||||
- RSS metadata is optional and supplemental if retained.
|
||||
- Postgres sink persists outlook runs and outlook rows.
|
||||
- Sample config and public docs describe the new kind, driver, schema, and storage contract.
|
||||
@@ -58,7 +58,8 @@ Diagnostic: compare the configured driver with the source driver table in
|
||||
Safe fix: correct the driver name. Current drivers include `nws_observation`,
|
||||
`nws_alerts`, `nws_forecast_hourly`, `nws_forecast_narrative`,
|
||||
`nws_forecast_discussion`, `nws_weatherstories`, `openmeteo_observation`,
|
||||
`openmeteo_forecast`, and `openweather_observation`.
|
||||
`openmeteo_forecast`, `openweather_observation`, and
|
||||
`spc_convective_outlook`.
|
||||
|
||||
## `unknown sink driver`
|
||||
|
||||
@@ -94,6 +95,20 @@ Diagnostic: inspect the named source in the error and check its `params`.
|
||||
Safe fix: add non-empty `url` and `user_agent` values. See
|
||||
[HTTP source params](config.md#http-source-params).
|
||||
|
||||
SPC convective outlook sources do not use `params.url`; they require
|
||||
`latitude`, `longitude`, and `user_agent`. See
|
||||
[SPC convective outlook params](config.md#spc-convective-outlook-params).
|
||||
|
||||
## `params.latitude is required` Or `params.longitude is required`
|
||||
|
||||
Symptom: startup exits for an `spc_convective_outlook` source.
|
||||
|
||||
Likely cause: the source is missing one of the configured point coordinates.
|
||||
|
||||
Diagnostic: inspect the named SPC source in the error and check its `params`.
|
||||
|
||||
Safe fix: add numeric `latitude` and `longitude` values in decimal degrees.
|
||||
|
||||
## `url must include units=metric`
|
||||
|
||||
Symptom: startup exits for an `openweather_observation` source.
|
||||
|
||||
@@ -18,6 +18,18 @@ sources:
|
||||
url: "https://api.weather.gov/alerts?point=38.6239,-90.3571&limit=20"
|
||||
user_agent: "weatherfeeder example (operator@example.com)"
|
||||
|
||||
- name: SPCConvectiveOutlookSTL
|
||||
mode: poll
|
||||
kinds: ["outlook"]
|
||||
driver: spc_convective_outlook
|
||||
every: 30m
|
||||
params:
|
||||
latitude: 38.6239
|
||||
longitude: -90.3571
|
||||
location_id: "stl"
|
||||
location_name: "St. Louis, MO"
|
||||
user_agent: "weatherfeeder example (operator@example.com)"
|
||||
|
||||
sinks:
|
||||
- name: nats_weather
|
||||
driver: nats
|
||||
@@ -27,4 +39,4 @@ sinks:
|
||||
|
||||
routes:
|
||||
- sink: nats_weather
|
||||
kinds: ["observation", "alert"]
|
||||
kinds: ["observation", "alert", "outlook"]
|
||||
|
||||
@@ -18,6 +18,18 @@ sources:
|
||||
url: "https://api.open-meteo.com/v1/forecast?latitude=38.6239&longitude=-90.3571&hourly=temperature_2m,relative_humidity_2m,dew_point_2m,apparent_temperature,precipitation_probability,precipitation,snowfall,weather_code,surface_pressure,wind_speed_10m,wind_direction_10m&forecast_days=3"
|
||||
user_agent: "weatherfeeder example (operator@example.com)"
|
||||
|
||||
- name: SPCConvectiveOutlookSTL
|
||||
mode: poll
|
||||
kinds: ["outlook"]
|
||||
driver: spc_convective_outlook
|
||||
every: 30m
|
||||
params:
|
||||
latitude: 38.6239
|
||||
longitude: -90.3571
|
||||
location_id: "stl"
|
||||
location_name: "St. Louis, MO"
|
||||
user_agent: "weatherfeeder example (operator@example.com)"
|
||||
|
||||
sinks:
|
||||
- name: pg_weather
|
||||
driver: postgres
|
||||
@@ -29,4 +41,4 @@ sinks:
|
||||
|
||||
routes:
|
||||
- sink: pg_weather
|
||||
kinds: ["observation", "forecast"]
|
||||
kinds: ["observation", "forecast", "outlook"]
|
||||
|
||||
107
internal/geo/geojson.go
Normal file
107
internal/geo/geojson.go
Normal file
@@ -0,0 +1,107 @@
|
||||
package geo
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
type geometry struct {
|
||||
Type string `json:"type"`
|
||||
Coordinates json.RawMessage `json:"coordinates"`
|
||||
}
|
||||
|
||||
// ContainsPoint reports whether a GeoJSON Polygon or MultiPolygon contains p.
|
||||
// GeoJSON coordinate order is [longitude, latitude].
|
||||
func ContainsPoint(raw []byte, p Point) (bool, error) {
|
||||
if len(raw) == 0 {
|
||||
return false, fmt.Errorf("geojson geometry is empty")
|
||||
}
|
||||
|
||||
var geom geometry
|
||||
if err := json.Unmarshal(raw, &geom); err != nil {
|
||||
return false, fmt.Errorf("decode geojson geometry: %w", err)
|
||||
}
|
||||
|
||||
switch geom.Type {
|
||||
case "Polygon":
|
||||
polygon, err := decodePolygon(geom.Coordinates)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("decode polygon: %w", err)
|
||||
}
|
||||
return polygonContainsPoint(polygon, p), nil
|
||||
case "MultiPolygon":
|
||||
multiPolygon, err := decodeMultiPolygon(geom.Coordinates)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("decode multipolygon: %w", err)
|
||||
}
|
||||
for _, polygon := range multiPolygon {
|
||||
if polygonContainsPoint(polygon, p) {
|
||||
return true, nil
|
||||
}
|
||||
}
|
||||
return false, nil
|
||||
case "":
|
||||
return false, fmt.Errorf("geojson geometry type is required")
|
||||
default:
|
||||
return false, fmt.Errorf("unsupported geojson geometry type %q", geom.Type)
|
||||
}
|
||||
}
|
||||
|
||||
func decodePolygon(raw json.RawMessage) (Polygon, error) {
|
||||
var coords [][][]float64
|
||||
if err := json.Unmarshal(raw, &coords); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return polygonFromCoordinates(coords)
|
||||
}
|
||||
|
||||
func decodeMultiPolygon(raw json.RawMessage) ([]Polygon, error) {
|
||||
var coords [][][][]float64
|
||||
if err := json.Unmarshal(raw, &coords); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(coords) == 0 {
|
||||
return nil, fmt.Errorf("multipolygon has no polygons")
|
||||
}
|
||||
|
||||
out := make([]Polygon, 0, len(coords))
|
||||
for i, polygonCoords := range coords {
|
||||
polygon, err := polygonFromCoordinates(polygonCoords)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("polygons[%d]: %w", i, err)
|
||||
}
|
||||
out = append(out, polygon)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func polygonFromCoordinates(coords [][][]float64) (Polygon, error) {
|
||||
if len(coords) == 0 {
|
||||
return nil, fmt.Errorf("polygon has no rings")
|
||||
}
|
||||
|
||||
polygon := make(Polygon, 0, len(coords))
|
||||
for i, ringCoords := range coords {
|
||||
ring, err := ringFromCoordinates(ringCoords)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("rings[%d]: %w", i, err)
|
||||
}
|
||||
polygon = append(polygon, ring)
|
||||
}
|
||||
return polygon, nil
|
||||
}
|
||||
|
||||
func ringFromCoordinates(coords [][]float64) (Ring, error) {
|
||||
if len(coords) == 0 {
|
||||
return nil, fmt.Errorf("ring has no points")
|
||||
}
|
||||
|
||||
ring := make(Ring, 0, len(coords))
|
||||
for i, pair := range coords {
|
||||
if len(pair) < 2 {
|
||||
return nil, fmt.Errorf("points[%d] has %d values, need longitude and latitude", i, len(pair))
|
||||
}
|
||||
ring = append(ring, Point{Longitude: pair[0], Latitude: pair[1]})
|
||||
}
|
||||
return ring, nil
|
||||
}
|
||||
105
internal/geo/point.go
Normal file
105
internal/geo/point.go
Normal file
@@ -0,0 +1,105 @@
|
||||
package geo
|
||||
|
||||
import "math"
|
||||
|
||||
const epsilon = 1e-9
|
||||
|
||||
// Point is a geographic coordinate in decimal degrees.
|
||||
type Point struct {
|
||||
Longitude float64
|
||||
Latitude float64
|
||||
}
|
||||
|
||||
// Ring is one GeoJSON linear ring.
|
||||
type Ring []Point
|
||||
|
||||
// Polygon is a GeoJSON polygon. The first ring is the exterior ring; subsequent
|
||||
// rings are holes.
|
||||
type Polygon []Ring
|
||||
|
||||
func polygonContainsPoint(polygon Polygon, p Point) bool {
|
||||
if len(polygon) == 0 {
|
||||
return false
|
||||
}
|
||||
if pointOnRing(polygon[0], p) {
|
||||
return true
|
||||
}
|
||||
if !ringContainsPoint(polygon[0], p) {
|
||||
return false
|
||||
}
|
||||
for _, hole := range polygon[1:] {
|
||||
if pointOnRing(hole, p) {
|
||||
return true
|
||||
}
|
||||
if ringContainsPoint(hole, p) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func ringContainsPoint(ring Ring, p Point) bool {
|
||||
inside := false
|
||||
n := len(ring)
|
||||
if n == 0 {
|
||||
return false
|
||||
}
|
||||
|
||||
for i, j := 0, n-1; i < n; j, i = i, i+1 {
|
||||
a := ring[j]
|
||||
b := ring[i]
|
||||
if pointOnSegment(p, a, b) {
|
||||
return true
|
||||
}
|
||||
|
||||
intersects := (a.Latitude > p.Latitude) != (b.Latitude > p.Latitude)
|
||||
if intersects {
|
||||
x := (b.Longitude-a.Longitude)*(p.Latitude-a.Latitude)/(b.Latitude-a.Latitude) + a.Longitude
|
||||
if almostEqual(x, p.Longitude) {
|
||||
return true
|
||||
}
|
||||
if x > p.Longitude {
|
||||
inside = !inside
|
||||
}
|
||||
}
|
||||
}
|
||||
return inside
|
||||
}
|
||||
|
||||
func pointOnRing(ring Ring, p Point) bool {
|
||||
n := len(ring)
|
||||
if n == 0 {
|
||||
return false
|
||||
}
|
||||
for i, j := 0, n-1; i < n; j, i = i, i+1 {
|
||||
if pointOnSegment(p, ring[j], ring[i]) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func pointOnSegment(p, a, b Point) bool {
|
||||
cross := (p.Latitude-a.Latitude)*(b.Longitude-a.Longitude) - (p.Longitude-a.Longitude)*(b.Latitude-a.Latitude)
|
||||
if math.Abs(cross) > epsilon {
|
||||
return false
|
||||
}
|
||||
|
||||
minLon, maxLon := minMax(a.Longitude, b.Longitude)
|
||||
minLat, maxLat := minMax(a.Latitude, b.Latitude)
|
||||
return p.Longitude >= minLon-epsilon &&
|
||||
p.Longitude <= maxLon+epsilon &&
|
||||
p.Latitude >= minLat-epsilon &&
|
||||
p.Latitude <= maxLat+epsilon
|
||||
}
|
||||
|
||||
func minMax(a, b float64) (float64, float64) {
|
||||
if a < b {
|
||||
return a, b
|
||||
}
|
||||
return b, a
|
||||
}
|
||||
|
||||
func almostEqual(a, b float64) bool {
|
||||
return math.Abs(a-b) <= epsilon
|
||||
}
|
||||
181
internal/geo/point_test.go
Normal file
181
internal/geo/point_test.go
Normal file
@@ -0,0 +1,181 @@
|
||||
package geo
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
const squarePolygon = `{
|
||||
"type": "Polygon",
|
||||
"coordinates": [[
|
||||
[-91.0, 38.0],
|
||||
[-90.0, 38.0],
|
||||
[-90.0, 39.0],
|
||||
[-91.0, 39.0],
|
||||
[-91.0, 38.0]
|
||||
]]
|
||||
}`
|
||||
|
||||
func TestContainsPointInsideSimplePolygon(t *testing.T) {
|
||||
got, err := ContainsPoint([]byte(squarePolygon), Point{Longitude: -90.5, Latitude: 38.5})
|
||||
if err != nil {
|
||||
t.Fatalf("ContainsPoint() error = %v", err)
|
||||
}
|
||||
if !got {
|
||||
t.Fatalf("ContainsPoint() = false, want true")
|
||||
}
|
||||
}
|
||||
|
||||
func TestContainsPointAcceptsRawMessage(t *testing.T) {
|
||||
got, err := ContainsPoint(json.RawMessage(squarePolygon), Point{Longitude: -90.5, Latitude: 38.5})
|
||||
if err != nil {
|
||||
t.Fatalf("ContainsPoint() error = %v", err)
|
||||
}
|
||||
if !got {
|
||||
t.Fatalf("ContainsPoint() = false, want true")
|
||||
}
|
||||
}
|
||||
|
||||
func TestContainsPointOutsideSimplePolygon(t *testing.T) {
|
||||
got, err := ContainsPoint([]byte(squarePolygon), Point{Longitude: -89.5, Latitude: 38.5})
|
||||
if err != nil {
|
||||
t.Fatalf("ContainsPoint() error = %v", err)
|
||||
}
|
||||
if got {
|
||||
t.Fatalf("ContainsPoint() = true, want false")
|
||||
}
|
||||
}
|
||||
|
||||
func TestContainsPointOnBoundary(t *testing.T) {
|
||||
got, err := ContainsPoint([]byte(squarePolygon), Point{Longitude: -91.0, Latitude: 38.5})
|
||||
if err != nil {
|
||||
t.Fatalf("ContainsPoint() error = %v", err)
|
||||
}
|
||||
if !got {
|
||||
t.Fatalf("ContainsPoint() = false, want true")
|
||||
}
|
||||
}
|
||||
|
||||
func TestContainsPointInHoleReturnsFalse(t *testing.T) {
|
||||
const polygonWithHole = `{
|
||||
"type": "Polygon",
|
||||
"coordinates": [
|
||||
[[0,0],[10,0],[10,10],[0,10],[0,0]],
|
||||
[[4,4],[6,4],[6,6],[4,6],[4,4]]
|
||||
]
|
||||
}`
|
||||
|
||||
got, err := ContainsPoint([]byte(polygonWithHole), Point{Longitude: 5, Latitude: 5})
|
||||
if err != nil {
|
||||
t.Fatalf("ContainsPoint() error = %v", err)
|
||||
}
|
||||
if got {
|
||||
t.Fatalf("ContainsPoint() = true, want false")
|
||||
}
|
||||
}
|
||||
|
||||
func TestContainsPointOnHoleBoundaryReturnsTrue(t *testing.T) {
|
||||
const polygonWithHole = `{
|
||||
"type": "Polygon",
|
||||
"coordinates": [
|
||||
[[0,0],[10,0],[10,10],[0,10],[0,0]],
|
||||
[[4,4],[6,4],[6,6],[4,6],[4,4]]
|
||||
]
|
||||
}`
|
||||
|
||||
got, err := ContainsPoint([]byte(polygonWithHole), Point{Longitude: 4, Latitude: 5})
|
||||
if err != nil {
|
||||
t.Fatalf("ContainsPoint() error = %v", err)
|
||||
}
|
||||
if !got {
|
||||
t.Fatalf("ContainsPoint() = false, want true")
|
||||
}
|
||||
}
|
||||
|
||||
func TestContainsPointInsideOneMultiPolygonMember(t *testing.T) {
|
||||
const multiPolygon = `{
|
||||
"type": "MultiPolygon",
|
||||
"coordinates": [
|
||||
[[[0,0],[1,0],[1,1],[0,1],[0,0]]],
|
||||
[[[10,10],[12,10],[12,12],[10,12],[10,10]]]
|
||||
]
|
||||
}`
|
||||
|
||||
got, err := ContainsPoint([]byte(multiPolygon), Point{Longitude: 11, Latitude: 11})
|
||||
if err != nil {
|
||||
t.Fatalf("ContainsPoint() error = %v", err)
|
||||
}
|
||||
if !got {
|
||||
t.Fatalf("ContainsPoint() = false, want true")
|
||||
}
|
||||
}
|
||||
|
||||
func TestContainsPointUsesLongitudeLatitudeOrder(t *testing.T) {
|
||||
const narrowPolygon = `{
|
||||
"type": "Polygon",
|
||||
"coordinates": [[
|
||||
[-91.0, 38.0],
|
||||
[-90.0, 38.0],
|
||||
[-90.0, 39.0],
|
||||
[-91.0, 39.0],
|
||||
[-91.0, 38.0]
|
||||
]]
|
||||
}`
|
||||
|
||||
got, err := ContainsPoint([]byte(narrowPolygon), Point{Longitude: -90.5, Latitude: 38.5})
|
||||
if err != nil {
|
||||
t.Fatalf("ContainsPoint() error = %v", err)
|
||||
}
|
||||
if !got {
|
||||
t.Fatalf("ContainsPoint() = false, want true")
|
||||
}
|
||||
|
||||
got, err = ContainsPoint([]byte(narrowPolygon), Point{Longitude: 38.5, Latitude: -90.5})
|
||||
if err != nil {
|
||||
t.Fatalf("ContainsPoint() reversed error = %v", err)
|
||||
}
|
||||
if got {
|
||||
t.Fatalf("ContainsPoint() with reversed coordinate values = true, want false")
|
||||
}
|
||||
}
|
||||
|
||||
func TestContainsPointUnsupportedGeometryError(t *testing.T) {
|
||||
_, err := ContainsPoint([]byte(`{"type":"Point","coordinates":[-90,38]}`), Point{Longitude: -90, Latitude: 38})
|
||||
if err == nil {
|
||||
t.Fatalf("ContainsPoint() error = nil, want error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), `unsupported geojson geometry type "Point"`) {
|
||||
t.Fatalf("ContainsPoint() error = %q", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestContainsPointInvalidJSONError(t *testing.T) {
|
||||
_, err := ContainsPoint([]byte(`{"type":"Polygon"`), Point{Longitude: -90, Latitude: 38})
|
||||
if err == nil {
|
||||
t.Fatalf("ContainsPoint() error = nil, want error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "decode geojson geometry") {
|
||||
t.Fatalf("ContainsPoint() error = %q", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestContainsPointMalformedCoordinatesError(t *testing.T) {
|
||||
_, err := ContainsPoint([]byte(`{"type":"Polygon","coordinates":[[[1]]]}`), Point{Longitude: 1, Latitude: 1})
|
||||
if err == nil {
|
||||
t.Fatalf("ContainsPoint() error = nil, want error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "need longitude and latitude") {
|
||||
t.Fatalf("ContainsPoint() error = %q", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestContainsPointEmptyRingError(t *testing.T) {
|
||||
_, err := ContainsPoint([]byte(`{"type":"Polygon","coordinates":[[]]}`), Point{Longitude: 1, Latitude: 1})
|
||||
if err == nil {
|
||||
t.Fatalf("ContainsPoint() error = nil, want error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "ring has no points") {
|
||||
t.Fatalf("ContainsPoint() error = %q", err)
|
||||
}
|
||||
}
|
||||
@@ -7,12 +7,14 @@ import (
|
||||
"gitea.maximumdirect.net/ejr/weatherfeeder/internal/normalizers/nws"
|
||||
"gitea.maximumdirect.net/ejr/weatherfeeder/internal/normalizers/openmeteo"
|
||||
"gitea.maximumdirect.net/ejr/weatherfeeder/internal/normalizers/openweather"
|
||||
"gitea.maximumdirect.net/ejr/weatherfeeder/internal/normalizers/spc"
|
||||
)
|
||||
|
||||
var builtinRegistrations = []func([]fknormalize.Normalizer) []fknormalize.Normalizer{
|
||||
nws.Register,
|
||||
openmeteo.Register,
|
||||
openweather.Register,
|
||||
spc.Register,
|
||||
}
|
||||
|
||||
// RegisterBuiltins registers all normalizers shipped with this binary.
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"gitea.maximumdirect.net/ejr/weatherfeeder/internal/normalizers/nws"
|
||||
"gitea.maximumdirect.net/ejr/weatherfeeder/internal/normalizers/openmeteo"
|
||||
"gitea.maximumdirect.net/ejr/weatherfeeder/internal/normalizers/openweather"
|
||||
"gitea.maximumdirect.net/ejr/weatherfeeder/internal/normalizers/spc"
|
||||
)
|
||||
|
||||
func TestRegisterBuiltinsOrder(t *testing.T) {
|
||||
@@ -25,6 +26,7 @@ func TestRegisterBuiltinsOrder(t *testing.T) {
|
||||
openmeteo.ObservationNormalizer{},
|
||||
openmeteo.ForecastNormalizer{},
|
||||
openweather.ObservationNormalizer{},
|
||||
spc.ConvectiveOutlookNormalizer{},
|
||||
}
|
||||
|
||||
if len(got) != len(want) {
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/ejr/feedkit/event"
|
||||
"gitea.maximumdirect.net/ejr/weatherfeeder/standards"
|
||||
)
|
||||
|
||||
func TestFinalizeRoundsWeatherPayloadFloats(t *testing.T) {
|
||||
@@ -14,7 +15,7 @@ func TestFinalizeRoundsWeatherPayloadFloats(t *testing.T) {
|
||||
|
||||
in := event.Event{
|
||||
ID: "evt-1",
|
||||
Kind: event.Kind("observation"),
|
||||
Kind: event.Kind(standards.KindObservation),
|
||||
Source: "source-a",
|
||||
EmittedAt: time.Date(2026, 3, 28, 12, 0, 0, 0, time.UTC),
|
||||
Schema: "raw.example.v1",
|
||||
|
||||
@@ -18,6 +18,18 @@ import (
|
||||
// Errors include a small amount of operation context ("extract payload", "decode raw payload").
|
||||
// Callers typically wrap these with a provider/kind label.
|
||||
func DecodeJSONPayload[T any](in event.Event) (T, error) {
|
||||
var zero T
|
||||
|
||||
if typed, ok := in.Payload.(T); ok {
|
||||
return typed, nil
|
||||
}
|
||||
if ptr, ok := in.Payload.(*T); ok {
|
||||
if ptr == nil {
|
||||
return zero, fmt.Errorf("extract payload: payload pointer is nil")
|
||||
}
|
||||
return *ptr, nil
|
||||
}
|
||||
|
||||
return fknormalize.DecodeJSONPayload[T](in)
|
||||
}
|
||||
|
||||
|
||||
39
internal/normalizers/common/json_test.go
Normal file
39
internal/normalizers/common/json_test.go
Normal file
@@ -0,0 +1,39 @@
|
||||
package common
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/ejr/feedkit/event"
|
||||
)
|
||||
|
||||
func TestDecodeJSONPayloadAcceptsTypedPayload(t *testing.T) {
|
||||
type rawPayload struct {
|
||||
Value string `json:"value"`
|
||||
}
|
||||
|
||||
got, err := DecodeJSONPayload[rawPayload](event.Event{
|
||||
Payload: rawPayload{Value: "ok"},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("DecodeJSONPayload() error = %v", err)
|
||||
}
|
||||
if got.Value != "ok" {
|
||||
t.Fatalf("Value = %q, want ok", got.Value)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecodeJSONPayloadAcceptsTypedPointerPayload(t *testing.T) {
|
||||
type rawPayload struct {
|
||||
Value string `json:"value"`
|
||||
}
|
||||
|
||||
got, err := DecodeJSONPayload[rawPayload](event.Event{
|
||||
Payload: &rawPayload{Value: "ok"},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("DecodeJSONPayload() error = %v", err)
|
||||
}
|
||||
if got.Value != "ok" {
|
||||
t.Fatalf("Value = %q, want ok", got.Value)
|
||||
}
|
||||
}
|
||||
@@ -93,12 +93,8 @@ func buildAlerts(parsed nwsAlertsResponse, fallbackAsOf time.Time) (model.Weathe
|
||||
sent := nwscommon.ParseTimePtr(p.Sent)
|
||||
effective := nwscommon.ParseTimePtr(p.Effective)
|
||||
onset := nwscommon.ParseTimePtr(p.Onset)
|
||||
|
||||
// Expires: prefer "expires"; fall back to "ends" if present.
|
||||
ends := nwscommon.ParseTimePtr(p.Ends)
|
||||
expires := nwscommon.ParseTimePtr(p.Expires)
|
||||
if expires == nil {
|
||||
expires = nwscommon.ParseTimePtr(p.Ends)
|
||||
}
|
||||
|
||||
refs := parseNWSAlertReferences(p.References)
|
||||
|
||||
@@ -123,6 +119,7 @@ func buildAlerts(parsed nwsAlertsResponse, fallbackAsOf time.Time) (model.Weathe
|
||||
Sent: sent,
|
||||
Effective: effective,
|
||||
Onset: onset,
|
||||
Ends: ends,
|
||||
Expires: expires,
|
||||
|
||||
AreaDescription: strings.TrimSpace(p.AreaDesc),
|
||||
|
||||
136
internal/normalizers/nws/alerts_test.go
Normal file
136
internal/normalizers/nws/alerts_test.go
Normal file
@@ -0,0 +1,136 @@
|
||||
package nws
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/ejr/feedkit/event"
|
||||
"gitea.maximumdirect.net/ejr/weatherfeeder/model"
|
||||
"gitea.maximumdirect.net/ejr/weatherfeeder/standards"
|
||||
)
|
||||
|
||||
func TestAlertsNormalizerMapsEndsSeparatelyFromExpires(t *testing.T) {
|
||||
raw := []byte(`{
|
||||
"updated":"2026-06-16T10:00:00+00:00",
|
||||
"title":"Current watches, warnings, and advisories for St. Louis",
|
||||
"features":[{
|
||||
"id":"https://api.weather.gov/alerts/alert-1",
|
||||
"properties":{
|
||||
"event":"Flood Warning",
|
||||
"headline":"Flood Warning issued",
|
||||
"sent":"2026-06-16T09:55:00+00:00",
|
||||
"effective":"2026-06-16T10:00:00+00:00",
|
||||
"onset":"2026-06-16T10:15:00+00:00",
|
||||
"ends":"2026-06-16T14:00:00+00:00",
|
||||
"expires":"2026-06-16T11:00:00+00:00"
|
||||
}
|
||||
}]
|
||||
}`)
|
||||
|
||||
out, err := AlertsNormalizer{}.Normalize(context.Background(), alertRawEvent(raw))
|
||||
if err != nil {
|
||||
t.Fatalf("Normalize() error = %v", err)
|
||||
}
|
||||
|
||||
run := decodeAlertRun(t, out)
|
||||
if len(run.Alerts) != 1 {
|
||||
t.Fatalf("expected 1 alert, got %d", len(run.Alerts))
|
||||
}
|
||||
alert := run.Alerts[0]
|
||||
wantEnds := time.Date(2026, 6, 16, 14, 0, 0, 0, time.UTC)
|
||||
wantExpires := time.Date(2026, 6, 16, 11, 0, 0, 0, time.UTC)
|
||||
if alert.Ends == nil || !alert.Ends.Equal(wantEnds) {
|
||||
t.Fatalf("ends = %v, want %s", alert.Ends, wantEnds)
|
||||
}
|
||||
if alert.Expires == nil || !alert.Expires.Equal(wantExpires) {
|
||||
t.Fatalf("expires = %v, want %s", alert.Expires, wantExpires)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAlertsNormalizerDoesNotFallbackExpiresToEnds(t *testing.T) {
|
||||
raw := []byte(`{
|
||||
"updated":"2026-06-16T10:00:00+00:00",
|
||||
"features":[{
|
||||
"id":"alert-ends-only",
|
||||
"properties":{
|
||||
"event":"Heat Advisory",
|
||||
"ends":"2026-06-16T22:00:00+00:00"
|
||||
}
|
||||
}]
|
||||
}`)
|
||||
|
||||
out, err := AlertsNormalizer{}.Normalize(context.Background(), alertRawEvent(raw))
|
||||
if err != nil {
|
||||
t.Fatalf("Normalize() error = %v", err)
|
||||
}
|
||||
|
||||
run := decodeAlertRun(t, out)
|
||||
alert := run.Alerts[0]
|
||||
if alert.Ends == nil {
|
||||
t.Fatal("expected ends to be populated")
|
||||
}
|
||||
if alert.Expires != nil {
|
||||
t.Fatalf("expected expires nil when upstream expires is absent, got %v", alert.Expires)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAlertsNormalizerIgnoresInvalidEnds(t *testing.T) {
|
||||
raw := []byte(`{
|
||||
"updated":"2026-06-16T10:00:00+00:00",
|
||||
"features":[{
|
||||
"id":"alert-invalid-ends",
|
||||
"properties":{
|
||||
"event":"Special Weather Statement",
|
||||
"ends":"not-a-time",
|
||||
"expires":"2026-06-16T11:00:00+00:00"
|
||||
}
|
||||
}]
|
||||
}`)
|
||||
|
||||
out, err := AlertsNormalizer{}.Normalize(context.Background(), alertRawEvent(raw))
|
||||
if err != nil {
|
||||
t.Fatalf("Normalize() error = %v", err)
|
||||
}
|
||||
|
||||
run := decodeAlertRun(t, out)
|
||||
alert := run.Alerts[0]
|
||||
if alert.Ends != nil {
|
||||
t.Fatalf("expected invalid ends to map nil, got %v", alert.Ends)
|
||||
}
|
||||
if alert.Expires == nil {
|
||||
t.Fatal("expected expires to remain populated")
|
||||
}
|
||||
}
|
||||
|
||||
func alertRawEvent(raw []byte) event.Event {
|
||||
emittedAt := time.Date(2026, 6, 16, 10, 5, 0, 0, time.UTC)
|
||||
return event.Event{
|
||||
ID: "raw-alerts",
|
||||
Kind: event.Kind(standards.KindAlert),
|
||||
Source: "NWSAlerts",
|
||||
Schema: standards.SchemaRawNWSAlertsV1,
|
||||
EmittedAt: emittedAt,
|
||||
Payload: json.RawMessage(raw),
|
||||
}
|
||||
}
|
||||
|
||||
func decodeAlertRun(t *testing.T, e *event.Event) model.WeatherAlertRun {
|
||||
t.Helper()
|
||||
if e == nil {
|
||||
t.Fatal("expected normalized event")
|
||||
}
|
||||
if e.Schema != standards.SchemaWeatherAlertV1 {
|
||||
t.Fatalf("schema = %q, want %q", e.Schema, standards.SchemaWeatherAlertV1)
|
||||
}
|
||||
var run model.WeatherAlertRun
|
||||
raw, err := json.Marshal(e.Payload)
|
||||
if err != nil {
|
||||
t.Fatalf("marshal alert payload: %v", err)
|
||||
}
|
||||
if err := json.Unmarshal(raw, &run); err != nil {
|
||||
t.Fatalf("decode alert payload: %v", err)
|
||||
}
|
||||
return run
|
||||
}
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -18,7 +19,7 @@ func TestForecastDiscussionNormalizerProducesCanonicalSchema(t *testing.T) {
|
||||
|
||||
out, err := (ForecastDiscussionNormalizer{}).Normalize(nil, event.Event{
|
||||
ID: "evt-discussion-1",
|
||||
Kind: event.Kind("forecast_discussion"),
|
||||
Kind: event.Kind(standards.KindForecastDiscussion),
|
||||
Source: "nws-discussion-test",
|
||||
EmittedAt: time.Date(2026, 3, 28, 19, 25, 0, 0, time.UTC),
|
||||
Schema: standards.SchemaRawNWSForecastDiscussionV1,
|
||||
@@ -33,7 +34,7 @@ func TestForecastDiscussionNormalizerProducesCanonicalSchema(t *testing.T) {
|
||||
if out.Schema != standards.SchemaWeatherForecastDiscussionV1 {
|
||||
t.Fatalf("Schema = %q, want %q", out.Schema, standards.SchemaWeatherForecastDiscussionV1)
|
||||
}
|
||||
if out.Kind != event.Kind("forecast_discussion") {
|
||||
if out.Kind != event.Kind(standards.KindForecastDiscussion) {
|
||||
t.Fatalf("Kind = %q, want forecast_discussion", out.Kind)
|
||||
}
|
||||
|
||||
@@ -71,10 +72,281 @@ func TestForecastDiscussionNormalizerProducesCanonicalSchema(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestForecastDiscussionNormalizerSupportsMixedHeadingFormats(t *testing.T) {
|
||||
out, err := (ForecastDiscussionNormalizer{}).Normalize(nil, event.Event{
|
||||
ID: "evt-discussion-mixed-format",
|
||||
Kind: event.Kind(standards.KindForecastDiscussion),
|
||||
Source: "nws-discussion-test",
|
||||
EmittedAt: time.Date(2026, 3, 28, 19, 25, 0, 0, time.UTC),
|
||||
Schema: standards.SchemaRawNWSForecastDiscussionV1,
|
||||
Payload: loadMixedFormatForecastDiscussionSampleHTML(t),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Normalize() error = %v", err)
|
||||
}
|
||||
if out == nil {
|
||||
t.Fatalf("Normalize() returned nil output")
|
||||
}
|
||||
if out.Kind != event.Kind(standards.KindForecastDiscussion) {
|
||||
t.Fatalf("Kind = %q, want forecast_discussion", out.Kind)
|
||||
}
|
||||
if out.Schema != standards.SchemaWeatherForecastDiscussionV1 {
|
||||
t.Fatalf("Schema = %q, want %q", out.Schema, standards.SchemaWeatherForecastDiscussionV1)
|
||||
}
|
||||
wantEffectiveAt := time.Date(2026, 3, 28, 19, 24, 0, 0, time.UTC)
|
||||
if out.EffectiveAt == nil || !out.EffectiveAt.Equal(wantEffectiveAt) {
|
||||
t.Fatalf("EffectiveAt = %v, want %s", out.EffectiveAt, wantEffectiveAt.Format(time.RFC3339))
|
||||
}
|
||||
|
||||
payload, ok := out.Payload.(model.WeatherForecastDiscussion)
|
||||
if !ok {
|
||||
t.Fatalf("Payload type = %T, want model.WeatherForecastDiscussion", out.Payload)
|
||||
}
|
||||
if payload.ShortTerm == nil || payload.LongTerm == nil {
|
||||
t.Fatalf("ShortTerm=%v LongTerm=%v, want both populated", payload.ShortTerm, payload.LongTerm)
|
||||
}
|
||||
if payload.ShortTerm.Qualifier != "Through Late Sunday Night" {
|
||||
t.Fatalf("ShortTerm.Qualifier = %q", payload.ShortTerm.Qualifier)
|
||||
}
|
||||
if !strings.Contains(payload.ShortTerm.Text, "After a chilly morning") {
|
||||
t.Fatalf("ShortTerm.Text missing expected prose: %q", payload.ShortTerm.Text)
|
||||
}
|
||||
if payload.LongTerm.Qualifier != "Monday through Next Saturday" {
|
||||
t.Fatalf("LongTerm.Qualifier = %q", payload.LongTerm.Qualifier)
|
||||
}
|
||||
if !strings.Contains(payload.LongTerm.Text, "The peak of the warmth arrives Monday and Tuesday") {
|
||||
t.Fatalf("LongTerm.Text missing expected prose: %q", payload.LongTerm.Text)
|
||||
}
|
||||
if strings.Contains(payload.LongTerm.Text, "AVIATION") || strings.Contains(payload.LongTerm.Text, "VFR conditions are expected") {
|
||||
t.Fatalf("LongTerm.Text includes aviation content: %q", payload.LongTerm.Text)
|
||||
}
|
||||
|
||||
b, err := json.Marshal(out.Payload)
|
||||
if err != nil {
|
||||
t.Fatalf("json.Marshal(payload) error = %v", err)
|
||||
}
|
||||
var fields map[string]any
|
||||
if err := json.Unmarshal(b, &fields); err != nil {
|
||||
t.Fatalf("json.Unmarshal(payload) error = %v", err)
|
||||
}
|
||||
for _, key := range []string{"aviation", "sections"} {
|
||||
if _, ok := fields[key]; ok {
|
||||
t.Fatalf("unexpected key %q in canonical payload", key)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestForecastDiscussionNormalizerSupportsCrossOfficeLayout(t *testing.T) {
|
||||
in := event.Event{
|
||||
ID: "evt-discussion-bou",
|
||||
Kind: event.Kind(standards.KindForecastDiscussion),
|
||||
Source: "nws-discussion-bou-test",
|
||||
EmittedAt: time.Date(2026, 4, 7, 19, 1, 0, 0, time.UTC),
|
||||
Schema: standards.SchemaRawNWSForecastDiscussionV1,
|
||||
Payload: loadForecastDiscussionBOUSampleHTML(t),
|
||||
}
|
||||
|
||||
out, err := (ForecastDiscussionNormalizer{}).Normalize(nil, in)
|
||||
if err != nil {
|
||||
t.Fatalf("Normalize() error = %v", err)
|
||||
}
|
||||
if out == nil {
|
||||
t.Fatalf("Normalize() returned nil output")
|
||||
}
|
||||
if out.ID != in.ID || out.Source != in.Source || !out.EmittedAt.Equal(in.EmittedAt) {
|
||||
t.Fatalf("envelope = %#v, want ID/source/emittedAt from input", out)
|
||||
}
|
||||
if out.Kind != event.Kind(standards.KindForecastDiscussion) {
|
||||
t.Fatalf("Kind = %q, want forecast_discussion", out.Kind)
|
||||
}
|
||||
if out.Schema != standards.SchemaWeatherForecastDiscussionV1 {
|
||||
t.Fatalf("Schema = %q, want %q", out.Schema, standards.SchemaWeatherForecastDiscussionV1)
|
||||
}
|
||||
wantEffectiveAt := time.Date(2026, 4, 7, 19, 0, 0, 0, time.UTC)
|
||||
if out.EffectiveAt == nil || !out.EffectiveAt.Equal(wantEffectiveAt) {
|
||||
t.Fatalf("EffectiveAt = %v, want %s", out.EffectiveAt, wantEffectiveAt.Format(time.RFC3339))
|
||||
}
|
||||
|
||||
payload, ok := out.Payload.(model.WeatherForecastDiscussion)
|
||||
if !ok {
|
||||
t.Fatalf("Payload type = %T, want model.WeatherForecastDiscussion", out.Payload)
|
||||
}
|
||||
if payload.OfficeID != "BOU" || payload.OfficeName != "National Weather Service Denver CO" {
|
||||
t.Fatalf("OfficeID=%q OfficeName=%q", payload.OfficeID, payload.OfficeName)
|
||||
}
|
||||
wantMessages := []string{
|
||||
"Strong winds are expected along the Front Range this evening.",
|
||||
"Cooler temperatures arrive on Wednesday.",
|
||||
}
|
||||
if len(payload.KeyMessages) != len(wantMessages) {
|
||||
t.Fatalf("KeyMessages = %#v, want %#v", payload.KeyMessages, wantMessages)
|
||||
}
|
||||
for i := range wantMessages {
|
||||
if payload.KeyMessages[i] != wantMessages[i] {
|
||||
t.Fatalf("KeyMessages[%d] = %q, want %q", i, payload.KeyMessages[i], wantMessages[i])
|
||||
}
|
||||
}
|
||||
if payload.ShortTerm == nil || payload.LongTerm == nil {
|
||||
t.Fatalf("ShortTerm=%v LongTerm=%v, want both populated", payload.ShortTerm, payload.LongTerm)
|
||||
}
|
||||
if payload.ShortTerm.Qualifier != "(Tonight through Wednesday)" || payload.ShortTerm.Text != "Gusty west winds will continue through the evening before decreasing overnight." {
|
||||
t.Fatalf("ShortTerm = %#v", payload.ShortTerm)
|
||||
}
|
||||
if payload.LongTerm.Qualifier != "(Thursday through Saturday)" || payload.LongTerm.Text != "Warmer and drier conditions return Thursday, followed by a chance of showers Friday." {
|
||||
t.Fatalf("LongTerm = %#v", payload.LongTerm)
|
||||
}
|
||||
if payload.ShortTerm.IssuedAt == nil || payload.LongTerm.IssuedAt == nil ||
|
||||
!payload.ShortTerm.IssuedAt.Equal(wantEffectiveAt) || !payload.LongTerm.IssuedAt.Equal(wantEffectiveAt) {
|
||||
t.Fatalf("section issue times = short %v long %v, want %s", payload.ShortTerm.IssuedAt, payload.LongTerm.IssuedAt, wantEffectiveAt.Format(time.RFC3339))
|
||||
}
|
||||
|
||||
b, err := json.Marshal(out.Payload)
|
||||
if err != nil {
|
||||
t.Fatalf("json.Marshal(payload) error = %v", err)
|
||||
}
|
||||
var fields map[string]any
|
||||
if err := json.Unmarshal(b, &fields); err != nil {
|
||||
t.Fatalf("json.Unmarshal(payload) error = %v", err)
|
||||
}
|
||||
for _, key := range []string{"aviation", "discussion", "sections"} {
|
||||
if _, ok := fields[key]; ok {
|
||||
t.Fatalf("unexpected key %q in canonical payload", key)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestForecastDiscussionNormalizerSupportsCrossOfficeKeyMessageFixtures(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
filename string
|
||||
id string
|
||||
source string
|
||||
emittedAt time.Time
|
||||
effectiveAt time.Time
|
||||
messages []string
|
||||
}{
|
||||
{
|
||||
name: "numbered key messages",
|
||||
filename: "forecast_discussion_bgm_numbered_sample.html",
|
||||
id: "evt-discussion-bgm",
|
||||
source: "nws-discussion-bgm-test",
|
||||
emittedAt: time.Date(2026, 4, 10, 17, 31, 0, 0, time.UTC),
|
||||
effectiveAt: time.Date(2026, 4, 10, 17, 30, 0, 0, time.UTC),
|
||||
messages: []string{
|
||||
"Periods of rain are expected through Saturday, with locally heavier amounts possible.",
|
||||
"Cooler temperatures return late this weekend.",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "key points alias",
|
||||
filename: "forecast_discussion_mfr_key_points_sample.html",
|
||||
id: "evt-discussion-mfr",
|
||||
source: "nws-discussion-mfr-test",
|
||||
emittedAt: time.Date(2026, 4, 10, 19, 46, 0, 0, time.UTC),
|
||||
effectiveAt: time.Date(2026, 4, 10, 19, 45, 0, 0, time.UTC),
|
||||
messages: []string{
|
||||
"Gusty winds will develop over exposed ridges, especially during the afternoon.",
|
||||
"Inland valleys remain dry through Saturday.",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "as of preamble",
|
||||
filename: "forecast_discussion_rah_as_of_sample.html",
|
||||
id: "evt-discussion-rah",
|
||||
source: "nws-discussion-rah-test",
|
||||
emittedAt: time.Date(2026, 8, 2, 16, 36, 0, 0, time.UTC),
|
||||
effectiveAt: time.Date(2026, 8, 2, 16, 35, 0, 0, time.UTC),
|
||||
messages: []string{
|
||||
"Scattered storms may produce locally heavy rain this afternoon.",
|
||||
"Drier weather arrives Monday.",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "parenthesized numeric markers",
|
||||
filename: "forecast_discussion_lwx_parenthesized_number_sample.html",
|
||||
id: "evt-discussion-lwx",
|
||||
source: "nws-discussion-lwx-test",
|
||||
emittedAt: time.Date(2026, 8, 2, 18, 1, 0, 0, time.UTC),
|
||||
effectiveAt: time.Date(2026, 8, 2, 18, 0, 0, 0, time.UTC),
|
||||
messages: []string{
|
||||
"Thunderstorms remain possible near the Blue Ridge this evening.",
|
||||
"Seasonably warm conditions continue Monday.",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "embedded key messages in previous discussion",
|
||||
filename: "forecast_discussion_mfr_prev_discussion_sample.html",
|
||||
id: "evt-discussion-mfr-previous",
|
||||
source: "nws-discussion-mfr-previous-test",
|
||||
emittedAt: time.Date(2026, 8, 2, 22, 20, 0, 0, time.UTC),
|
||||
effectiveAt: time.Date(2026, 8, 2, 22, 19, 0, 0, time.UTC),
|
||||
messages: []string{
|
||||
"Heat returns to inland valleys Monday.",
|
||||
"Gusty afternoon winds develop east of the Cascades.",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
in := event.Event{
|
||||
ID: tt.id,
|
||||
Kind: event.Kind(standards.KindForecastDiscussion),
|
||||
Source: tt.source,
|
||||
EmittedAt: tt.emittedAt,
|
||||
Schema: standards.SchemaRawNWSForecastDiscussionV1,
|
||||
Payload: loadForecastDiscussionFixtureHTML(t, tt.filename),
|
||||
}
|
||||
|
||||
out, err := (ForecastDiscussionNormalizer{}).Normalize(nil, in)
|
||||
if err != nil {
|
||||
t.Fatalf("Normalize() error = %v", err)
|
||||
}
|
||||
if out == nil {
|
||||
t.Fatalf("Normalize() returned nil output")
|
||||
}
|
||||
if out.ID != in.ID || out.Source != in.Source || !out.EmittedAt.Equal(in.EmittedAt) {
|
||||
t.Fatalf("envelope = %#v, want ID/source/emittedAt from input", out)
|
||||
}
|
||||
if out.Kind != event.Kind(standards.KindForecastDiscussion) {
|
||||
t.Fatalf("Kind = %q, want forecast_discussion", out.Kind)
|
||||
}
|
||||
if out.Schema != standards.SchemaWeatherForecastDiscussionV1 {
|
||||
t.Fatalf("Schema = %q, want %q", out.Schema, standards.SchemaWeatherForecastDiscussionV1)
|
||||
}
|
||||
if out.EffectiveAt == nil || !out.EffectiveAt.Equal(tt.effectiveAt) {
|
||||
t.Fatalf("EffectiveAt = %v, want %s", out.EffectiveAt, tt.effectiveAt.Format(time.RFC3339))
|
||||
}
|
||||
|
||||
payload, ok := out.Payload.(model.WeatherForecastDiscussion)
|
||||
if !ok {
|
||||
t.Fatalf("Payload type = %T, want model.WeatherForecastDiscussion", out.Payload)
|
||||
}
|
||||
if !reflect.DeepEqual(payload.KeyMessages, tt.messages) {
|
||||
t.Fatalf("KeyMessages = %#v, want %#v", payload.KeyMessages, tt.messages)
|
||||
}
|
||||
|
||||
b, err := json.Marshal(payload)
|
||||
if err != nil {
|
||||
t.Fatalf("json.Marshal(payload) error = %v", err)
|
||||
}
|
||||
var fields map[string]any
|
||||
if err := json.Unmarshal(b, &fields); err != nil {
|
||||
t.Fatalf("json.Unmarshal(payload) error = %v", err)
|
||||
}
|
||||
for _, key := range []string{"nearTerm", "discussion", "aviation", "sections"} {
|
||||
if _, ok := fields[key]; ok {
|
||||
t.Fatalf("unexpected key %q in canonical payload", key)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestForecastDiscussionNormalizerRejectsMissingIssueTime(t *testing.T) {
|
||||
_, err := (ForecastDiscussionNormalizer{}).Normalize(nil, event.Event{
|
||||
ID: "evt-discussion-bad",
|
||||
Kind: event.Kind("forecast_discussion"),
|
||||
Kind: event.Kind(standards.KindForecastDiscussion),
|
||||
Source: "nws-discussion-test",
|
||||
EmittedAt: time.Date(2026, 3, 28, 19, 25, 0, 0, time.UTC),
|
||||
Schema: standards.SchemaRawNWSForecastDiscussionV1,
|
||||
@@ -93,7 +365,7 @@ func TestForecastDiscussionNormalizerWireShapeHasNoUnexpectedKeys(t *testing.T)
|
||||
|
||||
out, err := (ForecastDiscussionNormalizer{}).Normalize(nil, event.Event{
|
||||
ID: "evt-discussion-2",
|
||||
Kind: event.Kind("forecast_discussion"),
|
||||
Kind: event.Kind(standards.KindForecastDiscussion),
|
||||
Source: "nws-discussion-test",
|
||||
EmittedAt: time.Date(2026, 3, 28, 19, 25, 0, 0, time.UTC),
|
||||
Schema: standards.SchemaRawNWSForecastDiscussionV1,
|
||||
@@ -128,3 +400,56 @@ func loadForecastDiscussionSampleHTML(t *testing.T) string {
|
||||
}
|
||||
return string(b)
|
||||
}
|
||||
|
||||
func loadForecastDiscussionBOUSampleHTML(t *testing.T) string {
|
||||
t.Helper()
|
||||
|
||||
path := filepath.Join("..", "..", "providers", "nws", "testdata", "forecast_discussion_bou_sample.html")
|
||||
b, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatalf("os.ReadFile(%q) error = %v", path, err)
|
||||
}
|
||||
return string(b)
|
||||
}
|
||||
|
||||
func loadForecastDiscussionFixtureHTML(t *testing.T, filename string) string {
|
||||
t.Helper()
|
||||
|
||||
path := filepath.Join("..", "..", "providers", "nws", "testdata", filename)
|
||||
b, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatalf("os.ReadFile(%q) error = %v", path, err)
|
||||
}
|
||||
return string(b)
|
||||
}
|
||||
|
||||
func loadMixedFormatForecastDiscussionSampleHTML(t *testing.T) string {
|
||||
t.Helper()
|
||||
|
||||
raw := loadForecastDiscussionSampleHTML(t)
|
||||
replacements := []struct {
|
||||
original string
|
||||
replacement string
|
||||
}{
|
||||
{
|
||||
original: ".SHORT TERM... (Through Late Sunday Night)",
|
||||
replacement: ".SHORT TERM /Through Late Sunday Night/...",
|
||||
},
|
||||
{
|
||||
original: ".LONG TERM... (Monday through Next Saturday)",
|
||||
replacement: ".LONG TERM /Monday through Next Saturday/...",
|
||||
},
|
||||
{
|
||||
original: ".AVIATION... (For the 18z TAFs through 18z Sunday Afternoon)",
|
||||
replacement: ".AVIATION /For the 18z TAFs through 18z Sunday Afternoon/...",
|
||||
},
|
||||
}
|
||||
for _, replacement := range replacements {
|
||||
if !strings.Contains(raw, replacement.original) {
|
||||
t.Fatalf("fixture missing heading %q", replacement.original)
|
||||
}
|
||||
raw = strings.Replace(raw, replacement.original, replacement.replacement, 1)
|
||||
}
|
||||
|
||||
return raw
|
||||
}
|
||||
|
||||
@@ -182,7 +182,7 @@ func TestNormalizeForecastEventBySchemaProducesCanonicalWeatherForecastSchema(t
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
out, err := normalizeForecastEventBySchema(event.Event{
|
||||
ID: "evt-1",
|
||||
Kind: event.Kind("forecast"),
|
||||
Kind: event.Kind(standards.KindForecast),
|
||||
Source: "nws-test",
|
||||
EmittedAt: time.Date(2026, 3, 16, 18, 0, 0, 0, time.UTC),
|
||||
Schema: tt.schema,
|
||||
|
||||
@@ -22,7 +22,7 @@ func TestWeatherStoriesNormalizerProducesCanonicalSchemaAndMapsSample(t *testing
|
||||
if out.Schema != standards.SchemaWeatherStoryV1 {
|
||||
t.Fatalf("Schema = %q, want %q", out.Schema, standards.SchemaWeatherStoryV1)
|
||||
}
|
||||
if out.Kind != event.Kind("weather_story") {
|
||||
if out.Kind != event.Kind(standards.KindWeatherStory) {
|
||||
t.Fatalf("Kind = %q, want weather_story", out.Kind)
|
||||
}
|
||||
|
||||
@@ -118,7 +118,7 @@ func TestWeatherStoriesNormalizerMatch(t *testing.T) {
|
||||
func weatherStoriesRawEvent(payload string) event.Event {
|
||||
return event.Event{
|
||||
ID: "evt-weatherstories-1",
|
||||
Kind: event.Kind("weather_story"),
|
||||
Kind: event.Kind(standards.KindWeatherStory),
|
||||
Source: "nws-weatherstories-test",
|
||||
EmittedAt: time.Date(2026, 5, 30, 9, 5, 0, 0, time.UTC),
|
||||
Schema: standards.SchemaRawNWSWeatherStoriesV1,
|
||||
|
||||
365
internal/normalizers/spc/convective_outlook.go
Normal file
365
internal/normalizers/spc/convective_outlook.go
Normal file
@@ -0,0 +1,365 @@
|
||||
package spc
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"math"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/ejr/feedkit/event"
|
||||
"gitea.maximumdirect.net/ejr/weatherfeeder/internal/geo"
|
||||
normcommon "gitea.maximumdirect.net/ejr/weatherfeeder/internal/normalizers/common"
|
||||
spcprovider "gitea.maximumdirect.net/ejr/weatherfeeder/internal/providers/spc"
|
||||
"gitea.maximumdirect.net/ejr/weatherfeeder/model"
|
||||
"gitea.maximumdirect.net/ejr/weatherfeeder/standards"
|
||||
)
|
||||
|
||||
const (
|
||||
providerSPC = "spc"
|
||||
productConvective = "convective"
|
||||
outlookNormalizer = "spc convective outlook"
|
||||
outlookTypeUnknown = 99
|
||||
)
|
||||
|
||||
var idTokenRE = regexp.MustCompile(`[^a-z0-9]+`)
|
||||
|
||||
// ConvectiveOutlookNormalizer converts:
|
||||
//
|
||||
// standards.SchemaRawSPCConvectiveOutlookV1 -> standards.SchemaWeatherOutlookV2
|
||||
//
|
||||
// It maps SPC GeoJSON outlook features containing the configured location into
|
||||
// canonical outlook polygons and adds matching day-level print-page discussions.
|
||||
type ConvectiveOutlookNormalizer struct{}
|
||||
|
||||
func (ConvectiveOutlookNormalizer) Match(e event.Event) bool {
|
||||
return strings.TrimSpace(e.Schema) == standards.SchemaRawSPCConvectiveOutlookV1
|
||||
}
|
||||
|
||||
func (ConvectiveOutlookNormalizer) Normalize(ctx context.Context, in event.Event) (*event.Event, error) {
|
||||
_ = ctx
|
||||
|
||||
fallbackAsOf := in.EmittedAt.UTC()
|
||||
if in.EffectiveAt != nil && !in.EffectiveAt.IsZero() {
|
||||
fallbackAsOf = in.EffectiveAt.UTC()
|
||||
}
|
||||
|
||||
return normcommon.NormalizeJSON(
|
||||
in,
|
||||
outlookNormalizer,
|
||||
standards.SchemaWeatherOutlookV2,
|
||||
func(parsed spcprovider.RawConvectiveOutlookBundle) (model.WeatherOutlookRun, time.Time, error) {
|
||||
return buildConvectiveOutlook(parsed, fallbackAsOf)
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
func buildConvectiveOutlook(bundle spcprovider.RawConvectiveOutlookBundle, fallbackAsOf time.Time) (model.WeatherOutlookRun, time.Time, error) {
|
||||
if err := validateCoordinates(bundle.Latitude, bundle.Longitude); err != nil {
|
||||
return model.WeatherOutlookRun{}, time.Time{}, err
|
||||
}
|
||||
|
||||
discussions, latestDiscussionUpdated, err := parseDiscussions(bundle.Discussions)
|
||||
if err != nil {
|
||||
return model.WeatherOutlookRun{}, time.Time{}, err
|
||||
}
|
||||
|
||||
products := orderedProducts(bundle.Products)
|
||||
point := geo.Point{Latitude: bundle.Latitude, Longitude: bundle.Longitude}
|
||||
outlooks := make([]model.WeatherOutlook, 0)
|
||||
var latestIssue time.Time
|
||||
|
||||
for _, product := range products {
|
||||
if err := validateProductMetadata(product); err != nil {
|
||||
return model.WeatherOutlookRun{}, time.Time{}, err
|
||||
}
|
||||
if _, ok := discussions[product.Day]; !ok {
|
||||
return model.WeatherOutlookRun{}, time.Time{}, fmt.Errorf("product %s: discussion for day %d is required", product.Key, product.Day)
|
||||
}
|
||||
|
||||
collection, err := spcprovider.DecodeGeoJSON(product.Body)
|
||||
if err != nil {
|
||||
return model.WeatherOutlookRun{}, time.Time{}, fmt.Errorf("product %s: %w", product.Key, err)
|
||||
}
|
||||
|
||||
for i, feature := range collection.Features {
|
||||
if spcprovider.IsEmptyGeometryCollection(feature.Geometry) {
|
||||
issuedAt, err := parseRequiredSPCTime(feature.Properties.IssueISO, fmt.Sprintf("product %s feature %d.ISSUE_ISO", product.Key, i))
|
||||
if err != nil {
|
||||
return model.WeatherOutlookRun{}, time.Time{}, err
|
||||
}
|
||||
if latestIssue.IsZero() || issuedAt.After(latestIssue) {
|
||||
latestIssue = issuedAt
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
outlook, err := mapFeature(product, feature, i, point)
|
||||
if err != nil {
|
||||
return model.WeatherOutlookRun{}, time.Time{}, err
|
||||
}
|
||||
if latestIssue.IsZero() || outlook.IssuedAt.After(latestIssue) {
|
||||
latestIssue = outlook.IssuedAt
|
||||
}
|
||||
if !outlook.ContainsLocation {
|
||||
continue
|
||||
}
|
||||
outlooks = append(outlooks, outlook)
|
||||
}
|
||||
}
|
||||
|
||||
runDiscussions, err := buildOutlookDiscussions(outlooks, discussions)
|
||||
if err != nil {
|
||||
return model.WeatherOutlookRun{}, time.Time{}, err
|
||||
}
|
||||
|
||||
asOf := latestIssue
|
||||
if asOf.IsZero() {
|
||||
asOf = latestDiscussionUpdated
|
||||
}
|
||||
if asOf.IsZero() {
|
||||
asOf = fallbackAsOf.UTC()
|
||||
}
|
||||
|
||||
var issuedAt *time.Time
|
||||
if !latestIssue.IsZero() {
|
||||
t := latestIssue.UTC()
|
||||
issuedAt = &t
|
||||
}
|
||||
|
||||
lat := bundle.Latitude
|
||||
lon := bundle.Longitude
|
||||
run := model.WeatherOutlookRun{
|
||||
LocationID: strings.TrimSpace(bundle.LocationID),
|
||||
LocationName: strings.TrimSpace(bundle.LocationName),
|
||||
Latitude: &lat,
|
||||
Longitude: &lon,
|
||||
AsOf: asOf.UTC(),
|
||||
IssuedAt: issuedAt,
|
||||
Outlooks: outlooks,
|
||||
Discussions: runDiscussions,
|
||||
}
|
||||
return run, run.AsOf, nil
|
||||
}
|
||||
|
||||
type parsedDiscussion struct {
|
||||
Headline string
|
||||
Summary string
|
||||
Discussion string
|
||||
UpdatedAt *time.Time
|
||||
}
|
||||
|
||||
func parseDiscussions(pages []spcprovider.RawDiscussionPage) (map[int]parsedDiscussion, time.Time, error) {
|
||||
out := map[int]parsedDiscussion{}
|
||||
var latestUpdated time.Time
|
||||
for _, page := range pages {
|
||||
parsed, err := spcprovider.ParseDiscussionHTML(page.Body)
|
||||
if err != nil {
|
||||
return nil, time.Time{}, fmt.Errorf("discussion %s: %w", page.Key, err)
|
||||
}
|
||||
day := page.Day
|
||||
if day == 0 {
|
||||
if meta, ok := spcprovider.DiscussionProductByKey(page.Key); ok {
|
||||
day = meta.Day
|
||||
}
|
||||
}
|
||||
if day < 1 || day > 3 {
|
||||
return nil, time.Time{}, fmt.Errorf("discussion %s: day must be 1, 2, or 3, got %d", page.Key, page.Day)
|
||||
}
|
||||
disc := parsedDiscussion{
|
||||
Headline: strings.TrimSpace(parsed.Headline),
|
||||
Summary: strings.TrimSpace(parsed.Summary),
|
||||
Discussion: strings.TrimSpace(parsed.Discussion),
|
||||
UpdatedAt: parsed.UpdatedAt,
|
||||
}
|
||||
out[day] = disc
|
||||
if parsed.UpdatedAt != nil && (latestUpdated.IsZero() || parsed.UpdatedAt.After(latestUpdated)) {
|
||||
latestUpdated = parsed.UpdatedAt.UTC()
|
||||
}
|
||||
}
|
||||
return out, latestUpdated, nil
|
||||
}
|
||||
|
||||
func buildOutlookDiscussions(outlooks []model.WeatherOutlook, discussions map[int]parsedDiscussion) ([]model.WeatherOutlookDiscussion, error) {
|
||||
daysWithOutlooks := map[int]bool{}
|
||||
for _, outlook := range outlooks {
|
||||
daysWithOutlooks[outlook.Day] = true
|
||||
}
|
||||
|
||||
days := make([]int, 0, len(daysWithOutlooks))
|
||||
for day := range daysWithOutlooks {
|
||||
days = append(days, day)
|
||||
}
|
||||
sort.Ints(days)
|
||||
|
||||
out := make([]model.WeatherOutlookDiscussion, 0, len(days))
|
||||
for _, day := range days {
|
||||
disc, ok := discussions[day]
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("discussion for retained day %d is required", day)
|
||||
}
|
||||
out = append(out, model.WeatherOutlookDiscussion{
|
||||
Day: day,
|
||||
Headline: disc.Headline,
|
||||
Summary: disc.Summary,
|
||||
Discussion: disc.Discussion,
|
||||
UpdatedAt: utcTimePtr(disc.UpdatedAt),
|
||||
})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func utcTimePtr(t *time.Time) *time.Time {
|
||||
if t == nil {
|
||||
return nil
|
||||
}
|
||||
tt := t.UTC()
|
||||
return &tt
|
||||
}
|
||||
|
||||
func orderedProducts(products []spcprovider.RawOutlookProduct) []spcprovider.RawOutlookProduct {
|
||||
out := make([]spcprovider.RawOutlookProduct, len(products))
|
||||
copy(out, products)
|
||||
sort.SliceStable(out, func(i, j int) bool {
|
||||
if out[i].Day != out[j].Day {
|
||||
return out[i].Day < out[j].Day
|
||||
}
|
||||
left := outlookTypeOrder(out[i].OutlookType)
|
||||
right := outlookTypeOrder(out[j].OutlookType)
|
||||
if left != right {
|
||||
return left < right
|
||||
}
|
||||
return out[i].Key < out[j].Key
|
||||
})
|
||||
return out
|
||||
}
|
||||
|
||||
func outlookTypeOrder(outlookType string) int {
|
||||
switch strings.TrimSpace(outlookType) {
|
||||
case spcprovider.OutlookTypeCategorical:
|
||||
return 0
|
||||
case spcprovider.OutlookTypeTornado:
|
||||
return 1
|
||||
case spcprovider.OutlookTypeHail:
|
||||
return 2
|
||||
case spcprovider.OutlookTypeWind:
|
||||
return 3
|
||||
default:
|
||||
return outlookTypeUnknown
|
||||
}
|
||||
}
|
||||
|
||||
func validateProductMetadata(product spcprovider.RawOutlookProduct) error {
|
||||
if product.Day < 1 || product.Day > 3 {
|
||||
return fmt.Errorf("product %s: day must be 1, 2, or 3, got %d", product.Key, product.Day)
|
||||
}
|
||||
switch strings.TrimSpace(product.OutlookType) {
|
||||
case spcprovider.OutlookTypeCategorical, spcprovider.OutlookTypeTornado, spcprovider.OutlookTypeHail, spcprovider.OutlookTypeWind:
|
||||
return nil
|
||||
default:
|
||||
return fmt.Errorf("product %s: unsupported outlook type %q", product.Key, product.OutlookType)
|
||||
}
|
||||
}
|
||||
|
||||
func mapFeature(product spcprovider.RawOutlookProduct, feature spcprovider.GeoJSONFeature, index int, point geo.Point) (model.WeatherOutlook, error) {
|
||||
fieldPrefix := fmt.Sprintf("product %s feature %d", product.Key, index)
|
||||
props := feature.Properties
|
||||
|
||||
validFrom, err := parseRequiredSPCTime(props.ValidISO, fieldPrefix+".VALID_ISO")
|
||||
if err != nil {
|
||||
return model.WeatherOutlook{}, err
|
||||
}
|
||||
validTo, err := parseRequiredSPCTime(props.ExpireISO, fieldPrefix+".EXPIRE_ISO")
|
||||
if err != nil {
|
||||
return model.WeatherOutlook{}, err
|
||||
}
|
||||
issuedAt, err := parseRequiredSPCTime(props.IssueISO, fieldPrefix+".ISSUE_ISO")
|
||||
if err != nil {
|
||||
return model.WeatherOutlook{}, err
|
||||
}
|
||||
label := strings.TrimSpace(props.Label)
|
||||
if label == "" {
|
||||
return model.WeatherOutlook{}, fmt.Errorf("%s.LABEL is required", fieldPrefix)
|
||||
}
|
||||
if len(feature.Geometry) == 0 {
|
||||
return model.WeatherOutlook{}, fmt.Errorf("%s.geometry is required", fieldPrefix)
|
||||
}
|
||||
containsLocation, err := geo.ContainsPoint(feature.Geometry, point)
|
||||
if err != nil {
|
||||
return model.WeatherOutlook{}, fmt.Errorf("%s.geometry: %w", fieldPrefix, err)
|
||||
}
|
||||
|
||||
geometry := make(json.RawMessage, len(feature.Geometry))
|
||||
copy(geometry, feature.Geometry)
|
||||
|
||||
return model.WeatherOutlook{
|
||||
ID: outlookID(product.Day, product.OutlookType, label, issuedAt, validFrom, index),
|
||||
Provider: providerSPC,
|
||||
Product: productConvective,
|
||||
Day: product.Day,
|
||||
OutlookType: strings.TrimSpace(product.OutlookType),
|
||||
Label: label,
|
||||
LabelText: strings.TrimSpace(props.Label2),
|
||||
SeverityRank: props.DN,
|
||||
ValidFrom: validFrom,
|
||||
ValidTo: validTo,
|
||||
IssuedAt: issuedAt,
|
||||
ExpiresAt: validTo,
|
||||
Forecaster: strings.TrimSpace(props.Forecaster),
|
||||
SourceURL: strings.TrimSpace(product.URL),
|
||||
ImageURL: "",
|
||||
ContainsLocation: containsLocation,
|
||||
Geometry: geometry,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func parseRequiredSPCTime(value, field string) (time.Time, error) {
|
||||
if strings.TrimSpace(value) == "" {
|
||||
return time.Time{}, fmt.Errorf("%s is required", field)
|
||||
}
|
||||
t, err := spcprovider.ParseISOTimestamp(value)
|
||||
if err != nil {
|
||||
return time.Time{}, fmt.Errorf("%s: %w", field, err)
|
||||
}
|
||||
return t.UTC(), nil
|
||||
}
|
||||
|
||||
func outlookID(day int, outlookType, label string, issuedAt time.Time, validFrom time.Time, index int) string {
|
||||
return fmt.Sprintf(
|
||||
"spc-convective-day%d-%s-%s-%s-%s-%d",
|
||||
day,
|
||||
safeIDToken(outlookType),
|
||||
safeIDToken(label),
|
||||
issuedAt.UTC().Format(time.RFC3339),
|
||||
validFrom.UTC().Format(time.RFC3339),
|
||||
index,
|
||||
)
|
||||
}
|
||||
|
||||
func safeIDToken(value string) string {
|
||||
value = strings.ToLower(strings.TrimSpace(value))
|
||||
value = idTokenRE.ReplaceAllString(value, "-")
|
||||
value = strings.Trim(value, "-")
|
||||
if value == "" {
|
||||
return "unknown"
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
func validateCoordinates(latitude, longitude float64) error {
|
||||
switch {
|
||||
case math.IsNaN(latitude) || math.IsInf(latitude, 0):
|
||||
return fmt.Errorf("latitude must be finite")
|
||||
case math.IsNaN(longitude) || math.IsInf(longitude, 0):
|
||||
return fmt.Errorf("longitude must be finite")
|
||||
case latitude < -90 || latitude > 90:
|
||||
return fmt.Errorf("latitude must be between -90 and 90, got %v", latitude)
|
||||
case longitude < -180 || longitude > 180:
|
||||
return fmt.Errorf("longitude must be between -180 and 180, got %v", longitude)
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
526
internal/normalizers/spc/convective_outlook_test.go
Normal file
526
internal/normalizers/spc/convective_outlook_test.go
Normal file
@@ -0,0 +1,526 @@
|
||||
package spc
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/ejr/feedkit/event"
|
||||
spcprovider "gitea.maximumdirect.net/ejr/weatherfeeder/internal/providers/spc"
|
||||
"gitea.maximumdirect.net/ejr/weatherfeeder/model"
|
||||
"gitea.maximumdirect.net/ejr/weatherfeeder/standards"
|
||||
)
|
||||
|
||||
func TestConvectiveOutlookNormalizerMatch(t *testing.T) {
|
||||
n := ConvectiveOutlookNormalizer{}
|
||||
if !n.Match(event.Event{Schema: standards.SchemaRawSPCConvectiveOutlookV1}) {
|
||||
t.Fatalf("Match(raw SPC outlook) = false, want true")
|
||||
}
|
||||
if n.Match(event.Event{Schema: standards.SchemaRawNWSAlertsV1}) {
|
||||
t.Fatalf("Match(raw NWS alerts) = true, want false")
|
||||
}
|
||||
}
|
||||
|
||||
func TestConvectiveOutlookNormalizerProducesCanonicalSchemaAndMapsSample(t *testing.T) {
|
||||
out, err := (ConvectiveOutlookNormalizer{}).Normalize(nil, spcRawEvent(t, spcBundle(t, 38.5, -90.5)))
|
||||
if err != nil {
|
||||
t.Fatalf("Normalize() error = %v", err)
|
||||
}
|
||||
if out.Schema != standards.SchemaWeatherOutlookV2 {
|
||||
t.Fatalf("Schema = %q, want %q", out.Schema, standards.SchemaWeatherOutlookV2)
|
||||
}
|
||||
if out.Kind != event.Kind(standards.KindOutlook) {
|
||||
t.Fatalf("Kind = %q, want outlook", out.Kind)
|
||||
}
|
||||
|
||||
run, ok := out.Payload.(model.WeatherOutlookRun)
|
||||
if !ok {
|
||||
t.Fatalf("Payload type = %T, want model.WeatherOutlookRun", out.Payload)
|
||||
}
|
||||
wantAsOf := time.Date(2026, 6, 11, 19, 45, 0, 0, time.UTC)
|
||||
if !run.AsOf.Equal(wantAsOf) {
|
||||
t.Fatalf("AsOf = %s, want %s", run.AsOf, wantAsOf)
|
||||
}
|
||||
if run.IssuedAt == nil || !run.IssuedAt.Equal(wantAsOf) {
|
||||
t.Fatalf("IssuedAt = %v, want %s", run.IssuedAt, wantAsOf)
|
||||
}
|
||||
if out.EffectiveAt == nil || !out.EffectiveAt.Equal(wantAsOf) {
|
||||
t.Fatalf("EffectiveAt = %v, want %s", out.EffectiveAt, wantAsOf)
|
||||
}
|
||||
if run.LocationID != "stl" || run.LocationName != "St. Louis, MO" {
|
||||
t.Fatalf("location metadata = %q/%q", run.LocationID, run.LocationName)
|
||||
}
|
||||
if run.Latitude == nil || *run.Latitude != 38.5 || run.Longitude == nil || *run.Longitude != -90.5 {
|
||||
t.Fatalf("coordinates = %v,%v", run.Latitude, run.Longitude)
|
||||
}
|
||||
if len(run.Outlooks) != 4 {
|
||||
t.Fatalf("Outlooks length = %d, want 4", len(run.Outlooks))
|
||||
}
|
||||
assertAllOutlooksContainLocation(t, run.Outlooks)
|
||||
assertDiscussionDays(t, run.Discussions, 1)
|
||||
day1Discussion := run.Discussions[0]
|
||||
if day1Discussion.Headline != "Day 1 Convective Outlook" {
|
||||
t.Fatalf("day 1 Headline = %q", day1Discussion.Headline)
|
||||
}
|
||||
if !strings.Contains(day1Discussion.Summary, "central Plains") {
|
||||
t.Fatalf("day 1 Summary = %q", day1Discussion.Summary)
|
||||
}
|
||||
if !strings.Contains(day1Discussion.Discussion, "...DISCUSSION...") {
|
||||
t.Fatalf("day 1 Discussion missing product text: %q", day1Discussion.Discussion)
|
||||
}
|
||||
if !strings.HasPrefix(day1Discussion.Discussion, "SPC AC 111234") {
|
||||
t.Fatalf("day 1 Discussion = %q, want SPC product code prefix", day1Discussion.Discussion)
|
||||
}
|
||||
|
||||
got := run.Outlooks[0]
|
||||
if got.Provider != "spc" || got.Product != "convective" {
|
||||
t.Fatalf("provider/product = %q/%q", got.Provider, got.Product)
|
||||
}
|
||||
if got.Day != 1 || got.OutlookType != spcprovider.OutlookTypeCategorical {
|
||||
t.Fatalf("day/type = %d/%q", got.Day, got.OutlookType)
|
||||
}
|
||||
if got.Label != "SLGT" || got.LabelText != "Slight Risk" {
|
||||
t.Fatalf("label fields = %q/%q", got.Label, got.LabelText)
|
||||
}
|
||||
if got.SeverityRank == nil || *got.SeverityRank != 3 {
|
||||
t.Fatalf("SeverityRank = %v, want 3", got.SeverityRank)
|
||||
}
|
||||
assertTime(t, "ValidFrom", got.ValidFrom, 2026, 6, 11, 13, 0, 0)
|
||||
assertTime(t, "ValidTo", got.ValidTo, 2026, 6, 12, 12, 0, 0)
|
||||
assertTime(t, "IssuedAt", got.IssuedAt, 2026, 6, 11, 12, 34, 56)
|
||||
assertTime(t, "ExpiresAt", got.ExpiresAt, 2026, 6, 12, 12, 0, 0)
|
||||
if got.Forecaster != "SMITH" {
|
||||
t.Fatalf("Forecaster = %q, want SMITH", got.Forecaster)
|
||||
}
|
||||
if got.SourceURL != "https://example.invalid/day1_categorical.geojson" {
|
||||
t.Fatalf("SourceURL = %q", got.SourceURL)
|
||||
}
|
||||
wantGeometry := `{"type":"Polygon","coordinates":[[[-91.0,38.0],[-90.0,38.0],[-90.0,39.0],[-91.0,39.0],[-91.0,38.0]]]}`
|
||||
if string(got.Geometry) != wantGeometry {
|
||||
t.Fatalf("Geometry = %s, want %s", got.Geometry, wantGeometry)
|
||||
}
|
||||
if !got.ContainsLocation {
|
||||
t.Fatalf("ContainsLocation = false, want true")
|
||||
}
|
||||
if got.ID != "spc-convective-day1-categorical-slgt-2026-06-11T12:34:56Z-2026-06-11T13:00:00Z-0" {
|
||||
t.Fatalf("ID = %q", got.ID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConvectiveOutlookNormalizerAcceptsTypedSourcePayload(t *testing.T) {
|
||||
bundle := spcBundle(t, 38.5, -90.5)
|
||||
in := spcRawEvent(t, bundle)
|
||||
in.Payload = bundle
|
||||
|
||||
out, err := (ConvectiveOutlookNormalizer{}).Normalize(nil, in)
|
||||
if err != nil {
|
||||
t.Fatalf("Normalize() error = %v", err)
|
||||
}
|
||||
run := out.Payload.(model.WeatherOutlookRun)
|
||||
if len(run.Outlooks) != 4 {
|
||||
t.Fatalf("Outlooks length = %d, want 4", len(run.Outlooks))
|
||||
}
|
||||
assertAllOutlooksContainLocation(t, run.Outlooks)
|
||||
assertDiscussionDays(t, run.Discussions, 1)
|
||||
}
|
||||
|
||||
func TestConvectiveOutlookNormalizerEmitsEmptyLocalRunOutsidePolygons(t *testing.T) {
|
||||
out, err := (ConvectiveOutlookNormalizer{}).Normalize(nil, spcRawEvent(t, spcBundle(t, 0, 0)))
|
||||
if err != nil {
|
||||
t.Fatalf("Normalize() error = %v", err)
|
||||
}
|
||||
run := out.Payload.(model.WeatherOutlookRun)
|
||||
if len(run.Outlooks) != 0 {
|
||||
t.Fatalf("Outlooks length = %d, want 0", len(run.Outlooks))
|
||||
}
|
||||
if len(run.Discussions) != 0 {
|
||||
t.Fatalf("Discussions length = %d, want 0", len(run.Discussions))
|
||||
}
|
||||
wantAsOf := time.Date(2026, 6, 11, 19, 45, 0, 0, time.UTC)
|
||||
if !run.AsOf.Equal(wantAsOf) {
|
||||
t.Fatalf("AsOf = %s, want latest product issue time %s", run.AsOf, wantAsOf)
|
||||
}
|
||||
if run.IssuedAt == nil || !run.IssuedAt.Equal(wantAsOf) {
|
||||
t.Fatalf("IssuedAt = %v, want latest product issue time %s", run.IssuedAt, wantAsOf)
|
||||
}
|
||||
if out.EffectiveAt == nil || !out.EffectiveAt.Equal(run.AsOf) {
|
||||
t.Fatalf("EffectiveAt = %v, want run AsOf %s", out.EffectiveAt, run.AsOf)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConvectiveOutlookNormalizerOrdersProductsByDayAndType(t *testing.T) {
|
||||
bundle := spcBundle(t, 38.5, -90.5)
|
||||
for i, j := 0, len(bundle.Products)-1; i < j; i, j = i+1, j-1 {
|
||||
bundle.Products[i], bundle.Products[j] = bundle.Products[j], bundle.Products[i]
|
||||
}
|
||||
|
||||
out, err := (ConvectiveOutlookNormalizer{}).Normalize(nil, spcRawEvent(t, bundle))
|
||||
if err != nil {
|
||||
t.Fatalf("Normalize() error = %v", err)
|
||||
}
|
||||
run := out.Payload.(model.WeatherOutlookRun)
|
||||
got := []string{
|
||||
run.Outlooks[0].OutlookType,
|
||||
run.Outlooks[1].OutlookType,
|
||||
run.Outlooks[2].OutlookType,
|
||||
run.Outlooks[3].OutlookType,
|
||||
}
|
||||
want := []string{
|
||||
spcprovider.OutlookTypeCategorical,
|
||||
spcprovider.OutlookTypeTornado,
|
||||
spcprovider.OutlookTypeHail,
|
||||
spcprovider.OutlookTypeWind,
|
||||
}
|
||||
for i := range want {
|
||||
if got[i] != want[i] || run.Outlooks[i].Day != 1 {
|
||||
t.Fatalf("outlook[%d] = day %d type %q, want day 1 type %q", i, run.Outlooks[i].Day, got[i], want[i])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestConvectiveOutlookNormalizerMapsProbabilisticOutlookTypes(t *testing.T) {
|
||||
out, err := (ConvectiveOutlookNormalizer{}).Normalize(nil, spcRawEvent(t, spcBundle(t, 38.5, -90.5)))
|
||||
if err != nil {
|
||||
t.Fatalf("Normalize() error = %v", err)
|
||||
}
|
||||
run := out.Payload.(model.WeatherOutlookRun)
|
||||
assertAllOutlooksContainLocation(t, run.Outlooks)
|
||||
for _, outlookType := range []string{
|
||||
spcprovider.OutlookTypeTornado,
|
||||
spcprovider.OutlookTypeHail,
|
||||
spcprovider.OutlookTypeWind,
|
||||
} {
|
||||
if findOutlook(run.Outlooks, 1, outlookType) == nil {
|
||||
t.Fatalf("missing day 1 outlook type %q", outlookType)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestConvectiveOutlookNormalizerSkipsEmptyGeometryCollectionPlaceholder(t *testing.T) {
|
||||
bundle := spcBundle(t, 36, -99)
|
||||
replaced := false
|
||||
for i := range bundle.Products {
|
||||
if bundle.Products[i].Day == 2 && bundle.Products[i].OutlookType == spcprovider.OutlookTypeTornado {
|
||||
bundle.Products[i].Body = json.RawMessage(emptyGeometryCollectionGeoJSON())
|
||||
replaced = true
|
||||
}
|
||||
}
|
||||
if !replaced {
|
||||
t.Fatalf("test setup did not find day 2 tornado product")
|
||||
}
|
||||
|
||||
out, err := (ConvectiveOutlookNormalizer{}).Normalize(nil, spcRawEvent(t, bundle))
|
||||
if err != nil {
|
||||
t.Fatalf("Normalize() error = %v", err)
|
||||
}
|
||||
run := out.Payload.(model.WeatherOutlookRun)
|
||||
if len(run.Outlooks) != 3 {
|
||||
t.Fatalf("Outlooks length = %d, want 3", len(run.Outlooks))
|
||||
}
|
||||
assertAllOutlooksContainLocation(t, run.Outlooks)
|
||||
assertDiscussionDays(t, run.Discussions, 2)
|
||||
if got := findOutlook(run.Outlooks, 2, spcprovider.OutlookTypeTornado); got != nil {
|
||||
t.Fatalf("day 2 tornado outlook = %+v, want nil placeholder skipped", *got)
|
||||
}
|
||||
wantAsOf := time.Date(2026, 6, 12, 10, 0, 0, 0, time.UTC)
|
||||
if !run.AsOf.Equal(wantAsOf) {
|
||||
t.Fatalf("AsOf = %s, want placeholder ISSUE_ISO %s", run.AsOf, wantAsOf)
|
||||
}
|
||||
if out.EffectiveAt == nil || !out.EffectiveAt.Equal(wantAsOf) {
|
||||
t.Fatalf("EffectiveAt = %v, want placeholder ISSUE_ISO %s", out.EffectiveAt, wantAsOf)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConvectiveOutlookNormalizerIncludesOnlyDayWithContainingPolygons(t *testing.T) {
|
||||
out, err := (ConvectiveOutlookNormalizer{}).Normalize(nil, spcRawEvent(t, spcBundle(t, 36, -99)))
|
||||
if err != nil {
|
||||
t.Fatalf("Normalize() error = %v", err)
|
||||
}
|
||||
run := out.Payload.(model.WeatherOutlookRun)
|
||||
if len(run.Outlooks) == 0 {
|
||||
t.Fatalf("Outlooks length = 0, want retained day 2 outlooks")
|
||||
}
|
||||
assertAllOutlooksContainLocation(t, run.Outlooks)
|
||||
for i, outlook := range run.Outlooks {
|
||||
if outlook.Day != 2 {
|
||||
t.Fatalf("outlook[%d].Day = %d, want 2", i, outlook.Day)
|
||||
}
|
||||
}
|
||||
assertDiscussionDays(t, run.Discussions, 2)
|
||||
}
|
||||
|
||||
func TestConvectiveOutlookNormalizerUsesOneDiscussionForMultipleSameDayOutlooks(t *testing.T) {
|
||||
out, err := (ConvectiveOutlookNormalizer{}).Normalize(nil, spcRawEvent(t, spcBundle(t, 38.5, -90.5)))
|
||||
if err != nil {
|
||||
t.Fatalf("Normalize() error = %v", err)
|
||||
}
|
||||
run := out.Payload.(model.WeatherOutlookRun)
|
||||
if got := countOutlooksByDay(run.Outlooks, 1); got < 2 {
|
||||
t.Fatalf("day 1 outlook count = %d, want multiple", got)
|
||||
}
|
||||
assertAllOutlooksContainLocation(t, run.Outlooks)
|
||||
assertDiscussionDays(t, run.Discussions, 1)
|
||||
}
|
||||
|
||||
func TestConvectiveOutlookNormalizerPreservesCorrectionMarker(t *testing.T) {
|
||||
bundle := spcBundle(t, 36, -99)
|
||||
out, err := (ConvectiveOutlookNormalizer{}).Normalize(nil, spcRawEvent(t, bundle))
|
||||
if err != nil {
|
||||
t.Fatalf("Normalize() error = %v", err)
|
||||
}
|
||||
run := out.Payload.(model.WeatherOutlookRun)
|
||||
got := findOutlook(run.Outlooks, 2, spcprovider.OutlookTypeTornado)
|
||||
if got == nil {
|
||||
t.Fatalf("missing day 2 tornado outlook")
|
||||
}
|
||||
assertDiscussionDays(t, run.Discussions, 2)
|
||||
if !strings.Contains(run.Discussions[0].Headline, "CORR 1") {
|
||||
t.Fatalf("day 2 headline = %q, want correction marker", run.Discussions[0].Headline)
|
||||
}
|
||||
if !strings.Contains(run.Discussions[0].Discussion, "CORR 1") {
|
||||
t.Fatalf("day 2 discussion = %q, want correction marker", run.Discussions[0].Discussion)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConvectiveOutlookNormalizerMissingRSSNormalizes(t *testing.T) {
|
||||
bundle := spcBundle(t, 0, 0)
|
||||
bundle.RSS = nil
|
||||
if _, err := (ConvectiveOutlookNormalizer{}).Normalize(nil, spcRawEvent(t, bundle)); err != nil {
|
||||
t.Fatalf("Normalize() error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConvectiveOutlookNormalizerInvalidRequiredTimestampFailsWithContext(t *testing.T) {
|
||||
bundle := spcBundle(t, 0, 0)
|
||||
bundle.Products[0].Body = json.RawMessage(strings.Replace(
|
||||
string(bundle.Products[0].Body),
|
||||
`"ISSUE_ISO": "2026-06-11T12:34:56Z"`,
|
||||
`"ISSUE_ISO": "bad"`,
|
||||
1,
|
||||
))
|
||||
|
||||
_, err := (ConvectiveOutlookNormalizer{}).Normalize(nil, spcRawEvent(t, bundle))
|
||||
if err == nil {
|
||||
t.Fatalf("Normalize() error = nil, want error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "product day1_categorical feature 0.ISSUE_ISO") {
|
||||
t.Fatalf("error = %q, want product and feature context", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConvectiveOutlookNormalizerInvalidGeometryFailsWithContext(t *testing.T) {
|
||||
bundle := spcBundle(t, 0, 0)
|
||||
bundle.Products[0].Body = json.RawMessage(strings.Replace(
|
||||
string(bundle.Products[0].Body),
|
||||
`"geometry": {`,
|
||||
`"geometry": {"type":"LineString","coordinates":[[-91,38],[-90,39]]}, "oldGeometry": {`,
|
||||
1,
|
||||
))
|
||||
|
||||
_, err := (ConvectiveOutlookNormalizer{}).Normalize(nil, spcRawEvent(t, bundle))
|
||||
if err == nil {
|
||||
t.Fatalf("Normalize() error = nil, want error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "product day1_categorical feature 0.geometry") {
|
||||
t.Fatalf("error = %q, want product and feature context", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConvectiveOutlookNormalizerRejectsMissingLabel(t *testing.T) {
|
||||
bundle := spcBundle(t, 0, 0)
|
||||
bundle.Products[0].Body = json.RawMessage(strings.Replace(
|
||||
string(bundle.Products[0].Body),
|
||||
`"LABEL": "SLGT"`,
|
||||
`"LABEL": ""`,
|
||||
1,
|
||||
))
|
||||
|
||||
_, err := (ConvectiveOutlookNormalizer{}).Normalize(nil, spcRawEvent(t, bundle))
|
||||
if err == nil {
|
||||
t.Fatalf("Normalize() error = nil, want error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "product day1_categorical feature 0.LABEL") {
|
||||
t.Fatalf("error = %q, want label context", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConvectiveOutlookNormalizerRejectsMissingDiscussion(t *testing.T) {
|
||||
bundle := spcBundle(t, 38.5, -90.5)
|
||||
bundle.Discussions = bundle.Discussions[1:]
|
||||
|
||||
_, err := (ConvectiveOutlookNormalizer{}).Normalize(nil, spcRawEvent(t, bundle))
|
||||
if err == nil {
|
||||
t.Fatalf("Normalize() error = nil, want error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "product day1_categorical: discussion for day 1 is required") {
|
||||
t.Fatalf("error = %q, want missing discussion context", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConvectiveOutlookNormalizerOutputJSONShape(t *testing.T) {
|
||||
out, err := (ConvectiveOutlookNormalizer{}).Normalize(nil, spcRawEvent(t, spcBundle(t, 38.5, -90.5)))
|
||||
if err != nil {
|
||||
t.Fatalf("Normalize() error = %v", err)
|
||||
}
|
||||
raw, err := json.Marshal(out.Payload)
|
||||
if err != nil {
|
||||
t.Fatalf("Marshal(payload) error = %v", err)
|
||||
}
|
||||
got := string(raw)
|
||||
for _, want := range []string{`"asOf"`, `"outlooks"`, `"discussions"`, `"containsLocation"`, `"geometry"`} {
|
||||
if !strings.Contains(got, want) {
|
||||
t.Fatalf("payload JSON missing %s: %s", want, got)
|
||||
}
|
||||
}
|
||||
outlookStart := strings.Index(got, `"outlooks"`)
|
||||
discussionStart := strings.Index(got, `"discussions"`)
|
||||
if outlookStart == -1 || discussionStart == -1 || discussionStart <= outlookStart {
|
||||
t.Fatalf("payload JSON has unexpected outlook/discussion order: %s", got)
|
||||
}
|
||||
outlookJSON := got[outlookStart:discussionStart]
|
||||
for _, unwanted := range []string{`"headline"`, `"summary"`, `"discussion"`} {
|
||||
if strings.Contains(outlookJSON, unwanted) {
|
||||
t.Fatalf("outlook JSON exposed polygon-level prose key %s: %s", unwanted, got)
|
||||
}
|
||||
}
|
||||
for _, unwanted := range []string{`"products"`, `"fetchedAt"`, `"body"`} {
|
||||
if strings.Contains(got, unwanted) {
|
||||
t.Fatalf("payload JSON exposed raw key %s: %s", unwanted, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func emptyGeometryCollectionGeoJSON() []byte {
|
||||
return []byte(`{
|
||||
"type": "FeatureCollection",
|
||||
"features": [
|
||||
{
|
||||
"type": "Feature",
|
||||
"properties": {
|
||||
"VALID_ISO": "2026-06-12T12:00:00Z",
|
||||
"EXPIRE_ISO": "2026-06-13T12:00:00Z",
|
||||
"ISSUE_ISO": "2026-06-12T10:00:00Z",
|
||||
"FORECASTER": "DOE",
|
||||
"LABEL": "Less Than 2% All Areas",
|
||||
"LABEL2": "",
|
||||
"DN": 0
|
||||
},
|
||||
"geometry": {
|
||||
"type": "GeometryCollection",
|
||||
"geometries": []
|
||||
}
|
||||
}
|
||||
]
|
||||
}`)
|
||||
}
|
||||
|
||||
func spcRawEvent(t *testing.T, bundle spcprovider.RawConvectiveOutlookBundle) event.Event {
|
||||
t.Helper()
|
||||
raw, err := json.Marshal(bundle)
|
||||
if err != nil {
|
||||
t.Fatalf("Marshal(bundle) error = %v", err)
|
||||
}
|
||||
effectiveAt := time.Date(2026, 6, 11, 19, 45, 0, 0, time.UTC)
|
||||
return event.Event{
|
||||
ID: "evt-spc-outlook-1",
|
||||
Kind: event.Kind(standards.KindOutlook),
|
||||
Source: "spc-test",
|
||||
EmittedAt: time.Date(2026, 6, 11, 20, 5, 0, 0, time.UTC),
|
||||
EffectiveAt: &effectiveAt,
|
||||
Schema: standards.SchemaRawSPCConvectiveOutlookV1,
|
||||
Payload: json.RawMessage(raw),
|
||||
}
|
||||
}
|
||||
|
||||
func spcBundle(t *testing.T, latitude, longitude float64) spcprovider.RawConvectiveOutlookBundle {
|
||||
t.Helper()
|
||||
fetchedAt := time.Date(2026, 6, 11, 20, 0, 0, 0, time.UTC)
|
||||
products := make([]spcprovider.RawOutlookProduct, 0, len(spcprovider.GeoJSONProducts()))
|
||||
for _, product := range spcprovider.GeoJSONProducts() {
|
||||
products = append(products, spcprovider.RawOutlookProduct{
|
||||
Key: product.Key,
|
||||
Day: product.Day,
|
||||
OutlookType: product.OutlookType,
|
||||
URL: "https://example.invalid/" + product.Key + ".geojson",
|
||||
FetchedAt: fetchedAt,
|
||||
Body: json.RawMessage(geoJSONFixtureForProduct(t, product.Key)),
|
||||
})
|
||||
}
|
||||
return spcprovider.RawConvectiveOutlookBundle{
|
||||
LocationID: "stl",
|
||||
LocationName: "St. Louis, MO",
|
||||
Latitude: latitude,
|
||||
Longitude: longitude,
|
||||
FetchedAt: fetchedAt,
|
||||
Products: products,
|
||||
Discussions: []spcprovider.RawDiscussionPage{
|
||||
{Key: "day1", Day: 1, URL: "https://example.invalid/day1.html", FetchedAt: fetchedAt, Body: string(readSPCTestFixture(t, "day1_prt.html"))},
|
||||
{Key: "day2", Day: 2, URL: "https://example.invalid/day2.html", FetchedAt: fetchedAt, Body: string(readSPCTestFixture(t, "day2_prt_corr.html"))},
|
||||
{Key: "day3", Day: 3, URL: "https://example.invalid/day3.html", FetchedAt: fetchedAt, Body: string(readSPCTestFixture(t, "day3_prt.html"))},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func geoJSONFixtureForProduct(t *testing.T, key string) []byte {
|
||||
t.Helper()
|
||||
switch {
|
||||
case strings.HasPrefix(key, "day1_"):
|
||||
return readSPCTestFixture(t, "day1_cat.geojson")
|
||||
case strings.HasPrefix(key, "day2_"):
|
||||
return readSPCTestFixture(t, "day2_torn.geojson")
|
||||
case strings.HasPrefix(key, "day3_"):
|
||||
return readSPCTestFixture(t, "day3_cat.geojson")
|
||||
default:
|
||||
t.Fatalf("unknown product key %q", key)
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func findOutlook(outlooks []model.WeatherOutlook, day int, outlookType string) *model.WeatherOutlook {
|
||||
for i := range outlooks {
|
||||
if outlooks[i].Day == day && outlooks[i].OutlookType == outlookType {
|
||||
return &outlooks[i]
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func countOutlooksByDay(outlooks []model.WeatherOutlook, day int) int {
|
||||
count := 0
|
||||
for _, outlook := range outlooks {
|
||||
if outlook.Day == day {
|
||||
count++
|
||||
}
|
||||
}
|
||||
return count
|
||||
}
|
||||
|
||||
func assertAllOutlooksContainLocation(t *testing.T, outlooks []model.WeatherOutlook) {
|
||||
t.Helper()
|
||||
for i, outlook := range outlooks {
|
||||
if !outlook.ContainsLocation {
|
||||
t.Fatalf("outlook[%d].ContainsLocation = false, want true", i)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func assertDiscussionDays(t *testing.T, discussions []model.WeatherOutlookDiscussion, want ...int) {
|
||||
t.Helper()
|
||||
if len(discussions) != len(want) {
|
||||
t.Fatalf("Discussions length = %d, want %d", len(discussions), len(want))
|
||||
}
|
||||
for i, day := range want {
|
||||
if discussions[i].Day != day {
|
||||
t.Fatalf("Discussions[%d].Day = %d, want %d", i, discussions[i].Day, day)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func assertTime(t *testing.T, name string, got time.Time, year int, month time.Month, day int, hour int, minute int, second int) {
|
||||
t.Helper()
|
||||
want := time.Date(year, month, day, hour, minute, second, 0, time.UTC)
|
||||
if !got.Equal(want) {
|
||||
t.Fatalf("%s = %s, want %s", name, got, want)
|
||||
}
|
||||
}
|
||||
17
internal/normalizers/spc/fixture_test.go
Normal file
17
internal/normalizers/spc/fixture_test.go
Normal file
@@ -0,0 +1,17 @@
|
||||
package spc
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func readSPCTestFixture(t *testing.T, name string) []byte {
|
||||
t.Helper()
|
||||
path := filepath.Join("..", "..", "providers", "spc", "testdata", name)
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatalf("read fixture %s: %v", path, err)
|
||||
}
|
||||
return raw
|
||||
}
|
||||
14
internal/normalizers/spc/register.go
Normal file
14
internal/normalizers/spc/register.go
Normal file
@@ -0,0 +1,14 @@
|
||||
package spc
|
||||
|
||||
import (
|
||||
fknormalize "gitea.maximumdirect.net/ejr/feedkit/processors/normalize"
|
||||
)
|
||||
|
||||
var builtins = []fknormalize.Normalizer{
|
||||
ConvectiveOutlookNormalizer{},
|
||||
}
|
||||
|
||||
// Register appends SPC normalizers in stable order.
|
||||
func Register(in []fknormalize.Normalizer) []fknormalize.Normalizer {
|
||||
return append(in, builtins...)
|
||||
}
|
||||
@@ -27,11 +27,34 @@ type ForecastDiscussionSection struct {
|
||||
Text string
|
||||
}
|
||||
|
||||
type forecastDiscussionSectionRole uint8
|
||||
|
||||
const (
|
||||
forecastDiscussionSectionRoleKeyMessages forecastDiscussionSectionRole = iota
|
||||
forecastDiscussionSectionRoleShortTerm
|
||||
forecastDiscussionSectionRoleLongTerm
|
||||
)
|
||||
|
||||
type forecastDiscussionSectionHeading struct {
|
||||
section string
|
||||
qualifier string
|
||||
}
|
||||
|
||||
type forecastDiscussionSectionBlock struct {
|
||||
heading forecastDiscussionSectionHeading
|
||||
body []string
|
||||
}
|
||||
|
||||
var (
|
||||
forecastDiscussionHeaderRE = regexp.MustCompile(`^\.(KEY MESSAGES|SHORT TERM|LONG TERM|AVIATION)\.\.\.(.*)$`)
|
||||
forecastDiscussionAFDRE = regexp.MustCompile(`^AFD([A-Z]{3})$`)
|
||||
forecastDiscussionWMORE = regexp.MustCompile(`\bK([A-Z]{3})\b`)
|
||||
forecastDiscussionSigRE = regexp.MustCompile(`^[A-Z]{2,6}$`)
|
||||
forecastDiscussionSectionRoles = map[string]forecastDiscussionSectionRole{
|
||||
"KEY MESSAGES": forecastDiscussionSectionRoleKeyMessages,
|
||||
"KEY POINTS": forecastDiscussionSectionRoleKeyMessages,
|
||||
"SHORT TERM": forecastDiscussionSectionRoleShortTerm,
|
||||
"LONG TERM": forecastDiscussionSectionRoleLongTerm,
|
||||
}
|
||||
forecastDiscussionAFDRE = regexp.MustCompile(`^AFD([A-Z]{3})$`)
|
||||
forecastDiscussionWMORE = regexp.MustCompile(`\bK([A-Z]{3})\b`)
|
||||
forecastDiscussionSigRE = regexp.MustCompile(`^[A-Z]{2,6}$`)
|
||||
)
|
||||
|
||||
func ParseForecastDiscussionHTML(raw string) (ForecastDiscussion, error) {
|
||||
@@ -99,22 +122,30 @@ func ParseForecastDiscussionText(text string) (ForecastDiscussion, error) {
|
||||
IssuedAt: issuedAt.UTC(),
|
||||
}
|
||||
|
||||
if block, ok := extractForecastDiscussionSection(lines, "KEY MESSAGES"); ok {
|
||||
out.KeyMessages = parseForecastDiscussionKeyMessages(block)
|
||||
}
|
||||
if block, ok := extractForecastDiscussionSection(lines, "SHORT TERM"); ok {
|
||||
section, err := parseForecastDiscussionTextSection(block)
|
||||
if err != nil {
|
||||
return ForecastDiscussion{}, fmt.Errorf("parse SHORT TERM: %w", err)
|
||||
seenRoles := make(map[forecastDiscussionSectionRole]bool, len(forecastDiscussionSectionRoles))
|
||||
for _, block := range parseForecastDiscussionSectionBlocks(lines) {
|
||||
role, ok := forecastDiscussionSectionRoles[block.heading.section]
|
||||
if !ok || seenRoles[role] {
|
||||
continue
|
||||
}
|
||||
out.ShortTerm = §ion
|
||||
}
|
||||
if block, ok := extractForecastDiscussionSection(lines, "LONG TERM"); ok {
|
||||
section, err := parseForecastDiscussionTextSection(block)
|
||||
if err != nil {
|
||||
return ForecastDiscussion{}, fmt.Errorf("parse LONG TERM: %w", err)
|
||||
seenRoles[role] = true
|
||||
|
||||
switch role {
|
||||
case forecastDiscussionSectionRoleKeyMessages:
|
||||
out.KeyMessages = parseForecastDiscussionKeyMessages(block.body)
|
||||
case forecastDiscussionSectionRoleShortTerm:
|
||||
section, err := parseForecastDiscussionTextSection(block)
|
||||
if err != nil {
|
||||
return ForecastDiscussion{}, fmt.Errorf("parse %s: %w", block.heading.section, err)
|
||||
}
|
||||
out.ShortTerm = §ion
|
||||
case forecastDiscussionSectionRoleLongTerm:
|
||||
section, err := parseForecastDiscussionTextSection(block)
|
||||
if err != nil {
|
||||
return ForecastDiscussion{}, fmt.Errorf("parse %s: %w", block.heading.section, err)
|
||||
}
|
||||
out.LongTerm = §ion
|
||||
}
|
||||
out.LongTerm = §ion
|
||||
}
|
||||
|
||||
return out, nil
|
||||
@@ -285,8 +316,9 @@ func parseForecastDiscussionHeader(lines []string) (string, time.Time, error) {
|
||||
|
||||
func parseForecastDiscussionIssueTime(line string) (time.Time, error) {
|
||||
line = strings.TrimSpace(line)
|
||||
line = strings.TrimPrefix(line, "Issued at ")
|
||||
line = strings.TrimSpace(line)
|
||||
if isForecastDiscussionIssuedAtLine(line) {
|
||||
line = strings.TrimSpace(line[len("Issued at"):])
|
||||
}
|
||||
|
||||
parts := strings.Fields(line)
|
||||
if len(parts) != 7 {
|
||||
@@ -386,40 +418,217 @@ func forecastDiscussionLocation(abbrev string) (*time.Location, error) {
|
||||
return time.FixedZone(abbr, offset), nil
|
||||
}
|
||||
|
||||
func extractForecastDiscussionSection(lines []string, section string) ([]string, bool) {
|
||||
target := "." + section + "..."
|
||||
for i, raw := range lines {
|
||||
func parseForecastDiscussionSectionHeading(line string) (forecastDiscussionSectionHeading, bool) {
|
||||
line = strings.TrimSpace(line)
|
||||
if len(line) < 2 || line[0] != '.' {
|
||||
return forecastDiscussionSectionHeading{}, false
|
||||
}
|
||||
|
||||
if strings.HasSuffix(line, "/...") {
|
||||
return parseForecastDiscussionSlashQualifiedHeading(line)
|
||||
}
|
||||
if strings.HasSuffix(line, "...") {
|
||||
if heading, ok := parseForecastDiscussionParenthesizedTerminalHeading(line); ok {
|
||||
return heading, true
|
||||
}
|
||||
}
|
||||
return parseForecastDiscussionEllipsisHeading(line)
|
||||
}
|
||||
|
||||
func parseForecastDiscussionSlashQualifiedHeading(line string) (forecastDiscussionSectionHeading, bool) {
|
||||
content := strings.TrimSuffix(line[1:], "/...")
|
||||
separator := -1
|
||||
for i := 1; i < len(content); i++ {
|
||||
if content[i] == '/' && isForecastDiscussionHorizontalWhitespace(content[i-1]) {
|
||||
separator = i
|
||||
break
|
||||
}
|
||||
}
|
||||
if separator < 0 {
|
||||
return forecastDiscussionSectionHeading{}, false
|
||||
}
|
||||
|
||||
section, ok := normalizeForecastDiscussionSectionIdentity(content[:separator])
|
||||
if !ok {
|
||||
return forecastDiscussionSectionHeading{}, false
|
||||
}
|
||||
qualifier := strings.TrimSpace(content[separator+1:])
|
||||
if qualifier == "" {
|
||||
return forecastDiscussionSectionHeading{}, false
|
||||
}
|
||||
return forecastDiscussionSectionHeading{section: section, qualifier: qualifier}, true
|
||||
}
|
||||
|
||||
func parseForecastDiscussionParenthesizedTerminalHeading(line string) (forecastDiscussionSectionHeading, bool) {
|
||||
if len(line) < 4 || line[0] != '.' || !strings.HasSuffix(line, "...") {
|
||||
return forecastDiscussionSectionHeading{}, false
|
||||
}
|
||||
|
||||
content := strings.TrimRight(line[1:len(line)-3], " \t")
|
||||
if !strings.HasSuffix(content, ")") {
|
||||
return forecastDiscussionSectionHeading{}, false
|
||||
}
|
||||
|
||||
separator := -1
|
||||
for i := 1; i < len(content); i++ {
|
||||
if content[i] == '(' && isForecastDiscussionHorizontalWhitespace(content[i-1]) {
|
||||
separator = i
|
||||
break
|
||||
}
|
||||
}
|
||||
if separator < 0 {
|
||||
return forecastDiscussionSectionHeading{}, false
|
||||
}
|
||||
|
||||
section, ok := normalizeForecastDiscussionSectionIdentity(content[:separator])
|
||||
if !ok {
|
||||
return forecastDiscussionSectionHeading{}, false
|
||||
}
|
||||
qualifier := content[separator:]
|
||||
if len(qualifier) <= 2 || strings.TrimSpace(qualifier[1:len(qualifier)-1]) == "" {
|
||||
return forecastDiscussionSectionHeading{}, false
|
||||
}
|
||||
return forecastDiscussionSectionHeading{section: section, qualifier: qualifier}, true
|
||||
}
|
||||
|
||||
func parseForecastDiscussionEllipsisHeading(line string) (forecastDiscussionSectionHeading, bool) {
|
||||
content := line[1:]
|
||||
delimiter := strings.Index(content, "...")
|
||||
if delimiter < 0 {
|
||||
return forecastDiscussionSectionHeading{}, false
|
||||
}
|
||||
|
||||
section, ok := normalizeForecastDiscussionSectionIdentity(content[:delimiter])
|
||||
if !ok {
|
||||
return forecastDiscussionSectionHeading{}, false
|
||||
}
|
||||
return forecastDiscussionSectionHeading{
|
||||
section: section,
|
||||
qualifier: strings.TrimSpace(content[delimiter+3:]),
|
||||
}, true
|
||||
}
|
||||
|
||||
func normalizeForecastDiscussionSectionIdentity(raw string) (string, bool) {
|
||||
var normalized strings.Builder
|
||||
pendingSpace := false
|
||||
hasLetterOrDigit := false
|
||||
|
||||
for i := 0; i < len(raw); i++ {
|
||||
b := raw[i]
|
||||
switch {
|
||||
case isForecastDiscussionIdentityLetterOrDigit(b):
|
||||
hasLetterOrDigit = true
|
||||
case b == ' ' || b == '\t':
|
||||
pendingSpace = normalized.Len() > 0
|
||||
continue
|
||||
case b == '/' && i > 0 && isForecastDiscussionHorizontalWhitespace(raw[i-1]):
|
||||
return "", false
|
||||
case b != '/' && b != '&' && b != '\'' && b != '-':
|
||||
return "", false
|
||||
}
|
||||
|
||||
if pendingSpace {
|
||||
normalized.WriteByte(' ')
|
||||
pendingSpace = false
|
||||
}
|
||||
normalized.WriteByte(b)
|
||||
}
|
||||
if !hasLetterOrDigit {
|
||||
return "", false
|
||||
}
|
||||
return normalized.String(), true
|
||||
}
|
||||
|
||||
func isForecastDiscussionIdentityLetterOrDigit(b byte) bool {
|
||||
return b >= 'A' && b <= 'Z' || b >= '0' && b <= '9'
|
||||
}
|
||||
|
||||
func isForecastDiscussionHorizontalWhitespace(b byte) bool {
|
||||
return b == ' ' || b == '\t'
|
||||
}
|
||||
|
||||
func parseForecastDiscussionSectionBlocks(lines []string) []forecastDiscussionSectionBlock {
|
||||
var blocks []forecastDiscussionSectionBlock
|
||||
var active *forecastDiscussionSectionBlock
|
||||
embeddedHeadings := false
|
||||
|
||||
finish := func() {
|
||||
if active == nil {
|
||||
return
|
||||
}
|
||||
blocks = append(blocks, *active)
|
||||
active = nil
|
||||
}
|
||||
|
||||
for _, raw := range lines {
|
||||
line := strings.TrimSpace(raw)
|
||||
if !strings.HasPrefix(line, target) {
|
||||
if line == "$$" {
|
||||
finish()
|
||||
break
|
||||
}
|
||||
if line == "&&" || strings.Contains(line, "WATCHES/WARNINGS/ADVISORIES") {
|
||||
finish()
|
||||
embeddedHeadings = false
|
||||
continue
|
||||
}
|
||||
|
||||
out := []string{line}
|
||||
for j := i + 1; j < len(lines); j++ {
|
||||
next := strings.TrimSpace(lines[j])
|
||||
if next == "&&" || next == "$$" || strings.Contains(next, "WATCHES/WARNINGS/ADVISORIES") {
|
||||
break
|
||||
}
|
||||
if j > i+1 && isForecastDiscussionSectionHeader(next) {
|
||||
break
|
||||
}
|
||||
out = append(out, lines[j])
|
||||
heading, ok := parseForecastDiscussionSectionHeading(raw)
|
||||
if ok {
|
||||
finish()
|
||||
active = &forecastDiscussionSectionBlock{heading: heading}
|
||||
embeddedHeadings = isForecastDiscussionEmbeddedSectionWrapper(heading.section)
|
||||
continue
|
||||
}
|
||||
if embeddedHeadings {
|
||||
heading, ok = parseForecastDiscussionEmbeddedSectionHeading(raw)
|
||||
if ok {
|
||||
finish()
|
||||
active = &forecastDiscussionSectionBlock{heading: heading}
|
||||
continue
|
||||
}
|
||||
}
|
||||
if active != nil {
|
||||
active.body = append(active.body, raw)
|
||||
}
|
||||
return out, true
|
||||
}
|
||||
return nil, false
|
||||
finish()
|
||||
|
||||
return blocks
|
||||
}
|
||||
|
||||
func isForecastDiscussionSectionHeader(line string) bool {
|
||||
return forecastDiscussionHeaderRE.MatchString(strings.TrimSpace(line))
|
||||
func isForecastDiscussionEmbeddedSectionWrapper(section string) bool {
|
||||
return section == "PREV DISCUSSION"
|
||||
}
|
||||
|
||||
func parseForecastDiscussionKeyMessages(block []string) []string {
|
||||
if len(block) <= 1 {
|
||||
func parseForecastDiscussionEmbeddedSectionHeading(line string) (forecastDiscussionSectionHeading, bool) {
|
||||
line = strings.TrimSpace(line)
|
||||
if line == "" || line[0] == '.' {
|
||||
return forecastDiscussionSectionHeading{}, false
|
||||
}
|
||||
return parseForecastDiscussionSectionHeading("." + line)
|
||||
}
|
||||
|
||||
func parseForecastDiscussionKeyMessages(body []string) []string {
|
||||
body = removeForecastDiscussionPresentationMarkers(body)
|
||||
body = trimBlankLines(body)
|
||||
if len(body) > 0 && isForecastDiscussionKeyMessageMetadataLine(body[0]) {
|
||||
body = trimBlankLines(body[1:])
|
||||
}
|
||||
if len(body) == 0 {
|
||||
return nil
|
||||
}
|
||||
hasMarkers := false
|
||||
for _, raw := range body {
|
||||
line := strings.TrimSpace(raw)
|
||||
if line == "" {
|
||||
continue
|
||||
}
|
||||
if _, ok := stripForecastDiscussionKeyMessageMarker(line); ok {
|
||||
hasMarkers = true
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
body := trimBlankLines(block[1:])
|
||||
var messages []string
|
||||
var current strings.Builder
|
||||
|
||||
@@ -431,15 +640,21 @@ func parseForecastDiscussionKeyMessages(block []string) []string {
|
||||
current.Reset()
|
||||
}
|
||||
|
||||
seenMarker := false
|
||||
for _, raw := range body {
|
||||
line := strings.TrimSpace(raw)
|
||||
if line == "" {
|
||||
if !hasMarkers || !seenMarker {
|
||||
flush()
|
||||
}
|
||||
continue
|
||||
}
|
||||
if strings.HasPrefix(line, "-") {
|
||||
if stripped, ok := stripForecastDiscussionKeyMessageMarker(line); ok {
|
||||
flush()
|
||||
line = strings.TrimSpace(strings.TrimPrefix(line, "-"))
|
||||
current.WriteString(line)
|
||||
seenMarker = true
|
||||
line = stripped
|
||||
}
|
||||
if line == "" {
|
||||
continue
|
||||
}
|
||||
if current.Len() > 0 {
|
||||
@@ -452,25 +667,24 @@ func parseForecastDiscussionKeyMessages(block []string) []string {
|
||||
return messages
|
||||
}
|
||||
|
||||
func parseForecastDiscussionTextSection(block []string) (ForecastDiscussionSection, error) {
|
||||
if len(block) == 0 {
|
||||
return ForecastDiscussionSection{}, fmt.Errorf("empty section")
|
||||
}
|
||||
|
||||
func parseForecastDiscussionTextSection(block forecastDiscussionSectionBlock) (ForecastDiscussionSection, error) {
|
||||
section := ForecastDiscussionSection{
|
||||
Qualifier: parseForecastDiscussionQualifier(strings.TrimSpace(block[0])),
|
||||
Qualifier: block.heading.qualifier,
|
||||
}
|
||||
|
||||
body := trimBlankLines(block[1:])
|
||||
body := trimBlankLines(removeForecastDiscussionPresentationMarkers(block.body))
|
||||
if section.Qualifier == "" && len(body) > 0 && isForecastDiscussionStandaloneParenthetical(body[0]) {
|
||||
section.Qualifier = strings.TrimSpace(body[0])
|
||||
body = trimBlankLines(body[1:])
|
||||
}
|
||||
if len(body) == 0 {
|
||||
return section, nil
|
||||
}
|
||||
|
||||
first := strings.TrimSpace(body[0])
|
||||
if strings.HasPrefix(first, "Issued at ") {
|
||||
issuedAt, err := parseForecastDiscussionIssueTime(first)
|
||||
if isForecastDiscussionIssuedAtLine(body[0]) {
|
||||
issuedAt, err := parseForecastDiscussionIssueTime(body[0])
|
||||
if err != nil {
|
||||
return ForecastDiscussionSection{}, fmt.Errorf("parse section issuedAt %q: %w", first, err)
|
||||
return ForecastDiscussionSection{}, fmt.Errorf("parse section issuedAt %q: %w", strings.TrimSpace(body[0]), err)
|
||||
}
|
||||
tt := issuedAt.UTC()
|
||||
section.IssuedAt = &tt
|
||||
@@ -482,12 +696,147 @@ func parseForecastDiscussionTextSection(block []string) (ForecastDiscussionSecti
|
||||
return section, nil
|
||||
}
|
||||
|
||||
func parseForecastDiscussionQualifier(header string) string {
|
||||
m := forecastDiscussionHeaderRE.FindStringSubmatch(header)
|
||||
if len(m) != 3 {
|
||||
return ""
|
||||
func isForecastDiscussionPresentationMarker(line string) bool {
|
||||
switch {
|
||||
case strings.EqualFold(strings.TrimSpace(line), "-- Changed Discussion --"):
|
||||
return true
|
||||
case strings.EqualFold(strings.TrimSpace(line), "-- End Changed Discussion --"):
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
return strings.TrimSpace(m[2])
|
||||
}
|
||||
|
||||
func removeForecastDiscussionPresentationMarkers(lines []string) []string {
|
||||
body := make([]string, 0, len(lines))
|
||||
for _, line := range lines {
|
||||
if !isForecastDiscussionPresentationMarker(line) {
|
||||
body = append(body, line)
|
||||
}
|
||||
}
|
||||
return body
|
||||
}
|
||||
|
||||
func isForecastDiscussionStandaloneParenthetical(line string) bool {
|
||||
line = strings.TrimSpace(line)
|
||||
return len(line) > 2 && line[0] == '(' && line[len(line)-1] == ')' && strings.TrimSpace(line[1:len(line)-1]) != ""
|
||||
}
|
||||
|
||||
func isForecastDiscussionIssuedAtLine(line string) bool {
|
||||
line = strings.TrimSpace(line)
|
||||
return len(line) > len("Issued at") &&
|
||||
strings.EqualFold(line[:len("Issued at")], "Issued at") &&
|
||||
isForecastDiscussionHorizontalWhitespace(line[len("Issued at")])
|
||||
}
|
||||
|
||||
func isForecastDiscussionKeyMessageMetadataLine(line string) bool {
|
||||
line = strings.TrimSpace(line)
|
||||
for _, label := range []string{"Issued at", "Updated at"} {
|
||||
if !hasForecastDiscussionASCIIPrefix(line, label) || len(line) == len(label) || !isForecastDiscussionHorizontalWhitespace(line[len(label)]) {
|
||||
continue
|
||||
}
|
||||
if _, err := parseForecastDiscussionIssueTime(strings.TrimSpace(line[len(label):])); err == nil {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return isForecastDiscussionKeyMessageAsOfLine(line)
|
||||
}
|
||||
|
||||
func isForecastDiscussionKeyMessageAsOfLine(line string) bool {
|
||||
const label = "As of"
|
||||
|
||||
if !hasForecastDiscussionASCIIPrefix(line, label) || len(line) == len(label) || !isForecastDiscussionHorizontalWhitespace(line[len(label)]) {
|
||||
return false
|
||||
}
|
||||
remainder := strings.TrimSpace(line[len(label):])
|
||||
if !strings.HasSuffix(remainder, "...") {
|
||||
return false
|
||||
}
|
||||
fields := strings.Fields(strings.TrimSpace(strings.TrimSuffix(remainder, "...")))
|
||||
if len(fields) != 3 {
|
||||
return false
|
||||
}
|
||||
if _, _, err := parseForecastDiscussionClock(fields[0], fields[1]); err != nil {
|
||||
return false
|
||||
}
|
||||
switch strings.ToLower(fields[2]) {
|
||||
case "monday", "tuesday", "wednesday", "thursday", "friday", "saturday", "sunday":
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func stripForecastDiscussionKeyMessageMarker(line string) (string, bool) {
|
||||
if line == "" {
|
||||
return "", false
|
||||
}
|
||||
|
||||
if line[0] == '-' || line[0] == '*' {
|
||||
content := line[1:]
|
||||
hadWhitespace := len(content) > 0 && isForecastDiscussionHorizontalWhitespace(content[0])
|
||||
content = strings.TrimLeft(content, " \t")
|
||||
if hadWhitespace {
|
||||
if stripped, ok := stripForecastDiscussionKeyMessageNumericMarker(content); ok {
|
||||
content = stripped
|
||||
}
|
||||
}
|
||||
return content, true
|
||||
}
|
||||
|
||||
return stripForecastDiscussionKeyMessageNumericMarker(line)
|
||||
}
|
||||
|
||||
func stripForecastDiscussionKeyMessageNumericMarker(line string) (string, bool) {
|
||||
digitStart := 0
|
||||
digitEnd := 0
|
||||
parenthesized := len(line) > 0 && line[0] == '('
|
||||
if parenthesized {
|
||||
digitStart = 1
|
||||
digitEnd = 1
|
||||
}
|
||||
for digitEnd < len(line) && line[digitEnd] >= '0' && line[digitEnd] <= '9' {
|
||||
digitEnd++
|
||||
}
|
||||
if digitEnd == digitStart || digitEnd == len(line) {
|
||||
return "", false
|
||||
}
|
||||
if parenthesized && line[digitEnd] != ')' {
|
||||
return "", false
|
||||
}
|
||||
if !parenthesized && line[digitEnd] != ')' && line[digitEnd] != '.' {
|
||||
return "", false
|
||||
}
|
||||
|
||||
markerEnd := digitEnd + 1
|
||||
if markerEnd < len(line) && !isForecastDiscussionHorizontalWhitespace(line[markerEnd]) {
|
||||
return "", false
|
||||
}
|
||||
value, err := strconv.ParseUint(line[digitStart:digitEnd], 10, 0)
|
||||
if err != nil || value == 0 {
|
||||
return "", false
|
||||
}
|
||||
return strings.TrimLeft(line[markerEnd:], " \t"), true
|
||||
}
|
||||
|
||||
func hasForecastDiscussionASCIIPrefix(line, prefix string) bool {
|
||||
if len(line) < len(prefix) {
|
||||
return false
|
||||
}
|
||||
for i := range prefix {
|
||||
actual := line[i]
|
||||
if actual >= 'A' && actual <= 'Z' {
|
||||
actual += 'a' - 'A'
|
||||
}
|
||||
expected := prefix[i]
|
||||
if expected >= 'A' && expected <= 'Z' {
|
||||
expected += 'a' - 'A'
|
||||
}
|
||||
if actual != expected {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func trimBlankLines(lines []string) []string {
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
26
internal/providers/nws/testdata/forecast_discussion_bgm_numbered_sample.html
vendored
Normal file
26
internal/providers/nws/testdata/forecast_discussion_bgm_numbered_sample.html
vendored
Normal file
@@ -0,0 +1,26 @@
|
||||
<!DOCTYPE html>
|
||||
<!-- Representative BGM/CTP-style layout; prose is concise edited test data, not an archived product. -->
|
||||
<html>
|
||||
<body>
|
||||
<pre class="glossaryProduct">
|
||||
FXUS61 KBGM 101730
|
||||
AFDBGM
|
||||
|
||||
Area Forecast Discussion
|
||||
National Weather Service Binghamton NY
|
||||
130 PM EDT Fri Apr 10 2026
|
||||
|
||||
.KEY MESSAGES...
|
||||
1) Periods of rain are expected through Saturday,
|
||||
with locally heavier amounts possible.
|
||||
2. Cooler temperatures return late this weekend.
|
||||
|
||||
.DISCUSSION...
|
||||
Discussion details remain boundary-only content.
|
||||
|
||||
$$
|
||||
|
||||
WFO BGM
|
||||
</pre>
|
||||
</body>
|
||||
</html>
|
||||
48
internal/providers/nws/testdata/forecast_discussion_bou_sample.html
vendored
Normal file
48
internal/providers/nws/testdata/forecast_discussion_bou_sample.html
vendored
Normal file
@@ -0,0 +1,48 @@
|
||||
<!DOCTYPE html>
|
||||
<html>
|
||||
<body>
|
||||
<pre class="glossaryProduct">
|
||||
FXUS65 KBOU 071900
|
||||
AFDBOU
|
||||
|
||||
Area Forecast Discussion
|
||||
National Weather Service Denver CO
|
||||
100 PM MDT Tue Apr 7 2026
|
||||
|
||||
.KEY MESSAGES...
|
||||
-- Changed Discussion --
|
||||
Updated at 100 PM MDT Tue Apr 7 2026
|
||||
- Strong winds are expected along the Front Range this evening.
|
||||
- Cooler temperatures arrive on Wednesday.
|
||||
-- End Changed Discussion --
|
||||
|
||||
&&
|
||||
|
||||
.SHORT TERM...
|
||||
(Tonight through Wednesday)
|
||||
Issued at 100 PM MDT Tue Apr 7 2026
|
||||
|
||||
Gusty west winds will continue through the evening before decreasing overnight.
|
||||
|
||||
&&
|
||||
|
||||
.LONG TERM...
|
||||
(Thursday through Saturday)
|
||||
ISSUED AT 100 PM MDT Tue Apr 7 2026
|
||||
|
||||
Warmer and drier conditions return Thursday, followed by a chance of showers Friday.
|
||||
|
||||
&&
|
||||
|
||||
.AVIATION...
|
||||
|
||||
VFR conditions are expected at Denver-area terminals through Wednesday morning.
|
||||
|
||||
&&
|
||||
|
||||
$$
|
||||
|
||||
WFO BOU
|
||||
</pre>
|
||||
</body>
|
||||
</html>
|
||||
27
internal/providers/nws/testdata/forecast_discussion_lwx_parenthesized_number_sample.html
vendored
Normal file
27
internal/providers/nws/testdata/forecast_discussion_lwx_parenthesized_number_sample.html
vendored
Normal file
@@ -0,0 +1,27 @@
|
||||
<!DOCTYPE html>
|
||||
<!-- Representative LWX-style layout; prose is concise edited test data, not an archived product. -->
|
||||
<html>
|
||||
<body>
|
||||
<pre class="glossaryProduct">
|
||||
FXUS61 KLWX 021800
|
||||
AFDLWX
|
||||
|
||||
Area Forecast Discussion
|
||||
National Weather Service Baltimore MD/Washington DC
|
||||
200 PM EDT Sun Aug 2 2026
|
||||
|
||||
.KEY MESSAGES...
|
||||
- (1) Thunderstorms remain possible near the Blue Ridge this evening.
|
||||
- (2) Seasonably warm conditions continue Monday.
|
||||
|
||||
&&
|
||||
|
||||
.AVIATION...
|
||||
Aviation details remain boundary-only content.
|
||||
|
||||
$$
|
||||
|
||||
WFO LWX
|
||||
</pre>
|
||||
</body>
|
||||
</html>
|
||||
25
internal/providers/nws/testdata/forecast_discussion_mfr_key_points_sample.html
vendored
Normal file
25
internal/providers/nws/testdata/forecast_discussion_mfr_key_points_sample.html
vendored
Normal file
@@ -0,0 +1,25 @@
|
||||
<!DOCTYPE html>
|
||||
<!-- Representative MFR-style layout; prose is concise edited test data, not an archived product. -->
|
||||
<html>
|
||||
<body>
|
||||
<pre class="glossaryProduct">
|
||||
FXUS66 KMFR 101945
|
||||
AFDMFR
|
||||
|
||||
Area Forecast Discussion
|
||||
National Weather Service Medford OR
|
||||
1245 PM PDT Fri Apr 10 2026
|
||||
|
||||
.KEY POINTS...
|
||||
* Gusty winds will develop over exposed ridges,
|
||||
especially during the afternoon.
|
||||
* Inland valleys remain dry through Saturday.
|
||||
.DISCUSSION (Today through Thursday)...
|
||||
Discussion details must not be included with key points.
|
||||
|
||||
$$
|
||||
|
||||
WFO MFR
|
||||
</pre>
|
||||
</body>
|
||||
</html>
|
||||
33
internal/providers/nws/testdata/forecast_discussion_mfr_prev_discussion_sample.html
vendored
Normal file
33
internal/providers/nws/testdata/forecast_discussion_mfr_prev_discussion_sample.html
vendored
Normal file
@@ -0,0 +1,33 @@
|
||||
<!DOCTYPE html>
|
||||
<!-- Representative current MFR previous-discussion wrapper layout; prose is concise edited test data, not an archived product. -->
|
||||
<html>
|
||||
<body>
|
||||
<pre class="glossaryProduct">
|
||||
FXUS66 KMFR 022219
|
||||
AFDMFR
|
||||
|
||||
Area Forecast Discussion
|
||||
National Weather Service Medford OR
|
||||
319 PM PDT Sun Aug 2 2026
|
||||
|
||||
.PREV DISCUSSION... /Issued 319 PM PDT Sun Aug 2 2026/
|
||||
|
||||
KEY MESSAGES...
|
||||
|
||||
* Heat returns to inland valleys Monday.
|
||||
* Gusty afternoon winds develop east of the Cascades.
|
||||
|
||||
DISCUSSION...
|
||||
Discussion details must not be included with key messages.
|
||||
|
||||
&&
|
||||
|
||||
.MFR WATCHES/WARNINGS/ADVISORIES...
|
||||
None.
|
||||
|
||||
$$
|
||||
|
||||
WFO MFR
|
||||
</pre>
|
||||
</body>
|
||||
</html>
|
||||
29
internal/providers/nws/testdata/forecast_discussion_rah_as_of_sample.html
vendored
Normal file
29
internal/providers/nws/testdata/forecast_discussion_rah_as_of_sample.html
vendored
Normal file
@@ -0,0 +1,29 @@
|
||||
<!DOCTYPE html>
|
||||
<!-- Representative RAH-style layout; prose is concise edited test data, not an archived product. -->
|
||||
<html>
|
||||
<body>
|
||||
<pre class="glossaryProduct">
|
||||
FXUS62 KRAH 021635
|
||||
AFDRAH
|
||||
|
||||
Area Forecast Discussion
|
||||
National Weather Service Raleigh NC
|
||||
1235 PM EDT Sun Aug 2 2026
|
||||
|
||||
.KEY MESSAGES...
|
||||
As of 1235 PM Sunday...
|
||||
|
||||
1) Scattered storms may produce locally heavy rain this afternoon.
|
||||
2) Drier weather arrives Monday.
|
||||
|
||||
&&
|
||||
|
||||
.DISCUSSION...
|
||||
Discussion details remain boundary-only content.
|
||||
|
||||
$$
|
||||
|
||||
WFO RAH
|
||||
</pre>
|
||||
</body>
|
||||
</html>
|
||||
188
internal/providers/spc/discussion.go
Normal file
188
internal/providers/spc/discussion.go
Normal file
@@ -0,0 +1,188 @@
|
||||
package spc
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"html"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
var (
|
||||
scriptBlockRE = regexp.MustCompile(`(?is)<script\b[^>]*>.*?</script>`)
|
||||
preBlockRE = regexp.MustCompile(`(?is)<pre\b[^>]*>(.*?)</pre>`)
|
||||
tagRE = regexp.MustCompile(`(?is)<[^>]+>`)
|
||||
updatedRE = regexp.MustCompile(`(?im)^\s*Updated:\s*(.+?)\s*$`)
|
||||
pageUpdatedRE = regexp.MustCompile(`(?i)\bUpdated:\s*((?:\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z)|(?:[A-Z][a-z]{2}\s+[A-Z][a-z]{2}\s+\d{1,2}\s+\d{2}:\d{2}:\d{2}\s+UTC\s+\d{4})|(?:\d{4}\s+UTC\s+[A-Z][a-z]{2}\s+[A-Z][a-z]{2}\s+\d{1,2}\s+\d{4})|(?:\d{4}Z\s+[A-Z][a-z]{2}\s+[A-Z][a-z]{2}\s+\d{1,2}\s+\d{4}))`)
|
||||
productCodeRE = regexp.MustCompile(`(?i)^SPC\s+AC\s+\d+\s*$`)
|
||||
sectionRE = regexp.MustCompile(`^\s*\.\.\.[A-Z0-9 /-]+\.{3}\s*$`)
|
||||
)
|
||||
|
||||
// DiscussionText contains parsed text from an SPC print page.
|
||||
type DiscussionText struct {
|
||||
ProductTitle string
|
||||
Headline string
|
||||
Summary string
|
||||
Discussion string
|
||||
UpdatedAt *time.Time
|
||||
}
|
||||
|
||||
// ExtractProductText extracts and cleans the first useful preformatted SPC
|
||||
// product text block from a print page.
|
||||
func ExtractProductText(rawHTML string) (string, error) {
|
||||
matches := preBlockRE.FindAllStringSubmatch(rawHTML, -1)
|
||||
for _, match := range matches {
|
||||
if len(match) < 2 {
|
||||
continue
|
||||
}
|
||||
text := cleanHTMLText(match[1])
|
||||
if strings.TrimSpace(text) != "" {
|
||||
return text, nil
|
||||
}
|
||||
}
|
||||
return "", fmt.Errorf("no useful pre block found")
|
||||
}
|
||||
|
||||
// ParseDiscussionHTML extracts SPC product text and page-level metadata from a
|
||||
// print-page HTML document.
|
||||
func ParseDiscussionHTML(rawHTML string) (DiscussionText, error) {
|
||||
text, err := ExtractProductText(rawHTML)
|
||||
if err != nil {
|
||||
return DiscussionText{}, err
|
||||
}
|
||||
parsed := ParseDiscussionText(text)
|
||||
if updatedAt := ParsePageUpdatedTimestamp(rawHTML); updatedAt != nil {
|
||||
parsed.UpdatedAt = updatedAt
|
||||
}
|
||||
return parsed, nil
|
||||
}
|
||||
|
||||
// ParseDiscussionText extracts common SPC narrative metadata from cleaned
|
||||
// product text.
|
||||
func ParseDiscussionText(text string) DiscussionText {
|
||||
text = trimBlankLines(normalizeNewlines(text))
|
||||
title := ParseProductTitle(text)
|
||||
return DiscussionText{
|
||||
ProductTitle: title,
|
||||
Headline: title,
|
||||
Summary: ExtractSummary(text),
|
||||
Discussion: text,
|
||||
UpdatedAt: ParseUpdatedTimestamp(text),
|
||||
}
|
||||
}
|
||||
|
||||
// ParsePageUpdatedTimestamp parses the page-level Updated row from an SPC print
|
||||
// page. SPC currently places this outside the product <pre> block.
|
||||
func ParsePageUpdatedTimestamp(rawHTML string) *time.Time {
|
||||
text := cleanHTMLText(rawHTML)
|
||||
text = strings.ReplaceAll(text, "\u00a0", " ")
|
||||
text = strings.Join(strings.Fields(text), " ")
|
||||
match := pageUpdatedRE.FindStringSubmatch(text)
|
||||
if len(match) != 2 {
|
||||
return nil
|
||||
}
|
||||
return parseUpdatedValue(match[1])
|
||||
}
|
||||
|
||||
// ParseUpdatedTimestamp parses an SPC print-page Updated line when present.
|
||||
func ParseUpdatedTimestamp(text string) *time.Time {
|
||||
match := updatedRE.FindStringSubmatch(normalizeNewlines(text))
|
||||
if len(match) != 2 {
|
||||
return nil
|
||||
}
|
||||
return parseUpdatedValue(match[1])
|
||||
}
|
||||
|
||||
// ParseProductTitle returns the first non-empty product line from cleaned text.
|
||||
func ParseProductTitle(text string) string {
|
||||
for _, line := range strings.Split(normalizeNewlines(text), "\n") {
|
||||
line = strings.TrimSpace(line)
|
||||
if line == "" || strings.HasPrefix(line, "Updated:") || productCodeRE.MatchString(line) {
|
||||
continue
|
||||
}
|
||||
return line
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// ParseHeadline returns the human-facing headline from cleaned text.
|
||||
func ParseHeadline(text string) string {
|
||||
return ParseProductTitle(text)
|
||||
}
|
||||
|
||||
// ExtractSummary returns text under the ...SUMMARY... section through the next
|
||||
// SPC section heading.
|
||||
func ExtractSummary(text string) string {
|
||||
lines := strings.Split(normalizeNewlines(text), "\n")
|
||||
start := -1
|
||||
for i, line := range lines {
|
||||
if strings.EqualFold(strings.TrimSpace(line), "...SUMMARY...") {
|
||||
start = i + 1
|
||||
break
|
||||
}
|
||||
}
|
||||
if start < 0 {
|
||||
return ""
|
||||
}
|
||||
|
||||
var out []string
|
||||
for _, line := range lines[start:] {
|
||||
if sectionRE.MatchString(line) {
|
||||
break
|
||||
}
|
||||
out = append(out, line)
|
||||
}
|
||||
return trimBlankLines(strings.Join(out, "\n"))
|
||||
}
|
||||
|
||||
func cleanHTMLText(raw string) string {
|
||||
raw = scriptBlockRE.ReplaceAllString(raw, "")
|
||||
raw = tagRE.ReplaceAllString(raw, "")
|
||||
raw = html.UnescapeString(raw)
|
||||
raw = normalizeNewlines(raw)
|
||||
return trimBlankLines(raw)
|
||||
}
|
||||
|
||||
func normalizeNewlines(text string) string {
|
||||
text = strings.ReplaceAll(text, "\r\n", "\n")
|
||||
text = strings.ReplaceAll(text, "\r", "\n")
|
||||
return text
|
||||
}
|
||||
|
||||
func trimBlankLines(text string) string {
|
||||
lines := strings.Split(normalizeNewlines(text), "\n")
|
||||
start := 0
|
||||
for start < len(lines) && strings.TrimSpace(lines[start]) == "" {
|
||||
start++
|
||||
}
|
||||
end := len(lines)
|
||||
for end > start && strings.TrimSpace(lines[end-1]) == "" {
|
||||
end--
|
||||
}
|
||||
return strings.Join(lines[start:end], "\n")
|
||||
}
|
||||
|
||||
func parseUpdatedValue(value string) *time.Time {
|
||||
value = strings.TrimSpace(value)
|
||||
if value == "" {
|
||||
return nil
|
||||
}
|
||||
if t := parseOptionalISOTimestamp(value); t != nil {
|
||||
return t
|
||||
}
|
||||
for _, layout := range []string{
|
||||
"Mon Jan 2 15:04:05 UTC 2006",
|
||||
"1504 UTC Mon Jan 2 2006",
|
||||
"1504Z Mon Jan 2 2006",
|
||||
"3:04 PM UTC Mon Jan 2 2006",
|
||||
time.RFC1123,
|
||||
time.RFC1123Z,
|
||||
} {
|
||||
t, err := time.Parse(layout, value)
|
||||
if err == nil {
|
||||
tt := t.UTC()
|
||||
return &tt
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
109
internal/providers/spc/discussion_test.go
Normal file
109
internal/providers/spc/discussion_test.go
Normal file
@@ -0,0 +1,109 @@
|
||||
package spc
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestExtractProductTextCleansPreBlock(t *testing.T) {
|
||||
raw := string(readTestFile(t, "day1_prt.html"))
|
||||
|
||||
got, err := ExtractProductText(raw)
|
||||
if err != nil {
|
||||
t.Fatalf("ExtractProductText() error = %v", err)
|
||||
}
|
||||
if strings.Contains(got, "<script") || strings.Contains(got, "<pre") {
|
||||
t.Fatalf("ExtractProductText() retained HTML: %q", got)
|
||||
}
|
||||
if strings.Contains(got, "ignore me") {
|
||||
t.Fatalf("ExtractProductText() retained script content: %q", got)
|
||||
}
|
||||
if !strings.Contains(got, "Day 1 Convective Outlook") {
|
||||
t.Fatalf("ExtractProductText() missing headline: %q", got)
|
||||
}
|
||||
if !strings.HasPrefix(got, "SPC AC 111234") {
|
||||
t.Fatalf("ExtractProductText() = %q, want product code prefix", got)
|
||||
}
|
||||
if strings.HasPrefix(got, "\n") || strings.HasSuffix(got, "\n") {
|
||||
t.Fatalf("ExtractProductText() retained surrounding blank lines: %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseDiscussionHTMLExtractsHeadlineSummaryAndUpdated(t *testing.T) {
|
||||
got, err := ParseDiscussionHTML(string(readTestFile(t, "day1_prt.html")))
|
||||
if err != nil {
|
||||
t.Fatalf("ParseDiscussionHTML() error = %v", err)
|
||||
}
|
||||
if got.ProductTitle != "Day 1 Convective Outlook" {
|
||||
t.Fatalf("ProductTitle = %q", got.ProductTitle)
|
||||
}
|
||||
if got.Headline != "Day 1 Convective Outlook" {
|
||||
t.Fatalf("Headline = %q", got.Headline)
|
||||
}
|
||||
wantSummary := "Severe thunderstorms are possible across parts of the central Plains\nand mid Mississippi Valley this afternoon and evening."
|
||||
if got.Summary != wantSummary {
|
||||
t.Fatalf("Summary = %q, want %q", got.Summary, wantSummary)
|
||||
}
|
||||
if !strings.Contains(got.Discussion, "...DISCUSSION...") {
|
||||
t.Fatalf("Discussion missing full text: %q", got.Discussion)
|
||||
}
|
||||
if !strings.HasPrefix(got.Discussion, "SPC AC 111234") {
|
||||
t.Fatalf("Discussion = %q, want product code prefix", got.Discussion)
|
||||
}
|
||||
wantUpdated := time.Date(2026, 6, 11, 12, 45, 0, 0, time.UTC)
|
||||
if got.UpdatedAt == nil || !got.UpdatedAt.Equal(wantUpdated) {
|
||||
t.Fatalf("UpdatedAt = %v, want %s", got.UpdatedAt, wantUpdated)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseProductTitleSkipsSPCProductCode(t *testing.T) {
|
||||
got := ParseProductTitle("SPC AC 101959\nDay 1 Convective Outlook\nNWS Storm Prediction Center Norman OK")
|
||||
if got != "Day 1 Convective Outlook" {
|
||||
t.Fatalf("ParseProductTitle() = %q, want Day 1 Convective Outlook", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseDiscussionTextPreservesCorrectionMarker(t *testing.T) {
|
||||
got, err := ParseDiscussionHTML(string(readTestFile(t, "day2_prt_corr.html")))
|
||||
if err != nil {
|
||||
t.Fatalf("ParseDiscussionHTML() error = %v", err)
|
||||
}
|
||||
|
||||
if !strings.Contains(got.Headline, "CORR 1") {
|
||||
t.Fatalf("Headline = %q, want correction marker", got.Headline)
|
||||
}
|
||||
if !strings.Contains(got.Discussion, "CORR 1") {
|
||||
t.Fatalf("Discussion = %q, want correction marker", got.Discussion)
|
||||
}
|
||||
wantUpdated := time.Date(2026, 6, 11, 17, 30, 0, 0, time.UTC)
|
||||
if got.UpdatedAt == nil || !got.UpdatedAt.Equal(wantUpdated) {
|
||||
t.Fatalf("UpdatedAt = %v, want %s", got.UpdatedAt, wantUpdated)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseUpdatedTimestampReturnsNilWhenAbsent(t *testing.T) {
|
||||
text, err := ExtractProductText(string(readTestFile(t, "day2_prt_corr.html")))
|
||||
if err != nil {
|
||||
t.Fatalf("ExtractProductText() error = %v", err)
|
||||
}
|
||||
if got := ParseUpdatedTimestamp(text); got != nil {
|
||||
t.Fatalf("ParseUpdatedTimestamp() = %v, want nil", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseUpdatedTimestampAcceptsSPCUTCFormat(t *testing.T) {
|
||||
got := ParseUpdatedTimestamp("Updated: 1945 UTC Thu Jun 11 2026")
|
||||
want := time.Date(2026, 6, 11, 19, 45, 0, 0, time.UTC)
|
||||
if got == nil || !got.Equal(want) {
|
||||
t.Fatalf("ParseUpdatedTimestamp() = %v, want %s", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParsePageUpdatedTimestampAcceptsLiveSPCShape(t *testing.T) {
|
||||
got := ParsePageUpdatedTimestamp(string(readTestFile(t, "day3_prt.html")))
|
||||
want := time.Date(2026, 6, 11, 20, 0, 0, 0, time.UTC)
|
||||
if got == nil || !got.Equal(want) {
|
||||
t.Fatalf("ParsePageUpdatedTimestamp() = %v, want %s", got, want)
|
||||
}
|
||||
}
|
||||
8
internal/providers/spc/doc.go
Normal file
8
internal/providers/spc/doc.go
Normal file
@@ -0,0 +1,8 @@
|
||||
// Package spc contains deterministic helper code for Storm Prediction Center
|
||||
// products used by sources and normalizers.
|
||||
//
|
||||
// Rules:
|
||||
// - No network I/O here.
|
||||
// - Keep helpers deterministic and easy to unit test.
|
||||
// - Preserve upstream payload fragments needed for canonical mapping.
|
||||
package spc
|
||||
17
internal/providers/spc/fixture_test.go
Normal file
17
internal/providers/spc/fixture_test.go
Normal file
@@ -0,0 +1,17 @@
|
||||
package spc
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func readTestFile(t *testing.T, name string) []byte {
|
||||
t.Helper()
|
||||
path := filepath.Join("testdata", name)
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatalf("read %s: %v", path, err)
|
||||
}
|
||||
return raw
|
||||
}
|
||||
120
internal/providers/spc/geojson.go
Normal file
120
internal/providers/spc/geojson.go
Normal file
@@ -0,0 +1,120 @@
|
||||
package spc
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// GeoJSONFeatureCollection is the minimal SPC outlook FeatureCollection shape
|
||||
// needed by weatherfeeder.
|
||||
type GeoJSONFeatureCollection struct {
|
||||
Type string `json:"type"`
|
||||
Features []GeoJSONFeature `json:"features"`
|
||||
}
|
||||
|
||||
// GeoJSONFeature preserves typed SPC properties and compact raw geometry.
|
||||
type GeoJSONFeature struct {
|
||||
Type string `json:"type"`
|
||||
Properties GeoJSONProperties `json:"properties"`
|
||||
Geometry json.RawMessage `json:"geometry"`
|
||||
}
|
||||
|
||||
// GeoJSONProperties contains the SPC fields used by canonical mapping.
|
||||
type GeoJSONProperties struct {
|
||||
ValidISO string `json:"VALID_ISO"`
|
||||
ExpireISO string `json:"EXPIRE_ISO"`
|
||||
IssueISO string `json:"ISSUE_ISO"`
|
||||
Forecaster string `json:"FORECASTER"`
|
||||
Label string `json:"LABEL"`
|
||||
Label2 string `json:"LABEL2"`
|
||||
DN *int `json:"DN"`
|
||||
}
|
||||
|
||||
type geometryMetadata struct {
|
||||
Type string `json:"type"`
|
||||
Geometries []json.RawMessage `json:"geometries"`
|
||||
}
|
||||
|
||||
// DecodeGeoJSON decodes an SPC GeoJSON outlook product and compacts feature
|
||||
// geometry JSON for stable downstream storage.
|
||||
func DecodeGeoJSON(raw []byte) (GeoJSONFeatureCollection, error) {
|
||||
var collection GeoJSONFeatureCollection
|
||||
if err := json.Unmarshal(raw, &collection); err != nil {
|
||||
return GeoJSONFeatureCollection{}, fmt.Errorf("decode geojson: %w", err)
|
||||
}
|
||||
for i := range collection.Features {
|
||||
geom, err := compactJSON(collection.Features[i].Geometry)
|
||||
if err != nil {
|
||||
return GeoJSONFeatureCollection{}, fmt.Errorf("features[%d].geometry: %w", i, err)
|
||||
}
|
||||
collection.Features[i].Geometry = geom
|
||||
}
|
||||
return collection, nil
|
||||
}
|
||||
|
||||
// IsEmptyGeometryCollection reports whether raw is SPC's no-polygon placeholder
|
||||
// geometry shape: a GeometryCollection with no child geometries.
|
||||
func IsEmptyGeometryCollection(raw json.RawMessage) bool {
|
||||
var meta geometryMetadata
|
||||
if err := json.Unmarshal(raw, &meta); err != nil {
|
||||
return false
|
||||
}
|
||||
return meta.Type == "GeometryCollection" && len(meta.Geometries) == 0
|
||||
}
|
||||
|
||||
func (p *GeoJSONProperties) UnmarshalJSON(raw []byte) error {
|
||||
type alias GeoJSONProperties
|
||||
var aux struct {
|
||||
alias
|
||||
DN any `json:"DN"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &aux); err != nil {
|
||||
return err
|
||||
}
|
||||
*p = GeoJSONProperties(aux.alias)
|
||||
dn, err := parseSeverityRank(aux.DN)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
p.DN = dn
|
||||
return nil
|
||||
}
|
||||
|
||||
func parseSeverityRank(value any) (*int, error) {
|
||||
switch v := value.(type) {
|
||||
case nil:
|
||||
return nil, nil
|
||||
case float64:
|
||||
rank := int(v)
|
||||
if float64(rank) != v {
|
||||
return nil, fmt.Errorf("DN must be an integer, got %v", v)
|
||||
}
|
||||
return &rank, nil
|
||||
case string:
|
||||
v = strings.TrimSpace(v)
|
||||
if v == "" {
|
||||
return nil, nil
|
||||
}
|
||||
rank, err := strconv.Atoi(v)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("DN must be an integer, got %q", v)
|
||||
}
|
||||
return &rank, nil
|
||||
default:
|
||||
return nil, fmt.Errorf("DN must be an integer or string, got %T", value)
|
||||
}
|
||||
}
|
||||
|
||||
func compactJSON(raw json.RawMessage) (json.RawMessage, error) {
|
||||
if len(raw) == 0 {
|
||||
return nil, fmt.Errorf("missing")
|
||||
}
|
||||
var buf bytes.Buffer
|
||||
if err := json.Compact(&buf, raw); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return json.RawMessage(buf.Bytes()), nil
|
||||
}
|
||||
115
internal/providers/spc/geojson_test.go
Normal file
115
internal/providers/spc/geojson_test.go
Normal file
@@ -0,0 +1,115 @@
|
||||
package spc
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestDecodeGeoJSONExposesSPCPropertiesAndCompactGeometry(t *testing.T) {
|
||||
raw := readTestFile(t, "day1_cat.geojson")
|
||||
|
||||
got, err := DecodeGeoJSON(raw)
|
||||
if err != nil {
|
||||
t.Fatalf("DecodeGeoJSON() error = %v", err)
|
||||
}
|
||||
if got.Type != "FeatureCollection" {
|
||||
t.Fatalf("Type = %q, want FeatureCollection", got.Type)
|
||||
}
|
||||
if len(got.Features) != 1 {
|
||||
t.Fatalf("Features length = %d, want 1", len(got.Features))
|
||||
}
|
||||
|
||||
feature := got.Features[0]
|
||||
props := feature.Properties
|
||||
if props.ValidISO != "2026-06-11T13:00:00Z" {
|
||||
t.Fatalf("VALID_ISO = %q", props.ValidISO)
|
||||
}
|
||||
if props.ExpireISO != "2026-06-12T12:00:00Z" {
|
||||
t.Fatalf("EXPIRE_ISO = %q", props.ExpireISO)
|
||||
}
|
||||
if props.IssueISO != "2026-06-11T12:34:56Z" {
|
||||
t.Fatalf("ISSUE_ISO = %q", props.IssueISO)
|
||||
}
|
||||
if props.Forecaster != "SMITH" {
|
||||
t.Fatalf("FORECASTER = %q", props.Forecaster)
|
||||
}
|
||||
if props.Label != "SLGT" {
|
||||
t.Fatalf("LABEL = %q", props.Label)
|
||||
}
|
||||
if props.Label2 != "Slight Risk" {
|
||||
t.Fatalf("LABEL2 = %q", props.Label2)
|
||||
}
|
||||
if props.DN == nil || *props.DN != 3 {
|
||||
t.Fatalf("DN = %v, want 3", props.DN)
|
||||
}
|
||||
|
||||
wantGeometry := `{"type":"Polygon","coordinates":[[[-91.0,38.0],[-90.0,38.0],[-90.0,39.0],[-91.0,39.0],[-91.0,38.0]]]}`
|
||||
if string(feature.Geometry) != wantGeometry {
|
||||
t.Fatalf("Geometry = %s, want %s", feature.Geometry, wantGeometry)
|
||||
}
|
||||
if strings.Contains(string(feature.Geometry), "\n") || strings.Contains(string(feature.Geometry), " ") {
|
||||
t.Fatalf("Geometry is not compact: %q", feature.Geometry)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecodeGeoJSONParsesSeverityRankString(t *testing.T) {
|
||||
raw := readTestFile(t, "day2_torn.geojson")
|
||||
|
||||
got, err := DecodeGeoJSON(raw)
|
||||
if err != nil {
|
||||
t.Fatalf("DecodeGeoJSON() error = %v", err)
|
||||
}
|
||||
props := got.Features[0].Properties
|
||||
if props.DN == nil || *props.DN != 5 {
|
||||
t.Fatalf("DN = %v, want 5", props.DN)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsEmptyGeometryCollection(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
raw string
|
||||
want bool
|
||||
}{
|
||||
{
|
||||
name: "empty geometry collection",
|
||||
raw: `{"type":"GeometryCollection","geometries":[]}`,
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "non-empty geometry collection",
|
||||
raw: `{"type":"GeometryCollection","geometries":[{"type":"Polygon","coordinates":[]} ]}`,
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "polygon",
|
||||
raw: `{"type":"Polygon","coordinates":[]}`,
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "invalid json",
|
||||
raw: `{`,
|
||||
want: false,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
if got := IsEmptyGeometryCollection([]byte(tt.raw)); got != tt.want {
|
||||
t.Fatalf("IsEmptyGeometryCollection() = %v, want %v", got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseISOTimestampTrimsAndReturnsUTC(t *testing.T) {
|
||||
got, err := ParseISOTimestamp(" 2026-06-11T12:34:56Z ")
|
||||
if err != nil {
|
||||
t.Fatalf("ParseISOTimestamp() error = %v", err)
|
||||
}
|
||||
want := time.Date(2026, 6, 11, 12, 34, 56, 0, time.UTC)
|
||||
if !got.Equal(want) {
|
||||
t.Fatalf("ParseISOTimestamp() = %s, want %s", got, want)
|
||||
}
|
||||
}
|
||||
86
internal/providers/spc/product.go
Normal file
86
internal/providers/spc/product.go
Normal file
@@ -0,0 +1,86 @@
|
||||
package spc
|
||||
|
||||
import "fmt"
|
||||
|
||||
const (
|
||||
OutlookTypeCategorical = "categorical"
|
||||
OutlookTypeTornado = "tornado"
|
||||
OutlookTypeHail = "hail"
|
||||
OutlookTypeWind = "wind"
|
||||
)
|
||||
|
||||
// GeoJSONProduct describes one required SPC convective outlook GeoJSON product.
|
||||
type GeoJSONProduct struct {
|
||||
Key string
|
||||
Day int
|
||||
OutlookType string
|
||||
URL string
|
||||
}
|
||||
|
||||
// DiscussionProduct describes one required SPC convective outlook print page.
|
||||
type DiscussionProduct struct {
|
||||
Key string
|
||||
Day int
|
||||
URL string
|
||||
}
|
||||
|
||||
var geoJSONProducts = []GeoJSONProduct{
|
||||
{Key: "day1_categorical", Day: 1, OutlookType: OutlookTypeCategorical, URL: "https://www.spc.noaa.gov/products/outlook/day1otlk_cat.nolyr.geojson"},
|
||||
{Key: "day1_tornado", Day: 1, OutlookType: OutlookTypeTornado, URL: "https://www.spc.noaa.gov/products/outlook/day1otlk_torn.nolyr.geojson"},
|
||||
{Key: "day1_hail", Day: 1, OutlookType: OutlookTypeHail, URL: "https://www.spc.noaa.gov/products/outlook/day1otlk_hail.nolyr.geojson"},
|
||||
{Key: "day1_wind", Day: 1, OutlookType: OutlookTypeWind, URL: "https://www.spc.noaa.gov/products/outlook/day1otlk_wind.nolyr.geojson"},
|
||||
{Key: "day2_categorical", Day: 2, OutlookType: OutlookTypeCategorical, URL: "https://www.spc.noaa.gov/products/outlook/day2otlk_cat.nolyr.geojson"},
|
||||
{Key: "day2_tornado", Day: 2, OutlookType: OutlookTypeTornado, URL: "https://www.spc.noaa.gov/products/outlook/day2otlk_torn.nolyr.geojson"},
|
||||
{Key: "day2_hail", Day: 2, OutlookType: OutlookTypeHail, URL: "https://www.spc.noaa.gov/products/outlook/day2otlk_hail.nolyr.geojson"},
|
||||
{Key: "day2_wind", Day: 2, OutlookType: OutlookTypeWind, URL: "https://www.spc.noaa.gov/products/outlook/day2otlk_wind.nolyr.geojson"},
|
||||
{Key: "day3_categorical", Day: 3, OutlookType: OutlookTypeCategorical, URL: "https://www.spc.noaa.gov/products/outlook/day3otlk_cat.nolyr.geojson"},
|
||||
}
|
||||
|
||||
var discussionProducts = []DiscussionProduct{
|
||||
{Key: "day1", Day: 1, URL: "https://www.spc.noaa.gov/products/outlook/day1otlk_prt.html"},
|
||||
{Key: "day2", Day: 2, URL: "https://www.spc.noaa.gov/products/outlook/day2otlk_prt.html"},
|
||||
{Key: "day3", Day: 3, URL: "https://www.spc.noaa.gov/products/outlook/day3otlk_prt.html"},
|
||||
}
|
||||
|
||||
// GeoJSONProducts returns the required SPC convective outlook GeoJSON products
|
||||
// in stable day/type order.
|
||||
func GeoJSONProducts() []GeoJSONProduct {
|
||||
out := make([]GeoJSONProduct, len(geoJSONProducts))
|
||||
copy(out, geoJSONProducts)
|
||||
return out
|
||||
}
|
||||
|
||||
// DiscussionProducts returns the required SPC convective outlook print pages in
|
||||
// stable day order.
|
||||
func DiscussionProducts() []DiscussionProduct {
|
||||
out := make([]DiscussionProduct, len(discussionProducts))
|
||||
copy(out, discussionProducts)
|
||||
return out
|
||||
}
|
||||
|
||||
// GeoJSONProductByKey returns product metadata for a configured product key.
|
||||
func GeoJSONProductByKey(key string) (GeoJSONProduct, bool) {
|
||||
for _, product := range geoJSONProducts {
|
||||
if product.Key == key {
|
||||
return product, true
|
||||
}
|
||||
}
|
||||
return GeoJSONProduct{}, false
|
||||
}
|
||||
|
||||
// DiscussionProductByKey returns discussion metadata for a configured day key.
|
||||
func DiscussionProductByKey(key string) (DiscussionProduct, bool) {
|
||||
for _, product := range discussionProducts {
|
||||
if product.Key == key {
|
||||
return product, true
|
||||
}
|
||||
}
|
||||
return DiscussionProduct{}, false
|
||||
}
|
||||
|
||||
func validateProductDay(day int) error {
|
||||
if day < 1 || day > 3 {
|
||||
return fmt.Errorf("day must be 1, 2, or 3, got %d", day)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
53
internal/providers/spc/product_test.go
Normal file
53
internal/providers/spc/product_test.go
Normal file
@@ -0,0 +1,53 @@
|
||||
package spc
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestGeoJSONProductsStableOrder(t *testing.T) {
|
||||
got := GeoJSONProducts()
|
||||
if len(got) != 9 {
|
||||
t.Fatalf("GeoJSONProducts() length = %d, want 9", len(got))
|
||||
}
|
||||
|
||||
wantKeys := []string{
|
||||
"day1_categorical",
|
||||
"day1_tornado",
|
||||
"day1_hail",
|
||||
"day1_wind",
|
||||
"day2_categorical",
|
||||
"day2_tornado",
|
||||
"day2_hail",
|
||||
"day2_wind",
|
||||
"day3_categorical",
|
||||
}
|
||||
for i, want := range wantKeys {
|
||||
if got[i].Key != want {
|
||||
t.Fatalf("GeoJSONProducts()[%d].Key = %q, want %q", i, got[i].Key, want)
|
||||
}
|
||||
if err := validateProductDay(got[i].Day); err != nil {
|
||||
t.Fatalf("GeoJSONProducts()[%d].Day invalid: %v", i, err)
|
||||
}
|
||||
if got[i].URL == "" {
|
||||
t.Fatalf("GeoJSONProducts()[%d].URL is empty", i)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDiscussionProductsStableOrder(t *testing.T) {
|
||||
got := DiscussionProducts()
|
||||
if len(got) != 3 {
|
||||
t.Fatalf("DiscussionProducts() length = %d, want 3", len(got))
|
||||
}
|
||||
|
||||
wantKeys := []string{"day1", "day2", "day3"}
|
||||
for i, want := range wantKeys {
|
||||
if got[i].Key != want {
|
||||
t.Fatalf("DiscussionProducts()[%d].Key = %q, want %q", i, got[i].Key, want)
|
||||
}
|
||||
if got[i].Day != i+1 {
|
||||
t.Fatalf("DiscussionProducts()[%d].Day = %d, want %d", i, got[i].Day, i+1)
|
||||
}
|
||||
if got[i].URL == "" {
|
||||
t.Fatalf("DiscussionProducts()[%d].URL is empty", i)
|
||||
}
|
||||
}
|
||||
}
|
||||
45
internal/providers/spc/raw.go
Normal file
45
internal/providers/spc/raw.go
Normal file
@@ -0,0 +1,45 @@
|
||||
package spc
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"time"
|
||||
)
|
||||
|
||||
// RawConvectiveOutlookBundle is the provider payload shape for SPC convective
|
||||
// outlook fetch bundles.
|
||||
type RawConvectiveOutlookBundle struct {
|
||||
LocationID string `json:"locationId,omitempty"`
|
||||
LocationName string `json:"locationName,omitempty"`
|
||||
Latitude float64 `json:"latitude"`
|
||||
Longitude float64 `json:"longitude"`
|
||||
FetchedAt time.Time `json:"fetchedAt"`
|
||||
Products []RawOutlookProduct `json:"products"`
|
||||
Discussions []RawDiscussionPage `json:"discussions"`
|
||||
RSS *RawRSSFeed `json:"rss,omitempty"`
|
||||
}
|
||||
|
||||
// RawOutlookProduct contains one fetched SPC GeoJSON product.
|
||||
type RawOutlookProduct struct {
|
||||
Key string `json:"key"`
|
||||
Day int `json:"day"`
|
||||
OutlookType string `json:"outlookType"`
|
||||
URL string `json:"url"`
|
||||
FetchedAt time.Time `json:"fetchedAt"`
|
||||
Body json.RawMessage `json:"body"`
|
||||
}
|
||||
|
||||
// RawDiscussionPage contains one fetched SPC print page.
|
||||
type RawDiscussionPage struct {
|
||||
Key string `json:"key"`
|
||||
Day int `json:"day"`
|
||||
URL string `json:"url"`
|
||||
FetchedAt time.Time `json:"fetchedAt"`
|
||||
Body string `json:"body"`
|
||||
}
|
||||
|
||||
// RawRSSFeed contains optional fetched SPC RSS metadata.
|
||||
type RawRSSFeed struct {
|
||||
URL string `json:"url"`
|
||||
FetchedAt time.Time `json:"fetchedAt"`
|
||||
Body string `json:"body"`
|
||||
}
|
||||
51
internal/providers/spc/raw_test.go
Normal file
51
internal/providers/spc/raw_test.go
Normal file
@@ -0,0 +1,51 @@
|
||||
package spc
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestRawConvectiveOutlookBundleJSONShape(t *testing.T) {
|
||||
fetchedAt := time.Date(2026, 6, 11, 20, 0, 0, 0, time.UTC)
|
||||
bundle := RawConvectiveOutlookBundle{
|
||||
LocationID: "stl",
|
||||
LocationName: "St. Louis, MO",
|
||||
Latitude: 38.6239,
|
||||
Longitude: -90.3571,
|
||||
FetchedAt: fetchedAt,
|
||||
Products: []RawOutlookProduct{{
|
||||
Key: "day1_categorical",
|
||||
Day: 1,
|
||||
OutlookType: OutlookTypeCategorical,
|
||||
URL: "https://example.invalid/day1.geojson",
|
||||
FetchedAt: fetchedAt,
|
||||
Body: json.RawMessage(`{"type":"FeatureCollection","features":[]}`),
|
||||
}},
|
||||
Discussions: []RawDiscussionPage{{
|
||||
Key: "day1",
|
||||
Day: 1,
|
||||
URL: "https://example.invalid/day1.html",
|
||||
FetchedAt: fetchedAt,
|
||||
Body: "Day 1 Convective Outlook",
|
||||
}},
|
||||
}
|
||||
|
||||
raw, err := json.Marshal(bundle)
|
||||
if err != nil {
|
||||
t.Fatalf("Marshal() error = %v", err)
|
||||
}
|
||||
|
||||
var got map[string]any
|
||||
if err := json.Unmarshal(raw, &got); err != nil {
|
||||
t.Fatalf("Unmarshal() error = %v", err)
|
||||
}
|
||||
for _, key := range []string{"locationId", "locationName", "latitude", "longitude", "fetchedAt", "products", "discussions"} {
|
||||
if _, ok := got[key]; !ok {
|
||||
t.Fatalf("marshaled bundle missing key %q in %s", key, raw)
|
||||
}
|
||||
}
|
||||
if _, ok := got["rss"]; ok {
|
||||
t.Fatalf("marshaled bundle included empty rss: %s", raw)
|
||||
}
|
||||
}
|
||||
81
internal/providers/spc/rss.go
Normal file
81
internal/providers/spc/rss.go
Normal file
@@ -0,0 +1,81 @@
|
||||
package spc
|
||||
|
||||
import (
|
||||
"encoding/xml"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// RSSFeed is a minimal view of the optional SPC RSS feed.
|
||||
type RSSFeed struct {
|
||||
Title string
|
||||
Link string
|
||||
Description string
|
||||
LastBuildDate *time.Time
|
||||
Items []RSSItem
|
||||
}
|
||||
|
||||
// RSSItem is a minimal view of one optional SPC RSS item.
|
||||
type RSSItem struct {
|
||||
Title string
|
||||
Link string
|
||||
Description string
|
||||
PubDate string
|
||||
GUID string
|
||||
}
|
||||
|
||||
// ParseRSSFeed decodes supplemental SPC RSS metadata.
|
||||
func ParseRSSFeed(raw string) (RSSFeed, error) {
|
||||
var doc struct {
|
||||
Channel struct {
|
||||
Title string `xml:"title"`
|
||||
Link string `xml:"link"`
|
||||
Description string `xml:"description"`
|
||||
LastBuildDate string `xml:"lastBuildDate"`
|
||||
Items []struct {
|
||||
Title string `xml:"title"`
|
||||
Link string `xml:"link"`
|
||||
Description string `xml:"description"`
|
||||
PubDate string `xml:"pubDate"`
|
||||
GUID string `xml:"guid"`
|
||||
} `xml:"item"`
|
||||
} `xml:"channel"`
|
||||
}
|
||||
if err := xml.Unmarshal([]byte(raw), &doc); err != nil {
|
||||
return RSSFeed{}, fmt.Errorf("decode rss: %w", err)
|
||||
}
|
||||
|
||||
feed := RSSFeed{
|
||||
Title: strings.TrimSpace(doc.Channel.Title),
|
||||
Link: strings.TrimSpace(doc.Channel.Link),
|
||||
Description: strings.TrimSpace(doc.Channel.Description),
|
||||
LastBuildDate: parseRSSDate(doc.Channel.LastBuildDate),
|
||||
Items: make([]RSSItem, 0, len(doc.Channel.Items)),
|
||||
}
|
||||
for _, item := range doc.Channel.Items {
|
||||
feed.Items = append(feed.Items, RSSItem{
|
||||
Title: strings.TrimSpace(item.Title),
|
||||
Link: strings.TrimSpace(item.Link),
|
||||
Description: strings.TrimSpace(item.Description),
|
||||
PubDate: strings.TrimSpace(item.PubDate),
|
||||
GUID: strings.TrimSpace(item.GUID),
|
||||
})
|
||||
}
|
||||
return feed, nil
|
||||
}
|
||||
|
||||
func parseRSSDate(value string) *time.Time {
|
||||
value = strings.TrimSpace(value)
|
||||
if value == "" {
|
||||
return nil
|
||||
}
|
||||
for _, layout := range []string{time.RFC1123Z, time.RFC1123} {
|
||||
t, err := time.Parse(layout, value)
|
||||
if err == nil {
|
||||
tt := t.UTC()
|
||||
return &tt
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
43
internal/providers/spc/rss_test.go
Normal file
43
internal/providers/spc/rss_test.go
Normal file
@@ -0,0 +1,43 @@
|
||||
package spc
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestParseRSSFeed(t *testing.T) {
|
||||
const raw = `<?xml version="1.0"?>
|
||||
<rss version="2.0">
|
||||
<channel>
|
||||
<title>SPC AC RSS</title>
|
||||
<link>https://www.spc.noaa.gov/products/</link>
|
||||
<description>SPC products</description>
|
||||
<lastBuildDate>Thu, 11 Jun 2026 19:00:00 +0000</lastBuildDate>
|
||||
<item>
|
||||
<title>Day 1 Convective Outlook</title>
|
||||
<link>https://www.spc.noaa.gov/products/outlook/day1otlk.html</link>
|
||||
<description>Outlook text</description>
|
||||
<pubDate>Thu, 11 Jun 2026 18:55:00 +0000</pubDate>
|
||||
<guid>day1</guid>
|
||||
</item>
|
||||
</channel>
|
||||
</rss>`
|
||||
|
||||
got, err := ParseRSSFeed(raw)
|
||||
if err != nil {
|
||||
t.Fatalf("ParseRSSFeed() error = %v", err)
|
||||
}
|
||||
if got.Title != "SPC AC RSS" {
|
||||
t.Fatalf("Title = %q", got.Title)
|
||||
}
|
||||
wantBuild := time.Date(2026, 6, 11, 19, 0, 0, 0, time.UTC)
|
||||
if got.LastBuildDate == nil || !got.LastBuildDate.Equal(wantBuild) {
|
||||
t.Fatalf("LastBuildDate = %v, want %s", got.LastBuildDate, wantBuild)
|
||||
}
|
||||
if len(got.Items) != 1 {
|
||||
t.Fatalf("Items length = %d, want 1", len(got.Items))
|
||||
}
|
||||
if got.Items[0].GUID != "day1" {
|
||||
t.Fatalf("Item GUID = %q", got.Items[0].GUID)
|
||||
}
|
||||
}
|
||||
29
internal/providers/spc/testdata/day1_cat.geojson
vendored
Normal file
29
internal/providers/spc/testdata/day1_cat.geojson
vendored
Normal file
@@ -0,0 +1,29 @@
|
||||
{
|
||||
"type": "FeatureCollection",
|
||||
"features": [
|
||||
{
|
||||
"type": "Feature",
|
||||
"properties": {
|
||||
"VALID_ISO": "2026-06-11T13:00:00Z",
|
||||
"EXPIRE_ISO": "2026-06-12T12:00:00Z",
|
||||
"ISSUE_ISO": "2026-06-11T12:34:56Z",
|
||||
"FORECASTER": "SMITH",
|
||||
"LABEL": "SLGT",
|
||||
"LABEL2": "Slight Risk",
|
||||
"DN": 3
|
||||
},
|
||||
"geometry": {
|
||||
"type": "Polygon",
|
||||
"coordinates": [
|
||||
[
|
||||
[-91.0, 38.0],
|
||||
[-90.0, 38.0],
|
||||
[-90.0, 39.0],
|
||||
[-91.0, 39.0],
|
||||
[-91.0, 38.0]
|
||||
]
|
||||
]
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
22
internal/providers/spc/testdata/day1_prt.html
vendored
Normal file
22
internal/providers/spc/testdata/day1_prt.html
vendored
Normal file
@@ -0,0 +1,22 @@
|
||||
<!doctype html>
|
||||
<html>
|
||||
<head><title>Day 1 Convective Outlook</title></head>
|
||||
<body>
|
||||
<table>
|
||||
<tr><td align="center" class="rpttext" nowrap>Updated: Thu Jun 11 12:45:00 UTC 2026 (<a href="archive/day1-geojson.zip">geojson</a>)</td></tr>
|
||||
</table>
|
||||
<pre>
|
||||
<script>window.bad = "<b>ignore me</b>";</script>
|
||||
SPC AC 111234
|
||||
Day 1 Convective Outlook
|
||||
NWS Storm Prediction Center Norman OK
|
||||
|
||||
...SUMMARY...
|
||||
Severe thunderstorms are possible across parts of the central Plains
|
||||
and mid Mississippi Valley this afternoon and evening.
|
||||
|
||||
...DISCUSSION...
|
||||
The primary threats will be damaging wind and large hail.
|
||||
</pre>
|
||||
</body>
|
||||
</html>
|
||||
19
internal/providers/spc/testdata/day2_prt_corr.html
vendored
Normal file
19
internal/providers/spc/testdata/day2_prt_corr.html
vendored
Normal file
@@ -0,0 +1,19 @@
|
||||
<!doctype html>
|
||||
<html>
|
||||
<body>
|
||||
<table>
|
||||
<tr><td class="rpttext">Updated: Thu Jun 11 17:30:00 UTC 2026 </td></tr>
|
||||
</table>
|
||||
<pre>
|
||||
SPC AC 111730
|
||||
Day 2 Convective Outlook CORR 1
|
||||
NWS Storm Prediction Center Norman OK
|
||||
|
||||
...SUMMARY...
|
||||
Scattered severe thunderstorms remain possible across the southern Plains.
|
||||
|
||||
...DISCUSSION...
|
||||
Corrected outlook text remains otherwise unchanged.
|
||||
</pre>
|
||||
</body>
|
||||
</html>
|
||||
29
internal/providers/spc/testdata/day2_torn.geojson
vendored
Normal file
29
internal/providers/spc/testdata/day2_torn.geojson
vendored
Normal file
@@ -0,0 +1,29 @@
|
||||
{
|
||||
"type": "FeatureCollection",
|
||||
"features": [
|
||||
{
|
||||
"type": "Feature",
|
||||
"properties": {
|
||||
"VALID_ISO": "2026-06-12T12:00:00Z",
|
||||
"EXPIRE_ISO": "2026-06-13T12:00:00Z",
|
||||
"ISSUE_ISO": "2026-06-11T17:30:00Z",
|
||||
"FORECASTER": "DOE",
|
||||
"LABEL": "5",
|
||||
"LABEL2": "5% Tornado",
|
||||
"DN": "5"
|
||||
},
|
||||
"geometry": {
|
||||
"type": "Polygon",
|
||||
"coordinates": [
|
||||
[
|
||||
[-100.0, 35.0],
|
||||
[-98.0, 35.0],
|
||||
[-98.0, 37.0],
|
||||
[-100.0, 37.0],
|
||||
[-100.0, 35.0]
|
||||
]
|
||||
]
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
31
internal/providers/spc/testdata/day3_cat.geojson
vendored
Normal file
31
internal/providers/spc/testdata/day3_cat.geojson
vendored
Normal file
@@ -0,0 +1,31 @@
|
||||
{
|
||||
"type": "FeatureCollection",
|
||||
"features": [
|
||||
{
|
||||
"type": "Feature",
|
||||
"properties": {
|
||||
"VALID_ISO": "2026-06-13T12:00:00Z",
|
||||
"EXPIRE_ISO": "2026-06-14T12:00:00Z",
|
||||
"ISSUE_ISO": "2026-06-11T19:45:00Z",
|
||||
"FORECASTER": "LEE",
|
||||
"LABEL": "MRGL",
|
||||
"LABEL2": "Marginal Risk",
|
||||
"DN": 2
|
||||
},
|
||||
"geometry": {
|
||||
"type": "MultiPolygon",
|
||||
"coordinates": [
|
||||
[
|
||||
[
|
||||
[-103.0, 34.0],
|
||||
[-101.0, 34.0],
|
||||
[-101.0, 36.0],
|
||||
[-103.0, 36.0],
|
||||
[-103.0, 34.0]
|
||||
]
|
||||
]
|
||||
]
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
19
internal/providers/spc/testdata/day3_prt.html
vendored
Normal file
19
internal/providers/spc/testdata/day3_prt.html
vendored
Normal file
@@ -0,0 +1,19 @@
|
||||
<!doctype html>
|
||||
<html>
|
||||
<body>
|
||||
<table>
|
||||
<tr><td class="rpttext">Updated: Thu Jun 11 20:00:00 UTC 2026 </td></tr>
|
||||
</table>
|
||||
<pre>
|
||||
SPC AC 112000
|
||||
Day 3 Convective Outlook
|
||||
NWS Storm Prediction Center Norman OK
|
||||
|
||||
...SUMMARY...
|
||||
A corridor of strong to severe storms may develop near a frontal zone.
|
||||
|
||||
...DISCUSSION...
|
||||
Confidence remains moderate for organized storms.
|
||||
</pre>
|
||||
</body>
|
||||
</html>
|
||||
24
internal/providers/spc/time.go
Normal file
24
internal/providers/spc/time.go
Normal file
@@ -0,0 +1,24 @@
|
||||
package spc
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// ParseISOTimestamp parses SPC ISO timestamps from GeoJSON properties.
|
||||
func ParseISOTimestamp(value string) (time.Time, error) {
|
||||
return time.Parse(time.RFC3339, strings.TrimSpace(value))
|
||||
}
|
||||
|
||||
func parseOptionalISOTimestamp(value string) *time.Time {
|
||||
value = strings.TrimSpace(value)
|
||||
if value == "" {
|
||||
return nil
|
||||
}
|
||||
t, err := ParseISOTimestamp(value)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
tt := t.UTC()
|
||||
return &tt
|
||||
}
|
||||
@@ -11,6 +11,7 @@
|
||||
// - weather.forecast_discussion.v1 -> model.WeatherForecastDiscussion
|
||||
// - weather.weather_story.v1 -> model.WeatherStoryRun
|
||||
// - weather.alert.v1 -> model.WeatherAlertRun
|
||||
// - weather.outlook.v2 -> model.WeatherOutlookRun
|
||||
//
|
||||
// Parent/child relationships:
|
||||
// - observations.event_id -> observation_present_weather.event_id
|
||||
@@ -19,9 +20,11 @@
|
||||
// - weather_story_runs.event_id -> weather_stories.run_event_id
|
||||
// - alert_runs.event_id -> alerts.run_event_id
|
||||
// - alerts.(run_event_id, alert_index) -> alert_references.(run_event_id, alert_index)
|
||||
// - outlook_runs.event_id -> outlooks.run_event_id
|
||||
// - outlook_runs.event_id -> outlook_discussions.run_event_id
|
||||
//
|
||||
// Dedupe and retention behavior:
|
||||
// - Parent primary keys (event_id): observations, forecasts, alert_runs.
|
||||
// - Parent primary keys (event_id): observations, forecasts, alert_runs, outlook_runs.
|
||||
// - Child primary keys use positional indexes to preserve payload order.
|
||||
// - Prune columns:
|
||||
// - observations.observed_at
|
||||
@@ -35,11 +38,14 @@
|
||||
// - alert_runs.as_of
|
||||
// - alerts.as_of
|
||||
// - alert_references.as_of
|
||||
// - outlook_runs.as_of
|
||||
// - outlooks.as_of
|
||||
// - outlook_discussions.as_of
|
||||
//
|
||||
// Envelope field mapping (shared parent columns)
|
||||
//
|
||||
// These columns exist on parent tables such as observations, forecasts,
|
||||
// forecast_discussions, weather_story_runs, and alert_runs:
|
||||
// forecast_discussions, weather_story_runs, alert_runs, and outlook_runs:
|
||||
// - event_id TEXT -> event.id
|
||||
// - event_kind TEXT -> event.kind
|
||||
// - event_source TEXT -> event.source
|
||||
@@ -192,6 +198,7 @@
|
||||
// - sent TIMESTAMPTZ NULL -> payload.alerts[i].sent
|
||||
// - effective TIMESTAMPTZ NULL -> payload.alerts[i].effective
|
||||
// - onset TIMESTAMPTZ NULL -> payload.alerts[i].onset
|
||||
// - ends TIMESTAMPTZ NULL -> payload.alerts[i].ends
|
||||
// - expires TIMESTAMPTZ NULL -> payload.alerts[i].expires
|
||||
// - area_description TEXT NULL -> payload.alerts[i].areaDescription
|
||||
// - sender_name TEXT NULL -> payload.alerts[i].senderName
|
||||
@@ -208,6 +215,57 @@
|
||||
// - sender TEXT NULL -> payload.alerts[i].references[j].sender
|
||||
// - sent TIMESTAMPTZ NULL -> payload.alerts[i].references[j].sent
|
||||
//
|
||||
// 10. outlook_runs (PK: event_id)
|
||||
//
|
||||
// - event_id TEXT -> event.id
|
||||
// - event_kind TEXT -> event.kind
|
||||
// - event_source TEXT -> event.source
|
||||
// - event_schema TEXT -> event.schema
|
||||
// - event_emitted_at TIMESTAMPTZ -> event.emitted_at
|
||||
// - event_effective_at TIMESTAMPTZ NULL -> event.effective_at
|
||||
// - location_id TEXT NULL -> payload.locationId
|
||||
// - location_name TEXT NULL -> payload.locationName
|
||||
// - latitude DOUBLE PRECISION NULL -> payload.latitude
|
||||
// - longitude DOUBLE PRECISION NULL -> payload.longitude
|
||||
// - as_of TIMESTAMPTZ -> payload.asOf
|
||||
// - issued_at TIMESTAMPTZ NULL -> payload.issuedAt
|
||||
// - outlook_count INTEGER -> len(payload.outlooks)
|
||||
// - discussion_count INTEGER -> len(payload.discussions)
|
||||
//
|
||||
// 11. outlooks (PK: run_event_id, outlook_index)
|
||||
//
|
||||
// - run_event_id TEXT -> outlook_runs.event_id / payload.outlooks[i]
|
||||
// - outlook_index INTEGER -> i (array position in payload.outlooks)
|
||||
// - as_of TIMESTAMPTZ -> payload.asOf (copied from parent)
|
||||
// - outlook_id TEXT -> payload.outlooks[i].id
|
||||
// - provider TEXT -> payload.outlooks[i].provider
|
||||
// - product TEXT -> payload.outlooks[i].product
|
||||
// - day INTEGER -> payload.outlooks[i].day
|
||||
// - outlook_type TEXT -> payload.outlooks[i].outlookType
|
||||
// - label TEXT -> payload.outlooks[i].label
|
||||
// - label_text TEXT NULL -> payload.outlooks[i].labelText
|
||||
// - severity_rank INTEGER NULL -> payload.outlooks[i].severityRank
|
||||
// - valid_from TIMESTAMPTZ -> payload.outlooks[i].validFrom
|
||||
// - valid_to TIMESTAMPTZ -> payload.outlooks[i].validTo
|
||||
// - issued_at TIMESTAMPTZ -> payload.outlooks[i].issuedAt
|
||||
// - expires_at TIMESTAMPTZ -> payload.outlooks[i].expiresAt
|
||||
// - forecaster TEXT NULL -> payload.outlooks[i].forecaster
|
||||
// - source_url TEXT NULL -> payload.outlooks[i].sourceUrl
|
||||
// - image_url TEXT NULL -> payload.outlooks[i].imageUrl
|
||||
// - contains_location BOOLEAN -> payload.outlooks[i].containsLocation
|
||||
// - geometry_json TEXT -> compact JSON payload.outlooks[i].geometry
|
||||
//
|
||||
// 12. outlook_discussions (PK: run_event_id, discussion_index)
|
||||
//
|
||||
// - run_event_id TEXT -> outlook_runs.event_id / payload.discussions[i]
|
||||
// - discussion_index INTEGER -> i (array position in payload.discussions)
|
||||
// - as_of TIMESTAMPTZ -> payload.asOf (copied from parent)
|
||||
// - day INTEGER -> payload.discussions[i].day
|
||||
// - headline TEXT NULL -> payload.discussions[i].headline
|
||||
// - summary TEXT NULL -> payload.discussions[i].summary
|
||||
// - discussion TEXT NULL -> payload.discussions[i].discussion
|
||||
// - updated_at TIMESTAMPTZ NULL -> payload.discussions[i].updatedAt
|
||||
//
|
||||
// Reconstructing canonical JSON payloads
|
||||
//
|
||||
// - WeatherObservation:
|
||||
@@ -226,4 +284,9 @@
|
||||
// read one row from alert_runs, join alerts by run_event_id ordered by
|
||||
// alert_index, then join alert_references by (run_event_id, alert_index)
|
||||
// ordered by reference_index to rebuild references per alert.
|
||||
//
|
||||
// - WeatherOutlookRun:
|
||||
// read one row from outlook_runs, join outlooks by run_event_id ordered by
|
||||
// outlook_index to rebuild outlooks, then join outlook_discussions by
|
||||
// run_event_id ordered by discussion_index to rebuild discussions.
|
||||
package postgres
|
||||
|
||||
33
internal/sinks/postgres/docs_test.go
Normal file
33
internal/sinks/postgres/docs_test.go
Normal file
@@ -0,0 +1,33 @@
|
||||
package postgres
|
||||
|
||||
import (
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestDocumentedOutlookDiscussionStorage(t *testing.T) {
|
||||
for _, path := range []string{
|
||||
"../../../docs/integrations/postgres.md",
|
||||
"../../../docs/internal/postgres-sink.md",
|
||||
} {
|
||||
t.Run(path, func(t *testing.T) {
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatalf("ReadFile(%s) error = %v", path, err)
|
||||
}
|
||||
doc := string(raw)
|
||||
|
||||
for _, want := range []string{
|
||||
tableOutlookDiscussions,
|
||||
"discussion_count",
|
||||
"discussion_index",
|
||||
"weather.outlook.v2",
|
||||
} {
|
||||
if !strings.Contains(doc, want) {
|
||||
t.Fatalf("%s missing %q", path, want)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,6 +1,7 @@
|
||||
package postgres
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
@@ -26,6 +27,8 @@ func mapPostgresEvent(_ context.Context, e fkevent.Event) ([]fksinks.PostgresWri
|
||||
return mapWeatherStoryEvent(e)
|
||||
case standards.SchemaWeatherAlertV1:
|
||||
return mapAlertEvent(e)
|
||||
case standards.SchemaWeatherOutlookV2:
|
||||
return mapOutlookEvent(e)
|
||||
default:
|
||||
return nil, nil
|
||||
}
|
||||
@@ -45,13 +48,7 @@ func mapObservationEvent(e fkevent.Event) ([]fksinks.PostgresWrite, error) {
|
||||
|
||||
writes = append(writes, fksinks.PostgresWrite{
|
||||
Table: tableObservations,
|
||||
Values: map[string]any{
|
||||
"event_id": e.ID,
|
||||
"event_kind": string(e.Kind),
|
||||
"event_source": e.Source,
|
||||
"event_schema": e.Schema,
|
||||
"event_emitted_at": e.EmittedAt.UTC(),
|
||||
"event_effective_at": nullableTime(e.EffectiveAt),
|
||||
Values: parentEventValues(e, map[string]any{
|
||||
"station_id": nullableString(obs.StationID),
|
||||
"station_name": nullableString(obs.StationName),
|
||||
"observed_at": observedAt,
|
||||
@@ -67,7 +64,7 @@ func mapObservationEvent(e fkevent.Event) ([]fksinks.PostgresWrite, error) {
|
||||
"visibility_meters": nullableFloat64(obs.VisibilityMeters),
|
||||
"relative_humidity_percent": nullableFloat64(obs.RelativeHumidityPercent),
|
||||
"apparent_temperature_c": nullableFloat64(obs.ApparentTemperatureC),
|
||||
},
|
||||
}),
|
||||
})
|
||||
|
||||
for i, pw := range obs.PresentWeather {
|
||||
@@ -106,23 +103,17 @@ func mapForecastEvent(e fkevent.Event) ([]fksinks.PostgresWrite, error) {
|
||||
|
||||
writes = append(writes, fksinks.PostgresWrite{
|
||||
Table: tableForecasts,
|
||||
Values: map[string]any{
|
||||
"event_id": e.ID,
|
||||
"event_kind": string(e.Kind),
|
||||
"event_source": e.Source,
|
||||
"event_schema": e.Schema,
|
||||
"event_emitted_at": e.EmittedAt.UTC(),
|
||||
"event_effective_at": nullableTime(e.EffectiveAt),
|
||||
"location_id": nullableString(run.LocationID),
|
||||
"location_name": nullableString(run.LocationName),
|
||||
"issued_at": issuedAt,
|
||||
"updated_at": nullableTime(run.UpdatedAt),
|
||||
"product": string(run.Product),
|
||||
"latitude": nullableFloat64(run.Latitude),
|
||||
"longitude": nullableFloat64(run.Longitude),
|
||||
"elevation_meters": nullableFloat64(run.ElevationMeters),
|
||||
"period_count": len(run.Periods),
|
||||
},
|
||||
Values: parentEventValues(e, map[string]any{
|
||||
"location_id": nullableString(run.LocationID),
|
||||
"location_name": nullableString(run.LocationName),
|
||||
"issued_at": issuedAt,
|
||||
"updated_at": nullableTime(run.UpdatedAt),
|
||||
"product": string(run.Product),
|
||||
"latitude": nullableFloat64(run.Latitude),
|
||||
"longitude": nullableFloat64(run.Longitude),
|
||||
"elevation_meters": nullableFloat64(run.ElevationMeters),
|
||||
"period_count": len(run.Periods),
|
||||
}),
|
||||
})
|
||||
|
||||
for i, p := range run.Periods {
|
||||
@@ -183,13 +174,7 @@ func mapForecastDiscussionEvent(e fkevent.Event) ([]fksinks.PostgresWrite, error
|
||||
writes := make([]fksinks.PostgresWrite, 0, 1+len(run.KeyMessages))
|
||||
writes = append(writes, fksinks.PostgresWrite{
|
||||
Table: tableForecastDiscussions,
|
||||
Values: map[string]any{
|
||||
"event_id": e.ID,
|
||||
"event_kind": string(e.Kind),
|
||||
"event_source": e.Source,
|
||||
"event_schema": e.Schema,
|
||||
"event_emitted_at": e.EmittedAt.UTC(),
|
||||
"event_effective_at": nullableTime(e.EffectiveAt),
|
||||
Values: parentEventValues(e, map[string]any{
|
||||
"office_id": nullableString(run.OfficeID),
|
||||
"office_name": nullableString(run.OfficeName),
|
||||
"issued_at": issuedAt,
|
||||
@@ -202,7 +187,7 @@ func mapForecastDiscussionEvent(e fkevent.Event) ([]fksinks.PostgresWrite, error
|
||||
"long_term_issued_at": longTermIssuedAt,
|
||||
"long_term_text": longTermText,
|
||||
"key_message_count": len(run.KeyMessages),
|
||||
},
|
||||
}),
|
||||
})
|
||||
|
||||
for i, msg := range run.KeyMessages {
|
||||
@@ -233,17 +218,11 @@ func mapWeatherStoryEvent(e fkevent.Event) ([]fksinks.PostgresWrite, error) {
|
||||
writes := make([]fksinks.PostgresWrite, 0, 1+len(run.Stories))
|
||||
writes = append(writes, fksinks.PostgresWrite{
|
||||
Table: tableWeatherStoryRuns,
|
||||
Values: map[string]any{
|
||||
"event_id": e.ID,
|
||||
"event_kind": string(e.Kind),
|
||||
"event_source": e.Source,
|
||||
"event_schema": e.Schema,
|
||||
"event_emitted_at": e.EmittedAt.UTC(),
|
||||
"event_effective_at": nullableTime(e.EffectiveAt),
|
||||
"office_id": nullableString(run.OfficeID),
|
||||
"as_of": asOf,
|
||||
"story_count": len(run.Stories),
|
||||
},
|
||||
Values: parentEventValues(e, map[string]any{
|
||||
"office_id": nullableString(run.OfficeID),
|
||||
"as_of": asOf,
|
||||
"story_count": len(run.Stories),
|
||||
}),
|
||||
})
|
||||
|
||||
for i, story := range run.Stories {
|
||||
@@ -287,20 +266,14 @@ func mapAlertEvent(e fkevent.Event) ([]fksinks.PostgresWrite, error) {
|
||||
|
||||
writes = append(writes, fksinks.PostgresWrite{
|
||||
Table: tableAlertRuns,
|
||||
Values: map[string]any{
|
||||
"event_id": e.ID,
|
||||
"event_kind": string(e.Kind),
|
||||
"event_source": e.Source,
|
||||
"event_schema": e.Schema,
|
||||
"event_emitted_at": e.EmittedAt.UTC(),
|
||||
"event_effective_at": nullableTime(e.EffectiveAt),
|
||||
"location_id": nullableString(run.LocationID),
|
||||
"location_name": nullableString(run.LocationName),
|
||||
"as_of": asOf,
|
||||
"latitude": nullableFloat64(run.Latitude),
|
||||
"longitude": nullableFloat64(run.Longitude),
|
||||
"alert_count": len(run.Alerts),
|
||||
},
|
||||
Values: parentEventValues(e, map[string]any{
|
||||
"location_id": nullableString(run.LocationID),
|
||||
"location_name": nullableString(run.LocationName),
|
||||
"as_of": asOf,
|
||||
"latitude": nullableFloat64(run.Latitude),
|
||||
"longitude": nullableFloat64(run.Longitude),
|
||||
"alert_count": len(run.Alerts),
|
||||
}),
|
||||
})
|
||||
|
||||
for i, a := range run.Alerts {
|
||||
@@ -329,6 +302,7 @@ func mapAlertEvent(e fkevent.Event) ([]fksinks.PostgresWrite, error) {
|
||||
"sent": nullableTime(a.Sent),
|
||||
"effective": nullableTime(a.Effective),
|
||||
"onset": nullableTime(a.Onset),
|
||||
"ends": nullableTime(a.Ends),
|
||||
"expires": nullableTime(a.Expires),
|
||||
"area_description": nullableString(a.AreaDescription),
|
||||
"sender_name": nullableString(a.SenderName),
|
||||
@@ -356,6 +330,143 @@ func mapAlertEvent(e fkevent.Event) ([]fksinks.PostgresWrite, error) {
|
||||
return writes, nil
|
||||
}
|
||||
|
||||
func mapOutlookEvent(e fkevent.Event) ([]fksinks.PostgresWrite, error) {
|
||||
run, err := decodePayload[model.WeatherOutlookRun](e.Payload)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("decode outlook payload: %w", err)
|
||||
}
|
||||
if run.AsOf.IsZero() {
|
||||
return nil, fmt.Errorf("decode outlook payload: asOf is required")
|
||||
}
|
||||
|
||||
asOf := run.AsOf.UTC()
|
||||
if err := validateOutlookDiscussions(run.Discussions); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
writes := make([]fksinks.PostgresWrite, 0, 1+len(run.Outlooks)+len(run.Discussions))
|
||||
writes = append(writes, fksinks.PostgresWrite{
|
||||
Table: tableOutlookRuns,
|
||||
Values: parentEventValues(e, map[string]any{
|
||||
"location_id": nullableString(run.LocationID),
|
||||
"location_name": nullableString(run.LocationName),
|
||||
"latitude": nullableFloat64(run.Latitude),
|
||||
"longitude": nullableFloat64(run.Longitude),
|
||||
"as_of": asOf,
|
||||
"issued_at": nullableTime(run.IssuedAt),
|
||||
"outlook_count": len(run.Outlooks),
|
||||
"discussion_count": len(run.Discussions),
|
||||
}),
|
||||
})
|
||||
|
||||
for i, outlook := range run.Outlooks {
|
||||
if err := validateOutlook(outlook, i); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
geometryJSON, err := requiredCompactJSONText(outlook.Geometry)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("decode outlook payload: outlooks[%d].geometry: %w", i, err)
|
||||
}
|
||||
|
||||
writes = append(writes, fksinks.PostgresWrite{
|
||||
Table: tableOutlooks,
|
||||
Values: map[string]any{
|
||||
"run_event_id": e.ID,
|
||||
"outlook_index": i,
|
||||
"as_of": asOf,
|
||||
"outlook_id": outlook.ID,
|
||||
"provider": outlook.Provider,
|
||||
"product": outlook.Product,
|
||||
"day": outlook.Day,
|
||||
"outlook_type": outlook.OutlookType,
|
||||
"label": outlook.Label,
|
||||
"label_text": nullableString(outlook.LabelText),
|
||||
"severity_rank": nullableInt(outlook.SeverityRank),
|
||||
"valid_from": outlook.ValidFrom.UTC(),
|
||||
"valid_to": outlook.ValidTo.UTC(),
|
||||
"issued_at": outlook.IssuedAt.UTC(),
|
||||
"expires_at": outlook.ExpiresAt.UTC(),
|
||||
"forecaster": nullableString(outlook.Forecaster),
|
||||
"source_url": nullableString(outlook.SourceURL),
|
||||
"image_url": nullableString(outlook.ImageURL),
|
||||
"contains_location": outlook.ContainsLocation,
|
||||
"geometry_json": geometryJSON,
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
for i, discussion := range run.Discussions {
|
||||
writes = append(writes, fksinks.PostgresWrite{
|
||||
Table: tableOutlookDiscussions,
|
||||
Values: map[string]any{
|
||||
"run_event_id": e.ID,
|
||||
"discussion_index": i,
|
||||
"as_of": asOf,
|
||||
"day": discussion.Day,
|
||||
"headline": nullableString(discussion.Headline),
|
||||
"summary": nullableString(discussion.Summary),
|
||||
"discussion": nullableString(discussion.Discussion),
|
||||
"updated_at": nullableTime(discussion.UpdatedAt),
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
return writes, nil
|
||||
}
|
||||
|
||||
func validateOutlook(outlook model.WeatherOutlook, index int) error {
|
||||
if strings.TrimSpace(outlook.ID) == "" {
|
||||
return fmt.Errorf("decode outlook payload: outlooks[%d].id is required", index)
|
||||
}
|
||||
if strings.TrimSpace(outlook.Provider) == "" {
|
||||
return fmt.Errorf("decode outlook payload: outlooks[%d].provider is required", index)
|
||||
}
|
||||
if strings.TrimSpace(outlook.Product) == "" {
|
||||
return fmt.Errorf("decode outlook payload: outlooks[%d].product is required", index)
|
||||
}
|
||||
if outlook.Day == 0 {
|
||||
return fmt.Errorf("decode outlook payload: outlooks[%d].day is required", index)
|
||||
}
|
||||
if strings.TrimSpace(outlook.OutlookType) == "" {
|
||||
return fmt.Errorf("decode outlook payload: outlooks[%d].outlookType is required", index)
|
||||
}
|
||||
if strings.TrimSpace(outlook.Label) == "" {
|
||||
return fmt.Errorf("decode outlook payload: outlooks[%d].label is required", index)
|
||||
}
|
||||
if outlook.ValidFrom.IsZero() || outlook.ValidTo.IsZero() {
|
||||
return fmt.Errorf("decode outlook payload: outlooks[%d] validFrom/validTo are required", index)
|
||||
}
|
||||
if outlook.IssuedAt.IsZero() || outlook.ExpiresAt.IsZero() {
|
||||
return fmt.Errorf("decode outlook payload: outlooks[%d] issuedAt/expiresAt are required", index)
|
||||
}
|
||||
if len(outlook.Geometry) == 0 {
|
||||
return fmt.Errorf("decode outlook payload: outlooks[%d].geometry is required", index)
|
||||
}
|
||||
if !outlook.ContainsLocation {
|
||||
return fmt.Errorf("decode outlook payload: outlooks[%d].containsLocation must be true", index)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateOutlookDiscussions(discussions []model.WeatherOutlookDiscussion) error {
|
||||
seenDays := map[int]int{}
|
||||
for i, discussion := range discussions {
|
||||
if discussion.Day < 1 || discussion.Day > 3 {
|
||||
return fmt.Errorf("decode outlook payload: discussions[%d].day must be 1, 2, or 3", i)
|
||||
}
|
||||
if strings.TrimSpace(discussion.Headline) == "" &&
|
||||
strings.TrimSpace(discussion.Summary) == "" &&
|
||||
strings.TrimSpace(discussion.Discussion) == "" {
|
||||
return fmt.Errorf("decode outlook payload: discussions[%d] headline, summary, or discussion is required", i)
|
||||
}
|
||||
if first, ok := seenDays[discussion.Day]; ok {
|
||||
return fmt.Errorf("decode outlook payload: discussions[%d].day duplicates discussions[%d].day %d", i, first, discussion.Day)
|
||||
}
|
||||
seenDays[discussion.Day] = i
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func decodePayload[T any](payload any) (T, error) {
|
||||
var out T
|
||||
if payload == nil {
|
||||
@@ -382,6 +493,21 @@ func decodePayload[T any](payload any) (T, error) {
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func parentEventValues(e fkevent.Event, values map[string]any) map[string]any {
|
||||
out := map[string]any{
|
||||
"event_id": e.ID,
|
||||
"event_kind": string(e.Kind),
|
||||
"event_source": e.Source,
|
||||
"event_schema": e.Schema,
|
||||
"event_emitted_at": e.EmittedAt.UTC(),
|
||||
"event_effective_at": nullableTime(e.EffectiveAt),
|
||||
}
|
||||
for k, v := range values {
|
||||
out[k] = v
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func nullableDiscussionSection(section *model.WeatherForecastDiscussionSection) (any, any, any) {
|
||||
if section == nil {
|
||||
return nil, nil, nil
|
||||
@@ -418,6 +544,13 @@ func nullableBool(v *bool) any {
|
||||
return *v
|
||||
}
|
||||
|
||||
func nullableInt(v *int) any {
|
||||
if v == nil {
|
||||
return nil
|
||||
}
|
||||
return *v
|
||||
}
|
||||
|
||||
func nullableTime(v *time.Time) any {
|
||||
if v == nil || v.IsZero() {
|
||||
return nil
|
||||
@@ -445,3 +578,19 @@ func compactJSONText(v any) (any, error) {
|
||||
}
|
||||
return string(b), nil
|
||||
}
|
||||
|
||||
func requiredCompactJSONText(v any) (string, error) {
|
||||
compact, err := compactJSONText(v)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
s, ok := compact.(string)
|
||||
if !ok || strings.TrimSpace(s) == "" || strings.TrimSpace(s) == "null" {
|
||||
return "", fmt.Errorf("is required")
|
||||
}
|
||||
var buf bytes.Buffer
|
||||
if err := json.Compact(&buf, []byte(s)); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return buf.String(), nil
|
||||
}
|
||||
|
||||
@@ -27,7 +27,7 @@ func TestMapPostgresEventObservationStructPayload(t *testing.T) {
|
||||
PresentWeather: []model.PresentWeather{{Raw: map[string]any{"a": 1, "b": "x"}}},
|
||||
}
|
||||
|
||||
writes, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherObservationV1, "observation", obs))
|
||||
writes, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherObservationV1, standards.KindObservation, obs))
|
||||
if err != nil {
|
||||
t.Fatalf("mapPostgresEvent() error = %v", err)
|
||||
}
|
||||
@@ -74,7 +74,7 @@ func TestMapPostgresEventForecastStructPayload(t *testing.T) {
|
||||
},
|
||||
}
|
||||
|
||||
writes, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherForecastV1, "forecast", run))
|
||||
writes, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherForecastV1, standards.KindForecast, run))
|
||||
if err != nil {
|
||||
t.Fatalf("mapPostgresEvent() error = %v", err)
|
||||
}
|
||||
@@ -103,6 +103,8 @@ func TestMapPostgresEventForecastStructPayload(t *testing.T) {
|
||||
|
||||
func TestMapPostgresEventAlertStructPayload(t *testing.T) {
|
||||
sent := time.Date(2026, 3, 16, 17, 0, 0, 0, time.UTC)
|
||||
ends := time.Date(2026, 3, 16, 20, 0, 0, 0, time.UTC)
|
||||
expires := time.Date(2026, 3, 16, 18, 30, 0, 0, time.UTC)
|
||||
run := model.WeatherAlertRun{
|
||||
AsOf: time.Date(2026, 3, 16, 18, 0, 0, 0, time.UTC),
|
||||
Alerts: []model.WeatherAlert{
|
||||
@@ -110,6 +112,8 @@ func TestMapPostgresEventAlertStructPayload(t *testing.T) {
|
||||
ID: "urn:alert:1",
|
||||
Headline: "Winter Weather Advisory",
|
||||
Severity: "Moderate",
|
||||
Ends: &ends,
|
||||
Expires: &expires,
|
||||
References: []model.AlertReference{
|
||||
{ID: "urn:ref:1", Sent: &sent},
|
||||
{Identifier: "ref-two"},
|
||||
@@ -122,7 +126,7 @@ func TestMapPostgresEventAlertStructPayload(t *testing.T) {
|
||||
},
|
||||
}
|
||||
|
||||
writes, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherAlertV1, "alert", run))
|
||||
writes, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherAlertV1, standards.KindAlert, run))
|
||||
if err != nil {
|
||||
t.Fatalf("mapPostgresEvent() error = %v", err)
|
||||
}
|
||||
@@ -145,6 +149,20 @@ func TestMapPostgresEventAlertStructPayload(t *testing.T) {
|
||||
if got := firstAlert.Values["reference_count"]; got != 2 {
|
||||
t.Fatalf("alerts reference_count = %#v, want 2", got)
|
||||
}
|
||||
if got := firstAlert.Values["ends"]; got != ends {
|
||||
t.Fatalf("alerts ends = %#v, want %#v", got, ends)
|
||||
}
|
||||
if got := firstAlert.Values["expires"]; got != expires {
|
||||
t.Fatalf("alerts expires = %#v, want %#v", got, expires)
|
||||
}
|
||||
|
||||
alertWrites := writesForTable(writes, tableAlerts)
|
||||
if len(alertWrites) != 2 {
|
||||
t.Fatalf("alert writes len = %d, want 2", len(alertWrites))
|
||||
}
|
||||
if got := alertWrites[1].Values["ends"]; got != nil {
|
||||
t.Fatalf("second alert ends = %#v, want nil", got)
|
||||
}
|
||||
|
||||
assertAllWritesIncludeAllColumns(t, writes)
|
||||
}
|
||||
@@ -163,7 +181,7 @@ func TestMapPostgresEventForecastDiscussionStructPayload(t *testing.T) {
|
||||
LongTerm: &model.WeatherForecastDiscussionSection{Text: "Long term text"},
|
||||
}
|
||||
|
||||
writes, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherForecastDiscussionV1, "forecast_discussion", run))
|
||||
writes, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherForecastDiscussionV1, standards.KindForecastDiscussion, run))
|
||||
if err != nil {
|
||||
t.Fatalf("mapPostgresEvent() error = %v", err)
|
||||
}
|
||||
@@ -213,7 +231,7 @@ func TestMapPostgresEventWeatherStoryStructPayload(t *testing.T) {
|
||||
},
|
||||
}
|
||||
|
||||
writes, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherStoryV1, "weather_story", run))
|
||||
writes, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherStoryV1, standards.KindWeatherStory, run))
|
||||
if err != nil {
|
||||
t.Fatalf("mapPostgresEvent() error = %v", err)
|
||||
}
|
||||
@@ -239,8 +257,331 @@ func TestMapPostgresEventWeatherStoryStructPayload(t *testing.T) {
|
||||
assertAllWritesIncludeAllColumns(t, writes)
|
||||
}
|
||||
|
||||
func TestMapPostgresEventOutlookStructPayload(t *testing.T) {
|
||||
lat := 38.6239
|
||||
lon := -90.3571
|
||||
issuedAt := time.Date(2026, 6, 11, 19, 45, 0, 0, time.FixedZone("UTC-5", -5*60*60))
|
||||
updatedAt := time.Date(2026, 6, 11, 21, 15, 0, 0, time.FixedZone("UTC-5", -5*60*60))
|
||||
severity := 3
|
||||
run := model.WeatherOutlookRun{
|
||||
LocationID: "stl",
|
||||
LocationName: "St. Louis, MO",
|
||||
Latitude: &lat,
|
||||
Longitude: &lon,
|
||||
AsOf: time.Date(2026, 6, 12, 0, 45, 0, 0, time.UTC),
|
||||
IssuedAt: &issuedAt,
|
||||
Outlooks: []model.WeatherOutlook{
|
||||
{
|
||||
ID: "outlook-1",
|
||||
Provider: "spc",
|
||||
Product: "convective",
|
||||
Day: 1,
|
||||
OutlookType: "categorical",
|
||||
Label: "SLGT",
|
||||
LabelText: "Slight Risk",
|
||||
SeverityRank: &severity,
|
||||
ValidFrom: time.Date(2026, 6, 11, 13, 0, 0, 0, time.FixedZone("UTC-5", -5*60*60)),
|
||||
ValidTo: time.Date(2026, 6, 12, 12, 0, 0, 0, time.UTC),
|
||||
IssuedAt: issuedAt,
|
||||
ExpiresAt: time.Date(2026, 6, 12, 12, 0, 0, 0, time.UTC),
|
||||
Forecaster: "SMITH",
|
||||
SourceURL: "https://example.invalid/day1.geojson",
|
||||
ContainsLocation: true,
|
||||
Geometry: json.RawMessage(`{ "type" : "Polygon", "coordinates" : [ [ [ -91.0, 38.0 ], [ -90.0, 38.0 ], [ -90.0, 39.0 ], [ -91.0, 39.0 ], [ -91.0, 38.0 ] ] ] }`),
|
||||
},
|
||||
{
|
||||
ID: "outlook-2",
|
||||
Provider: "spc",
|
||||
Product: "convective",
|
||||
Day: 1,
|
||||
OutlookType: "wind",
|
||||
Label: "15",
|
||||
ValidFrom: time.Date(2026, 6, 11, 13, 0, 0, 0, time.UTC),
|
||||
ValidTo: time.Date(2026, 6, 12, 12, 0, 0, 0, time.UTC),
|
||||
IssuedAt: time.Date(2026, 6, 11, 19, 45, 0, 0, time.UTC),
|
||||
ExpiresAt: time.Date(2026, 6, 12, 12, 0, 0, 0, time.UTC),
|
||||
ContainsLocation: true,
|
||||
Geometry: json.RawMessage(`{"type":"Polygon","coordinates":[[[-100,35],[-98,35],[-98,37],[-100,37],[-100,35]]]}`),
|
||||
},
|
||||
},
|
||||
Discussions: []model.WeatherOutlookDiscussion{
|
||||
{
|
||||
Day: 1,
|
||||
Headline: "Day 1 Convective Outlook",
|
||||
Summary: "Severe thunderstorms are possible.",
|
||||
Discussion: "Full discussion text.",
|
||||
UpdatedAt: &updatedAt,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
writes, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherOutlookV2, standards.KindOutlook, run))
|
||||
if err != nil {
|
||||
t.Fatalf("mapPostgresEvent() error = %v", err)
|
||||
}
|
||||
if len(writes) != 4 {
|
||||
t.Fatalf("mapPostgresEvent() writes len = %d, want 4", len(writes))
|
||||
}
|
||||
if writes[0].Table != tableOutlookRuns {
|
||||
t.Fatalf("writes[0].Table = %q, want %q", writes[0].Table, tableOutlookRuns)
|
||||
}
|
||||
if got := writes[0].Values["outlook_count"]; got != 2 {
|
||||
t.Fatalf("outlook_runs outlook_count = %#v, want 2", got)
|
||||
}
|
||||
if got := writes[0].Values["discussion_count"]; got != 1 {
|
||||
t.Fatalf("outlook_runs discussion_count = %#v, want 1", got)
|
||||
}
|
||||
if got := writes[0].Values["issued_at"]; got != issuedAt.UTC() {
|
||||
t.Fatalf("outlook_runs issued_at = %#v, want UTC %s", got, issuedAt.UTC())
|
||||
}
|
||||
if writes[1].Table != tableOutlooks || writes[2].Table != tableOutlooks {
|
||||
t.Fatalf("outlook writes not in expected order")
|
||||
}
|
||||
if got := writes[1].Values["outlook_index"]; got != 0 {
|
||||
t.Fatalf("first outlook index = %#v, want 0", got)
|
||||
}
|
||||
if got := writes[1].Values["outlook_id"]; got != "outlook-1" {
|
||||
t.Fatalf("first outlook_id = %#v, want outlook-1", got)
|
||||
}
|
||||
if got := writes[1].Values["provider"]; got != "spc" {
|
||||
t.Fatalf("first provider = %#v, want spc", got)
|
||||
}
|
||||
if got := writes[1].Values["valid_from"]; got != run.Outlooks[0].ValidFrom.UTC() {
|
||||
t.Fatalf("first valid_from = %#v, want UTC %s", got, run.Outlooks[0].ValidFrom.UTC())
|
||||
}
|
||||
if got := writes[1].Values["geometry_json"]; got != `{"type":"Polygon","coordinates":[[[-91.0,38.0],[-90.0,38.0],[-90.0,39.0],[-91.0,39.0],[-91.0,38.0]]]}` {
|
||||
t.Fatalf("first geometry_json = %#v", got)
|
||||
}
|
||||
if got := writes[2].Values["contains_location"]; got != true {
|
||||
t.Fatalf("second contains_location = %#v, want true", got)
|
||||
}
|
||||
if writes[3].Table != tableOutlookDiscussions {
|
||||
t.Fatalf("writes[3].Table = %q, want %q", writes[3].Table, tableOutlookDiscussions)
|
||||
}
|
||||
if got := writes[3].Values["discussion_index"]; got != 0 {
|
||||
t.Fatalf("discussion_index = %#v, want 0", got)
|
||||
}
|
||||
if got := writes[3].Values["as_of"]; got != run.AsOf.UTC() {
|
||||
t.Fatalf("discussion as_of = %#v, want %s", got, run.AsOf.UTC())
|
||||
}
|
||||
if got := writes[3].Values["day"]; got != 1 {
|
||||
t.Fatalf("discussion day = %#v, want 1", got)
|
||||
}
|
||||
if got := writes[3].Values["headline"]; got != "Day 1 Convective Outlook" {
|
||||
t.Fatalf("discussion headline = %#v", got)
|
||||
}
|
||||
if got := writes[3].Values["summary"]; got != "Severe thunderstorms are possible." {
|
||||
t.Fatalf("discussion summary = %#v", got)
|
||||
}
|
||||
if got := writes[3].Values["discussion"]; got != "Full discussion text." {
|
||||
t.Fatalf("discussion text = %#v", got)
|
||||
}
|
||||
if got := writes[3].Values["updated_at"]; got != updatedAt.UTC() {
|
||||
t.Fatalf("discussion updated_at = %#v, want UTC %s", got, updatedAt.UTC())
|
||||
}
|
||||
|
||||
assertAllWritesIncludeAllColumns(t, writes)
|
||||
}
|
||||
|
||||
func TestMapPostgresEventOutlookEmptyLocalRun(t *testing.T) {
|
||||
run := model.WeatherOutlookRun{
|
||||
AsOf: time.Date(2026, 6, 11, 19, 45, 0, 0, time.UTC),
|
||||
}
|
||||
|
||||
writes, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherOutlookV2, standards.KindOutlook, run))
|
||||
if err != nil {
|
||||
t.Fatalf("mapPostgresEvent() error = %v", err)
|
||||
}
|
||||
if len(writes) != 1 {
|
||||
t.Fatalf("mapPostgresEvent() writes len = %d, want 1", len(writes))
|
||||
}
|
||||
if writes[0].Table != tableOutlookRuns {
|
||||
t.Fatalf("writes[0].Table = %q, want %q", writes[0].Table, tableOutlookRuns)
|
||||
}
|
||||
if got := writes[0].Values["outlook_count"]; got != 0 {
|
||||
t.Fatalf("outlook_runs outlook_count = %#v, want 0", got)
|
||||
}
|
||||
if got := writes[0].Values["discussion_count"]; got != 0 {
|
||||
t.Fatalf("outlook_runs discussion_count = %#v, want 0", got)
|
||||
}
|
||||
|
||||
assertAllWritesIncludeAllColumns(t, writes)
|
||||
}
|
||||
|
||||
func TestMapPostgresEventOutlookRejectsMissingAsOf(t *testing.T) {
|
||||
_, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherOutlookV2, standards.KindOutlook, model.WeatherOutlookRun{}))
|
||||
if err == nil {
|
||||
t.Fatalf("mapPostgresEvent() error = nil, want missing asOf error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "asOf is required") {
|
||||
t.Fatalf("error = %q, want asOf context", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMapPostgresEventOutlookRejectsMissingIDAndProvider(t *testing.T) {
|
||||
base := model.WeatherOutlook{
|
||||
ID: "outlook-1",
|
||||
Provider: "spc",
|
||||
Product: "convective",
|
||||
Day: 1,
|
||||
OutlookType: "categorical",
|
||||
Label: "SLGT",
|
||||
ValidFrom: time.Date(2026, 6, 11, 13, 0, 0, 0, time.UTC),
|
||||
ValidTo: time.Date(2026, 6, 12, 12, 0, 0, 0, time.UTC),
|
||||
IssuedAt: time.Date(2026, 6, 11, 19, 45, 0, 0, time.UTC),
|
||||
ExpiresAt: time.Date(2026, 6, 12, 12, 0, 0, 0, time.UTC),
|
||||
ContainsLocation: true,
|
||||
Geometry: json.RawMessage(`{"type":"Polygon","coordinates":[[[-91,38],[-90,38],[-90,39],[-91,39],[-91,38]]]}`),
|
||||
}
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
mutate func(*model.WeatherOutlook)
|
||||
wantErr string
|
||||
}{
|
||||
{
|
||||
name: "missing id",
|
||||
mutate: func(outlook *model.WeatherOutlook) { outlook.ID = "" },
|
||||
wantErr: "outlooks[0].id is required",
|
||||
},
|
||||
{
|
||||
name: "missing provider",
|
||||
mutate: func(outlook *model.WeatherOutlook) { outlook.Provider = "" },
|
||||
wantErr: "outlooks[0].provider is required",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
outlook := base
|
||||
tt.mutate(&outlook)
|
||||
run := model.WeatherOutlookRun{
|
||||
AsOf: time.Date(2026, 6, 11, 19, 45, 0, 0, time.UTC),
|
||||
Outlooks: []model.WeatherOutlook{outlook},
|
||||
}
|
||||
_, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherOutlookV2, standards.KindOutlook, run))
|
||||
if err == nil {
|
||||
t.Fatalf("mapPostgresEvent() error = nil, want %q", tt.wantErr)
|
||||
}
|
||||
if !strings.Contains(err.Error(), tt.wantErr) {
|
||||
t.Fatalf("error = %q, want %q", err, tt.wantErr)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestMapPostgresEventOutlookRejectsMissingRequiredTimes(t *testing.T) {
|
||||
run := model.WeatherOutlookRun{
|
||||
AsOf: time.Date(2026, 6, 11, 19, 45, 0, 0, time.UTC),
|
||||
Outlooks: []model.WeatherOutlook{{
|
||||
ID: "outlook-1",
|
||||
Provider: "spc",
|
||||
Product: "convective",
|
||||
Day: 1,
|
||||
OutlookType: "categorical",
|
||||
Label: "SLGT",
|
||||
ContainsLocation: true,
|
||||
Geometry: json.RawMessage(`{"type":"Polygon","coordinates":[[[-91,38],[-90,38],[-90,39],[-91,39],[-91,38]]]}`),
|
||||
}},
|
||||
}
|
||||
_, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherOutlookV2, standards.KindOutlook, run))
|
||||
if err == nil {
|
||||
t.Fatalf("mapPostgresEvent() error = nil, want missing time error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "outlooks[0] validFrom/validTo are required") {
|
||||
t.Fatalf("error = %q, want outlook time context", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMapPostgresEventOutlookRejectsEmptyGeometry(t *testing.T) {
|
||||
run := model.WeatherOutlookRun{
|
||||
AsOf: time.Date(2026, 6, 11, 19, 45, 0, 0, time.UTC),
|
||||
Outlooks: []model.WeatherOutlook{{
|
||||
ID: "outlook-1",
|
||||
Provider: "spc",
|
||||
Product: "convective",
|
||||
Day: 1,
|
||||
OutlookType: "categorical",
|
||||
Label: "SLGT",
|
||||
ValidFrom: time.Date(2026, 6, 11, 13, 0, 0, 0, time.UTC),
|
||||
ValidTo: time.Date(2026, 6, 12, 12, 0, 0, 0, time.UTC),
|
||||
IssuedAt: time.Date(2026, 6, 11, 19, 45, 0, 0, time.UTC),
|
||||
ExpiresAt: time.Date(2026, 6, 12, 12, 0, 0, 0, time.UTC),
|
||||
ContainsLocation: true,
|
||||
}},
|
||||
}
|
||||
_, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherOutlookV2, standards.KindOutlook, run))
|
||||
if err == nil {
|
||||
t.Fatalf("mapPostgresEvent() error = nil, want geometry error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "outlooks[0].geometry is required") {
|
||||
t.Fatalf("error = %q, want geometry context", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMapPostgresEventOutlookRejectsDuplicateDiscussionDay(t *testing.T) {
|
||||
run := model.WeatherOutlookRun{
|
||||
AsOf: time.Date(2026, 6, 11, 19, 45, 0, 0, time.UTC),
|
||||
Discussions: []model.WeatherOutlookDiscussion{
|
||||
{Day: 1, Discussion: "First day one discussion."},
|
||||
{Day: 1, Discussion: "Duplicate day one discussion."},
|
||||
},
|
||||
}
|
||||
_, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherOutlookV2, standards.KindOutlook, run))
|
||||
if err == nil {
|
||||
t.Fatalf("mapPostgresEvent() error = nil, want duplicate discussion day error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "discussions[1].day duplicates discussions[0].day 1") {
|
||||
t.Fatalf("error = %q, want duplicate discussion day context", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMapPostgresEventOutlookRejectsInvalidDiscussionDay(t *testing.T) {
|
||||
run := model.WeatherOutlookRun{
|
||||
AsOf: time.Date(2026, 6, 11, 19, 45, 0, 0, time.UTC),
|
||||
Discussions: []model.WeatherOutlookDiscussion{{Day: 4, Discussion: "Invalid day."}},
|
||||
}
|
||||
_, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherOutlookV2, standards.KindOutlook, run))
|
||||
if err == nil {
|
||||
t.Fatalf("mapPostgresEvent() error = nil, want invalid discussion day error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "discussions[0].day must be 1, 2, or 3") {
|
||||
t.Fatalf("error = %q, want invalid discussion day context", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMapPostgresEventOutlookRejectsEmptyDiscussionContent(t *testing.T) {
|
||||
run := model.WeatherOutlookRun{
|
||||
AsOf: time.Date(2026, 6, 11, 19, 45, 0, 0, time.UTC),
|
||||
Discussions: []model.WeatherOutlookDiscussion{{Day: 1}},
|
||||
}
|
||||
_, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherOutlookV2, standards.KindOutlook, run))
|
||||
if err == nil {
|
||||
t.Fatalf("mapPostgresEvent() error = nil, want empty discussion content error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "discussions[0] headline, summary, or discussion is required") {
|
||||
t.Fatalf("error = %q, want empty discussion content context", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMapPostgresEventOutlookRejectsContainsLocationFalse(t *testing.T) {
|
||||
run := model.WeatherOutlookRun{
|
||||
AsOf: time.Date(2026, 6, 11, 19, 45, 0, 0, time.UTC),
|
||||
Outlooks: []model.WeatherOutlook{validTestOutlook()},
|
||||
}
|
||||
run.Outlooks[0].ContainsLocation = false
|
||||
|
||||
_, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherOutlookV2, standards.KindOutlook, run))
|
||||
if err == nil {
|
||||
t.Fatalf("mapPostgresEvent() error = nil, want containsLocation error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "outlooks[0].containsLocation must be true") {
|
||||
t.Fatalf("error = %q, want containsLocation context", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMapPostgresEventWeatherStoryRejectsMissingAsOf(t *testing.T) {
|
||||
_, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherStoryV1, "weather_story", model.WeatherStoryRun{}))
|
||||
_, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherStoryV1, standards.KindWeatherStory, model.WeatherStoryRun{}))
|
||||
if err == nil {
|
||||
t.Fatalf("mapPostgresEvent() error = nil, want missing asOf error")
|
||||
}
|
||||
@@ -254,7 +595,7 @@ func TestMapPostgresEventWeatherStoryRejectsMissingStoryTimes(t *testing.T) {
|
||||
AsOf: time.Date(2026, 5, 30, 9, 0, 34, 0, time.UTC),
|
||||
Stories: []model.WeatherStory{{Title: "missing times"}},
|
||||
}
|
||||
_, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherStoryV1, "weather_story", run))
|
||||
_, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherStoryV1, standards.KindWeatherStory, run))
|
||||
if err == nil {
|
||||
t.Fatalf("mapPostgresEvent() error = nil, want missing story times error")
|
||||
}
|
||||
@@ -284,7 +625,7 @@ func TestMapPostgresEventMapPayload(t *testing.T) {
|
||||
t.Fatalf("json.Unmarshal() error = %v", err)
|
||||
}
|
||||
|
||||
writes, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherForecastV1, "forecast", payload))
|
||||
writes, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherForecastV1, standards.KindForecast, payload))
|
||||
if err != nil {
|
||||
t.Fatalf("mapPostgresEvent() error = %v", err)
|
||||
}
|
||||
@@ -299,7 +640,18 @@ func TestMapPostgresEventMapPayload(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestMapPostgresEventUnknownSchemaNoOp(t *testing.T) {
|
||||
writes, err := mapPostgresEvent(context.Background(), testEvent("weather.unknown.v1", "observation", map[string]any{"x": 1}))
|
||||
writes, err := mapPostgresEvent(context.Background(), testEvent("weather.unknown.v1", standards.KindObservation, map[string]any{"x": 1}))
|
||||
if err != nil {
|
||||
t.Fatalf("mapPostgresEvent() error = %v", err)
|
||||
}
|
||||
if len(writes) != 0 {
|
||||
t.Fatalf("mapPostgresEvent() writes len = %d, want 0", len(writes))
|
||||
}
|
||||
}
|
||||
|
||||
func TestMapPostgresEventLegacyOutlookSchemaNoOp(t *testing.T) {
|
||||
run := model.WeatherOutlookRun{AsOf: time.Date(2026, 6, 11, 19, 45, 0, 0, time.UTC)}
|
||||
writes, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherOutlookV1, standards.KindOutlook, run))
|
||||
if err != nil {
|
||||
t.Fatalf("mapPostgresEvent() error = %v", err)
|
||||
}
|
||||
@@ -309,7 +661,7 @@ func TestMapPostgresEventUnknownSchemaNoOp(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestMapPostgresEventMalformedPayload(t *testing.T) {
|
||||
_, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherForecastV1, "forecast", "bad"))
|
||||
_, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherForecastV1, standards.KindForecast, "bad"))
|
||||
if err == nil {
|
||||
t.Fatalf("mapPostgresEvent() expected error for malformed payload")
|
||||
}
|
||||
@@ -319,7 +671,7 @@ func TestMapPostgresEventMalformedPayload(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestMapPostgresEventForecastDiscussionMalformedPayload(t *testing.T) {
|
||||
_, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherForecastDiscussionV1, "forecast_discussion", "bad"))
|
||||
_, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherForecastDiscussionV1, standards.KindForecastDiscussion, "bad"))
|
||||
if err == nil {
|
||||
t.Fatalf("mapPostgresEvent() expected error for malformed payload")
|
||||
}
|
||||
@@ -328,6 +680,56 @@ func TestMapPostgresEventForecastDiscussionMalformedPayload(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestParentEventValuesAddsEnvelopeAndPreservesProductValues(t *testing.T) {
|
||||
emittedAt := time.Date(2026, 3, 16, 13, 31, 0, 0, time.FixedZone("CDT", -5*60*60))
|
||||
effectiveAt := time.Date(2026, 3, 16, 13, 30, 0, 0, time.FixedZone("CDT", -5*60*60))
|
||||
event := fkevent.Event{
|
||||
ID: "evt-envelope",
|
||||
Kind: fkevent.Kind(standards.KindForecast),
|
||||
Source: "test-source",
|
||||
Schema: standards.SchemaWeatherForecastV1,
|
||||
EmittedAt: emittedAt,
|
||||
EffectiveAt: &effectiveAt,
|
||||
}
|
||||
|
||||
got := parentEventValues(event, map[string]any{"product_col": "product-value"})
|
||||
|
||||
assertParentEnvelopeValues(t, got, event)
|
||||
if got["product_col"] != "product-value" {
|
||||
t.Fatalf("product_col = %#v, want product-value", got["product_col"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestParentEventValuesNullEffectiveAt(t *testing.T) {
|
||||
base := fkevent.Event{
|
||||
ID: "evt-envelope",
|
||||
Kind: fkevent.Kind(standards.KindObservation),
|
||||
Source: "test-source",
|
||||
Schema: standards.SchemaWeatherObservationV1,
|
||||
EmittedAt: time.Date(2026, 3, 16, 18, 31, 0, 0, time.UTC),
|
||||
}
|
||||
|
||||
for _, tt := range []struct {
|
||||
name string
|
||||
mut func(*fkevent.Event)
|
||||
}{
|
||||
{name: "nil", mut: func(*fkevent.Event) {}},
|
||||
{name: "zero", mut: func(event *fkevent.Event) {
|
||||
zero := time.Time{}
|
||||
event.EffectiveAt = &zero
|
||||
}},
|
||||
} {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
event := base
|
||||
tt.mut(&event)
|
||||
got := parentEventValues(event, nil)
|
||||
if got["event_effective_at"] != nil {
|
||||
t.Fatalf("event_effective_at = %#v, want nil", got["event_effective_at"])
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func testEvent(schema string, kind fkevent.Kind, payload any) fkevent.Event {
|
||||
effectiveAt := time.Date(2026, 3, 16, 18, 30, 0, 0, time.UTC)
|
||||
return fkevent.Event{
|
||||
@@ -341,6 +743,30 @@ func testEvent(schema string, kind fkevent.Kind, payload any) fkevent.Event {
|
||||
}
|
||||
}
|
||||
|
||||
func assertParentEnvelopeValues(t *testing.T, values map[string]any, event fkevent.Event) {
|
||||
t.Helper()
|
||||
|
||||
if got := values["event_id"]; got != event.ID {
|
||||
t.Fatalf("event_id = %#v, want %q", got, event.ID)
|
||||
}
|
||||
if got := values["event_kind"]; got != string(event.Kind) {
|
||||
t.Fatalf("event_kind = %#v, want %q", got, event.Kind)
|
||||
}
|
||||
if got := values["event_source"]; got != event.Source {
|
||||
t.Fatalf("event_source = %#v, want %q", got, event.Source)
|
||||
}
|
||||
if got := values["event_schema"]; got != event.Schema {
|
||||
t.Fatalf("event_schema = %#v, want %q", got, event.Schema)
|
||||
}
|
||||
if got := values["event_emitted_at"]; got != event.EmittedAt.UTC() {
|
||||
t.Fatalf("event_emitted_at = %#v, want %s", got, event.EmittedAt.UTC())
|
||||
}
|
||||
wantEffective := nullableTime(event.EffectiveAt)
|
||||
if got := values["event_effective_at"]; got != wantEffective {
|
||||
t.Fatalf("event_effective_at = %#v, want %#v", got, wantEffective)
|
||||
}
|
||||
}
|
||||
|
||||
func firstWriteForTable(writes []fksinks.PostgresWrite, table string) (fksinks.PostgresWrite, bool) {
|
||||
for _, w := range writes {
|
||||
if w.Table == table {
|
||||
@@ -350,6 +776,16 @@ func firstWriteForTable(writes []fksinks.PostgresWrite, table string) (fksinks.P
|
||||
return fksinks.PostgresWrite{}, false
|
||||
}
|
||||
|
||||
func writesForTable(writes []fksinks.PostgresWrite, table string) []fksinks.PostgresWrite {
|
||||
out := make([]fksinks.PostgresWrite, 0)
|
||||
for _, w := range writes {
|
||||
if w.Table == table {
|
||||
out = append(out, w)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func assertAllWritesIncludeAllColumns(t *testing.T, writes []fksinks.PostgresWrite) {
|
||||
t.Helper()
|
||||
colCounts := tableColumnCounts()
|
||||
@@ -373,6 +809,23 @@ func tableColumnCounts() map[string]int {
|
||||
return m
|
||||
}
|
||||
|
||||
func validTestOutlook() model.WeatherOutlook {
|
||||
return model.WeatherOutlook{
|
||||
ID: "outlook-1",
|
||||
Provider: "spc",
|
||||
Product: "convective",
|
||||
Day: 1,
|
||||
OutlookType: "categorical",
|
||||
Label: "SLGT",
|
||||
ValidFrom: time.Date(2026, 6, 11, 13, 0, 0, 0, time.UTC),
|
||||
ValidTo: time.Date(2026, 6, 12, 12, 0, 0, 0, time.UTC),
|
||||
IssuedAt: time.Date(2026, 6, 11, 19, 45, 0, 0, time.UTC),
|
||||
ExpiresAt: time.Date(2026, 6, 12, 12, 0, 0, 0, time.UTC),
|
||||
ContainsLocation: true,
|
||||
Geometry: json.RawMessage(`{"type":"Polygon","coordinates":[[[-91,38],[-90,38],[-90,39],[-91,39],[-91,38]]]}`),
|
||||
}
|
||||
}
|
||||
|
||||
func wmoCodePtr(v model.WMOCode) *model.WMOCode {
|
||||
out := v
|
||||
return &out
|
||||
|
||||
@@ -16,6 +16,9 @@ const (
|
||||
tableAlertRuns = "alert_runs"
|
||||
tableAlerts = "alerts"
|
||||
tableAlertReferences = "alert_references"
|
||||
tableOutlookRuns = "outlook_runs"
|
||||
tableOutlooks = "outlooks"
|
||||
tableOutlookDiscussions = "outlook_discussions"
|
||||
)
|
||||
|
||||
// PostgresSchema returns weatherfeeder's Postgres schema definition.
|
||||
@@ -24,13 +27,7 @@ func PostgresSchema() fksinks.PostgresSchema {
|
||||
Tables: []fksinks.PostgresTable{
|
||||
{
|
||||
Name: tableObservations,
|
||||
Columns: []fksinks.PostgresColumn{
|
||||
{Name: "event_id", Type: "TEXT", Nullable: false},
|
||||
{Name: "event_kind", Type: "TEXT", Nullable: false},
|
||||
{Name: "event_source", Type: "TEXT", Nullable: false},
|
||||
{Name: "event_schema", Type: "TEXT", Nullable: false},
|
||||
{Name: "event_emitted_at", Type: "TIMESTAMPTZ", Nullable: false},
|
||||
{Name: "event_effective_at", Type: "TIMESTAMPTZ", Nullable: true},
|
||||
Columns: parentEnvelopeColumns([]fksinks.PostgresColumn{
|
||||
{Name: "station_id", Type: "TEXT", Nullable: true},
|
||||
{Name: "station_name", Type: "TEXT", Nullable: true},
|
||||
{Name: "observed_at", Type: "TIMESTAMPTZ", Nullable: false},
|
||||
@@ -46,7 +43,7 @@ func PostgresSchema() fksinks.PostgresSchema {
|
||||
{Name: "visibility_meters", Type: "DOUBLE PRECISION", Nullable: true},
|
||||
{Name: "relative_humidity_percent", Type: "DOUBLE PRECISION", Nullable: true},
|
||||
{Name: "apparent_temperature_c", Type: "DOUBLE PRECISION", Nullable: true},
|
||||
},
|
||||
}...),
|
||||
PrimaryKey: []string{"event_id"},
|
||||
PruneColumn: "observed_at",
|
||||
Indexes: []fksinks.PostgresIndex{
|
||||
@@ -71,13 +68,7 @@ func PostgresSchema() fksinks.PostgresSchema {
|
||||
},
|
||||
{
|
||||
Name: tableForecasts,
|
||||
Columns: []fksinks.PostgresColumn{
|
||||
{Name: "event_id", Type: "TEXT", Nullable: false},
|
||||
{Name: "event_kind", Type: "TEXT", Nullable: false},
|
||||
{Name: "event_source", Type: "TEXT", Nullable: false},
|
||||
{Name: "event_schema", Type: "TEXT", Nullable: false},
|
||||
{Name: "event_emitted_at", Type: "TIMESTAMPTZ", Nullable: false},
|
||||
{Name: "event_effective_at", Type: "TIMESTAMPTZ", Nullable: true},
|
||||
Columns: parentEnvelopeColumns([]fksinks.PostgresColumn{
|
||||
{Name: "location_id", Type: "TEXT", Nullable: true},
|
||||
{Name: "location_name", Type: "TEXT", Nullable: true},
|
||||
{Name: "issued_at", Type: "TIMESTAMPTZ", Nullable: false},
|
||||
@@ -87,7 +78,7 @@ func PostgresSchema() fksinks.PostgresSchema {
|
||||
{Name: "longitude", Type: "DOUBLE PRECISION", Nullable: true},
|
||||
{Name: "elevation_meters", Type: "DOUBLE PRECISION", Nullable: true},
|
||||
{Name: "period_count", Type: "INTEGER", Nullable: false},
|
||||
},
|
||||
}...),
|
||||
PrimaryKey: []string{"event_id"},
|
||||
PruneColumn: "issued_at",
|
||||
Indexes: []fksinks.PostgresIndex{
|
||||
@@ -135,13 +126,7 @@ func PostgresSchema() fksinks.PostgresSchema {
|
||||
},
|
||||
{
|
||||
Name: tableForecastDiscussions,
|
||||
Columns: []fksinks.PostgresColumn{
|
||||
{Name: "event_id", Type: "TEXT", Nullable: false},
|
||||
{Name: "event_kind", Type: "TEXT", Nullable: false},
|
||||
{Name: "event_source", Type: "TEXT", Nullable: false},
|
||||
{Name: "event_schema", Type: "TEXT", Nullable: false},
|
||||
{Name: "event_emitted_at", Type: "TIMESTAMPTZ", Nullable: false},
|
||||
{Name: "event_effective_at", Type: "TIMESTAMPTZ", Nullable: true},
|
||||
Columns: parentEnvelopeColumns([]fksinks.PostgresColumn{
|
||||
{Name: "office_id", Type: "TEXT", Nullable: true},
|
||||
{Name: "office_name", Type: "TEXT", Nullable: true},
|
||||
{Name: "issued_at", Type: "TIMESTAMPTZ", Nullable: false},
|
||||
@@ -154,7 +139,7 @@ func PostgresSchema() fksinks.PostgresSchema {
|
||||
{Name: "long_term_issued_at", Type: "TIMESTAMPTZ", Nullable: true},
|
||||
{Name: "long_term_text", Type: "TEXT", Nullable: true},
|
||||
{Name: "key_message_count", Type: "INTEGER", Nullable: false},
|
||||
},
|
||||
}...),
|
||||
PrimaryKey: []string{"event_id"},
|
||||
PruneColumn: "issued_at",
|
||||
Indexes: []fksinks.PostgresIndex{
|
||||
@@ -178,17 +163,11 @@ func PostgresSchema() fksinks.PostgresSchema {
|
||||
},
|
||||
{
|
||||
Name: tableWeatherStoryRuns,
|
||||
Columns: []fksinks.PostgresColumn{
|
||||
{Name: "event_id", Type: "TEXT", Nullable: false},
|
||||
{Name: "event_kind", Type: "TEXT", Nullable: false},
|
||||
{Name: "event_source", Type: "TEXT", Nullable: false},
|
||||
{Name: "event_schema", Type: "TEXT", Nullable: false},
|
||||
{Name: "event_emitted_at", Type: "TIMESTAMPTZ", Nullable: false},
|
||||
{Name: "event_effective_at", Type: "TIMESTAMPTZ", Nullable: true},
|
||||
Columns: parentEnvelopeColumns([]fksinks.PostgresColumn{
|
||||
{Name: "office_id", Type: "TEXT", Nullable: true},
|
||||
{Name: "as_of", Type: "TIMESTAMPTZ", Nullable: false},
|
||||
{Name: "story_count", Type: "INTEGER", Nullable: false},
|
||||
},
|
||||
}...),
|
||||
PrimaryKey: []string{"event_id"},
|
||||
PruneColumn: "as_of",
|
||||
Indexes: []fksinks.PostgresIndex{
|
||||
@@ -223,20 +202,14 @@ func PostgresSchema() fksinks.PostgresSchema {
|
||||
},
|
||||
{
|
||||
Name: tableAlertRuns,
|
||||
Columns: []fksinks.PostgresColumn{
|
||||
{Name: "event_id", Type: "TEXT", Nullable: false},
|
||||
{Name: "event_kind", Type: "TEXT", Nullable: false},
|
||||
{Name: "event_source", Type: "TEXT", Nullable: false},
|
||||
{Name: "event_schema", Type: "TEXT", Nullable: false},
|
||||
{Name: "event_emitted_at", Type: "TIMESTAMPTZ", Nullable: false},
|
||||
{Name: "event_effective_at", Type: "TIMESTAMPTZ", Nullable: true},
|
||||
Columns: parentEnvelopeColumns([]fksinks.PostgresColumn{
|
||||
{Name: "location_id", Type: "TEXT", Nullable: true},
|
||||
{Name: "location_name", Type: "TEXT", Nullable: true},
|
||||
{Name: "as_of", Type: "TIMESTAMPTZ", Nullable: false},
|
||||
{Name: "latitude", Type: "DOUBLE PRECISION", Nullable: true},
|
||||
{Name: "longitude", Type: "DOUBLE PRECISION", Nullable: true},
|
||||
{Name: "alert_count", Type: "INTEGER", Nullable: false},
|
||||
},
|
||||
}...),
|
||||
PrimaryKey: []string{"event_id"},
|
||||
PruneColumn: "as_of",
|
||||
Indexes: []fksinks.PostgresIndex{
|
||||
@@ -265,6 +238,7 @@ func PostgresSchema() fksinks.PostgresSchema {
|
||||
{Name: "sent", Type: "TIMESTAMPTZ", Nullable: true},
|
||||
{Name: "effective", Type: "TIMESTAMPTZ", Nullable: true},
|
||||
{Name: "onset", Type: "TIMESTAMPTZ", Nullable: true},
|
||||
{Name: "ends", Type: "TIMESTAMPTZ", Nullable: true},
|
||||
{Name: "expires", Type: "TIMESTAMPTZ", Nullable: true},
|
||||
{Name: "area_description", Type: "TEXT", Nullable: true},
|
||||
{Name: "sender_name", Type: "TEXT", Nullable: true},
|
||||
@@ -297,7 +271,89 @@ func PostgresSchema() fksinks.PostgresSchema {
|
||||
{Name: "idx_wf_alert_refs_sent", Columns: []string{"sent"}},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: tableOutlookRuns,
|
||||
Columns: parentEnvelopeColumns([]fksinks.PostgresColumn{
|
||||
{Name: "location_id", Type: "TEXT", Nullable: true},
|
||||
{Name: "location_name", Type: "TEXT", Nullable: true},
|
||||
{Name: "latitude", Type: "DOUBLE PRECISION", Nullable: true},
|
||||
{Name: "longitude", Type: "DOUBLE PRECISION", Nullable: true},
|
||||
{Name: "as_of", Type: "TIMESTAMPTZ", Nullable: false},
|
||||
{Name: "issued_at", Type: "TIMESTAMPTZ", Nullable: true},
|
||||
{Name: "outlook_count", Type: "INTEGER", Nullable: false},
|
||||
{Name: "discussion_count", Type: "INTEGER", Nullable: false},
|
||||
}...),
|
||||
PrimaryKey: []string{"event_id"},
|
||||
PruneColumn: "as_of",
|
||||
Indexes: []fksinks.PostgresIndex{
|
||||
{Name: "idx_wf_outlook_run_location_as_of", Columns: []string{"location_id", "as_of"}},
|
||||
{Name: "idx_wf_outlook_run_as_of", Columns: []string{"as_of"}},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: tableOutlooks,
|
||||
Columns: []fksinks.PostgresColumn{
|
||||
{Name: "run_event_id", Type: "TEXT REFERENCES outlook_runs(event_id) ON DELETE CASCADE", Nullable: false},
|
||||
{Name: "outlook_index", Type: "INTEGER", Nullable: false},
|
||||
{Name: "as_of", Type: "TIMESTAMPTZ", Nullable: false},
|
||||
{Name: "outlook_id", Type: "TEXT", Nullable: false},
|
||||
{Name: "provider", Type: "TEXT", Nullable: false},
|
||||
{Name: "product", Type: "TEXT", Nullable: false},
|
||||
{Name: "day", Type: "INTEGER", Nullable: false},
|
||||
{Name: "outlook_type", Type: "TEXT", Nullable: false},
|
||||
{Name: "label", Type: "TEXT", Nullable: false},
|
||||
{Name: "label_text", Type: "TEXT", Nullable: true},
|
||||
{Name: "severity_rank", Type: "INTEGER", Nullable: true},
|
||||
{Name: "valid_from", Type: "TIMESTAMPTZ", Nullable: false},
|
||||
{Name: "valid_to", Type: "TIMESTAMPTZ", Nullable: false},
|
||||
{Name: "issued_at", Type: "TIMESTAMPTZ", Nullable: false},
|
||||
{Name: "expires_at", Type: "TIMESTAMPTZ", Nullable: false},
|
||||
{Name: "forecaster", Type: "TEXT", Nullable: true},
|
||||
{Name: "source_url", Type: "TEXT", Nullable: true},
|
||||
{Name: "image_url", Type: "TEXT", Nullable: true},
|
||||
{Name: "contains_location", Type: "BOOLEAN", Nullable: false},
|
||||
{Name: "geometry_json", Type: "TEXT", Nullable: false},
|
||||
},
|
||||
PrimaryKey: []string{"run_event_id", "outlook_index"},
|
||||
PruneColumn: "as_of",
|
||||
Indexes: []fksinks.PostgresIndex{
|
||||
{Name: "idx_wf_outlooks_contains_valid", Columns: []string{"contains_location", "valid_from", "valid_to"}},
|
||||
{Name: "idx_wf_outlooks_day_type_label", Columns: []string{"day", "outlook_type", "label"}},
|
||||
{Name: "idx_wf_outlooks_valid", Columns: []string{"valid_from", "valid_to"}},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: tableOutlookDiscussions,
|
||||
Columns: []fksinks.PostgresColumn{
|
||||
{Name: "run_event_id", Type: "TEXT REFERENCES outlook_runs(event_id) ON DELETE CASCADE", Nullable: false},
|
||||
{Name: "discussion_index", Type: "INTEGER", Nullable: false},
|
||||
{Name: "as_of", Type: "TIMESTAMPTZ", Nullable: false},
|
||||
{Name: "day", Type: "INTEGER", Nullable: false},
|
||||
{Name: "headline", Type: "TEXT", Nullable: true},
|
||||
{Name: "summary", Type: "TEXT", Nullable: true},
|
||||
{Name: "discussion", Type: "TEXT", Nullable: true},
|
||||
{Name: "updated_at", Type: "TIMESTAMPTZ", Nullable: true},
|
||||
},
|
||||
PrimaryKey: []string{"run_event_id", "discussion_index"},
|
||||
PruneColumn: "as_of",
|
||||
Indexes: []fksinks.PostgresIndex{
|
||||
{Name: "idx_wf_outlook_discussions_day_as_of", Columns: []string{"day", "as_of"}},
|
||||
{Name: "idx_wf_outlook_discussions_run_day", Columns: []string{"run_event_id", "day"}, Unique: true},
|
||||
},
|
||||
},
|
||||
},
|
||||
MapEvent: mapPostgresEvent,
|
||||
}
|
||||
}
|
||||
|
||||
func parentEnvelopeColumns(extra ...fksinks.PostgresColumn) []fksinks.PostgresColumn {
|
||||
columns := []fksinks.PostgresColumn{
|
||||
{Name: "event_id", Type: "TEXT", Nullable: false},
|
||||
{Name: "event_kind", Type: "TEXT", Nullable: false},
|
||||
{Name: "event_source", Type: "TEXT", Nullable: false},
|
||||
{Name: "event_schema", Type: "TEXT", Nullable: false},
|
||||
{Name: "event_emitted_at", Type: "TIMESTAMPTZ", Nullable: false},
|
||||
{Name: "event_effective_at", Type: "TIMESTAMPTZ", Nullable: true},
|
||||
}
|
||||
return append(columns, extra...)
|
||||
}
|
||||
|
||||
@@ -1,6 +1,12 @@
|
||||
package postgres
|
||||
|
||||
import "testing"
|
||||
import (
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
fksinks "gitea.maximumdirect.net/ejr/feedkit/sinks"
|
||||
)
|
||||
|
||||
func TestWeatherPostgresSchemaShape(t *testing.T) {
|
||||
s := PostgresSchema()
|
||||
@@ -20,6 +26,9 @@ func TestWeatherPostgresSchemaShape(t *testing.T) {
|
||||
tableAlertRuns: true,
|
||||
tableAlerts: true,
|
||||
tableAlertReferences: true,
|
||||
tableOutlookRuns: true,
|
||||
tableOutlooks: true,
|
||||
tableOutlookDiscussions: true,
|
||||
}
|
||||
|
||||
if len(s.Tables) != len(wantTables) {
|
||||
@@ -43,6 +52,54 @@ func TestWeatherPostgresSchemaShape(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestWeatherPostgresSchemaIncludesOutlookTables(t *testing.T) {
|
||||
runColumns := columnsForTable(t, tableOutlookRuns)
|
||||
for _, col := range []string{"event_id", "event_kind", "event_source", "event_schema", "event_emitted_at", "event_effective_at", "location_id", "location_name", "latitude", "longitude", "as_of", "issued_at", "outlook_count", "discussion_count"} {
|
||||
if !runColumns[col] {
|
||||
t.Fatalf("%s missing %s column", tableOutlookRuns, col)
|
||||
}
|
||||
}
|
||||
assertTablePrimaryKey(t, tableOutlookRuns, []string{"event_id"})
|
||||
assertTableIndex(t, tableOutlookRuns, "idx_wf_outlook_run_location_as_of", []string{"location_id", "as_of"})
|
||||
assertTableIndex(t, tableOutlookRuns, "idx_wf_outlook_run_as_of", []string{"as_of"})
|
||||
|
||||
outlookColumns := columnsForTable(t, tableOutlooks)
|
||||
for _, col := range []string{"run_event_id", "outlook_index", "as_of", "outlook_id", "provider", "product", "day", "outlook_type", "label", "label_text", "severity_rank", "valid_from", "valid_to", "issued_at", "expires_at", "forecaster", "source_url", "image_url", "contains_location", "geometry_json"} {
|
||||
if !outlookColumns[col] {
|
||||
t.Fatalf("%s missing %s column", tableOutlooks, col)
|
||||
}
|
||||
}
|
||||
for _, col := range []string{"headline", "summary", "discussion"} {
|
||||
if outlookColumns[col] {
|
||||
t.Fatalf("%s still includes legacy %s column", tableOutlooks, col)
|
||||
}
|
||||
}
|
||||
assertTablePrimaryKey(t, tableOutlooks, []string{"run_event_id", "outlook_index"})
|
||||
assertTableIndex(t, tableOutlooks, "idx_wf_outlooks_contains_valid", []string{"contains_location", "valid_from", "valid_to"})
|
||||
assertTableIndex(t, tableOutlooks, "idx_wf_outlooks_day_type_label", []string{"day", "outlook_type", "label"})
|
||||
assertTableIndex(t, tableOutlooks, "idx_wf_outlooks_valid", []string{"valid_from", "valid_to"})
|
||||
|
||||
discussionColumns := columnsForTable(t, tableOutlookDiscussions)
|
||||
for _, col := range []string{"run_event_id", "discussion_index", "as_of", "day", "headline", "summary", "discussion", "updated_at"} {
|
||||
if !discussionColumns[col] {
|
||||
t.Fatalf("%s missing %s column", tableOutlookDiscussions, col)
|
||||
}
|
||||
}
|
||||
assertTablePrimaryKey(t, tableOutlookDiscussions, []string{"run_event_id", "discussion_index"})
|
||||
assertTablePruneColumn(t, tableOutlookDiscussions, "as_of")
|
||||
assertTableIndex(t, tableOutlookDiscussions, "idx_wf_outlook_discussions_day_as_of", []string{"day", "as_of"})
|
||||
assertTableUniqueIndex(t, tableOutlookDiscussions, "idx_wf_outlook_discussions_run_day", []string{"run_event_id", "day"})
|
||||
}
|
||||
|
||||
func TestWeatherPostgresSchemaIncludesAlertEndsColumn(t *testing.T) {
|
||||
alertColumns := columnsForTable(t, tableAlerts)
|
||||
for _, col := range []string{"run_event_id", "alert_index", "as_of", "alert_id", "onset", "ends", "expires"} {
|
||||
if !alertColumns[col] {
|
||||
t.Fatalf("%s missing %s column", tableAlerts, col)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestWeatherPostgresSchemaIncludesWeatherStoryColumns(t *testing.T) {
|
||||
runColumns := columnsForTable(t, tableWeatherStoryRuns)
|
||||
if !runColumns["as_of"] {
|
||||
@@ -60,20 +117,102 @@ func TestWeatherPostgresSchemaIncludesWeatherStoryColumns(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func columnsForTable(t *testing.T, table string) map[string]bool {
|
||||
func TestWeatherPostgresSchemaParentTablesStartWithEnvelopeColumns(t *testing.T) {
|
||||
for _, table := range []string{
|
||||
tableObservations,
|
||||
tableForecasts,
|
||||
tableForecastDiscussions,
|
||||
tableWeatherStoryRuns,
|
||||
tableAlertRuns,
|
||||
tableOutlookRuns,
|
||||
} {
|
||||
t.Run(table, func(t *testing.T) {
|
||||
columns := orderedColumnsForTable(t, table)
|
||||
want := parentEnvelopeColumns()
|
||||
if len(columns) < len(want) {
|
||||
t.Fatalf("%s has %d columns, want at least %d", table, len(columns), len(want))
|
||||
}
|
||||
if !reflect.DeepEqual(columns[:len(want)], want) {
|
||||
t.Fatalf("%s envelope prefix = %#v, want %#v", table, columns[:len(want)], want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func assertTablePrimaryKey(t *testing.T, table string, want []string) {
|
||||
t.Helper()
|
||||
tbl := tableByName(t, table)
|
||||
if strings.Join(tbl.PrimaryKey, ",") != strings.Join(want, ",") {
|
||||
t.Fatalf("%s primary key = %#v, want %#v", table, tbl.PrimaryKey, want)
|
||||
}
|
||||
}
|
||||
|
||||
func assertTablePruneColumn(t *testing.T, table string, want string) {
|
||||
t.Helper()
|
||||
tbl := tableByName(t, table)
|
||||
if tbl.PruneColumn != want {
|
||||
t.Fatalf("%s prune column = %q, want %q", table, tbl.PruneColumn, want)
|
||||
}
|
||||
}
|
||||
|
||||
func assertTableIndex(t *testing.T, table string, name string, want []string) {
|
||||
t.Helper()
|
||||
assertTableIndexWithUnique(t, table, name, want, false)
|
||||
}
|
||||
|
||||
func assertTableUniqueIndex(t *testing.T, table string, name string, want []string) {
|
||||
t.Helper()
|
||||
assertTableIndexWithUnique(t, table, name, want, true)
|
||||
}
|
||||
|
||||
func assertTableIndexWithUnique(t *testing.T, table string, name string, want []string, unique bool) {
|
||||
t.Helper()
|
||||
tbl := tableByName(t, table)
|
||||
for _, idx := range tbl.Indexes {
|
||||
if idx.Name == name {
|
||||
if strings.Join(idx.Columns, ",") != strings.Join(want, ",") {
|
||||
t.Fatalf("%s index %s columns = %#v, want %#v", table, name, idx.Columns, want)
|
||||
}
|
||||
if idx.Unique != unique {
|
||||
t.Fatalf("%s index %s unique = %v, want %v", table, name, idx.Unique, unique)
|
||||
}
|
||||
return
|
||||
}
|
||||
}
|
||||
t.Fatalf("%s missing index %s", table, name)
|
||||
}
|
||||
|
||||
func tableByName(t *testing.T, table string) fksinks.PostgresTable {
|
||||
t.Helper()
|
||||
for _, tbl := range PostgresSchema().Tables {
|
||||
if tbl.Name == table {
|
||||
return tbl
|
||||
}
|
||||
}
|
||||
t.Fatalf("missing table %q", table)
|
||||
return fksinks.PostgresTable{}
|
||||
}
|
||||
|
||||
func orderedColumnsForTable(t *testing.T, table string) []fksinks.PostgresColumn {
|
||||
t.Helper()
|
||||
|
||||
schema := PostgresSchema()
|
||||
for _, tbl := range schema.Tables {
|
||||
if tbl.Name != table {
|
||||
continue
|
||||
if tbl.Name == table {
|
||||
return tbl.Columns
|
||||
}
|
||||
cols := make(map[string]bool, len(tbl.Columns))
|
||||
for _, col := range tbl.Columns {
|
||||
cols[col.Name] = true
|
||||
}
|
||||
return cols
|
||||
}
|
||||
t.Fatalf("missing table %q", table)
|
||||
return nil
|
||||
}
|
||||
|
||||
func columnsForTable(t *testing.T, table string) map[string]bool {
|
||||
t.Helper()
|
||||
|
||||
ordered := orderedColumnsForTable(t, table)
|
||||
cols := make(map[string]bool, len(ordered))
|
||||
for _, col := range ordered {
|
||||
cols[col.Name] = true
|
||||
}
|
||||
return cols
|
||||
}
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"gitea.maximumdirect.net/ejr/weatherfeeder/internal/sources/nws"
|
||||
"gitea.maximumdirect.net/ejr/weatherfeeder/internal/sources/openmeteo"
|
||||
"gitea.maximumdirect.net/ejr/weatherfeeder/internal/sources/openweather"
|
||||
"gitea.maximumdirect.net/ejr/weatherfeeder/internal/sources/spc"
|
||||
|
||||
"gitea.maximumdirect.net/ejr/feedkit/config"
|
||||
fksource "gitea.maximumdirect.net/ejr/feedkit/sources"
|
||||
@@ -15,19 +16,22 @@ type pollDriverRegistration struct {
|
||||
}
|
||||
|
||||
var pollDriverRegistrations = []pollDriverRegistration{
|
||||
{driver: "nws_observation", factory: func(cfg config.SourceConfig) (fksource.PollSource, error) { return nws.NewObservationSource(cfg) }},
|
||||
{driver: "nws_alerts", factory: func(cfg config.SourceConfig) (fksource.PollSource, error) { return nws.NewAlertsSource(cfg) }},
|
||||
{driver: "nws_forecast_hourly", factory: func(cfg config.SourceConfig) (fksource.PollSource, error) { return nws.NewHourlyForecastSource(cfg) }},
|
||||
{driver: "nws_forecast_narrative", factory: func(cfg config.SourceConfig) (fksource.PollSource, error) { return nws.NewNarrativeForecastSource(cfg) }},
|
||||
{driver: "nws_forecast_discussion", factory: func(cfg config.SourceConfig) (fksource.PollSource, error) {
|
||||
{driver: nws.DriverObservation, factory: func(cfg config.SourceConfig) (fksource.PollSource, error) { return nws.NewObservationSource(cfg) }},
|
||||
{driver: nws.DriverAlerts, factory: func(cfg config.SourceConfig) (fksource.PollSource, error) { return nws.NewAlertsSource(cfg) }},
|
||||
{driver: nws.DriverForecastHourly, factory: func(cfg config.SourceConfig) (fksource.PollSource, error) { return nws.NewHourlyForecastSource(cfg) }},
|
||||
{driver: nws.DriverForecastNarrative, factory: func(cfg config.SourceConfig) (fksource.PollSource, error) { return nws.NewNarrativeForecastSource(cfg) }},
|
||||
{driver: nws.DriverForecastDiscussion, factory: func(cfg config.SourceConfig) (fksource.PollSource, error) {
|
||||
return nws.NewForecastDiscussionSource(cfg)
|
||||
}},
|
||||
{driver: "nws_weatherstories", factory: func(cfg config.SourceConfig) (fksource.PollSource, error) { return nws.NewWeatherStoriesSource(cfg) }},
|
||||
{driver: "openmeteo_observation", factory: func(cfg config.SourceConfig) (fksource.PollSource, error) { return openmeteo.NewObservationSource(cfg) }},
|
||||
{driver: "openmeteo_forecast", factory: func(cfg config.SourceConfig) (fksource.PollSource, error) { return openmeteo.NewForecastSource(cfg) }},
|
||||
{driver: "openweather_observation", factory: func(cfg config.SourceConfig) (fksource.PollSource, error) {
|
||||
{driver: nws.DriverWeatherStories, factory: func(cfg config.SourceConfig) (fksource.PollSource, error) { return nws.NewWeatherStoriesSource(cfg) }},
|
||||
{driver: openmeteo.DriverObservation, factory: func(cfg config.SourceConfig) (fksource.PollSource, error) { return openmeteo.NewObservationSource(cfg) }},
|
||||
{driver: openmeteo.DriverForecast, factory: func(cfg config.SourceConfig) (fksource.PollSource, error) { return openmeteo.NewForecastSource(cfg) }},
|
||||
{driver: openweather.DriverObservation, factory: func(cfg config.SourceConfig) (fksource.PollSource, error) {
|
||||
return openweather.NewObservationSource(cfg)
|
||||
}},
|
||||
{driver: spc.DriverConvectiveOutlook, factory: func(cfg config.SourceConfig) (fksource.PollSource, error) {
|
||||
return spc.NewConvectiveOutlookSource(cfg)
|
||||
}},
|
||||
}
|
||||
|
||||
// RegisterBuiltins registers the source drivers that ship with this binary.
|
||||
|
||||
@@ -6,57 +6,29 @@ import (
|
||||
|
||||
"gitea.maximumdirect.net/ejr/feedkit/config"
|
||||
fksource "gitea.maximumdirect.net/ejr/feedkit/sources"
|
||||
"gitea.maximumdirect.net/ejr/weatherfeeder/internal/sources/openweather"
|
||||
"gitea.maximumdirect.net/ejr/weatherfeeder/internal/sources/spc"
|
||||
)
|
||||
|
||||
func TestRegisterBuiltinsRegistersNWSHourlyForecastDriver(t *testing.T) {
|
||||
func TestRegisterBuiltinsRegistersCurrentPollDrivers(t *testing.T) {
|
||||
reg := fksource.NewRegistry()
|
||||
RegisterBuiltins(reg)
|
||||
|
||||
in, err := reg.BuildInput(sourceConfigForDriver("nws_forecast_hourly"))
|
||||
if err != nil {
|
||||
t.Fatalf("BuildInput(nws_forecast_hourly) error = %v", err)
|
||||
if len(pollDriverRegistrations) == 0 {
|
||||
t.Fatalf("pollDriverRegistrations is empty")
|
||||
}
|
||||
if _, ok := in.(fksource.PollSource); !ok {
|
||||
t.Fatalf("BuildInput(nws_forecast_hourly) type = %T, want PollSource", in)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRegisterBuiltinsRegistersNWSNarrativeForecastDriver(t *testing.T) {
|
||||
reg := fksource.NewRegistry()
|
||||
RegisterBuiltins(reg)
|
||||
|
||||
in, err := reg.BuildInput(sourceConfigForDriver("nws_forecast_narrative"))
|
||||
if err != nil {
|
||||
t.Fatalf("BuildInput(nws_forecast_narrative) error = %v", err)
|
||||
}
|
||||
if _, ok := in.(fksource.PollSource); !ok {
|
||||
t.Fatalf("BuildInput(nws_forecast_narrative) type = %T, want PollSource", in)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRegisterBuiltinsRegistersNWSForecastDiscussionDriver(t *testing.T) {
|
||||
reg := fksource.NewRegistry()
|
||||
RegisterBuiltins(reg)
|
||||
|
||||
in, err := reg.BuildInput(sourceConfigForDriver("nws_forecast_discussion"))
|
||||
if err != nil {
|
||||
t.Fatalf("BuildInput(nws_forecast_discussion) error = %v", err)
|
||||
}
|
||||
if _, ok := in.(fksource.PollSource); !ok {
|
||||
t.Fatalf("BuildInput(nws_forecast_discussion) type = %T, want PollSource", in)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRegisterBuiltinsRegistersNWSWeatherStoriesDriver(t *testing.T) {
|
||||
reg := fksource.NewRegistry()
|
||||
RegisterBuiltins(reg)
|
||||
|
||||
in, err := reg.BuildInput(sourceConfigForDriver("nws_weatherstories"))
|
||||
if err != nil {
|
||||
t.Fatalf("BuildInput(nws_weatherstories) error = %v", err)
|
||||
}
|
||||
if _, ok := in.(fksource.PollSource); !ok {
|
||||
t.Fatalf("BuildInput(nws_weatherstories) type = %T, want PollSource", in)
|
||||
for _, tt := range pollDriverRegistrations {
|
||||
tt := tt
|
||||
t.Run(tt.driver, func(t *testing.T) {
|
||||
in, err := reg.BuildInput(sourceConfigForDriver(tt.driver))
|
||||
if err != nil {
|
||||
t.Fatalf("BuildInput(%q) error = %v", tt.driver, err)
|
||||
}
|
||||
if _, ok := in.(fksource.PollSource); !ok {
|
||||
t.Fatalf("BuildInput(%q) type = %T, want PollSource", tt.driver, in)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -73,45 +45,23 @@ func TestRegisterBuiltinsDoesNotRegisterLegacyNWSForecastDriver(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRegisterBuiltinsRegistersAllCurrentDrivers(t *testing.T) {
|
||||
reg := fksource.NewRegistry()
|
||||
RegisterBuiltins(reg)
|
||||
|
||||
drivers := []string{
|
||||
"nws_observation",
|
||||
"nws_alerts",
|
||||
"nws_forecast_hourly",
|
||||
"nws_forecast_narrative",
|
||||
"nws_forecast_discussion",
|
||||
"nws_weatherstories",
|
||||
"openmeteo_observation",
|
||||
"openmeteo_forecast",
|
||||
"openweather_observation",
|
||||
}
|
||||
|
||||
for _, driver := range drivers {
|
||||
in, err := reg.BuildInput(sourceConfigForDriver(driver))
|
||||
if err != nil {
|
||||
t.Fatalf("BuildInput(%s) error = %v", driver, err)
|
||||
}
|
||||
if _, ok := in.(fksource.PollSource); !ok {
|
||||
t.Fatalf("BuildInput(%s) type = %T, want PollSource", driver, in)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func sourceConfigForDriver(driver string) config.SourceConfig {
|
||||
url := "https://example.invalid"
|
||||
if driver == "openweather_observation" {
|
||||
if driver == openweather.DriverObservation {
|
||||
url = "https://example.invalid?units=metric"
|
||||
}
|
||||
params := map[string]any{
|
||||
"url": url,
|
||||
"user_agent": "test-agent",
|
||||
}
|
||||
if driver == spc.DriverConvectiveOutlook {
|
||||
params["latitude"] = 38.6239
|
||||
params["longitude"] = -90.3571
|
||||
}
|
||||
return config.SourceConfig{
|
||||
Name: "test-source",
|
||||
Driver: driver,
|
||||
Mode: config.SourceModePoll,
|
||||
Params: map[string]any{
|
||||
"url": url,
|
||||
"user_agent": "test-agent",
|
||||
},
|
||||
Params: params,
|
||||
}
|
||||
}
|
||||
|
||||
32
internal/sources/docs_test.go
Normal file
32
internal/sources/docs_test.go
Normal file
@@ -0,0 +1,32 @@
|
||||
package sources
|
||||
|
||||
import (
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestDocumentedRegisteredSourceDrivers(t *testing.T) {
|
||||
docs := map[string]string{
|
||||
"docs/config.md": readDoc(t, "../../docs/config.md"),
|
||||
"docs/internal/sources.md": readDoc(t, "../../docs/internal/sources.md"),
|
||||
}
|
||||
|
||||
for _, reg := range pollDriverRegistrations {
|
||||
for path, doc := range docs {
|
||||
if !strings.Contains(doc, reg.driver) {
|
||||
t.Fatalf("%s missing source driver %q", path, reg.driver)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func readDoc(t *testing.T, path string) string {
|
||||
t.Helper()
|
||||
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatalf("ReadFile(%s) error = %v", path, err)
|
||||
}
|
||||
return string(raw)
|
||||
}
|
||||
58
internal/sources/internal/httpconfig/config.go
Normal file
58
internal/sources/internal/httpconfig/config.go
Normal file
@@ -0,0 +1,58 @@
|
||||
package httpconfig
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/ejr/feedkit/config"
|
||||
"gitea.maximumdirect.net/ejr/feedkit/transport"
|
||||
)
|
||||
|
||||
// Settings contains common HTTP client config for sources that fetch multiple documents.
|
||||
type Settings struct {
|
||||
Name string
|
||||
UserAgent string
|
||||
Timeout time.Duration
|
||||
BodyLimitBytes int64
|
||||
}
|
||||
|
||||
func Parse(driver string, cfg config.SourceConfig) (Settings, error) {
|
||||
name := strings.TrimSpace(cfg.Name)
|
||||
if name == "" {
|
||||
return Settings{}, fmt.Errorf("%s: name is required", driver)
|
||||
}
|
||||
if cfg.Params == nil {
|
||||
return Settings{}, fmt.Errorf("%s %q: params are required", driver, name)
|
||||
}
|
||||
|
||||
userAgent, ok := cfg.ParamString("user_agent", "userAgent")
|
||||
if !ok {
|
||||
return Settings{}, fmt.Errorf("%s %q: params.user_agent is required", driver, name)
|
||||
}
|
||||
|
||||
timeout := transport.DefaultHTTPTimeout
|
||||
if _, exists := cfg.Params["http_timeout"]; exists {
|
||||
var ok bool
|
||||
timeout, ok = cfg.ParamDuration("http_timeout")
|
||||
if !ok || timeout <= 0 {
|
||||
return Settings{}, fmt.Errorf("source %q: params.http_timeout must be a positive duration", name)
|
||||
}
|
||||
}
|
||||
|
||||
bodyLimit := transport.DefaultHTTPResponseBodyLimitBytes
|
||||
if _, exists := cfg.Params["http_response_body_limit_bytes"]; exists {
|
||||
rawLimit, ok := cfg.ParamInt("http_response_body_limit_bytes")
|
||||
if !ok || rawLimit <= 0 {
|
||||
return Settings{}, fmt.Errorf("source %q: params.http_response_body_limit_bytes must be a positive integer", name)
|
||||
}
|
||||
bodyLimit = int64(rawLimit)
|
||||
}
|
||||
|
||||
return Settings{
|
||||
Name: name,
|
||||
UserAgent: userAgent,
|
||||
Timeout: timeout,
|
||||
BodyLimitBytes: bodyLimit,
|
||||
}, nil
|
||||
}
|
||||
115
internal/sources/internal/httpconfig/config_test.go
Normal file
115
internal/sources/internal/httpconfig/config_test.go
Normal file
@@ -0,0 +1,115 @@
|
||||
package httpconfig
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/ejr/feedkit/config"
|
||||
"gitea.maximumdirect.net/ejr/feedkit/transport"
|
||||
)
|
||||
|
||||
func TestParseUsesRequiredValuesAndDefaults(t *testing.T) {
|
||||
got, err := Parse("test_driver", config.SourceConfig{
|
||||
Name: " test-source ",
|
||||
Params: map[string]any{
|
||||
"user_agent": "test-agent",
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Parse() error = %v", err)
|
||||
}
|
||||
if got.Name != "test-source" {
|
||||
t.Fatalf("Name = %q, want test-source", got.Name)
|
||||
}
|
||||
if got.UserAgent != "test-agent" {
|
||||
t.Fatalf("UserAgent = %q, want test-agent", got.UserAgent)
|
||||
}
|
||||
if got.Timeout != transport.DefaultHTTPTimeout {
|
||||
t.Fatalf("Timeout = %s, want %s", got.Timeout, transport.DefaultHTTPTimeout)
|
||||
}
|
||||
if got.BodyLimitBytes != transport.DefaultHTTPResponseBodyLimitBytes {
|
||||
t.Fatalf("BodyLimitBytes = %d, want %d", got.BodyLimitBytes, transport.DefaultHTTPResponseBodyLimitBytes)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseUsesAliasesAndOverrides(t *testing.T) {
|
||||
got, err := Parse("test_driver", config.SourceConfig{
|
||||
Name: "test-source",
|
||||
Params: map[string]any{
|
||||
"userAgent": "test-agent",
|
||||
"http_timeout": "2s",
|
||||
"http_response_body_limit_bytes": 2048,
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Parse() error = %v", err)
|
||||
}
|
||||
if got.UserAgent != "test-agent" {
|
||||
t.Fatalf("UserAgent = %q, want test-agent", got.UserAgent)
|
||||
}
|
||||
if got.Timeout != 2*time.Second {
|
||||
t.Fatalf("Timeout = %s, want 2s", got.Timeout)
|
||||
}
|
||||
if got.BodyLimitBytes != 2048 {
|
||||
t.Fatalf("BodyLimitBytes = %d, want 2048", got.BodyLimitBytes)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseRejectsInvalidConfig(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
cfg config.SourceConfig
|
||||
wantErr string
|
||||
}{
|
||||
{
|
||||
name: "missing name",
|
||||
cfg: config.SourceConfig{Params: map[string]any{"user_agent": "test-agent"}},
|
||||
wantErr: "test_driver: name is required",
|
||||
},
|
||||
{
|
||||
name: "missing params",
|
||||
cfg: config.SourceConfig{Name: "test-source"},
|
||||
wantErr: `test_driver "test-source": params are required`,
|
||||
},
|
||||
{
|
||||
name: "missing user agent",
|
||||
cfg: config.SourceConfig{Name: "test-source", Params: map[string]any{}},
|
||||
wantErr: `test_driver "test-source": params.user_agent is required`,
|
||||
},
|
||||
{
|
||||
name: "invalid timeout",
|
||||
cfg: config.SourceConfig{
|
||||
Name: "test-source",
|
||||
Params: map[string]any{
|
||||
"user_agent": "test-agent",
|
||||
"http_timeout": "0s",
|
||||
},
|
||||
},
|
||||
wantErr: `source "test-source": params.http_timeout must be a positive duration`,
|
||||
},
|
||||
{
|
||||
name: "invalid body limit",
|
||||
cfg: config.SourceConfig{
|
||||
Name: "test-source",
|
||||
Params: map[string]any{
|
||||
"user_agent": "test-agent",
|
||||
"http_response_body_limit_bytes": 0,
|
||||
},
|
||||
},
|
||||
wantErr: `source "test-source": params.http_response_body_limit_bytes must be a positive integer`,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
_, err := Parse("test_driver", tt.cfg)
|
||||
if err == nil {
|
||||
t.Fatalf("Parse() error = nil, want %q", tt.wantErr)
|
||||
}
|
||||
if !strings.Contains(err.Error(), tt.wantErr) {
|
||||
t.Fatalf("Parse() error = %q, want %q", err, tt.wantErr)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -26,10 +26,8 @@ type AlertsSource struct {
|
||||
}
|
||||
|
||||
func NewAlertsSource(cfg config.SourceConfig) (*AlertsSource, error) {
|
||||
const driver = "nws_alerts"
|
||||
|
||||
// NWS alerts responses are GeoJSON-ish; allow fallback to plain JSON as well.
|
||||
hs, err := fksources.NewHTTPSource(driver, cfg, "application/geo+json, application/json")
|
||||
hs, err := fksources.NewHTTPSource(DriverAlerts, cfg, "application/geo+json, application/json")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -40,7 +38,7 @@ func NewAlertsSource(cfg config.SourceConfig) (*AlertsSource, error) {
|
||||
func (s *AlertsSource) Name() string { return s.http.Name }
|
||||
|
||||
// Kinds is used for routing/policy.
|
||||
func (s *AlertsSource) Kinds() []event.Kind { return []event.Kind{event.Kind("alert")} }
|
||||
func (s *AlertsSource) Kinds() []event.Kind { return []event.Kind{event.Kind(standards.KindAlert)} }
|
||||
|
||||
func (s *AlertsSource) Poll(ctx context.Context) ([]event.Event, error) {
|
||||
raw, meta, changed, err := s.fetchRaw(ctx)
|
||||
@@ -71,7 +69,7 @@ func (s *AlertsSource) Poll(ctx context.Context) ([]event.Event, error) {
|
||||
eventID := fksources.DefaultEventID("", s.http.Name, effectiveAt, emittedAt)
|
||||
|
||||
return fksources.SingleEvent(
|
||||
event.Kind("alert"),
|
||||
event.Kind(standards.KindAlert),
|
||||
s.http.Name,
|
||||
standards.SchemaRawNWSAlertsV1,
|
||||
eventID,
|
||||
|
||||
11
internal/sources/nws/driver.go
Normal file
11
internal/sources/nws/driver.go
Normal file
@@ -0,0 +1,11 @@
|
||||
package nws
|
||||
|
||||
// Source driver strings registered by weatherfeeder for NWS sources.
|
||||
const (
|
||||
DriverObservation = "nws_observation"
|
||||
DriverAlerts = "nws_alerts"
|
||||
DriverForecastHourly = "nws_forecast_hourly"
|
||||
DriverForecastNarrative = "nws_forecast_narrative"
|
||||
DriverForecastDiscussion = "nws_forecast_discussion"
|
||||
DriverWeatherStories = "nws_weatherstories"
|
||||
)
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
"gitea.maximumdirect.net/ejr/feedkit/event"
|
||||
fksources "gitea.maximumdirect.net/ejr/feedkit/sources"
|
||||
nwscommon "gitea.maximumdirect.net/ejr/weatherfeeder/internal/providers/nws"
|
||||
"gitea.maximumdirect.net/ejr/weatherfeeder/standards"
|
||||
)
|
||||
|
||||
const nwsForecastAccept = "application/geo+json, application/json"
|
||||
@@ -44,7 +45,9 @@ func newForecastSource(cfg config.SourceConfig, driver, rawSchema string) (*fore
|
||||
|
||||
func (s *forecastSource) Name() string { return s.http.Name }
|
||||
|
||||
func (s *forecastSource) Kinds() []event.Kind { return []event.Kind{event.Kind("forecast")} }
|
||||
func (s *forecastSource) Kinds() []event.Kind {
|
||||
return []event.Kind{event.Kind(standards.KindForecast)}
|
||||
}
|
||||
|
||||
func (s *forecastSource) Poll(ctx context.Context) ([]event.Event, error) {
|
||||
raw, meta, changed, err := s.fetchRaw(ctx)
|
||||
@@ -69,7 +72,7 @@ func (s *forecastSource) Poll(ctx context.Context) ([]event.Event, error) {
|
||||
eventID := fksources.DefaultEventID("", s.http.Name, effectiveAt, emittedAt)
|
||||
|
||||
return fksources.SingleEvent(
|
||||
event.Kind("forecast"),
|
||||
event.Kind(standards.KindForecast),
|
||||
s.http.Name,
|
||||
s.rawSchema,
|
||||
eventID,
|
||||
|
||||
@@ -20,9 +20,7 @@ type ForecastDiscussionSource struct {
|
||||
}
|
||||
|
||||
func NewForecastDiscussionSource(cfg config.SourceConfig) (*ForecastDiscussionSource, error) {
|
||||
const driver = "nws_forecast_discussion"
|
||||
|
||||
hs, err := fksources.NewHTTPSource(driver, cfg, "text/html, application/xhtml+xml")
|
||||
hs, err := fksources.NewHTTPSource(DriverForecastDiscussion, cfg, "text/html, application/xhtml+xml")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -33,7 +31,7 @@ func NewForecastDiscussionSource(cfg config.SourceConfig) (*ForecastDiscussionSo
|
||||
func (s *ForecastDiscussionSource) Name() string { return s.http.Name }
|
||||
|
||||
func (s *ForecastDiscussionSource) Kinds() []event.Kind {
|
||||
return []event.Kind{event.Kind("forecast_discussion")}
|
||||
return []event.Kind{event.Kind(standards.KindForecastDiscussion)}
|
||||
}
|
||||
|
||||
func (s *ForecastDiscussionSource) Poll(ctx context.Context) ([]event.Event, error) {
|
||||
@@ -57,7 +55,7 @@ func (s *ForecastDiscussionSource) Poll(ctx context.Context) ([]event.Event, err
|
||||
eventID := fksources.DefaultEventID("", s.http.Name, effectiveAt, emittedAt)
|
||||
|
||||
return fksources.SingleEvent(
|
||||
event.Kind("forecast_discussion"),
|
||||
event.Kind(standards.KindForecastDiscussion),
|
||||
s.http.Name,
|
||||
standards.SchemaRawNWSForecastDiscussionV1,
|
||||
eventID,
|
||||
|
||||
@@ -27,7 +27,7 @@ func TestForecastDiscussionSourcePollEmitsExpectedEvent(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("NewForecastDiscussionSource() error = %v", err)
|
||||
}
|
||||
if got := src.Kinds(); len(got) != 1 || got[0] != event.Kind("forecast_discussion") {
|
||||
if got := src.Kinds(); len(got) != 1 || got[0] != event.Kind(standards.KindForecastDiscussion) {
|
||||
t.Fatalf("Kinds() = %#v, want [forecast_discussion]", got)
|
||||
}
|
||||
|
||||
@@ -40,7 +40,7 @@ func TestForecastDiscussionSourcePollEmitsExpectedEvent(t *testing.T) {
|
||||
}
|
||||
|
||||
got := events[0]
|
||||
if got.Kind != event.Kind("forecast_discussion") {
|
||||
if got.Kind != event.Kind(standards.KindForecastDiscussion) {
|
||||
t.Fatalf("Kind = %q, want forecast_discussion", got.Kind)
|
||||
}
|
||||
if got.Schema != standards.SchemaRawNWSForecastDiscussionV1 {
|
||||
@@ -117,7 +117,7 @@ func TestForecastDiscussionSourcePollRejectsInvalidHTML(t *testing.T) {
|
||||
func forecastDiscussionSourceConfig(url string) config.SourceConfig {
|
||||
return config.SourceConfig{
|
||||
Name: "test-forecast-discussion-source",
|
||||
Driver: "nws_forecast_discussion",
|
||||
Driver: DriverForecastDiscussion,
|
||||
Mode: config.SourceModePoll,
|
||||
Params: map[string]any{
|
||||
"url": url,
|
||||
|
||||
@@ -18,8 +18,7 @@ type HourlyForecastSource struct {
|
||||
}
|
||||
|
||||
func NewHourlyForecastSource(cfg config.SourceConfig) (*HourlyForecastSource, error) {
|
||||
const driver = "nws_forecast_hourly"
|
||||
src, err := newForecastSource(cfg, driver, standards.SchemaRawNWSHourlyForecastV1)
|
||||
src, err := newForecastSource(cfg, DriverForecastHourly, standards.SchemaRawNWSHourlyForecastV1)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@@ -18,8 +18,7 @@ type NarrativeForecastSource struct {
|
||||
}
|
||||
|
||||
func NewNarrativeForecastSource(cfg config.SourceConfig) (*NarrativeForecastSource, error) {
|
||||
const driver = "nws_forecast_narrative"
|
||||
src, err := newForecastSource(cfg, driver, standards.SchemaRawNWSNarrativeForecastV1)
|
||||
src, err := newForecastSource(cfg, DriverForecastNarrative, standards.SchemaRawNWSNarrativeForecastV1)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@@ -26,7 +26,7 @@ func TestForecastSourcesEmitExpectedSchemaAndPreferGeneratedAt(t *testing.T) {
|
||||
}{
|
||||
{
|
||||
name: "hourly",
|
||||
driver: "nws_forecast_hourly",
|
||||
driver: DriverForecastHourly,
|
||||
wantSchema: standards.SchemaRawNWSHourlyForecastV1,
|
||||
newSource: func(cfg config.SourceConfig) (forecastPoller, error) {
|
||||
return NewHourlyForecastSource(cfg)
|
||||
@@ -34,7 +34,7 @@ func TestForecastSourcesEmitExpectedSchemaAndPreferGeneratedAt(t *testing.T) {
|
||||
},
|
||||
{
|
||||
name: "narrative",
|
||||
driver: "nws_forecast_narrative",
|
||||
driver: DriverForecastNarrative,
|
||||
wantSchema: standards.SchemaRawNWSNarrativeForecastV1,
|
||||
newSource: func(cfg config.SourceConfig) (forecastPoller, error) {
|
||||
return NewNarrativeForecastSource(cfg)
|
||||
@@ -55,7 +55,7 @@ func TestForecastSourcesEmitExpectedSchemaAndPreferGeneratedAt(t *testing.T) {
|
||||
}
|
||||
if ks, ok := src.(interface{ Kinds() []event.Kind }); !ok {
|
||||
t.Fatalf("source does not implement Kinds()")
|
||||
} else if gotKinds := ks.Kinds(); len(gotKinds) != 1 || gotKinds[0] != event.Kind("forecast") {
|
||||
} else if gotKinds := ks.Kinds(); len(gotKinds) != 1 || gotKinds[0] != event.Kind(standards.KindForecast) {
|
||||
t.Fatalf("Kinds() = %#v, want [forecast]", gotKinds)
|
||||
}
|
||||
|
||||
@@ -69,7 +69,7 @@ func TestForecastSourcesEmitExpectedSchemaAndPreferGeneratedAt(t *testing.T) {
|
||||
if got[0].Schema != tt.wantSchema {
|
||||
t.Fatalf("Poll() schema = %q, want %q", got[0].Schema, tt.wantSchema)
|
||||
}
|
||||
if got[0].Kind != event.Kind("forecast") {
|
||||
if got[0].Kind != event.Kind(standards.KindForecast) {
|
||||
t.Fatalf("Poll() kind = %q, want forecast", got[0].Kind)
|
||||
}
|
||||
|
||||
@@ -117,7 +117,7 @@ func TestForecastSourcePollEffectiveAtFallbackOrder(t *testing.T) {
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
src, err := NewHourlyForecastSource(forecastSourceConfig("nws_forecast_hourly", srv.URL))
|
||||
src, err := NewHourlyForecastSource(forecastSourceConfig(DriverForecastHourly, srv.URL))
|
||||
if err != nil {
|
||||
t.Fatalf("NewHourlyForecastSource() error = %v", err)
|
||||
}
|
||||
@@ -148,7 +148,7 @@ func TestForecastSourcePollMetadataDecodeFailureStillEmitsRawEvent(t *testing.T)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
src, err := NewNarrativeForecastSource(forecastSourceConfig("nws_forecast_narrative", srv.URL))
|
||||
src, err := NewNarrativeForecastSource(forecastSourceConfig(DriverForecastNarrative, srv.URL))
|
||||
if err != nil {
|
||||
t.Fatalf("NewNarrativeForecastSource() error = %v", err)
|
||||
}
|
||||
|
||||
@@ -20,9 +20,7 @@ type ObservationSource struct {
|
||||
}
|
||||
|
||||
func NewObservationSource(cfg config.SourceConfig) (*ObservationSource, error) {
|
||||
const driver = "nws_observation"
|
||||
|
||||
hs, err := fksources.NewHTTPSource(driver, cfg, "application/geo+json, application/json")
|
||||
hs, err := fksources.NewHTTPSource(DriverObservation, cfg, "application/geo+json, application/json")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -32,7 +30,9 @@ func NewObservationSource(cfg config.SourceConfig) (*ObservationSource, error) {
|
||||
|
||||
func (s *ObservationSource) Name() string { return s.http.Name }
|
||||
|
||||
func (s *ObservationSource) Kinds() []event.Kind { return []event.Kind{event.Kind("observation")} }
|
||||
func (s *ObservationSource) Kinds() []event.Kind {
|
||||
return []event.Kind{event.Kind(standards.KindObservation)}
|
||||
}
|
||||
|
||||
func (s *ObservationSource) Poll(ctx context.Context) ([]event.Event, error) {
|
||||
raw, meta, changed, err := s.fetchRaw(ctx)
|
||||
@@ -54,7 +54,7 @@ func (s *ObservationSource) Poll(ctx context.Context) ([]event.Event, error) {
|
||||
eventID := fksources.DefaultEventID(meta.ID, s.http.Name, effectiveAt, emittedAt)
|
||||
|
||||
return fksources.SingleEvent(
|
||||
event.Kind("observation"),
|
||||
event.Kind(standards.KindObservation),
|
||||
s.http.Name,
|
||||
standards.SchemaRawNWSObservationV1,
|
||||
eventID,
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
|
||||
"gitea.maximumdirect.net/ejr/feedkit/config"
|
||||
"gitea.maximumdirect.net/ejr/feedkit/event"
|
||||
"gitea.maximumdirect.net/ejr/weatherfeeder/standards"
|
||||
)
|
||||
|
||||
func TestObservationSourcePollReturnsNoEventsOn304(t *testing.T) {
|
||||
@@ -31,7 +32,7 @@ func TestObservationSourcePollReturnsNoEventsOn304(t *testing.T) {
|
||||
|
||||
src, err := NewObservationSource(config.SourceConfig{
|
||||
Name: "NWSObservationTest",
|
||||
Driver: "nws_observation",
|
||||
Driver: DriverObservation,
|
||||
Mode: config.SourceModePoll,
|
||||
Params: map[string]any{
|
||||
"url": srv.URL,
|
||||
@@ -41,7 +42,7 @@ func TestObservationSourcePollReturnsNoEventsOn304(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("NewObservationSource() error = %v", err)
|
||||
}
|
||||
if got := src.Kinds(); len(got) != 1 || got[0] != event.Kind("observation") {
|
||||
if got := src.Kinds(); len(got) != 1 || got[0] != event.Kind(standards.KindObservation) {
|
||||
t.Fatalf("Kinds() = %#v, want [observation]", got)
|
||||
}
|
||||
|
||||
@@ -52,7 +53,7 @@ func TestObservationSourcePollReturnsNoEventsOn304(t *testing.T) {
|
||||
if len(first) != 1 {
|
||||
t.Fatalf("first Poll() len = %d, want 1", len(first))
|
||||
}
|
||||
if first[0].Kind != event.Kind("observation") {
|
||||
if first[0].Kind != event.Kind(standards.KindObservation) {
|
||||
t.Fatalf("first Poll() kind = %q", first[0].Kind)
|
||||
}
|
||||
|
||||
|
||||
@@ -22,9 +22,7 @@ type WeatherStoriesSource struct {
|
||||
}
|
||||
|
||||
func NewWeatherStoriesSource(cfg config.SourceConfig) (*WeatherStoriesSource, error) {
|
||||
const driver = "nws_weatherstories"
|
||||
|
||||
hs, err := fksources.NewHTTPSource(driver, cfg, "application/geo+json, application/json")
|
||||
hs, err := fksources.NewHTTPSource(DriverWeatherStories, cfg, "application/geo+json, application/json")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -35,7 +33,7 @@ func NewWeatherStoriesSource(cfg config.SourceConfig) (*WeatherStoriesSource, er
|
||||
func (s *WeatherStoriesSource) Name() string { return s.http.Name }
|
||||
|
||||
func (s *WeatherStoriesSource) Kinds() []event.Kind {
|
||||
return []event.Kind{event.Kind("weather_story")}
|
||||
return []event.Kind{event.Kind(standards.KindWeatherStory)}
|
||||
}
|
||||
|
||||
func (s *WeatherStoriesSource) Poll(ctx context.Context) ([]event.Event, error) {
|
||||
@@ -61,7 +59,7 @@ func (s *WeatherStoriesSource) Poll(ctx context.Context) ([]event.Event, error)
|
||||
eventID := fksources.DefaultEventID("", s.http.Name, effectiveAt, emittedAt)
|
||||
|
||||
return fksources.SingleEvent(
|
||||
event.Kind("weather_story"),
|
||||
event.Kind(standards.KindWeatherStory),
|
||||
s.http.Name,
|
||||
standards.SchemaRawNWSWeatherStoriesV1,
|
||||
eventID,
|
||||
|
||||
@@ -28,7 +28,7 @@ func TestWeatherStoriesSourcePollEmitsExpectedEventAndPrefersLatestUpdateTime(t
|
||||
if err != nil {
|
||||
t.Fatalf("NewWeatherStoriesSource() error = %v", err)
|
||||
}
|
||||
if got := src.Kinds(); len(got) != 1 || got[0] != event.Kind("weather_story") {
|
||||
if got := src.Kinds(); len(got) != 1 || got[0] != event.Kind(standards.KindWeatherStory) {
|
||||
t.Fatalf("Kinds() = %#v, want [weather_story]", got)
|
||||
}
|
||||
|
||||
@@ -41,7 +41,7 @@ func TestWeatherStoriesSourcePollEmitsExpectedEventAndPrefersLatestUpdateTime(t
|
||||
}
|
||||
|
||||
got := events[0]
|
||||
if got.Kind != event.Kind("weather_story") {
|
||||
if got.Kind != event.Kind(standards.KindWeatherStory) {
|
||||
t.Fatalf("Kind = %q, want weather_story", got.Kind)
|
||||
}
|
||||
if got.Schema != standards.SchemaRawNWSWeatherStoriesV1 {
|
||||
@@ -148,7 +148,7 @@ func TestWeatherStoriesSourcePollMetadataDecodeFailureStillEmitsRawEvent(t *test
|
||||
func weatherStoriesSourceConfig(url string) config.SourceConfig {
|
||||
return config.SourceConfig{
|
||||
Name: "test-weatherstories-source",
|
||||
Driver: "nws_weatherstories",
|
||||
Driver: DriverWeatherStories,
|
||||
Mode: config.SourceModePoll,
|
||||
Params: map[string]any{
|
||||
"url": url,
|
||||
|
||||
7
internal/sources/openmeteo/driver.go
Normal file
7
internal/sources/openmeteo/driver.go
Normal file
@@ -0,0 +1,7 @@
|
||||
package openmeteo
|
||||
|
||||
// Source driver strings registered by weatherfeeder for Open-Meteo sources.
|
||||
const (
|
||||
DriverObservation = "openmeteo_observation"
|
||||
DriverForecast = "openmeteo_forecast"
|
||||
)
|
||||
@@ -19,9 +19,7 @@ type ForecastSource struct {
|
||||
}
|
||||
|
||||
func NewForecastSource(cfg config.SourceConfig) (*ForecastSource, error) {
|
||||
const driver = "openmeteo_forecast"
|
||||
|
||||
hs, err := fksources.NewHTTPSource(driver, cfg, "application/json")
|
||||
hs, err := fksources.NewHTTPSource(DriverForecast, cfg, "application/json")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -31,7 +29,9 @@ func NewForecastSource(cfg config.SourceConfig) (*ForecastSource, error) {
|
||||
|
||||
func (s *ForecastSource) Name() string { return s.http.Name }
|
||||
|
||||
func (s *ForecastSource) Kinds() []event.Kind { return []event.Kind{event.Kind("forecast")} }
|
||||
func (s *ForecastSource) Kinds() []event.Kind {
|
||||
return []event.Kind{event.Kind(standards.KindForecast)}
|
||||
}
|
||||
|
||||
func (s *ForecastSource) Poll(ctx context.Context) ([]event.Event, error) {
|
||||
raw, meta, changed, err := s.fetchRaw(ctx)
|
||||
@@ -55,7 +55,7 @@ func (s *ForecastSource) Poll(ctx context.Context) ([]event.Event, error) {
|
||||
eventID := fksources.DefaultEventID("", s.http.Name, effectiveAt, emittedAt)
|
||||
|
||||
return fksources.SingleEvent(
|
||||
event.Kind("forecast"),
|
||||
event.Kind(standards.KindForecast),
|
||||
s.http.Name,
|
||||
standards.SchemaRawOpenMeteoHourlyForecastV1,
|
||||
eventID,
|
||||
|
||||
@@ -19,9 +19,7 @@ type ObservationSource struct {
|
||||
}
|
||||
|
||||
func NewObservationSource(cfg config.SourceConfig) (*ObservationSource, error) {
|
||||
const driver = "openmeteo_observation"
|
||||
|
||||
hs, err := fksources.NewHTTPSource(driver, cfg, "application/json")
|
||||
hs, err := fksources.NewHTTPSource(DriverObservation, cfg, "application/json")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -31,7 +29,9 @@ func NewObservationSource(cfg config.SourceConfig) (*ObservationSource, error) {
|
||||
|
||||
func (s *ObservationSource) Name() string { return s.http.Name }
|
||||
|
||||
func (s *ObservationSource) Kinds() []event.Kind { return []event.Kind{event.Kind("observation")} }
|
||||
func (s *ObservationSource) Kinds() []event.Kind {
|
||||
return []event.Kind{event.Kind(standards.KindObservation)}
|
||||
}
|
||||
|
||||
func (s *ObservationSource) Poll(ctx context.Context) ([]event.Event, error) {
|
||||
raw, meta, changed, err := s.fetchRaw(ctx)
|
||||
@@ -52,7 +52,7 @@ func (s *ObservationSource) Poll(ctx context.Context) ([]event.Event, error) {
|
||||
eventID := fksources.DefaultEventID("", s.http.Name, effectiveAt, emittedAt)
|
||||
|
||||
return fksources.SingleEvent(
|
||||
event.Kind("observation"),
|
||||
event.Kind(standards.KindObservation),
|
||||
s.http.Name,
|
||||
standards.SchemaRawOpenMeteoCurrentV1,
|
||||
eventID,
|
||||
|
||||
@@ -5,12 +5,13 @@ import (
|
||||
|
||||
"gitea.maximumdirect.net/ejr/feedkit/config"
|
||||
"gitea.maximumdirect.net/ejr/feedkit/event"
|
||||
"gitea.maximumdirect.net/ejr/weatherfeeder/standards"
|
||||
)
|
||||
|
||||
func TestObservationSourceAdvertisesKinds(t *testing.T) {
|
||||
src, err := NewObservationSource(config.SourceConfig{
|
||||
Name: "openmeteo-observation-test",
|
||||
Driver: "openmeteo_observation",
|
||||
Driver: DriverObservation,
|
||||
Mode: config.SourceModePoll,
|
||||
Params: map[string]any{
|
||||
"url": "https://example.invalid",
|
||||
@@ -20,7 +21,7 @@ func TestObservationSourceAdvertisesKinds(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("NewObservationSource() error = %v", err)
|
||||
}
|
||||
if got := src.Kinds(); len(got) != 1 || got[0] != event.Kind("observation") {
|
||||
if got := src.Kinds(); len(got) != 1 || got[0] != event.Kind(standards.KindObservation) {
|
||||
t.Fatalf("Kinds() = %#v, want [observation]", got)
|
||||
}
|
||||
}
|
||||
@@ -28,7 +29,7 @@ func TestObservationSourceAdvertisesKinds(t *testing.T) {
|
||||
func TestForecastSourceAdvertisesKinds(t *testing.T) {
|
||||
src, err := NewForecastSource(config.SourceConfig{
|
||||
Name: "openmeteo-forecast-test",
|
||||
Driver: "openmeteo_forecast",
|
||||
Driver: DriverForecast,
|
||||
Mode: config.SourceModePoll,
|
||||
Params: map[string]any{
|
||||
"url": "https://example.invalid",
|
||||
@@ -38,7 +39,7 @@ func TestForecastSourceAdvertisesKinds(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("NewForecastSource() error = %v", err)
|
||||
}
|
||||
if got := src.Kinds(); len(got) != 1 || got[0] != event.Kind("forecast") {
|
||||
if got := src.Kinds(); len(got) != 1 || got[0] != event.Kind(standards.KindForecast) {
|
||||
t.Fatalf("Kinds() = %#v, want [forecast]", got)
|
||||
}
|
||||
}
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user