37 Commits

Author SHA1 Message Date
50215d2105 Add a new field to the alert schema to fix a mismatch between the prior schema and the upstream NWS API
All checks were successful
ci/woodpecker/push/build-image Pipeline was successful
2026-06-16 19:57:48 -05:00
74411e3f54 Clean up completed roadmap and documentation for the new outlook schema
All checks were successful
ci/woodpecker/push/build-image Pipeline was successful
2026-06-12 08:30:41 -05:00
4358a7cdce Verify outlook schema release readiness
Some checks failed
ci/woodpecker/push/build-image Pipeline failed
2026-06-12 04:40:33 +00:00
dcea5261ab Document current outlook schema behavior 2026-06-12 04:38:44 +00:00
97141c7a9b Add outlook schema transition guide 2026-06-12 04:32:52 +00:00
2e2d36024e Align normalizer docs with outlook v2 2026-06-12 04:31:15 +00:00
4d2cddf801 Update Postgres outlook storage for v2 2026-06-12 04:29:00 +00:00
21a35a5205 Emit location-filtered SPC outlook v2 runs 2026-06-12 04:24:12 +00:00
435d1ade07 Add outlook v2 model contract 2026-06-12 04:19:56 +00:00
819ac24aed Added an implementation plan for updates and revisions to the outlook code
All checks were successful
ci/woodpecker/push/build-image Pipeline was successful
2026-06-11 23:15:00 -05:00
2c472449e8 Fix a bug in the GeoJSON handling when there are no active polygons
All checks were successful
ci/woodpecker/push/build-image Pipeline was successful
2026-06-11 22:21:59 -05:00
5d7f604a2c Implement remaining cleanup items prior to the next release
All checks were successful
ci/woodpecker/push/build-image Pipeline was successful
2026-06-11 10:17:49 -05:00
8041f99782 Clean and update documentation
All checks were successful
ci/woodpecker/push/build-image Pipeline was successful
2026-06-11 10:00:15 -05:00
c417c892d9 Enhance JSON payload decoding to accept both typed and pointer payloads, and add corresponding tests
All checks were successful
ci/woodpecker/push/build-image Pipeline was successful
2026-06-10 22:11:38 -05:00
481215c5db Update .dockerignore to include docs files required by tests
All checks were successful
ci/woodpecker/push/build-image Pipeline was successful
2026-06-10 22:03:39 -05:00
5d94d3f32d Remove invalid SPC URLs for day 3 tornado/wind/hail risk
Some checks failed
ci/woodpecker/push/build-image Pipeline failed
2026-06-10 21:56:11 -05:00
f8f1b8d4a5 Update documentation
Some checks failed
ci/woodpecker/push/build-image Pipeline failed
2026-06-10 21:46:33 -05:00
06d5973746 Clean up stale internal literals
Some checks failed
ci/woodpecker/push/build-image Pipeline was canceled
2026-06-11 02:23:42 +00:00
8045b27173 Move SPC provider fixture helper 2026-06-11 02:20:25 +00:00
985468c1b9 Add documentation identifier consistency tests 2026-06-11 02:18:15 +00:00
6a0b30b7c7 Centralize Postgres event envelope mapping 2026-06-11 02:15:52 +00:00
86ce4eb68c Share HTTP config parsing for multi-document sources 2026-06-11 02:13:38 +00:00
33541a71fc Table-drive source registry tests 2026-06-11 02:10:13 +00:00
b7277e0c02 Centralize weather event and driver identifiers 2026-06-11 02:08:36 +00:00
dec05821bf Add a staged cleanup roadmap to address the code quality audit
All checks were successful
ci/woodpecker/push/build-image Pipeline was successful
2026-06-10 21:02:07 -05:00
1a9f462fbf Audit code quality and deduplication opportunities 2026-06-10 20:04:00 -05:00
a990da957b Finalize SPC outlook feature addition and clean up implemented roadmap documentation
All checks were successful
ci/woodpecker/push/build-image Pipeline was successful
2026-06-10 19:54:39 -05:00
a4cd63ca4e Update the SPC outlook implementation plan to identify remaining gaps and corrections
All checks were successful
ci/woodpecker/push/build-image Pipeline was successful
2026-06-10 19:40:46 -05:00
fba519cab0 Update documentation for SPC outlook support
All checks were successful
ci/woodpecker/push/build-image Pipeline was successful
2026-06-11 00:30:49 +00:00
da8ff81692 Verify SPC outlook implementation 2026-06-11 00:28:30 +00:00
f91a185f9d Document SPC outlook support 2026-06-11 00:27:23 +00:00
e966276c40 Add Postgres mapping for SPC outlooks 2026-06-11 00:24:12 +00:00
1e2db468ea Add SPC outlook normalization 2026-06-11 00:20:31 +00:00
cefd4dfc7c Add raw SPC convective outlook source 2026-06-11 00:15:40 +00:00
42c646c328 Add GeoJSON point containment helper 2026-06-11 00:10:01 +00:00
b2c429983c Add SPC provider parsing helpers 2026-06-11 00:06:55 +00:00
0f20d1e4cb Add AGENTS.md 2026-06-10 23:59:45 +00:00
106 changed files with 6188 additions and 1373 deletions

View File

@@ -1,7 +1,8 @@
.git
.gitignore
**/*.md
!docs/*.md
!docs/**/*.md
dist/
tmp/
.DS_Store

4
AGENTS.md Normal file
View File

@@ -0,0 +1,4 @@
Please carefully review the documents in `docs/policy` before making any changes to this repository.
- `architecture.md` provides the canonical high-level architecture policy for this repository.
- `development.md` provides more granular development policy for this repository.
- `documentation.md` provides the canonical documentation policy for this repository.

View File

@@ -5,9 +5,9 @@ provider-specific responses into canonical weather events, and routes those
events to configured sinks.
It currently supports NWS observations, alerts, hourly forecasts, narrative
forecasts, forecast discussions, and weather stories; Open-Meteo observations
and hourly forecasts; and OpenWeather observations. Implemented sinks are
stdout, NATS, and Postgres.
forecasts, forecast discussions, and weather stories; SPC Day 1-3 convective
outlooks; Open-Meteo observations and hourly forecasts; and OpenWeather
observations. Implemented sinks are stdout, NATS, and Postgres.
## Quickstart
@@ -29,9 +29,12 @@ current working directory.
- [Operations guide](docs/operations.md)
- [Troubleshooting guide](docs/troubleshooting.md)
- [Example configs](examples/)
- [Go consumer guide](docs/consumers/api.md)
- [Event wire contract](docs/integrations/events.md)
- [Postgres table contract](docs/integrations/postgres.md)
- [Feedkit integration notes](docs/integrations/feedkit.md)
- [NWS integration notes](docs/integrations/nws.md)
- [SPC integration notes](docs/integrations/spc.md)
- [Open-Meteo integration notes](docs/integrations/openmeteo.md)
- [OpenWeather integration notes](docs/integrations/openweather.md)
- [Architecture policy](docs/policy/architecture.md)

View File

@@ -99,6 +99,18 @@ sources:
url: "https://api.weather.gov/alerts?point=38.6239,-90.3571&limit=20"
user_agent: "HomeOps (eric@maximumdirect.net)"
- name: SPCConvectiveOutlookSTL
mode: poll
kinds: ["outlook"]
driver: spc_convective_outlook
every: 30m
params:
latitude: 38.6239
longitude: -90.3571
location_id: "stl"
location_name: "St. Louis, MO"
user_agent: "HomeOps (eric@maximumdirect.net)"
sinks:
- name: stdout
driver: stdout
@@ -126,13 +138,13 @@ sinks:
routes:
- sink: stdout
kinds: ["observation", "forecast", "forecast_discussion", "weather_story", "alert"]
kinds: ["observation", "forecast", "forecast_discussion", "weather_story", "alert", "outlook"]
- sink: nats_weatherfeeder
kinds: ["observation", "forecast", "forecast_discussion", "weather_story", "alert"]
kinds: ["observation", "forecast", "forecast_discussion", "weather_story", "alert", "outlook"]
# - sink: pg_weatherfeeder
# kinds: ["observation", "forecast", "forecast_discussion", "weather_story", "alert"]
# kinds: ["observation", "forecast", "forecast_discussion", "weather_story", "alert", "outlook"]
# - sink: logfile
# kinds: ["observation", "alert", "forecast", "forecast_discussion", "weather_story"]
# kinds: ["observation", "alert", "forecast", "forecast_discussion", "weather_story", "outlook"]

View File

@@ -20,6 +20,7 @@ import (
wfnormalizers "gitea.maximumdirect.net/ejr/weatherfeeder/internal/normalizers"
wfsources "gitea.maximumdirect.net/ejr/weatherfeeder/internal/sources"
"gitea.maximumdirect.net/ejr/weatherfeeder/standards"
)
type testInput struct {
@@ -36,10 +37,10 @@ type testKindsSource struct {
func (s testKindsSource) Kinds() []fkevent.Kind { return s.kinds }
func TestValidateSourceExpectedKindsSubsetAllowed(t *testing.T) {
sc := config.SourceConfig{Kinds: []string{"observation"}}
sc := config.SourceConfig{Kinds: []string{standards.KindObservation}}
in := testKindsSource{
testInput: testInput{name: "test"},
kinds: []fkevent.Kind{"observation", "forecast"},
kinds: []fkevent.Kind{fkevent.Kind(standards.KindObservation), fkevent.Kind(standards.KindForecast)},
}
if err := fksources.ValidateExpectedKinds(sc, in); err != nil {
@@ -48,10 +49,10 @@ func TestValidateSourceExpectedKindsSubsetAllowed(t *testing.T) {
}
func TestValidateSourceExpectedKindsMismatchFails(t *testing.T) {
sc := config.SourceConfig{Kinds: []string{"alert"}}
sc := config.SourceConfig{Kinds: []string{standards.KindAlert}}
in := testKindsSource{
testInput: testInput{name: "test"},
kinds: []fkevent.Kind{"observation", "forecast"},
kinds: []fkevent.Kind{fkevent.Kind(standards.KindObservation), fkevent.Kind(standards.KindForecast)},
}
err := fksources.ValidateExpectedKinds(sc, in)
@@ -64,7 +65,7 @@ func TestValidateSourceExpectedKindsMismatchFails(t *testing.T) {
}
func TestValidateSourceExpectedKindsNoMetadataSkipsCheck(t *testing.T) {
sc := config.SourceConfig{Kinds: []string{"alert"}}
sc := config.SourceConfig{Kinds: []string{standards.KindAlert}}
in := testInput{name: "test"}
if err := fksources.ValidateExpectedKinds(sc, in); err != nil {
@@ -111,6 +112,10 @@ func TestMaintainedConfigExamplesLoad(t *testing.T) {
func assertConfigSourcesBuildSchedulerJobs(t *testing.T, cfg *config.Config) {
t.Helper()
if len(cfg.Sources) == 0 {
t.Fatalf("config has no sources")
}
reg := fksources.NewRegistry()
wfsources.RegisterBuiltins(reg)
@@ -158,7 +163,7 @@ func TestNormalizeNoMatchPassThrough(t *testing.T) {
pl := &fkpipeline.Pipeline{Processors: chain}
in := fkevent.Event{
ID: "evt-no-match",
Kind: fkevent.Kind("observation"),
Kind: fkevent.Kind(standards.KindObservation),
Source: "test",
EmittedAt: time.Now().UTC(),
Schema: "raw.weatherfeeder.unknown.v1",
@@ -188,7 +193,7 @@ func TestDedupeDropsSecondEventWithSameID(t *testing.T) {
pl := &fkpipeline.Pipeline{Processors: chain}
in := fkevent.Event{
ID: "evt-dedupe-1",
Kind: fkevent.Kind("observation"),
Kind: fkevent.Kind(standards.KindObservation),
Source: "test",
EmittedAt: time.Now().UTC(),
Schema: "raw.weatherfeeder.unknown.v1",

View File

@@ -52,6 +52,18 @@ sources:
url: "https://api.weather.gov/alerts?point=38.6239,-90.3571&limit=20"
user_agent: "Example weatherfeeder operator (ops@example.com)"
- name: SPCConvectiveOutlookLocal
mode: poll
driver: spc_convective_outlook
every: 30m
kinds: ["outlook"]
params:
latitude: 38.6239
longitude: -90.3571
location_id: "local"
location_name: "Configured point"
user_agent: "Example weatherfeeder operator (ops@example.com)"
sinks:
- name: nats_weather
driver: nats
@@ -69,10 +81,10 @@ sinks:
routes:
- sink: nats_weather
kinds: ["observation", "forecast", "forecast_discussion", "weather_story", "alert"]
kinds: ["observation", "forecast", "forecast_discussion", "weather_story", "alert", "outlook"]
- sink: pg_weather
kinds: ["observation", "forecast", "forecast_discussion", "weather_story", "alert"]
kinds: ["observation", "forecast", "forecast_discussion", "weather_story", "alert", "outlook"]
```
Do not commit real API keys, database passwords, or personal contact addresses in
@@ -95,10 +107,10 @@ copyable configs.
| `mode` | no | `poll`, `stream`, or omitted for auto. Current weatherfeeder drivers are polling drivers. |
| `every` | yes | Poll interval for current weatherfeeder source drivers. |
| `kinds` | no | Expected event kinds. If present, startup verifies they match the source driver. |
| `params` | driver-specific | Driver parameters. Current source drivers require HTTP params. |
| `params` | driver-specific | Driver parameters. See the source-specific sections below. |
Current event kinds are `observation`, `forecast`, `forecast_discussion`,
`weather_story`, and `alert`.
`weather_story`, `alert`, and `outlook`.
## Source Drivers
@@ -113,10 +125,11 @@ Current event kinds are `observation`, `forecast`, `forecast_discussion`,
| `openmeteo_observation` | `observation` | Open-Meteo current conditions. |
| `openmeteo_forecast` | `forecast` | Open-Meteo hourly forecast. |
| `openweather_observation` | `observation` | OpenWeather current weather. |
| `spc_convective_outlook` | `outlook` | SPC Day 1-3 convective outlooks. |
## HTTP Source Params
All current source drivers use the shared HTTP polling helper.
Most source drivers use the shared HTTP polling helper.
| Param | Required | Description |
|---|:---:|---|
@@ -132,6 +145,44 @@ source emits no events for that poll.
OpenWeather observation URLs must include `units=metric`. Startup fails if the
URL omits it or sets another unit system.
## SPC Convective Outlook Params
`spc_convective_outlook` fetches the nine required Day 1-3 GeoJSON outlook
products and the three required Day 1-3 print pages as one atomic bundle.
| Param | Required | Description |
|---|:---:|---|
| `latitude` | yes | Location latitude in decimal degrees. |
| `longitude` | yes | Location longitude in decimal degrees. |
| `user_agent` | yes | User-Agent sent to SPC. `userAgent` is also accepted. |
| `location_id` | no | Operator-defined location identifier copied into canonical outlook runs. |
| `location_name` | no | Human location label copied into canonical outlook runs. |
| `geojson_urls` | no | Map of product key to override URL. Used for tests and upstream URL changes. |
| `discussion_urls` | no | Map of day key to override print-page URL. Used for tests and upstream URL changes. |
| `rss_url` | no | Optional RSS URL. RSS is not fetched unless this is configured. |
| `http_timeout` | no | Positive duration for the HTTP client timeout. |
| `http_response_body_limit_bytes` | no | Positive integer response body limit in bytes. |
GeoJSON product keys are `day1_categorical`, `day1_tornado`, `day1_hail`,
`day1_wind`, `day2_categorical`, `day2_tornado`, `day2_hail`, `day2_wind`,
and `day3_categorical`. SPC does not provide Day 3 tornado, hail, or wind
GeoJSON products. Discussion keys are `day1`, `day2`, and `day3`.
```yaml
sources:
- name: SPCConvectiveOutlookSTL
mode: poll
kinds: ["outlook"]
driver: spc_convective_outlook
every: 30m
params:
latitude: 38.6239
longitude: -90.3571
location_id: "stl"
location_name: "St. Louis, MO"
user_agent: "Example weatherfeeder operator (ops@example.com)"
```
## Sink Fields
| Field | Required | Description |

92
docs/consumers/api.md Normal file
View File

@@ -0,0 +1,92 @@
# Consumer API Guide
## Purpose
This guide is for developers and LLM coding agents integrating `weatherfeeder`
from another Go codebase.
`weatherfeeder` is primarily a daemon, not an SDK. Its public integration
surface is intentionally narrow:
- `model`: canonical weather payload structs.
- `standards`: schema strings, event kind strings, and shared WMO constants.
- JSON event output from stdout and NATS sinks.
- Postgres tables written by the optional Postgres sink.
Packages under `internal/` are implementation details and are not public
integration surfaces.
## Recommended Workflow
Consumers should switch on the event `schema` value and decode `payload` into
the matching `model` type.
Minimal example:
```go
package consumer
import (
"encoding/json"
"fmt"
"gitea.maximumdirect.net/ejr/weatherfeeder/model"
"gitea.maximumdirect.net/ejr/weatherfeeder/standards"
)
type Event struct {
ID string `json:"id"`
Kind string `json:"kind"`
Schema string `json:"schema"`
Payload json.RawMessage `json:"payload"`
}
func Decode(payload []byte) (any, error) {
var evt Event
if err := json.Unmarshal(payload, &evt); err != nil {
return nil, err
}
switch evt.Schema {
case standards.SchemaWeatherObservationV1:
var out model.WeatherObservation
return &out, json.Unmarshal(evt.Payload, &out)
case standards.SchemaWeatherForecastV1:
var out model.WeatherForecastRun
return &out, json.Unmarshal(evt.Payload, &out)
case standards.SchemaWeatherForecastDiscussionV1:
var out model.WeatherForecastDiscussion
return &out, json.Unmarshal(evt.Payload, &out)
case standards.SchemaWeatherStoryV1:
var out model.WeatherStoryRun
return &out, json.Unmarshal(evt.Payload, &out)
case standards.SchemaWeatherAlertV1:
var out model.WeatherAlertRun
return &out, json.Unmarshal(evt.Payload, &out)
case standards.SchemaWeatherOutlookV2:
var out model.WeatherOutlookRun
return &out, json.Unmarshal(evt.Payload, &out)
default:
return nil, fmt.Errorf("unsupported weatherfeeder schema %q", evt.Schema)
}
}
```
## Consumer Responsibilities
- Treat event IDs as opaque.
- Treat absent `omitempty` fields as unknown, not zero.
- Prefer schema constants from `standards` over string literals in Go code.
- Expect canonical numeric measurements to use metric units.
- Expect canonical timestamps from normalizers to be UTC unless a field-specific
contract says otherwise.
- Handle additive fields within the same schema version.
- Do not import `internal/...` packages.
## Canonical References
- Public payload package: [`pkg-model.md`](pkg-model.md).
- Public constants package: [`pkg-standards.md`](pkg-standards.md).
- JSON event wire contract: [`../integrations/events.md`](../integrations/events.md).
- Postgres table contract: [`../integrations/postgres.md`](../integrations/postgres.md).
- Runtime and adapter architecture: [`../policy/architecture.md`](../policy/architecture.md).

View File

@@ -0,0 +1,64 @@
# Package `model`
## Import Path
```go
import "gitea.maximumdirect.net/ejr/weatherfeeder/model"
```
## Purpose
Package `model` defines `weatherfeeder`'s canonical weather payload structs.
These structs are emitted as the `payload` of canonical `weather.*` events and
are also the domain types consumed by downstream applications such as
`weatherapi`.
The JSON field tags on these structs are part of the wire contract. For the full
field-by-field JSON contract, use the [event wire contract](../integrations/events.md).
## Payload Types
Current canonical schema families map to these public types:
| Schema | Primary type |
|---|---|
| `weather.observation.v1` | `WeatherObservation` |
| `weather.forecast.v1` | `WeatherForecastRun` |
| `weather.forecast_discussion.v1` | `WeatherForecastDiscussion` |
| `weather.weather_story.v1` | `WeatherStoryRun` |
| `weather.alert.v1` | `WeatherAlertRun` |
| `weather.outlook.v2` | `WeatherOutlookRun` |
Related child types include:
- `WeatherObservationPresentWeather`
- `WeatherForecastPeriod`
- `WeatherForecastDiscussionSection`
- `WeatherStory`
- `WeatherAlert`
- `WeatherAlertReference`
- `WeatherOutlook`
- `WeatherOutlookDiscussion`
- `WMOCode`
## Wire And Compatibility Rules
- JSON tags define canonical payload field names.
- Pointer fields and fields tagged `omitempty` are optional on the wire.
- Missing optional fields mean unknown or not applicable.
- Canonical measurements use metric units.
- Canonical timestamps are `time.Time` values encoded by Go's JSON encoder.
- Normalized canonical timestamps are UTC unless a field-specific contract says
otherwise.
- Additive fields are compatible within a schema version.
- Removing, renaming, or changing the meaning of a field requires a new schema
identifier.
## Boundaries
`model` should not depend on source adapters, sinks, SQL column names, provider
HTTP shapes, or runtime configuration.
Consumers should not rely on packages under `internal/...`. Use `model` with
schema constants from [`standards`](pkg-standards.md) and the JSON contract in
[`docs/integrations/events.md`](../integrations/events.md).

View File

@@ -0,0 +1,96 @@
# Package `standards`
## Import Path
```go
import "gitea.maximumdirect.net/ejr/weatherfeeder/standards"
```
## Purpose
Package `standards` defines stable identifiers and shared weather constants used
by `weatherfeeder` producers and Go consumers.
Use this package when switching on event schemas, comparing event kinds, or
working with canonical WMO condition codes.
## Event Kind Constants
Current event kind constants are:
| Constant | Value |
|---|---|
| `KindObservation` | `observation` |
| `KindForecast` | `forecast` |
| `KindForecastDiscussion` | `forecast_discussion` |
| `KindWeatherStory` | `weather_story` |
| `KindAlert` | `alert` |
| `KindOutlook` | `outlook` |
These are plain string constants. Convert them at adapter boundaries when using
feedkit's `event.Kind` type.
## Canonical Schema Constants
Canonical schemas emitted after normalization:
| Constant | Value |
|---|---|
| `SchemaWeatherObservationV1` | `weather.observation.v1` |
| `SchemaWeatherForecastV1` | `weather.forecast.v1` |
| `SchemaWeatherForecastDiscussionV1` | `weather.forecast_discussion.v1` |
| `SchemaWeatherStoryV1` | `weather.weather_story.v1` |
| `SchemaWeatherAlertV1` | `weather.alert.v1` |
| `SchemaWeatherOutlookV2` | `weather.outlook.v2` |
Historical canonical schema constant:
| Constant | Value |
|---|---|
| `SchemaWeatherOutlookV1` | `weather.outlook.v1` |
## Raw Schema Constants
Raw source schemas emitted by current registered sources:
| Constant | Value |
|---|---|
| `SchemaRawNWSObservationV1` | `raw.nws.observation.v1` |
| `SchemaRawOpenMeteoCurrentV1` | `raw.openmeteo.current.v1` |
| `SchemaRawOpenWeatherCurrentV1` | `raw.openweather.current.v1` |
| `SchemaRawNWSHourlyForecastV1` | `raw.nws.hourly.forecast.v1` |
| `SchemaRawNWSNarrativeForecastV1` | `raw.nws.narrative.forecast.v1` |
| `SchemaRawNWSForecastDiscussionV1` | `raw.nws.forecast_discussion.v1` |
| `SchemaRawNWSWeatherStoriesV1` | `raw.nws.weatherstories.v1` |
| `SchemaRawOpenMeteoHourlyForecastV1` | `raw.openmeteo.hourly.forecast.v1` |
| `SchemaRawNWSAlertsV1` | `raw.nws.alerts.v1` |
| `SchemaRawSPCConvectiveOutlookV1` | `raw.spc.convective_outlook.v1` |
Additional raw schema constant:
| Constant | Value |
|---|---|
| `SchemaRawOpenWeatherHourlyForecastV1` | `raw.openweather.hourly.forecast.v1` |
`SchemaRawOpenWeatherHourlyForecastV1` exists in code, but no current registered
source emits it. Consumers should not expect that raw schema unless a later
registered source documents it as part of the current event contract.
## WMO Constants And Text
`standards` also defines the canonical `WMOCode` vocabulary and text helpers
used by normalized observations and forecasts.
Consumer guidance:
- Treat `WMOUnknown` as unknown condition data.
- Observation `conditionCode` is required in the current event contract.
- Forecast period `conditionCode` is optional because some forecast products do
not provide a meaningful WMO condition.
- Prefer WMO constants and helper functions from this package instead of
duplicating code tables in consumers.
## Boundaries
`standards` is provider-agnostic. Provider-specific parsing belongs in
`weatherfeeder` internals, not in this package and not in consumers.

View File

@@ -37,6 +37,7 @@ Canonical schemas emitted after normalization:
| `forecast_discussion` | `weather.forecast_discussion.v1` | `WeatherForecastDiscussion` |
| `weather_story` | `weather.weather_story.v1` | `WeatherStoryRun` |
| `alert` | `weather.alert.v1` | `WeatherAlertRun` |
| `outlook` | `weather.outlook.v2` | `WeatherOutlookRun` |
Raw upstream schemas emitted by current sources:
@@ -51,6 +52,7 @@ Raw upstream schemas emitted by current sources:
| `weather_story` | `raw.nws.weatherstories.v1` | NWS weather stories JSON |
| `forecast` | `raw.openmeteo.hourly.forecast.v1` | Open-Meteo hourly forecast JSON |
| `alert` | `raw.nws.alerts.v1` | NWS alerts JSON |
| `outlook` | `raw.spc.convective_outlook.v1` | SPC convective outlook raw bundle |
`standards.SchemaRawOpenWeatherHourlyForecastV1` exists in code, but no current
registered source emits it.
@@ -209,8 +211,9 @@ Payload type: `WeatherAlertRun`.
| `instruction` | string | no | Alert instruction. |
| `sent` | timestamp | no | Provider sent time. |
| `effective` | timestamp | no | Effective time. |
| `onset` | timestamp | no | Onset time. |
| `expires` | timestamp | no | Expiration time. |
| `onset` | timestamp | no | Alert period start. |
| `ends` | timestamp | no | Alert period end. |
| `expires` | timestamp | no | Provider expiration metadata; not necessarily the alert period end. |
| `areaDescription` | string | no | Affected area description. |
| `senderName` | string | no | Provider sender name. |
| `references` | array | no | Related alerts. |
@@ -218,6 +221,92 @@ Payload type: `WeatherAlertRun`.
`references[]` entries contain optional `id`, `identifier`, `sender`, and
`sent` fields.
## `weather.outlook.v2`
Payload type: `WeatherOutlookRun`.
The current producer is the SPC convective outlook normalizer. It emits Day 1-3
convective outlook snapshots for categorical, tornado, hail, and wind products
that apply to the configured forecast point. Raw SPC bundles remain complete;
canonical outlook payloads are filtered to local polygons. All timestamps are
UTC.
| Field | Type | Required | Notes |
|---|---|:---:|---|
| `locationId` | string | no | Operator-configured location identifier. |
| `locationName` | string | no | Operator-configured location label. |
| `latitude` | number | no | Configured point latitude in decimal degrees. |
| `longitude` | number | no | Configured point longitude in decimal degrees. |
| `asOf` | timestamp | yes | Snapshot time. For SPC, this is the latest outlook issue time when available. |
| `issuedAt` | timestamp | no | Latest issue time across outlook features when any feature exists. |
| `outlooks` | array | yes | Ordered outlook polygons containing the configured point. |
| `discussions` | array | yes | Run-level day discussions for retained outlook days. |
`outlooks[]` entries:
| Field | Type | Required | Notes |
|---|---|:---:|---|
| `id` | string | yes | Deterministic weatherfeeder outlook identifier. |
| `provider` | string | yes | Current value is `spc`. |
| `product` | string | yes | Current value is `convective`. |
| `day` | integer | yes | SPC outlook day, currently `1`, `2`, or `3`. |
| `outlookType` | string | yes | `categorical`, `tornado`, `hail`, or `wind`. |
| `label` | string | yes | SPC outlook label such as `SLGT` or `15`. |
| `labelText` | string | no | Human label text from SPC, when present. |
| `severityRank` | integer | no | SPC `DN` value, when present. |
| `validFrom` | timestamp | yes | Valid period start. |
| `validTo` | timestamp | yes | Valid period end. |
| `issuedAt` | timestamp | yes | Feature issue time. |
| `expiresAt` | timestamp | yes | Expiration time; currently equal to `validTo`. |
| `forecaster` | string | no | SPC forecaster text, when present. |
| `sourceUrl` | string | no | GeoJSON product URL for this outlook feature. |
| `imageUrl` | string | no | Reserved for provider image URLs; currently empty. |
| `containsLocation` | boolean | yes | Always `true` for emitted v2 outlooks. |
| `geometry` | object | yes | Compact GeoJSON `Polygon` or `MultiPolygon` geometry. |
`geometry` preserves the SPC feature geometry as compact GeoJSON using
`[longitude, latitude]` coordinate order. `containsLocation` is computed with
that geometry and the configured source `latitude`/`longitude`; boundary points
count as contained. Polygons that do not contain the configured point are not
included in canonical v2 payloads.
`discussions[]` entries:
| Field | Type | Required | Notes |
|---|---|:---:|---|
| `day` | integer | yes | SPC outlook day, currently `1`, `2`, or `3`. |
| `headline` | string | no | Matching Day 1-3 print-page product title. |
| `summary` | string | no | Text from the print-page `...SUMMARY...` section. |
| `discussion` | string | no | Cleaned full print-page product text. |
| `updatedAt` | timestamp | no | Print-page update time, when present. |
When no SPC polygons apply locally, the run is still emitted with `outlooks: []`
and `discussions: []`. Discussions are included only for days represented by at
least one retained outlook, and multiple retained outlook types for the same day
share one discussion entry.
### SPC Outlook Supersession
Consumers should prefer latest-run semantics for current conditions: read the
latest `WeatherOutlookRun` for the configured location and use its `outlooks`
and `discussions` arrays together.
Historical SQL consumers that collapse older rows should identify superseded
outlooks by `provider`, `product`, `outlookType`, `validFrom`, and `validTo`,
then keep rows with the greatest `issuedAt`. `day` and `label` are not identity
fields. When multiple retained polygons share that latest `issuedAt`, preserve
the full group.
## Legacy `weather.outlook.v1`
`weather.outlook.v1` is a historical canonical schema retained as a standards
constant for older data and consumers. Current SPC normalization emits
`weather.outlook.v2`.
The v1 payload used `WeatherOutlookRun` and placed `headline`, `summary`, and
`discussion` on each `outlooks[]` polygon. It also represented the pre-v2 SPC
canonical behavior, where national polygons were preserved in canonical output.
## Compact Example
```json

View File

@@ -0,0 +1,106 @@
# Feedkit Integration
## Purpose
This document describes the feedkit runtime behavior that `weatherfeeder`
currently depends on. It is for maintainers and LLM coding agents changing
runtime wiring, config behavior, source construction, processing, routing, or
sink behavior.
Weather-domain behavior belongs in `weatherfeeder`. Generic daemon mechanics
belong to feedkit.
## Current Dependency
`weatherfeeder` imports feedkit as its daemon framework dependency. The exact
module version is declared in `go.mod`.
Feedkit provides:
- YAML config loading and validation.
- Source, processor, and sink registries.
- HTTP source helper behavior.
- Scheduler polling.
- Normalize and dedupe processors.
- Route compilation and sink dispatch.
- Built-in stdout, NATS, and Postgres sink mechanics.
## Config Contract
`cmd/weatherfeeder` calls feedkit config loading for `config.yml` in the current
working directory.
Implemented behavior relied on by weatherfeeder docs and tests:
- Top-level config contains `sources`, `sinks`, and optional `routes`.
- Config struct fields are decoded strictly, so misspelled struct fields fail
startup.
- Driver-specific `params` maps are decoded generically and validated by the
source or sink constructor that consumes them.
- Source `kinds` can be validated against a source's advertised `Kinds()`.
## Source And HTTP Contract
Most weatherfeeder sources use feedkit's single-document HTTP source helper for:
- request construction;
- `User-Agent` and `Accept` headers;
- optional conditional GET validators;
- response body size limits;
- context-aware HTTP work;
- unchanged `304 Not Modified` responses that emit no events.
The SPC convective outlook source fetches multiple documents itself, but it uses
feedkit transport helpers for HTTP clients and response body limits.
## Scheduler And Processing Contract
Weatherfeeder builds feedkit scheduler jobs from source configs. Current source
drivers are polling drivers and use the configured `every` interval.
Events flow through a feedkit pipeline in this order:
1. normalize processor;
2. dedupe processor.
The normalize processor is configured with `RequireMatch=false`, so unmatched
schemas pass through unchanged. Weatherfeeder registers its built-in normalizers
and owns the provider-to-canonical mapping.
The dedupe processor stores a bounded in-memory set of recent event IDs. The
bound is configured in `cmd/weatherfeeder`.
## Dispatch And Sink Contract
Feedkit compiles routes from config and dispatches processed events to matching
sinks. If `routes` is omitted, every configured sink receives every event kind.
Feedkit owns sink fanout mechanics, per-sink workers, queueing, context-aware
shutdown, and sink error logging. Weatherfeeder owns the event kinds and schemas
that make routes meaningful.
Built-in feedkit sinks used by weatherfeeder:
- `stdout`: validates and writes JSON events to stdout.
- `nats`: publishes JSON events to a configured subject.
- generic `postgres` sink factory: opens the database, ensures tables/indexes,
runs transactions, inserts mapped rows, and prunes when configured.
Weatherfeeder supplies its Postgres table schema and event mapper to feedkit's
Postgres sink factory. The table contract is documented in
[`postgres.md`](postgres.md).
## Boundaries
Do not move weather-domain policy into feedkit. Weatherfeeder owns:
- provider source drivers;
- raw and canonical schema constants;
- event kind meaning;
- canonical payload structs;
- normalizers;
- Postgres table shape and row mapping.
Do not duplicate generic feedkit mechanics in weatherfeeder unless there is a
narrow weather-specific reason. Runtime composition details are documented in
[`../internal/runtime.md`](../internal/runtime.md).

View File

@@ -44,6 +44,9 @@ normalizer uses fields under `properties` such as `stationId`, `stationName`,
`nws_alerts` expects an alerts FeatureCollection. The normalizer uses the
collection `updated` timestamp, `title`, each feature ID, alert classification
fields, narrative fields, timing fields, sender fields, and references.
`properties.onset` and `properties.ends` map to the canonical alert period
start and end. `properties.expires` maps only to canonical `expires` provider
metadata and is not treated as the alert period end.
`nws_forecast_hourly` and `nws_forecast_narrative` expect gridpoint forecast
GeoJSON with `properties.generatedAt`, `properties.updateTime`, elevation,
@@ -98,8 +101,9 @@ unset. Forecast temperatures are converted to Celsius when NWS supplies
Fahrenheit, and wind speed strings are converted to kilometers per hour.
Alert timing fields are parsed best-effort. Invalid per-alert timestamps are
left unset rather than failing the whole alert run. Missing alert IDs are
synthesized from the run snapshot time and array position.
left unset rather than failing the whole alert run. NWS `ends` is preserved
separately from `expires`; `expires` does not fall back to `ends`. Missing alert
IDs are synthesized from the run snapshot time and array position.
Forecast discussion parsing requires an issue time. Weather story entries require
start time, end time, and update time.

View File

@@ -23,6 +23,7 @@ Events are mapped only for canonical weather schemas:
- `weather.forecast_discussion.v1`
- `weather.weather_story.v1`
- `weather.alert.v1`
- `weather.outlook.v2`
Unsupported schemas produce no writes for this sink. Mapped events are inserted
transactionally. Inserts use ordinary `INSERT`; duplicate primary keys fail the
@@ -56,6 +57,9 @@ Parent tables store the feed event envelope:
| `alert_runs` | `event_id` | `as_of` |
| `alerts` | `run_event_id`, `alert_index` | `as_of` |
| `alert_references` | `run_event_id`, `alert_index`, `reference_index` | `as_of` |
| `outlook_runs` | `event_id` | `as_of` |
| `outlooks` | `run_event_id`, `outlook_index` | `as_of` |
| `outlook_discussions` | `run_event_id`, `discussion_index` | `as_of` |
## Table Contract
@@ -349,6 +353,7 @@ Indexes:
| `sent` | `TIMESTAMPTZ` | yes | `payload.alerts[].sent` |
| `effective` | `TIMESTAMPTZ` | yes | `payload.alerts[].effective` |
| `onset` | `TIMESTAMPTZ` | yes | `payload.alerts[].onset` |
| `ends` | `TIMESTAMPTZ` | yes | `payload.alerts[].ends` |
| `expires` | `TIMESTAMPTZ` | yes | `payload.alerts[].expires` |
| `area_description` | `TEXT` | yes | `payload.alerts[].areaDescription` |
| `sender_name` | `TEXT` | yes | `payload.alerts[].senderName` |
@@ -379,6 +384,97 @@ Indexes:
| `sender` | `TEXT` | yes | `payload.alerts[].references[].sender` |
| `sent` | `TIMESTAMPTZ` | yes | `payload.alerts[].references[].sent` |
### `outlook_runs`
Primary key: `event_id`
Prune column: `as_of`
Indexes:
- `idx_wf_outlook_run_location_as_of` on `location_id`, `as_of`
- `idx_wf_outlook_run_as_of` on `as_of`
| Column | Type | Null | Source |
|---|---|:---:|---|
| `event_id` | `TEXT` | no | `event.id` |
| `event_kind` | `TEXT` | no | `event.kind` |
| `event_source` | `TEXT` | no | `event.source` |
| `event_schema` | `TEXT` | no | `event.schema` |
| `event_emitted_at` | `TIMESTAMPTZ` | no | `event.emitted_at` |
| `event_effective_at` | `TIMESTAMPTZ` | yes | `event.effective_at` |
| `location_id` | `TEXT` | yes | `payload.locationId` |
| `location_name` | `TEXT` | yes | `payload.locationName` |
| `latitude` | `DOUBLE PRECISION` | yes | `payload.latitude` |
| `longitude` | `DOUBLE PRECISION` | yes | `payload.longitude` |
| `as_of` | `TIMESTAMPTZ` | no | `payload.asOf` |
| `issued_at` | `TIMESTAMPTZ` | yes | `payload.issuedAt` |
| `outlook_count` | `INTEGER` | no | `len(payload.outlooks)` |
| `discussion_count` | `INTEGER` | no | `len(payload.discussions)` |
### `outlooks`
Primary key: `run_event_id`, `outlook_index`
Prune column: `as_of`
Foreign key: `run_event_id` references `outlook_runs(event_id)` with cascade
delete.
Indexes:
- `idx_wf_outlooks_contains_valid` on `contains_location`, `valid_from`, `valid_to`
- `idx_wf_outlooks_day_type_label` on `day`, `outlook_type`, `label`
- `idx_wf_outlooks_valid` on `valid_from`, `valid_to`
| Column | Type | Null | Source |
|---|---|:---:|---|
| `run_event_id` | `TEXT REFERENCES outlook_runs(event_id) ON DELETE CASCADE` | no | Parent event ID. |
| `outlook_index` | `INTEGER` | no | `payload.outlooks[]` index. |
| `as_of` | `TIMESTAMPTZ` | no | Parent `payload.asOf` |
| `outlook_id` | `TEXT` | no | `payload.outlooks[].id` |
| `provider` | `TEXT` | no | `payload.outlooks[].provider` |
| `product` | `TEXT` | no | `payload.outlooks[].product` |
| `day` | `INTEGER` | no | `payload.outlooks[].day` |
| `outlook_type` | `TEXT` | no | `payload.outlooks[].outlookType` |
| `label` | `TEXT` | no | `payload.outlooks[].label` |
| `label_text` | `TEXT` | yes | `payload.outlooks[].labelText` |
| `severity_rank` | `INTEGER` | yes | `payload.outlooks[].severityRank` |
| `valid_from` | `TIMESTAMPTZ` | no | `payload.outlooks[].validFrom` |
| `valid_to` | `TIMESTAMPTZ` | no | `payload.outlooks[].validTo` |
| `issued_at` | `TIMESTAMPTZ` | no | `payload.outlooks[].issuedAt` |
| `expires_at` | `TIMESTAMPTZ` | no | `payload.outlooks[].expiresAt` |
| `forecaster` | `TEXT` | yes | `payload.outlooks[].forecaster` |
| `source_url` | `TEXT` | yes | `payload.outlooks[].sourceUrl` |
| `image_url` | `TEXT` | yes | `payload.outlooks[].imageUrl` |
| `contains_location` | `BOOLEAN` | no | `payload.outlooks[].containsLocation` |
| `geometry_json` | `TEXT` | no | Compact JSON from `payload.outlooks[].geometry` |
### `outlook_discussions`
Primary key: `run_event_id`, `discussion_index`
Prune column: `as_of`
Foreign key: `run_event_id` references `outlook_runs(event_id)` with cascade
delete.
Indexes:
- `idx_wf_outlook_discussions_day_as_of` on `day`, `as_of`
- unique `idx_wf_outlook_discussions_run_day` on `run_event_id`, `day`
| Column | Type | Null | Source |
|---|---|:---:|---|
| `run_event_id` | `TEXT REFERENCES outlook_runs(event_id) ON DELETE CASCADE` | no | Parent event ID. |
| `discussion_index` | `INTEGER` | no | `payload.discussions[]` index. |
| `as_of` | `TIMESTAMPTZ` | no | Parent `payload.asOf` |
| `day` | `INTEGER` | no | `payload.discussions[].day` |
| `headline` | `TEXT` | yes | `payload.discussions[].headline` |
| `summary` | `TEXT` | yes | `payload.discussions[].summary` |
| `discussion` | `TEXT` | yes | `payload.discussions[].discussion` |
| `updated_at` | `TIMESTAMPTZ` | yes | `payload.discussions[].updatedAt` |
## Retention
When sink param `prune` is set, every successful write transaction deletes rows
@@ -401,3 +497,6 @@ binary does not provide CLI commands for them.
- `WeatherAlertRun`: read `alert_runs`, join `alerts` by `run_event_id` ordered
by `alert_index`, then join `alert_references` by `run_event_id` and
`alert_index` ordered by `reference_index`.
- `WeatherOutlookRun`: read `outlook_runs`, join `outlooks` by `run_event_id`
ordered by `outlook_index`, then join `outlook_discussions` by
`run_event_id` ordered by `discussion_index`.

144
docs/integrations/spc.md Normal file
View File

@@ -0,0 +1,144 @@
# SPC Integration Notes
## Purpose
This document describes the Storm Prediction Center convective outlook usage
implemented by `weatherfeeder`. It is for developers and operators maintaining
SPC source configuration, provider helpers, fixtures, and tests.
General config syntax belongs in [configuration](../config.md). Emitted JSON
events are documented in [event wire contract](events.md).
## Implemented Driver
| Driver | Kind | Raw schema | Canonical schema |
| --- | --- | --- | --- |
| `spc_convective_outlook` | `outlook` | `raw.spc.convective_outlook.v1` | `weather.outlook.v2` |
## Config Requirements
The driver requires:
- `latitude`
- `longitude`
- `user_agent`
Optional params are:
- `location_id`
- `location_name`
- `geojson_urls`
- `discussion_urls`
- `rss_url`
- `http_timeout`
- `http_response_body_limit_bytes`
RSS is not fetched unless `rss_url` is configured. URL override maps are intended
for tests and upstream URL changes; the default driver configuration owns the
current Day 1-3 SPC product URLs.
## Upstream Products Used
The source fetches nine required GeoJSON products every poll:
- Day 1 categorical, tornado, hail, and wind
- Day 2 categorical, tornado, hail, and wind
- Day 3 categorical
It also fetches three required print pages:
- Day 1 convective outlook print page
- Day 2 convective outlook print page
- Day 3 convective outlook print page
GeoJSON products are authoritative for outlook polygons, valid windows, issue
times, labels, and severity rank. Print pages are authoritative for run-level
day discussion headline, summary, and discussion text.
## Accept Headers
GeoJSON requests use:
```text
application/geo+json, application/json
```
Print-page requests use:
```text
text/html, application/xhtml+xml
```
RSS requests, when configured, use:
```text
application/rss+xml, application/xml, text/xml
```
## Polling And Raw Events
The source polls all required GeoJSON and print-page products as one bundle. If
any required request fails or returns a non-2xx response, the poll returns an
error and emits no partial event.
The raw payload contains fetched bodies plus configured location metadata and
per-product metadata. The source parses only the timestamp metadata needed for
event effective time selection; canonical mapping belongs to the normalizer.
The source emits no event when a complete fetched bundle is unchanged from the
previous successful poll. It does this with a source-local hash of the fetched
document bodies.
## Time Handling
Raw source `effective_at` prefers:
1. the latest valid GeoJSON `ISSUE_ISO`;
2. the latest print-page `Updated:` timestamp;
3. RSS `lastBuildDate` when RSS was fetched and parseable;
4. fetch time.
The normalizer sets canonical `asOf` and normalized event `effective_at` from
the latest valid GeoJSON issue time across the complete raw bundle, including
empty no-risk placeholders, with fallback to print-page update time and then the
incoming event envelope.
## Mapping Notes
The raw source fetches and envelopes the complete SPC bundle. The normalizer
decodes every configured GeoJSON product, skips empty no-risk
`GeometryCollection` placeholders, and emits canonical outlooks only when the
configured point is inside or on the boundary of a real feature. Products are
ordered by day, then categorical, tornado, hail, and wind. Retained feature order
is preserved within each product.
The normalizer computes `containsLocation` with the configured latitude and
longitude against compact GeoJSON `Polygon` or `MultiPolygon` geometry.
Coordinates use GeoJSON order, `[longitude, latitude]`, and boundary points
count as contained.
Canonical outlook runs are emitted even when no polygons apply locally. In that
case the payload contains empty `outlooks` and `discussions` arrays.
Print-page prose is represented as run-level day discussions. Discussions are
included only for days represented by at least one retained outlook. Multiple
retained outlook types for the same day share one discussion entry.
For downstream current-state and historical supersession guidance, see the
[event wire contract](events.md#spc-outlook-supersession).
## Failure Behavior
Constructor validation failures stop daemon startup. Polling failures are
returned to the scheduler and emit no event for that poll.
Normalization fails when required GeoJSON timestamps, labels, geometry, or
configured coordinates are missing or invalid. Print-page extraction failures
also fail normalization because print pages are required inputs.
## Tests To Inspect
- `internal/providers/spc/*_test.go`
- `internal/sources/spc/*_test.go`
- `internal/normalizers/spc/*_test.go`
- fixtures under `internal/providers/spc/testdata`

View File

@@ -27,6 +27,7 @@ Current mappings:
| `raw.nws.forecast_discussion.v1` | `weather.forecast_discussion.v1` |
| `raw.nws.weatherstories.v1` | `weather.weather_story.v1` |
| `raw.nws.alerts.v1` | `weather.alert.v1` |
| `raw.spc.convective_outlook.v1` | `weather.outlook.v2` |
## Boundaries
@@ -54,9 +55,18 @@ Weatherfeeder registers normalizers in a stable order:
1. NWS
2. Open-Meteo
3. OpenWeather
4. SPC
The current normalizers avoid ambiguous matches by using schema equality.
The SPC outlook normalizer decodes the raw multi-document bundle, maps
location-containing GeoJSON features to canonical outlooks, and adds one
run-level print-page discussion per retained outlook day. It preserves compact
GeoJSON feature geometry and computes `containsLocation` with
`internal/geo.ContainsPoint` using the source-configured point. Boundary points
count as contained. Polygons that do not contain the point are omitted from the
canonical run.
## State
Normalizers should be stateless. Shared helpers should be deterministic and free
@@ -78,6 +88,7 @@ to four digits after the decimal point.
- provider normalizer tests under `internal/normalizers/nws`
- provider normalizer tests under `internal/normalizers/openmeteo`
- provider normalizer tests under `internal/normalizers/openweather`
- provider normalizer tests under `internal/normalizers/spc`
- common helper tests under `internal/normalizers/common`
## Invariants

View File

@@ -19,10 +19,16 @@ Inputs are canonical feed events. The mapper currently handles these schemas:
- `weather.forecast_discussion.v1`
- `weather.weather_story.v1`
- `weather.alert.v1`
- `weather.outlook.v2`
Outputs are feedkit `PostgresWrite` values for weatherfeeder-owned tables.
Unsupported schemas produce no writes and no error.
Outlook runs are written to `outlook_runs`, retained local polygons are written
to `outlooks`, and run-level day discussions are written to
`outlook_discussions`. The parent run row stores `outlook_count` and
`discussion_count`.
## Boundaries
- Weatherfeeder owns table definitions in `schema.go`.
@@ -80,6 +86,8 @@ Child rows use positional indexes to preserve canonical array order:
- `story_index`
- `alert_index`
- `reference_index`
- `outlook_index`
- `discussion_index`
Required canonical fields are validated before writes are returned:
@@ -89,10 +97,17 @@ Required canonical fields are validated before writes are returned:
- forecast discussions require `issuedAt` and `product`;
- weather story runs require `asOf`, and each story requires `startTime`,
`endTime`, and `updatedAt`;
- alert runs require `asOf`, and each alert requires `id`.
- alert runs require `asOf`, and each alert requires `id`;
- outlook runs require `asOf`, and each outlook requires `id`, `provider`,
`product`, `day`, `outlookType`, `label`, `validFrom`, `validTo`, `issuedAt`,
`expiresAt`, `containsLocation: true`, and `geometry`;
- outlook discussions require day `1`, `2`, or `3`; at least one of
`headline`, `summary`, or `discussion`; and no duplicate discussion day in the
same run.
Nullable canonical values are converted to SQL nulls by mapper helpers.
Observation present-weather raw values are stored as compact JSON text.
Observation present-weather raw values and outlook geometry values are stored as
compact JSON text.
## Failure Behavior

View File

@@ -68,13 +68,28 @@ Runtime composition uses feedkit for:
Weatherfeeder registers its own source drivers and its Postgres schema mapper.
Responsibility split:
| Runtime concern | Owner |
| --- | --- |
| Config loading and generic validation | feedkit |
| Source, processor, and sink registries | feedkit mechanics; weatherfeeder registrations |
| Source polling and stream supervision | feedkit scheduler |
| Raw weather data fetching | weatherfeeder source adapters |
| Normalizer execution order and pass-through behavior | feedkit normalize processor |
| Weather raw-to-canonical mapping | weatherfeeder normalizers |
| Dedupe mechanics | feedkit dedupe processor |
| Route compilation and sink fanout | feedkit dispatch |
| Weather Postgres table shape and row mapping | weatherfeeder Postgres adapter |
| Postgres connection, DDL, inserts, transactions, and pruning | feedkit Postgres sink |
## State
Weatherfeeder-owned runtime state is in process:
- event channel contents;
- the bounded dedupe key set;
- source instances and their HTTP conditional validators;
- source instances and their in-memory unchanged-content state;
- scheduler and dispatcher goroutines.
There is no weatherfeeder-owned durable scheduler state, checkpoint, replay log,

View File

@@ -28,6 +28,7 @@ Current drivers:
| `openmeteo_observation` | `observation` | `raw.openmeteo.current.v1` |
| `openmeteo_forecast` | `forecast` | `raw.openmeteo.hourly.forecast.v1` |
| `openweather_observation` | `observation` | `raw.openweather.current.v1` |
| `spc_convective_outlook` | `outlook` | `raw.spc.convective_outlook.v1` |
## Boundaries
@@ -40,7 +41,7 @@ Current drivers:
## Config Fields Used
All current source drivers use feedkit `HTTPSource`.
Most source drivers use feedkit `HTTPSource`.
Required params:
@@ -56,12 +57,18 @@ Optional params:
OpenWeather observation sources additionally require the configured URL to use
metric units. This is enforced by `internal/providers/openweather`.
The SPC convective outlook source is a multi-document poll source rather than a
single-URL `HTTPSource`. It requires `latitude`, `longitude`, and `user_agent`;
accepts optional `location_id`, `location_name`, `geojson_urls`,
`discussion_urls`, and `rss_url`; and supports `http_timeout` and
`http_response_body_limit_bytes`.
Source-level `kinds`, when configured, are validated against the source's
advertised `Kinds()`.
## External Adapters Used
Sources use feedkit's HTTP helper for:
Most sources use feedkit's HTTP helper for:
- request construction;
- `User-Agent` and `Accept` headers;
@@ -71,11 +78,15 @@ Sources use feedkit's HTTP helper for:
NWS helpers parse NWS timestamps. Open-Meteo helpers parse provider-local times
with timezone or UTC-offset data. OpenWeather helpers enforce metric-unit URLs.
SPC helpers define Day 1-3 product metadata, parse GeoJSON timestamps, extract
cleaned print-page discussion text, and parse optional RSS metadata.
## State
HTTP conditional validators are held in each source instance. They are not
persisted across process restarts.
HTTP conditional validators are held in each single-document HTTP source
instance. They are not persisted across process restarts. The SPC outlook source
keeps only a source-local hash of the most recent complete required product
bundle and emits no event when a later complete bundle is unchanged.
## Failure Behavior
@@ -88,12 +99,18 @@ to default ID/effective-time behavior from feedkit source helpers.
Unchanged conditional responses return no events and no error.
The SPC outlook source fetches all configured Day 1-3 GeoJSON products and print
pages atomically. If any required GeoJSON or print-page request fails, the poll
returns an error and emits no partial bundle. RSS is fetched only when `rss_url`
is configured.
## Tests To Inspect
- `internal/sources/builtins_test.go`
- provider source tests under `internal/sources/nws`
- provider source tests under `internal/sources/openmeteo`
- provider source tests under `internal/sources/openweather`
- provider source tests under `internal/sources/spc`
- provider helper tests under `internal/providers`
## Invariants

View File

@@ -64,21 +64,25 @@ and before each interval tick. If no jitter is configured in code, feedkit uses
Poll failures are logged and do not stop the daemon. A failed poll emits no
events for that source until a subsequent poll succeeds.
## Conditional HTTP Fetches
## Unchanged Upstream Content
All current sources use feedkit's HTTP polling helper. By default,
`params.conditional` is `true`, so the helper keeps ETag and Last-Modified
validators in memory for each source instance.
Most sources use feedkit's single-document HTTP polling helper. By default,
`params.conditional` is `true` for those sources, so the helper keeps ETag and
Last-Modified validators in memory for each source instance.
If the upstream returns `304 Not Modified`, the source emits no events for that
poll. Validator state is in memory only; restarting the process starts with no
cached validators.
The SPC convective outlook source polls multiple required documents as one
bundle. It emits no event when a later complete bundle has the same fetched
document bodies as the previous successful poll.
## Processing And Dedupe
Every event passes through normalization first and dedupe second.
Normalizers match raw source schemas and produce canonical `weather.*.v1`
Normalizers match raw source schemas and produce versioned canonical `weather.*`
payloads. If an event has no matching normalizer, the normalize processor passes
it through unchanged.
@@ -128,6 +132,7 @@ table in that same transaction. See the
- scheduler goroutines and timers;
- HTTP conditional request validators;
- source-local unchanged-content state;
- event channel buffers;
- per-sink fanout queues;
- the dedupe ID set.

View File

@@ -23,10 +23,32 @@ The implemented runtime flow is:
Canonical payload structs live in `model`. Schema identifiers and cross-provider wire conventions live in `standards`. Source adapters live under `internal/sources`. Normalizers live under `internal/normalizers`. Provider-specific parsing helpers shared by sources and normalizers live under `internal/providers`. Sink-specific persistence mapping lives under `internal/sinks`.
## Architecture Style
`weatherfeeder` uses a pragmatic ports-and-adapters architecture rather than a
formal framework. Provider APIs, config loading, scheduling, dispatch, and sinks
sit outside the weather domain model and normalization rules.
The implementation style is:
- Pipeline-oriented: events flow from source polling through normalization,
dedupe, routing, and sink fanout.
- Schema-routed: normalizers select raw payloads by explicit schema strings, not
source names or configured routes.
- Provider-isolated: NWS, Open-Meteo, OpenWeather, and SPC quirks stay in
provider-specific source, provider-helper, and normalizer packages.
- Registry-based: built-in source drivers, normalizers, processors, and sinks
are assembled explicitly through registries instead of dynamic plugin loading.
- Adapter-clean: persistence and external-system details stay behind source and
sink adapters, not in `model` or normalizers.
- Direct Go: prefer small package-level constructors and straightforward code
over broad abstractions.
## Core Design Principles
- Hexagonal boundaries: provider APIs, config loading, scheduling, dispatch, and sinks are external mechanisms around the weather domain model and normalization logic.
- Raw-to-canonical flow: sources should fetch and envelope raw provider payloads; normalizers should own provider-to-canonical mapping.
- Location-focused canonical data: canonical weather events represent data relevant to the configured forecast location or configured provider object; `weatherfeeder` is not a national provider-data archive.
- Schema-based routing: normalizers match on event schema, not source name or event kind.
- Composable registries: source drivers, normalizers, processors, and sinks are assembled explicitly through registries.
- Bounded concurrency: scheduling and sink fanout are concurrent, but the application should keep queues, goroutine ownership, logging, and cancellation behavior visible.
@@ -57,6 +79,23 @@ Tests and examples:
- The sample `cmd/weatherfeeder/config.yml` is executable test input and is load-tested.
- Tests should keep exercising package contracts directly rather than relying only on full-daemon execution.
## Feedkit Boundary
`feedkit` provides reusable daemon infrastructure. `weatherfeeder` provides the
weather-domain adapters, models, schemas, and mapping policy.
| Area | Feedkit owns | Weatherfeeder owns |
| --- | --- | --- |
| Config | Generic YAML shape: sources, sinks, routes, modes, cadence, and params. | Driver-specific config rules such as NWS `user_agent`, OpenWeather `units=metric`, and SPC coordinates. |
| Events | Domain-agnostic event envelope: ID, kind, source, emitted/effective times, schema, and payload. | Event kind meaning, schema strings, and canonical weather payloads. |
| Sources | Source interfaces, registry, expected-kind validation, HTTP helper, and default event ID helper. | NWS/Open-Meteo/OpenWeather/SPC source drivers and raw schema emission. |
| Processing | Processor registry, normalize processor, dedupe processor, and pipeline execution. | Weather normalizers and schema-specific raw-to-canonical mapping. |
| Dispatch | Route compilation and sink fanout mechanics. | Which weather event kinds are configured and meaningful. |
| Sinks | Generic stdout, NATS, and Postgres sink mechanics. | Weather-specific Postgres schema and canonical event-to-row mapping. |
Do not move weather-domain policy into `feedkit`, and do not duplicate generic
daemon mechanics in `weatherfeeder` when feedkit already provides the boundary.
## Modules Or Processing Steps
The implemented processing steps are source polling, normalization, dedupe, and sink dispatch.

View File

@@ -24,7 +24,8 @@ docs, not here.
normalizers.
- `internal/sinks/postgres/`: weatherfeeder-owned Postgres schema and canonical
event mapper.
- `docs/`: current behavior, policies, integration contracts, and roadmap files.
- `docs/`: current behavior, consumer guides, integration contracts, policies,
and roadmap files.
- `examples/`: maintained, copyable configuration examples.
## Build And Test
@@ -70,6 +71,32 @@ Use fixtures, local test servers, and package-level tests.
- Prefer explicit registries and small package-level constructors over hidden
global behavior.
## Architecture-Preserving Changes
When changing `weatherfeeder`, preserve the split between feedkit
infrastructure and weather-domain behavior.
Do:
- keep generic scheduling, dispatch, processor, config, and sink mechanics in
feedkit;
- keep weather provider rules in source adapters, provider helpers, and
normalizers;
- keep canonical weather payloads in `model` and schema/wire identifiers in
`standards`;
- keep Postgres table and row mapping under `internal/sinks/postgres`;
- use explicit registries for built-in sources and normalizers.
Do not:
- move provider parsing, WMO mapping, or canonical weather policy into
`cmd/weatherfeeder`;
- move weather-specific constants, schemas, or validation rules into feedkit;
- put database column metadata or sink-specific tags on canonical model structs;
- replace explicit registries with dynamic plugin loading;
- introduce broad abstractions when a small provider-specific helper preserves
clarity.
## Dependency Policy
Prefer the Go standard library unless a dependency materially improves
@@ -183,6 +210,7 @@ When behavior changes, update the canonical docs in the same change:
- CLI behavior: `docs/cli.md`;
- operations and recovery: `docs/operations.md`;
- troubleshooting: `docs/troubleshooting.md`;
- public Go package consumption: `docs/consumers/`;
- external contracts: `docs/integrations/`;
- internal component behavior: `docs/internal/`;
- copyable configs: `examples/`.

View File

@@ -2,12 +2,13 @@
## Purpose
Project documentation must help four audiences:
Project documentation must help five audiences:
1. users who need to run the application;
2. administrators/operators who need to configure and operate it;
3. developers who need to understand and change it safely;
4. LLM coding agents that need clear scope, boundaries, and invariants.
4. LLM coding agents that need clear scope, boundaries, and invariants;
5. developers and LLM coding agents integrating this project from another codebase.
Docs should be accurate, concise, task-oriented, and organized by audience. Prefer links to canonical docs over repetition.
@@ -42,11 +43,14 @@ Canonical homes:
- project purpose and quickstart: `README.md`
- development principles: `docs/policy/architecture.md`
- public HTTP API reference: `docs/api.md`
- configuration reference: `docs/config.md`
- CLI reference: `docs/cli.md`
- operations and recovery: `docs/operations.md`
- troubleshooting: `docs/troubleshooting.md`
- public API/package consumer guidance: `docs/consumers/`
- implemented internals: `docs/internal/`
- external protocol, service, and file-format contracts: `docs/integrations/`
- future work: `docs/roadmap/`
- contributor workflow: `docs/policy/development.md`
- copyable examples: `examples/`
@@ -106,7 +110,7 @@ Recommended:
- `examples/`
- `docs/policy/development.md`
### Modular, staged, service-oriented, or orchestration application
### Modular, service-oriented, or orchestration application
Required:
- `docs/cli.md`, if CLI-based
@@ -119,6 +123,31 @@ Recommended:
- `docs/troubleshooting.md`
- validated examples under `examples/`
### Public HTTP API service
Required:
- `docs/api.md`
- `docs/cli.md`, if CLI-based
- `docs/config.md`, if config-driven
- `docs/operations.md`
- `docs/internal/`
- `docs/policy/development.md`
Recommended:
- `docs/troubleshooting.md`
- `docs/consumers/`, for task-oriented client integration guides
- `docs/integrations/`, for upstream/downstream service contracts
- validated examples under `examples/`
### Project with public packages or consumer APIs
Required:
- `docs/consumers/api.md`
- one `docs/consumers/pkg-<name>.md` file per public package, if public packages exist
Recommended:
- copyable consumer examples under `examples/`, if practical
## Required Documents
### README.md
@@ -161,6 +190,32 @@ It should include:
For small projects, this file may be brief. It may simply state that the project is intentionally narrow, monolithic, and dependency-light.
### docs/api.md
**Audience:** external HTTP API consumers, developers, LLM coding agents integrating by HTTP
Required for projects whose primary public interface is HTTP.
`docs/api.md` is the canonical public HTTP API contract. It should be normative for external consumers and should not be duplicated by README, operations docs, consumer guides, or integration docs.
It should include:
1. base URL conventions;
2. authentication and authorization behavior, if implemented;
3. response envelope;
4. supported media types and content negotiation behavior;
5. shared query parameters;
6. endpoint reference grouped by route family;
7. request parameters and validation rules;
8. response fields, units, nullability, and optionality;
9. error response shape and status codes;
10. pagination, caching, rate-limit, idempotency, and retry behavior, if implemented;
11. compact request and response examples.
It must document only implemented endpoints and behavior. Planned endpoints, proposed fields, future filters, and experimental response shapes belong only under `docs/roadmap/`.
For HTTP API projects, `docs/consumers/` may provide task-oriented client integration guides, but those guides should link to `docs/api.md` for the authoritative endpoint contract.
### docs/policy/development.md
**Audience:** developers, LLM coding agents
@@ -175,7 +230,7 @@ It should include:
- dependency policy;
- how to add config fields;
- how to add CLI flags;
- how to add stages/modules/adapters, if applicable;
- how to add modules or adapters, if applicable;
- how to update examples;
- documentation update expectations.
@@ -216,7 +271,7 @@ Explain when commands are useful, not just their syntax.
**Audience:** administrators, operators
Required for applications that maintain state, support resume behavior, run multiple stages, write durable artifacts, use remote storage, or require recovery procedures.
Required for applications that maintain state, support resume behavior, run multi-step workflows, write durable artifacts, use remote storage, or require recovery procedures.
It should cover:
@@ -244,11 +299,40 @@ Each entry should include:
- safe fix;
- relevant links.
### docs/consumers/
**Audience:** developers and LLM coding agents integrating this project from another codebase
Required for projects with public packages, SDKs, client APIs, plugin APIs, or other application-facing integration surfaces.
This directory describes how an external codebase should consume the project's public API. It should be task-oriented and copyable where useful. It is not the place for internal implementation details or operator procedures.
For projects whose public API is HTTP, `docs/consumers/` is not required, and it should not duplicate the endpoint reference in `docs/api.md`. If present, it may provide practical integration workflows, client-specific examples, or migration notes that link back to `docs/api.md`.
`docs/consumers/api.md` should provide the consumer-facing overview and primary implementation workflow. It should include:
1. intended consumer audience and use cases;
2. required inputs supplied by operators or deployment configuration;
3. recommended public package or API workflow;
4. minimal copyable example;
5. consumer responsibilities and boundaries;
6. retry, idempotency, or status behavior, if applicable;
7. links to package-specific docs and canonical integration contracts.
Package-specific docs should be named `pkg-<name>.md` and should include:
1. import path;
2. intended use cases;
3. primary types and functions needed by consumers;
4. minimal examples;
5. validation, error, retry, and boundary behavior;
6. links to canonical file-format or wire-protocol contracts.
### docs/internal/
**Audience:** developers, LLM coding agents
Required for modular, staged, service-oriented, or orchestration projects.
Required for modular, service-oriented, or orchestration projects.
This directory describes implemented internal components. It is not the roadmap.
@@ -289,7 +373,9 @@ Roadmap docs should not be confused with current behavior.
Required for projects that depend on external CLIs, APIs, services, protocols, or file formats where the integration contract is important to maintain.
This directory contains concise, versioned reference notes for external integration contracts. It should document only the parts of the external system that this project actually uses.
This directory contains concise, versioned reference notes for external integration contracts. It should document only the parts of the external system that this project actually uses or exposes.
For public HTTP API services, `docs/integrations/` should document upstream, downstream, storage, protocol, or runtime contracts that the service depends on or bridges. It should not become a second copy of the public HTTP endpoint reference; that belongs in `docs/api.md`.
Use one file per integration where useful.
@@ -346,8 +432,10 @@ Before merging documentation changes, verify:
- README is concise and orientation-focused.
- `docs/policy/architecture.md` describes development principles.
- `docs/api.md` is the canonical HTTP contract for HTTP API services.
- Future work appears only under `docs/roadmap/`.
- User-facing docs avoid unnecessary internals.
- Consumer-facing docs explain public APIs without duplicating HTTP endpoint or integration contracts.
- Developer-facing docs preserve boundaries and invariants.
- Config examples match the schema.
- CLI examples match real commands and flags.

60
docs/roadmap/future.md Normal file
View File

@@ -0,0 +1,60 @@
# Future Work
## Purpose
This document is the catch-all roadmap for planned, deferred, aspirational, experimental, or unimplemented weatherfeeder work. Current behavior belongs in the canonical docs outside `docs/roadmap/`.
## SPC Convective Outlook Follow-Ups
### Weatherapi Outlook Endpoints
Expose persisted SPC convective outlooks through `weatherapi` after the weatherfeeder storage contract is stable.
Likely endpoints:
- `GET /outlooks/convective`
- `GET /outlooks/convective/active`
- `GET /outlooks/convective/location`
Recommended behavior:
- Return the latest outlook run by default.
- Support active outlook filtering by current time and `containsLocation=true`.
- Consider optional query filters for `day`, `outlookType`, and `containsLocation`.
- Preserve canonical outlook geometry for downstream display and audit use.
### SPC Day 4-8 Outlooks
Add SPC Day 4-8 convective outlook support as a schema-compatible extension only after Day 1-3 operation is proven.
Notes:
- Day 4-8 products have different semantics from Day 1-3 categorical/tornado/hail/wind products.
- Avoid forcing Day 4-8 assumptions into the current Day 1-3 model until the source shapes and consumer needs are reviewed.
- Prefer reusing `weather.outlook.v2` if the fields remain accurate; otherwise write a separate roadmap before changing the canonical contract.
### Degraded SPC Bundle Mode
Evaluate whether the SPC source should support degraded partial bundles when one required upstream product fails.
Current behavior should remain atomic: if a required GeoJSON or print-page fetch fails, emit no event for that poll.
Future degraded mode would need a clear contract for:
- distinguishing "no risk polygon" from "product missing";
- exposing per-product fetch errors without leaking raw provider internals into canonical events;
- deciding whether downstream sinks and APIs should store or serve partial snapshots.
### Richer SPC Page Assets And Tables
Evaluate whether to parse additional SPC print-page metadata beyond the current discussion text.
Possible additions:
- archive GeoJSON/shapefile/KML links;
- image URLs;
- page risk tables;
- city tables;
- richer discussion section metadata.
Keep GeoJSON products authoritative for polygons, validity windows, and point matching unless a future roadmap explicitly changes that contract.

View File

@@ -1,697 +0,0 @@
# SPC Convective Outlook Implementation Plan
## Purpose
Implement `weatherfeeder` support for Storm Prediction Center Day 1-3 convective outlooks described in [`docs/roadmap/spc.md`](spc.md). This plan is written for an LLM coding agent and should be followed stage by stage.
This is a planning document only. The implementation must preserve the existing weatherfeeder architecture: sources emit raw provider events, normalizers map raw payloads into canonical model types, and sinks persist canonical schemas.
## Decisions
The following choices are fixed for this implementation:
- Scope is Day 1-3 SPC convective outlooks only.
- Day 4-8 outlooks are out of scope.
- GeoJSON files are authoritative for polygons, validity windows, issue times, outlook labels, and severity ranking.
- Day 1-3 print pages are authoritative for discussion text.
- RSS is optional supplemental metadata only and must not be required for correctness.
- Do not fetch RSS by default. Include RSS only when an optional `rss_url` source param is configured.
- A poll is atomic for required products. If any configured GeoJSON or print-page URL fails or returns a non-2xx response, return an error and emit no event.
- Use compact GeoJSON geometry in the canonical payload for auditability and downstream display.
- Use standard-library-first parsing. Do not add an HTML parsing dependency unless string-based extraction proves unmaintainable during implementation.
- Keep all new planned behavior inside `weatherfeeder`; do not make `weatherapi` changes in this pass.
## Public Contract
Add schema constants in `standards/schema.go`:
- `SchemaRawSPCConvectiveOutlookV1 = "raw.spc.convective_outlook.v1"`
- `SchemaWeatherOutlookV1 = "weather.outlook.v1"`
Add source driver:
- `spc_convective_outlook`
Add event kind:
- `outlook`
Add canonical model types:
- `model.WeatherOutlookRun`
- `model.WeatherOutlook`
Canonical run fields:
```go
type WeatherOutlookRun struct {
LocationID string `json:"locationId,omitempty"`
LocationName string `json:"locationName,omitempty"`
Latitude *float64 `json:"latitude,omitempty"`
Longitude *float64 `json:"longitude,omitempty"`
AsOf time.Time `json:"asOf"`
IssuedAt *time.Time `json:"issuedAt,omitempty"`
Outlooks []WeatherOutlook `json:"outlooks"`
}
```
Canonical outlook fields:
```go
type WeatherOutlook struct {
ID string `json:"id"`
Provider string `json:"provider"`
Product string `json:"product"`
Day int `json:"day"`
OutlookType string `json:"outlookType"`
Label string `json:"label"`
LabelText string `json:"labelText,omitempty"`
SeverityRank *int `json:"severityRank,omitempty"`
ValidFrom time.Time `json:"validFrom"`
ValidTo time.Time `json:"validTo"`
IssuedAt time.Time `json:"issuedAt"`
ExpiresAt time.Time `json:"expiresAt"`
Forecaster string `json:"forecaster,omitempty"`
Headline string `json:"headline,omitempty"`
Summary string `json:"summary,omitempty"`
Discussion string `json:"discussion,omitempty"`
SourceURL string `json:"sourceUrl,omitempty"`
ImageURL string `json:"imageUrl,omitempty"`
ContainsLocation bool `json:"containsLocation"`
Geometry json.RawMessage `json:"geometry"`
}
```
Required canonical fields:
- Run: `asOf`, `outlooks`.
- Outlook: `id`, `provider`, `product`, `day`, `outlookType`, `label`, `validFrom`, `validTo`, `issuedAt`, `expiresAt`, `containsLocation`, `geometry`.
Canonical values:
- `provider` is `spc`.
- `product` is `convective`.
- `outlookType` is one of `categorical`, `tornado`, `hail`, `wind`.
- `day` is one of `1`, `2`, `3`.
## Source Inputs
Default required GeoJSON products:
- `https://www.spc.noaa.gov/products/outlook/day1otlk_cat.nolyr.geojson`
- `https://www.spc.noaa.gov/products/outlook/day1otlk_torn.nolyr.geojson`
- `https://www.spc.noaa.gov/products/outlook/day1otlk_hail.nolyr.geojson`
- `https://www.spc.noaa.gov/products/outlook/day1otlk_wind.nolyr.geojson`
- `https://www.spc.noaa.gov/products/outlook/day2otlk_cat.nolyr.geojson`
- `https://www.spc.noaa.gov/products/outlook/day2otlk_torn.nolyr.geojson`
- `https://www.spc.noaa.gov/products/outlook/day2otlk_hail.nolyr.geojson`
- `https://www.spc.noaa.gov/products/outlook/day2otlk_wind.nolyr.geojson`
- `https://www.spc.noaa.gov/products/outlook/day3otlk_cat.nolyr.geojson`
- `https://www.spc.noaa.gov/products/outlook/day3otlk_torn.nolyr.geojson`
- `https://www.spc.noaa.gov/products/outlook/day3otlk_hail.nolyr.geojson`
- `https://www.spc.noaa.gov/products/outlook/day3otlk_wind.nolyr.geojson`
Default required print-page products:
- `https://www.spc.noaa.gov/products/outlook/day1otlk_prt.html`
- `https://www.spc.noaa.gov/products/outlook/day2otlk_prt.html`
- `https://www.spc.noaa.gov/products/outlook/day3otlk_prt.html`
Optional RSS product:
- `https://www.spc.noaa.gov/products/spcacrss.xml`
Recommended config shape:
```yaml
- name: SPCConvectiveOutlookSTL
mode: poll
kinds: ["outlook"]
driver: spc_convective_outlook
every: 30m
params:
latitude: 38.6239
longitude: -90.3571
location_id: "stl"
location_name: "St. Louis, MO"
user_agent: "HomeOps (eric@maximumdirect.net)"
```
Optional source params:
- `geojson_urls`: map from product key to URL, used by tests and future upstream changes.
- `discussion_urls`: map from day key to URL, used by tests and future upstream changes.
- `rss_url`: string; when non-empty, fetch RSS as supplemental metadata.
Product keys for `geojson_urls`:
- `day1_categorical`, `day1_tornado`, `day1_hail`, `day1_wind`
- `day2_categorical`, `day2_tornado`, `day2_hail`, `day2_wind`
- `day3_categorical`, `day3_tornado`, `day3_hail`, `day3_wind`
Discussion keys for `discussion_urls`:
- `day1`, `day2`, `day3`
## Raw Bundle Shape
Create a provider raw bundle type under `internal/providers/spc` or `internal/normalizers/spc` and use it consistently between source tests and normalizer tests. Prefer `internal/providers/spc` if source metadata extraction and normalizer parsing share helpers.
Suggested raw payload shape:
```go
type RawConvectiveOutlookBundle struct {
LocationID string `json:"locationId,omitempty"`
LocationName string `json:"locationName,omitempty"`
Latitude float64 `json:"latitude"`
Longitude float64 `json:"longitude"`
FetchedAt time.Time `json:"fetchedAt"`
Products []RawOutlookProduct `json:"products"`
Discussions []RawDiscussionPage `json:"discussions"`
RSS *RawRSSFeed `json:"rss,omitempty"`
}
```
```go
type RawOutlookProduct struct {
Key string `json:"key"`
Day int `json:"day"`
OutlookType string `json:"outlookType"`
URL string `json:"url"`
FetchedAt time.Time `json:"fetchedAt"`
Body json.RawMessage `json:"body"`
}
```
```go
type RawDiscussionPage struct {
Key string `json:"key"`
Day int `json:"day"`
URL string `json:"url"`
FetchedAt time.Time `json:"fetchedAt"`
Body string `json:"body"`
}
```
```go
type RawRSSFeed struct {
URL string `json:"url"`
FetchedAt time.Time `json:"fetchedAt"`
Body string `json:"body"`
}
```
Do not put parsed canonical fields into the raw bundle except configured metadata and product keys needed to identify each fetched upstream document. Source-level timestamp parsing is allowed only for event `effectiveAt` selection.
## Stage 1: Provider Helpers And Fixtures
Goal: add deterministic SPC parsing primitives and test fixtures before wiring the source or normalizer.
Files to add:
- `internal/providers/spc/doc.go`
- `internal/providers/spc/time.go`
- `internal/providers/spc/product.go`
- `internal/providers/spc/geojson.go`
- `internal/providers/spc/discussion.go`
- `internal/providers/spc/rss.go`, only if optional RSS parsing is implemented
- `internal/providers/spc/testdata/day1_cat.geojson`
- `internal/providers/spc/testdata/day2_torn.geojson`
- `internal/providers/spc/testdata/day3_wind.geojson`
- `internal/providers/spc/testdata/day1_prt.html`
- `internal/providers/spc/testdata/day2_prt_corr.html`
- `internal/providers/spc/testdata/day3_prt.html`
Provider helper behavior:
- Define stable product metadata for the 12 required GeoJSON products.
- Define stable discussion metadata for the 3 required print-page products.
- Parse SPC ISO timestamps from GeoJSON properties using `time.Parse(time.RFC3339, value)` after trimming whitespace.
- Decode enough GeoJSON to expose feature properties and raw geometry without owning canonical mapping.
- Preserve raw geometry as compact JSON bytes for later canonical use.
- Extract print-page product text from the first useful `<pre>` block.
- Strip embedded `<script>` blocks and remaining tags from extracted `<pre>` content.
- Use `html.UnescapeString`, normalize CRLF to LF, and trim surrounding blank lines.
- Parse print-page `Updated:` timestamps when present.
- Parse discussion headline/product title from product text.
- Parse `...SUMMARY...` content through the next section heading.
- Preserve full product text as discussion text after cleanup.
- Preserve correction markers such as `CORR 1` in headline or metadata when present.
Dependency rule:
- Start with `strings`, `regexp`, `encoding/json`, `encoding/xml`, and `html` from the standard library.
- Do not add `golang.org/x/net/html` or another HTML parser unless tests show the string extraction is too brittle.
Tests:
- `go test ./internal/providers/spc`
- Product metadata contains exactly 12 GeoJSON products in day/type order.
- Discussion metadata contains exactly 3 print pages in day order.
- GeoJSON fixture decode exposes `VALID_ISO`, `EXPIRE_ISO`, `ISSUE_ISO`, `FORECASTER`, `LABEL`, `LABEL2`, `DN`, and geometry.
- Print-page fixture extraction returns product text without scripts or tags.
- Day 2 correction fixture preserves `CORR 1`.
- Summary extraction returns only the summary paragraph content.
- Updated timestamp parser returns UTC time when present and nil when absent.
Stage completion criteria:
- Provider helper tests pass.
- No source, normalizer, model, standards, sink, config, or docs current-behavior files are changed in this stage unless needed for package compilation.
## Stage 2: Geometry Helper
Goal: implement point-in-polygon support independent of SPC parsing.
Files to add:
- `internal/geo/geojson.go`
- `internal/geo/point.go`
- `internal/geo/point_test.go`
Behavior:
- Support GeoJSON `Polygon` and `MultiPolygon` only.
- Accept geometry as `json.RawMessage` or `[]byte`.
- Interpret GeoJSON coordinate order as `[longitude, latitude]`.
- Treat the first ring as the exterior ring.
- Treat subsequent rings as holes.
- Count boundary points as inside.
- Return a clear error for unsupported geometry types, malformed coordinates, empty rings, or invalid JSON.
- Use planar ray casting. This is sufficient for operational point-in-polygon checks at SPC polygon scale.
Tests:
- Point inside simple polygon returns true.
- Point outside simple polygon returns false.
- Point on polygon boundary returns true.
- Point in a hole returns false.
- Point inside one `MultiPolygon` member returns true.
- Longitude/latitude ordering regression test fails if coordinates are reversed.
- Unsupported geometry returns a useful error.
Verification:
```sh
go test ./internal/geo
```
Stage completion criteria:
- Geometry helper has no dependency on SPC, source, normalizer, or sink packages.
## Stage 3: Source Driver And Raw Schema
Goal: emit raw SPC outlook bundles from a new poll source.
Files to update:
- `standards/schema.go`
- `internal/sources/builtins.go`
- `internal/sources/builtins_test.go`
Files to add:
- `internal/sources/spc/convective_outlook.go`
- `internal/sources/spc/convective_outlook_test.go`
Source constructor:
- Export `NewConvectiveOutlookSource(cfg config.SourceConfig) (*ConvectiveOutlookSource, error)`.
- Register driver `spc_convective_outlook` in `internal/sources/builtins.go`.
- Validate required params `latitude` and `longitude`.
- Accept optional params `location_id`, `location_name`, `geojson_urls`, `discussion_urls`, and `rss_url`.
- Require `user_agent` when the existing HTTP source conventions require it.
- Advertise `Kinds() []event.Kind{event.Kind("outlook")}`.
Fetch behavior:
- Fetch all required GeoJSON URLs and print-page URLs every poll cycle.
- Fetch optional RSS only when `rss_url` is configured and non-empty.
- Use Accept `application/geo+json, application/json` for GeoJSON requests.
- Use Accept `text/html, application/xhtml+xml` for print pages.
- Use Accept `application/rss+xml, application/xml, text/xml` for RSS.
- Respect `context.Context` on all requests.
- Do not emit partial bundles.
- If the complete raw bundle is unchanged from the previous successful poll, return no events. Implement source-local change detection by hashing the combined fetched bodies if feedkit HTTP conditional helpers do not directly support multi-document polling.
- Build the raw bundle with fetched bodies and metadata.
- Emit one raw event with kind `outlook` and schema `standards.SchemaRawSPCConvectiveOutlookV1`.
- Use `fksources.DefaultEventID("", sourceName, effectiveAt, emittedAt)`.
Effective time policy:
- Prefer latest valid `ISSUE_ISO` across all GeoJSON features.
- Fallback to latest valid print-page `Updated:` timestamp.
- Fallback to RSS `lastBuildDate` if RSS was fetched and parseable.
- Fallback to fetch time if no provider timestamp is parseable.
- Keep fetch-time fallback as the last source effective-time option so downstream consumers still receive a stable effective time when provider metadata is absent.
Tests:
- Builtin registry builds `spc_convective_outlook` as a `PollSource`.
- `Kinds()` returns `outlook`.
- Constructor rejects missing `latitude` or `longitude`.
- Poll emits one raw event with schema `raw.spc.convective_outlook.v1`.
- Raw bundle includes 12 products and 3 discussions when defaults are overridden to test-server URLs.
- Poll effectiveAt prefers latest GeoJSON `ISSUE_ISO`.
- Poll effectiveAt falls back to print-page `Updated:` when GeoJSON issue times are absent.
- Poll includes RSS only when `rss_url` is configured.
- Unchanged response emits no events on a second poll.
- A configured GeoJSON fetch failure returns error and emits no event.
- A configured print-page fetch failure returns error and emits no event.
- Tests use `httptest.Server`; do not call live SPC services.
Verification:
```sh
go test ./internal/providers/spc ./internal/sources
```
Stage completion criteria:
- Source emits raw bundles only.
- No canonical model, normalizer, or sink mapping is added in this stage except schema constants required for compilation.
## Stage 4: Canonical Model And Normalizer
Goal: convert raw SPC bundles into `weather.outlook.v1` events.
Files to update:
- `standards/schema.go`
- `internal/normalizers/builtins_test.go`
Files to add:
- `model/outlook.go`
- `internal/normalizers/spc/convective_outlook.go`
- `internal/normalizers/spc/register.go`
- `internal/normalizers/spc/convective_outlook_test.go`
Files to update for registration:
- `internal/normalizers/builtins.go`
Normalizer behavior:
- Add `SPCConvectiveOutlookNormalizer`.
- Match only `standards.SchemaRawSPCConvectiveOutlookV1`.
- Decode `RawConvectiveOutlookBundle`.
- Build one `model.WeatherOutlookRun` per raw bundle.
- Map each GeoJSON feature to one `model.WeatherOutlook`.
- Preserve feature order within each product.
- Order products by day then outlook type in this order: `categorical`, `tornado`, `hail`, `wind`.
- Compute `containsLocation` with `internal/geo` using the configured bundle latitude and longitude.
- Store compact feature geometry JSON in `WeatherOutlook.Geometry`.
- Set output schema to `standards.SchemaWeatherOutlookV1`.
- Set output effectiveAt to run `AsOf`.
- Use `internal/normalizers/common.Finalize` or existing normalizer helper patterns so envelope handling remains consistent.
GeoJSON property mapping:
- `VALID_ISO` maps to `validFrom` and is required.
- `EXPIRE_ISO` maps to `validTo` and `expiresAt` and is required.
- `ISSUE_ISO` maps to `issuedAt` and is required.
- `FORECASTER` maps to `forecaster` and is optional.
- `LABEL` maps to `label` and is required.
- `LABEL2` maps to `labelText` and is optional.
- `DN` maps to `severityRank` and is optional.
Derived mapping:
- `provider` is `spc`.
- `product` is `convective`.
- `day` and `outlookType` come from raw product metadata, not from brittle URL parsing when product metadata is available.
- `id` is deterministic: join day, outlook type, normalized label, issuedAt UTC in RFC3339 format, validFrom UTC in RFC3339 format, and product-local feature index. Use a stable ASCII-safe format.
- `sourceUrl` is the GeoJSON product URL for geometry/probability features.
- `imageUrl` is empty for v1.
Discussion enrichment:
- Parse each print-page discussion with `internal/providers/spc` helpers.
- Attach Day 1 print-page headline, summary, and discussion to Day 1 outlooks, Day 2 to Day 2 outlooks, and Day 3 to Day 3 outlooks.
- Use the same discussion text for categorical, tornado, hail, and wind outlooks for the same day.
- If a print page is fetched but discussion extraction fails, normalization should fail because print pages are required v1 inputs.
- RSS metadata must not be required for discussion enrichment.
Run timestamp policy:
- `AsOf` is latest valid `issuedAt` across all outlook features.
- Fallback to latest print-page `Updated:` timestamp.
- Fallback to input event `EffectiveAt`.
- Fallback to input event `EmittedAt`.
- `IssuedAt` is latest valid `issuedAt` across all outlook features when any feature exists; otherwise nil.
Error behavior:
- Fail normalization if required GeoJSON timestamps are missing or unparseable.
- Fail normalization if required `LABEL` is empty.
- Fail normalization if required geometry is missing or invalid.
- Fail normalization if latitude or longitude is missing or invalid in the raw bundle.
- Include product key and feature index in errors.
Tests:
- Normalizer matches only `raw.spc.convective_outlook.v1`.
- Builtin normalizer order includes the SPC normalizer after existing provider normalizers unless a specific order is needed.
- Canonical schema is `weather.outlook.v1`.
- Categorical fixture maps expected day, type, label, label text, severity rank, valid times, issue time, forecaster, source URL, and geometry.
- Probabilistic fixtures for tornado, hail, and wind map expected `outlookType` values.
- `containsLocation` is true for a fixture point inside a polygon.
- `containsLocation` is false for a fixture point outside a polygon.
- Print-page text maps to headline, summary, and discussion.
- Day 2 correction marker `CORR 1` is preserved in headline or discussion metadata.
- Missing optional RSS still normalizes successfully.
- Invalid required timestamp fails with product key and feature index context.
- Invalid geometry fails with product key and feature index context.
- Output JSON uses the intended field names and does not expose raw bundle internals.
Verification:
```sh
go test ./model ./internal/geo ./internal/providers/spc ./internal/normalizers ./internal/normalizers/spc
```
Stage completion criteria:
- A raw SPC bundle normalizes into canonical `model.WeatherOutlookRun`.
- No Postgres schema or current-behavior docs are updated in this stage unless the canonical schema documentation is intentionally updated with implementation.
## Stage 5: Postgres Sink
Goal: persist canonical outlook events through the weatherfeeder Postgres sink.
Files to update:
- `internal/sinks/postgres/schema.go`
- `internal/sinks/postgres/map.go`
- `internal/sinks/postgres/schema_test.go`
- `internal/sinks/postgres/map_test.go`
- `internal/sinks/postgres/doc.go`
Tables:
- `outlook_runs`
- `outlooks`
Add table constants:
- `tableOutlookRuns = "outlook_runs"`
- `tableOutlooks = "outlooks"`
`outlook_runs` columns:
- `event_id TEXT NOT NULL`
- `event_kind TEXT NOT NULL`
- `event_source TEXT NOT NULL`
- `event_schema TEXT NOT NULL`
- `event_emitted_at TIMESTAMPTZ NOT NULL`
- `event_effective_at TIMESTAMPTZ NULL`
- `location_id TEXT NULL`
- `location_name TEXT NULL`
- `latitude DOUBLE PRECISION NULL`
- `longitude DOUBLE PRECISION NULL`
- `as_of TIMESTAMPTZ NOT NULL`
- `issued_at TIMESTAMPTZ NULL`
- `outlook_count INTEGER NOT NULL`
`outlook_runs` keys and indexes:
- Primary key: `event_id`
- Prune column: `as_of`
- Index `idx_wf_outlook_run_location_as_of` on `location_id, as_of`
- Index `idx_wf_outlook_run_as_of` on `as_of`
`outlooks` columns:
- `run_event_id TEXT REFERENCES outlook_runs(event_id) ON DELETE CASCADE NOT NULL`
- `outlook_index INTEGER NOT NULL`
- `as_of TIMESTAMPTZ NOT NULL`
- `product TEXT NOT NULL`
- `day INTEGER NOT NULL`
- `outlook_type TEXT NOT NULL`
- `label TEXT NOT NULL`
- `label_text TEXT NULL`
- `severity_rank INTEGER NULL`
- `valid_from TIMESTAMPTZ NOT NULL`
- `valid_to TIMESTAMPTZ NOT NULL`
- `issued_at TIMESTAMPTZ NOT NULL`
- `expires_at TIMESTAMPTZ NOT NULL`
- `forecaster TEXT NULL`
- `headline TEXT NULL`
- `summary TEXT NULL`
- `discussion TEXT NULL`
- `source_url TEXT NULL`
- `image_url TEXT NULL`
- `contains_location BOOLEAN NOT NULL`
- `geometry_json TEXT NOT NULL`
`outlooks` keys and indexes:
- Primary key: `run_event_id, outlook_index`
- Prune column: `as_of`
- Index `idx_wf_outlooks_contains_valid` on `contains_location, valid_from, valid_to`
- Index `idx_wf_outlooks_day_type_label` on `day, outlook_type, label`
- Index `idx_wf_outlooks_valid` on `valid_from, valid_to`
Mapper behavior:
- Extend `mapPostgresEvent` for `standards.SchemaWeatherOutlookV1`.
- Decode `model.WeatherOutlookRun`.
- Require run `AsOf`.
- Map envelope columns exactly like existing parent run tables.
- Store all times as UTC.
- Write one `outlook_runs` row and one `outlooks` row per outlook.
- Use `outlook_index` as the zero-based slice index.
- Require outlook `ID`, `Provider`, `Product`, `Day`, `OutlookType`, `Label`, `ValidFrom`, `ValidTo`, `IssuedAt`, `ExpiresAt`, and non-empty `Geometry`.
- Store compact geometry JSON text in `geometry_json` using the existing compact JSON helper or a similar helper.
- Preserve all outlook polygons, not only those containing the configured point.
Tests:
- Schema includes `outlook_runs` and `outlooks`.
- Schema includes required columns and indexes.
- Mapper writes one run row plus one row per outlook.
- Mapper stores UTC times.
- Mapper stores compact geometry JSON.
- Mapper preserves `contains_location` false as false, not nil.
- Mapper rejects missing run `asOf`.
- Mapper rejects missing required outlook times.
- Mapper rejects empty geometry.
- Mapper ignores unrelated schemas unchanged.
Verification:
```sh
go test ./internal/sinks/postgres
```
Stage completion criteria:
- Postgres schema and mapper tests pass.
- No weatherapi read path is added in this stage.
## Stage 6: Config, Runtime Wiring, And Current-Behavior Docs
Goal: expose the completed source/normalizer/sink behavior in maintained configuration and documentation after implementation exists.
Files to update:
- `cmd/weatherfeeder/config.yml`
- `cmd/weatherfeeder/main_test.go`, if config load expectations need updates
- `docs/config.md`
- `docs/integrations/events.md`
- `docs/integrations/postgres.md`
- `docs/internal/sources.md`
- `docs/internal/normalizers.md`
- `docs/internal/postgres-sink.md`
- `README.md`
Config changes:
- Add sample source `SPCConvectiveOutlookSTL` with driver `spc_convective_outlook`, kind `outlook`, `every: 30m`, latitude, longitude, location metadata, and `user_agent`.
- Add `outlook` route examples for stdout, NATS, and Postgres where the existing config style includes kind lists.
- Keep optional RSS disabled in sample config unless there is a specific operator reason to include it.
Docs changes:
- Update current-behavior docs only after the code for that behavior exists.
- `docs/config.md` should document required and optional source params.
- `docs/integrations/events.md` should document `weather.outlook.v1`, including field definitions, required fields, optional fields, geometry semantics, and `containsLocation` semantics.
- `docs/integrations/postgres.md` should document `outlook_runs` and `outlooks`.
- Internal docs should explain provider boundaries, print-page discussion parsing, and point-in-polygon behavior where useful for maintainers.
- `README.md` should only briefly list SPC convective outlook support and link to canonical docs.
- Do not document weatherapi endpoints in weatherfeeder current-behavior docs.
Tests:
- Existing config load tests pass.
- Add config test coverage if the sample config is expected to build scheduler jobs for the new source.
- Documentation examples should use the real driver name and event kind.
Verification:
```sh
go test ./cmd/weatherfeeder ./internal/sources ./internal/normalizers ./internal/sinks/postgres
```
Stage completion criteria:
- Sample config remains loadable.
- Current-behavior docs match implemented code.
- No roadmap-only claims leak into non-roadmap docs beyond the implemented behavior.
## Stage 7: Full Verification And Cleanup
Goal: validate the complete feature and remove implementation-only rough edges.
Run focused tests:
```sh
go test ./internal/providers/spc ./internal/geo ./internal/sources ./internal/normalizers/... ./internal/sinks/postgres ./model ./cmd/weatherfeeder
```
Run full tests:
```sh
go test ./...
```
Manual review checklist:
- `standards/schema.go` contains raw and canonical SPC schema constants.
- `model/outlook.go` uses stable JSON tags and no provider-specific names except canonical strings.
- Source registry includes `spc_convective_outlook`.
- Normalizer registry includes the SPC normalizer.
- Source tests do not use live SPC services.
- Normalizer tests use fixtures and cover discussion parsing, geometry, and timestamp failures.
- Postgres tests cover schema shape and mapper validation.
- Docs follow `docs/policy/documentation.md` and use canonical homes.
- No weatherapi files are changed.
- No broad dependency was added without clear justification.
Acceptance criteria:
- A configured SPC source emits one raw bundled `outlook` event per changed complete product snapshot.
- Normalization produces `weather.outlook.v1` events.
- All Day 1-3 categorical, tornado, hail, and wind products are represented.
- Each outlook includes `containsLocation` for the configured latitude and longitude.
- Each outlook preserves compact GeoJSON geometry.
- Day 1-3 print-page discussion text is preserved in canonical headline, summary, and discussion fields where parseable.
- RSS metadata is optional and supplemental only.
- Postgres sink persists outlook runs and outlook rows.
- Sample config and current-behavior docs describe the implemented kind, driver, schema, and storage contract.
## Suggested Prompt Boundaries
This feature is too broad for a single safe implementation prompt. Use these implementation prompts in order:
1. Implement Stage 1 provider helpers and fixtures only.
2. Implement Stage 2 geometry helper only.
3. Implement Stage 3 source driver and raw schema only.
4. Implement Stage 4 canonical model and normalizer only.
5. Implement Stage 5 Postgres sink mapping only.
6. Implement Stage 6 config and current-behavior documentation only.
7. Run Stage 7 verification and perform targeted fixes only.
Each prompt should run its stage-specific tests before moving on. Do not proceed to the next stage with failing tests unless the failure is unrelated and explicitly documented.

View File

@@ -0,0 +1,87 @@
# SPC Outlook Postgres Schema Transition
## Purpose
This document describes how to reset existing Postgres outlook tables from the
`weather.outlook.v1` storage shape to the `weather.outlook.v2` compatible
storage shape.
Updated `weatherfeeder` versions create outlook tables with run-level
discussion storage. Existing databases that already contain the old outlook
table family need a manual reset because the Postgres sink creates tables with
`CREATE TABLE IF NOT EXISTS`.
## Scope
This reset drops only the outlook table family and lets updated `weatherfeeder`
recreate it:
- `outlook_discussions`
- `outlooks`
- `outlook_runs`
Other weather tables are not affected.
## Warning
These commands delete stored SPC outlook history. Existing `weather.outlook.v1`
outlook rows are intentionally removed. Downstream readers should be updated
intentionally for the new outlook shape.
## Deployment Order
1. Stop `weatherfeeder`.
2. Drop the existing outlook tables.
3. Deploy updated `weatherfeeder`.
4. Start `weatherfeeder` so the Postgres sink recreates the new outlook tables.
5. Deploy updated downstream consumers such as `weatherapi`.
## Reset SQL
```sql
DROP TABLE IF EXISTS outlook_discussions;
DROP TABLE IF EXISTS outlooks;
DROP TABLE IF EXISTS outlook_runs;
```
## Verification SQL
Before or after the updated daemon starts, this query shows which outlook tables
exist:
```sql
SELECT table_name
FROM information_schema.tables
WHERE table_name IN ('outlook_runs', 'outlooks', 'outlook_discussions')
ORDER BY table_name;
```
After the updated daemon has started and recreated the tables, verify the new
run column:
```sql
SELECT column_name, is_nullable, data_type
FROM information_schema.columns
WHERE table_name = 'outlook_runs'
AND column_name = 'discussion_count';
```
Verify the discussion table indexes:
```sql
SELECT indexname
FROM pg_indexes
WHERE tablename = 'outlook_discussions'
ORDER BY indexname;
```
Verify that legacy polygon-level prose columns are gone from `outlooks`:
```sql
SELECT column_name
FROM information_schema.columns
WHERE table_name = 'outlooks'
AND column_name IN ('headline', 'summary', 'discussion');
```
The final query should return zero rows.

View File

@@ -1,349 +0,0 @@
# SPC Convective Outlook Support Roadmap
## Summary
Add `weatherfeeder` support for Storm Prediction Center convective outlooks as a new canonical outlook domain. The feature should poll SPC GeoJSON outlook products, enrich them with Day 1-3 print-page discussion text, compute whether the configured forecast point is inside each active outlook polygon, normalize the result into a provider-independent schema, and persist it through existing sinks.
This is a new domain, not an extension of `weather.alert.v1`. SPC outlooks describe probabilistic/categorical risk areas over a validity window; alerts describe active actionable hazard messages.
## Public Contract
Add schema constants:
- Raw schema: `raw.spc.convective_outlook.v1`
- Canonical schema: `weather.outlook.v1`
Add source driver:
- `spc_convective_outlook`
Add event kind:
- `outlook`
Add canonical model types:
- `model.WeatherOutlookRun`
- `model.WeatherOutlook`
Recommended canonical run fields:
- `locationId`, `locationName`
- `latitude`, `longitude`
- `asOf`
- `issuedAt`
- `outlooks`
Recommended canonical outlook fields:
- `id`
- `provider`
- `product`
- `day`
- `outlookType`
- `label`
- `labelText`
- `severityRank`
- `validFrom`
- `validTo`
- `issuedAt`
- `expiresAt`
- `forecaster`
- `headline`
- `summary`
- `discussion`
- `sourceUrl`
- `imageUrl`
- `containsLocation`
- `geometry`
Contract defaults:
- `product` should be `convective`.
- `outlookType` should be one of `categorical`, `tornado`, `hail`, `wind`.
- `day` should be `1`, `2`, or `3` for this first implementation.
- `containsLocation` is computed against configured forecast coordinates.
- `geometry` should preserve compact GeoJSON geometry for auditability and future API use.
- Day 4-8 outlooks are out of scope for this first implementation. They have different product semantics and should be added in a later schema-compatible extension only after Day 1-3 support is proven.
## Source Scope
The source should fetch a bundle of SPC products in one poll cycle and emit one raw event containing fetched GeoJSON products, fetched Day 1-3 print pages, optional RSS metadata, configured point metadata, and per-product fetch metadata. The source may decode minimal timestamps for event identity/effective time, but provider-to-canonical mapping belongs in the normalizer.
Poll these GeoJSON URLs:
- `https://www.spc.noaa.gov/products/outlook/day1otlk_cat.nolyr.geojson`
- `https://www.spc.noaa.gov/products/outlook/day1otlk_torn.nolyr.geojson`
- `https://www.spc.noaa.gov/products/outlook/day1otlk_hail.nolyr.geojson`
- `https://www.spc.noaa.gov/products/outlook/day1otlk_wind.nolyr.geojson`
- `https://www.spc.noaa.gov/products/outlook/day2otlk_cat.nolyr.geojson`
- `https://www.spc.noaa.gov/products/outlook/day2otlk_torn.nolyr.geojson`
- `https://www.spc.noaa.gov/products/outlook/day2otlk_hail.nolyr.geojson`
- `https://www.spc.noaa.gov/products/outlook/day2otlk_wind.nolyr.geojson`
- `https://www.spc.noaa.gov/products/outlook/day3otlk_cat.nolyr.geojson`
- `https://www.spc.noaa.gov/products/outlook/day3otlk_torn.nolyr.geojson`
- `https://www.spc.noaa.gov/products/outlook/day3otlk_hail.nolyr.geojson`
- `https://www.spc.noaa.gov/products/outlook/day3otlk_wind.nolyr.geojson`
Implementation note: the initial candidate list duplicated Day 3 wind and omitted Day 2 wind. Use the corrected 12-product matrix above.
Poll these print-page discussion URLs:
- `https://www.spc.noaa.gov/products/outlook/day1otlk_prt.html`
- `https://www.spc.noaa.gov/products/outlook/day2otlk_prt.html`
- `https://www.spc.noaa.gov/products/outlook/day3otlk_prt.html`
Print-page usage:
- Use print pages as the authoritative source for Day 1-3 outlook discussion text.
- Preserve source URL, page title/header where available, page-level `Updated:` timestamp where parseable, and the raw HTML or extracted `<pre>` product block in the raw bundle.
- Extract canonical `headline`, `summary`, `discussion`, and discussion metadata in the normalizer or shared SPC provider helper.
- Treat the GeoJSON files, not print pages, as authoritative for polygons and validity windows.
- Ignore page risk tables and image assets for v1 unless they are needed to support discussion parsing.
Optionally poll:
- `https://www.spc.noaa.gov/products/spcacrss.xml`
RSS usage:
- Use RSS as supplemental discovery or health metadata, not as the primary discussion or geometry source.
- Preserve item title, link, description text, pubDate, and guid where useful.
- Do not depend on RSS completeness for Day 1-3 discussion text. The RSS feed can omit current outlook products, including a current Day 2 discussion.
- Do not attempt to parse polygons from RSS HTML.
Polling cadence:
- Use a default cadence of `30m`, not daily/twice-daily. SPC current outlook files can update several times per day, and a 30-minute poll with HTTP caching is low cost and less likely to miss updates.
- Rely on ETag/Last-Modified handling from the HTTP source layer where available.
- Keep the cadence configurable via `every`.
Recommended config shape:
```yaml
- name: SPCConvectiveOutlookSTL
mode: poll
kinds: ["outlook"]
driver: spc_convective_outlook
every: 30m
params:
latitude: 38.6239
longitude: -90.3571
location_id: "stl"
location_name: "St. Louis, MO"
user_agent: "HomeOps (eric@maximumdirect.net)"
```
The source should own default SPC URLs, with optional params to override GeoJSON URLs, print-page URLs, and the RSS URL for tests or future provider changes.
## Implementation Stages
### Stage 1: Raw Source and Schema
- Add schema constants.
- Add `internal/sources/spc` package.
- Implement `spc_convective_outlook` as a poll source.
- Fetch all configured GeoJSON products and Day 1-3 print pages during a poll cycle.
- Fetch RSS only when configured or retained as an optional supplemental source.
- Emit one raw event with kind `outlook` and schema `raw.spc.convective_outlook.v1`.
- Use the latest valid `ISSUE_ISO`, print-page `Updated:` timestamp, RSS `lastBuildDate`, or fetch time for `effectiveAt`, in that order.
- Preserve partial fetch errors as source errors for the first implementation; do not emit incomplete outlook bundles unless a later explicit degraded-mode policy is added.
- Keep source tests deterministic with local test servers and fixtures; do not call live SPC services from unit tests.
Tests:
- source driver builds as a `PollSource`
- source advertises kind `outlook`
- source emits one raw bundled event
- source chooses expected effective time
- source uses configured location metadata
- source fails clearly on missing latitude/longitude
- source fails clearly when a configured GeoJSON or print-page fetch fails
### Stage 2: GeoJSON and Spatial Matching
- Add an internal geometry helper package, for example `internal/geo`.
- Support GeoJSON `Polygon` and `MultiPolygon`.
- Implement point-in-polygon with ring support:
- first ring is the exterior
- subsequent rings are holes
- boundary counts as inside
- GeoJSON coordinate order is `[longitude, latitude]`
- Use planar ray casting for this operational point-in-polygon check.
Tests:
- point inside polygon
- point outside polygon
- point on polygon boundary
- point inside a hole is outside
- point inside one multipolygon member is inside
- longitude/latitude order regression test
### Stage 3: Normalizer and Canonical Mapping
- Add `model.WeatherOutlookRun` and `model.WeatherOutlook`.
- Add `SPCConvectiveOutlookNormalizer`.
- Match only `raw.spc.convective_outlook.v1`.
- Decode the raw source bundle.
- Map each GeoJSON feature to one `WeatherOutlook`.
- Preserve SPC feature order within each product, then order products by day and type.
- Map properties:
- `VALID_ISO` -> `validFrom`
- `EXPIRE_ISO` -> `validTo` and `expiresAt`
- `ISSUE_ISO` -> `issuedAt`
- `FORECASTER` -> `forecaster`
- `LABEL` -> `label`
- `LABEL2` -> `labelText`
- `DN` -> `severityRank`
- Derive:
- `day` from product key or URL
- `outlookType` from product key or URL
- `id` from day, type, label, issuedAt, validFrom, and feature index
- `containsLocation` from configured point and GeoJSON geometry
- Enrich narrative fields from the matching Day 1-3 print page:
- parse the main product text from the `<pre>` block
- strip embedded tags and scripts before text parsing
- preserve line breaks in discussion text
- parse page/title metadata, product title, issued line, valid line, correction markers such as `CORR 1`, forecaster, summary, and full discussion where practical
- use print-page text for canonical `headline`, `summary`, and `discussion`
- Use RSS only as optional supplemental metadata when present; missing or incomplete RSS must not prevent GeoJSON plus print-page normalization.
- Set run `asOf` to the latest valid `issuedAt` across features, falling back to print-page `Updated:` timestamps, RSS `lastBuildDate`, then input event time.
- Set run `issuedAt` to the latest valid `issuedAt` across features.
- Set output event `effectiveAt` to run `asOf`.
Tests:
- normalizer routes only raw SPC schema
- categorical and probabilistic products map expected fields
- `containsLocation` is true for a known point inside a fixture polygon
- `containsLocation` is false outside
- print-page discussion text maps to canonical headline, summary, and discussion
- correction markers such as `CORR 1` are preserved where present
- missing optional RSS still permits GeoJSON and print-page normalization
- malformed required GeoJSON timestamps fail with useful context
- canonical JSON shape does not expose raw-provider-only bundle internals
### Stage 4: Postgres Sink
Add tables:
- `outlook_runs`
- `outlooks`
Suggested `outlook_runs` columns:
- event envelope columns
- `location_id`
- `location_name`
- `latitude`
- `longitude`
- `as_of`
- `issued_at`
- `outlook_count`
Suggested `outlooks` columns:
- `run_event_id`
- `outlook_index`
- `as_of`
- `product`
- `day`
- `outlook_type`
- `label`
- `label_text`
- `severity_rank`
- `valid_from`
- `valid_to`
- `issued_at`
- `expires_at`
- `forecaster`
- `headline`
- `summary`
- `discussion`
- `source_url`
- `image_url`
- `contains_location`
- `geometry_json`
Suggested indexes:
- `outlook_runs(location_id, as_of)`
- `outlooks(contains_location, valid_from, valid_to)`
- `outlooks(day, outlook_type, label)`
- `outlooks(valid_from, valid_to)`
Mapping rules:
- Store `geometry` as compact GeoJSON text in `geometry_json`.
- Require run `asOf`.
- Require outlook `validFrom`, `validTo`, `issuedAt`, `day`, `outlookType`, and `label`.
- Preserve all outlook polygons, not only polygons containing the configured point.
Tests:
- schema includes outlook tables and indexes
- mapper writes one run row plus one row per outlook
- mapper stores compact geometry JSON
- mapper rejects missing required run/outlook fields
### Stage 5: Config and Documentation
- Update sample config with `SPCConvectiveOutlookSTL`.
- Add `outlook` to route examples.
- Update current-behavior docs in the implementation change, not before behavior exists.
- Update `docs/config.md` with source params.
- Update `docs/integrations/events.md` with `weather.outlook.v1`.
- Update `docs/integrations/postgres.md` with outlook table contract.
- Update relevant internal docs under `docs/internal/` if source, normalizer, or sink behavior needs maintainer context.
- Update `README.md` only with brief implemented-provider capability and links to canonical docs.
Docs should clearly state:
- Print pages are used for Day 1-3 narrative/discussion text.
- RSS, if retained, is supplemental and not required for complete discussion coverage.
- GeoJSON is used for polygons, validity windows, and point matching.
- `containsLocation` is computed by weatherfeeder at ingestion time.
- Geometry is stored for downstream audit/display.
- Day 4-8 outlooks are not included in the initial implementation.
### Stage 6: Weatherapi Follow-Up
Do not include weatherapi changes in the first weatherfeeder implementation unless explicitly requested.
Likely future weatherapi endpoints:
- `GET /outlooks/convective`
- `GET /outlooks/convective/active`
- `GET /outlooks/convective/location`
Recommended API behavior:
- latest run by default
- active outlooks filtered by current time and `containsLocation=true`
- optional filter query params for `day`, `outlookType`, and `containsLocation`
## Verification Commands
Run focused tests:
```sh
go test ./internal/sources ./internal/normalizers/... ./internal/sinks/postgres ./model
```
Run full weatherfeeder tests:
```sh
go test ./...
```
## Acceptance Criteria
- A configured SPC source emits raw outlook bundles.
- Normalization produces `weather.outlook.v1` events.
- Each outlook indicates whether the configured forecast point is inside its polygon.
- All current Day 1-3 categorical/tornado/hail/wind products are represented.
- Day 1-3 print-page discussion text is preserved where available.
- RSS metadata is optional and supplemental if retained.
- Postgres sink persists outlook runs and outlook rows.
- Sample config and public docs describe the new kind, driver, schema, and storage contract.

View File

@@ -58,7 +58,8 @@ Diagnostic: compare the configured driver with the source driver table in
Safe fix: correct the driver name. Current drivers include `nws_observation`,
`nws_alerts`, `nws_forecast_hourly`, `nws_forecast_narrative`,
`nws_forecast_discussion`, `nws_weatherstories`, `openmeteo_observation`,
`openmeteo_forecast`, and `openweather_observation`.
`openmeteo_forecast`, `openweather_observation`, and
`spc_convective_outlook`.
## `unknown sink driver`
@@ -94,6 +95,20 @@ Diagnostic: inspect the named source in the error and check its `params`.
Safe fix: add non-empty `url` and `user_agent` values. See
[HTTP source params](config.md#http-source-params).
SPC convective outlook sources do not use `params.url`; they require
`latitude`, `longitude`, and `user_agent`. See
[SPC convective outlook params](config.md#spc-convective-outlook-params).
## `params.latitude is required` Or `params.longitude is required`
Symptom: startup exits for an `spc_convective_outlook` source.
Likely cause: the source is missing one of the configured point coordinates.
Diagnostic: inspect the named SPC source in the error and check its `params`.
Safe fix: add numeric `latitude` and `longitude` values in decimal degrees.
## `url must include units=metric`
Symptom: startup exits for an `openweather_observation` source.

View File

@@ -18,6 +18,18 @@ sources:
url: "https://api.weather.gov/alerts?point=38.6239,-90.3571&limit=20"
user_agent: "weatherfeeder example (operator@example.com)"
- name: SPCConvectiveOutlookSTL
mode: poll
kinds: ["outlook"]
driver: spc_convective_outlook
every: 30m
params:
latitude: 38.6239
longitude: -90.3571
location_id: "stl"
location_name: "St. Louis, MO"
user_agent: "weatherfeeder example (operator@example.com)"
sinks:
- name: nats_weather
driver: nats
@@ -27,4 +39,4 @@ sinks:
routes:
- sink: nats_weather
kinds: ["observation", "alert"]
kinds: ["observation", "alert", "outlook"]

View File

@@ -18,6 +18,18 @@ sources:
url: "https://api.open-meteo.com/v1/forecast?latitude=38.6239&longitude=-90.3571&hourly=temperature_2m,relative_humidity_2m,dew_point_2m,apparent_temperature,precipitation_probability,precipitation,snowfall,weather_code,surface_pressure,wind_speed_10m,wind_direction_10m&forecast_days=3"
user_agent: "weatherfeeder example (operator@example.com)"
- name: SPCConvectiveOutlookSTL
mode: poll
kinds: ["outlook"]
driver: spc_convective_outlook
every: 30m
params:
latitude: 38.6239
longitude: -90.3571
location_id: "stl"
location_name: "St. Louis, MO"
user_agent: "weatherfeeder example (operator@example.com)"
sinks:
- name: pg_weather
driver: postgres
@@ -29,4 +41,4 @@ sinks:
routes:
- sink: pg_weather
kinds: ["observation", "forecast"]
kinds: ["observation", "forecast", "outlook"]

107
internal/geo/geojson.go Normal file
View File

@@ -0,0 +1,107 @@
package geo
import (
"encoding/json"
"fmt"
)
type geometry struct {
Type string `json:"type"`
Coordinates json.RawMessage `json:"coordinates"`
}
// ContainsPoint reports whether a GeoJSON Polygon or MultiPolygon contains p.
// GeoJSON coordinate order is [longitude, latitude].
func ContainsPoint(raw []byte, p Point) (bool, error) {
if len(raw) == 0 {
return false, fmt.Errorf("geojson geometry is empty")
}
var geom geometry
if err := json.Unmarshal(raw, &geom); err != nil {
return false, fmt.Errorf("decode geojson geometry: %w", err)
}
switch geom.Type {
case "Polygon":
polygon, err := decodePolygon(geom.Coordinates)
if err != nil {
return false, fmt.Errorf("decode polygon: %w", err)
}
return polygonContainsPoint(polygon, p), nil
case "MultiPolygon":
multiPolygon, err := decodeMultiPolygon(geom.Coordinates)
if err != nil {
return false, fmt.Errorf("decode multipolygon: %w", err)
}
for _, polygon := range multiPolygon {
if polygonContainsPoint(polygon, p) {
return true, nil
}
}
return false, nil
case "":
return false, fmt.Errorf("geojson geometry type is required")
default:
return false, fmt.Errorf("unsupported geojson geometry type %q", geom.Type)
}
}
func decodePolygon(raw json.RawMessage) (Polygon, error) {
var coords [][][]float64
if err := json.Unmarshal(raw, &coords); err != nil {
return nil, err
}
return polygonFromCoordinates(coords)
}
func decodeMultiPolygon(raw json.RawMessage) ([]Polygon, error) {
var coords [][][][]float64
if err := json.Unmarshal(raw, &coords); err != nil {
return nil, err
}
if len(coords) == 0 {
return nil, fmt.Errorf("multipolygon has no polygons")
}
out := make([]Polygon, 0, len(coords))
for i, polygonCoords := range coords {
polygon, err := polygonFromCoordinates(polygonCoords)
if err != nil {
return nil, fmt.Errorf("polygons[%d]: %w", i, err)
}
out = append(out, polygon)
}
return out, nil
}
func polygonFromCoordinates(coords [][][]float64) (Polygon, error) {
if len(coords) == 0 {
return nil, fmt.Errorf("polygon has no rings")
}
polygon := make(Polygon, 0, len(coords))
for i, ringCoords := range coords {
ring, err := ringFromCoordinates(ringCoords)
if err != nil {
return nil, fmt.Errorf("rings[%d]: %w", i, err)
}
polygon = append(polygon, ring)
}
return polygon, nil
}
func ringFromCoordinates(coords [][]float64) (Ring, error) {
if len(coords) == 0 {
return nil, fmt.Errorf("ring has no points")
}
ring := make(Ring, 0, len(coords))
for i, pair := range coords {
if len(pair) < 2 {
return nil, fmt.Errorf("points[%d] has %d values, need longitude and latitude", i, len(pair))
}
ring = append(ring, Point{Longitude: pair[0], Latitude: pair[1]})
}
return ring, nil
}

105
internal/geo/point.go Normal file
View File

@@ -0,0 +1,105 @@
package geo
import "math"
const epsilon = 1e-9
// Point is a geographic coordinate in decimal degrees.
type Point struct {
Longitude float64
Latitude float64
}
// Ring is one GeoJSON linear ring.
type Ring []Point
// Polygon is a GeoJSON polygon. The first ring is the exterior ring; subsequent
// rings are holes.
type Polygon []Ring
func polygonContainsPoint(polygon Polygon, p Point) bool {
if len(polygon) == 0 {
return false
}
if pointOnRing(polygon[0], p) {
return true
}
if !ringContainsPoint(polygon[0], p) {
return false
}
for _, hole := range polygon[1:] {
if pointOnRing(hole, p) {
return true
}
if ringContainsPoint(hole, p) {
return false
}
}
return true
}
func ringContainsPoint(ring Ring, p Point) bool {
inside := false
n := len(ring)
if n == 0 {
return false
}
for i, j := 0, n-1; i < n; j, i = i, i+1 {
a := ring[j]
b := ring[i]
if pointOnSegment(p, a, b) {
return true
}
intersects := (a.Latitude > p.Latitude) != (b.Latitude > p.Latitude)
if intersects {
x := (b.Longitude-a.Longitude)*(p.Latitude-a.Latitude)/(b.Latitude-a.Latitude) + a.Longitude
if almostEqual(x, p.Longitude) {
return true
}
if x > p.Longitude {
inside = !inside
}
}
}
return inside
}
func pointOnRing(ring Ring, p Point) bool {
n := len(ring)
if n == 0 {
return false
}
for i, j := 0, n-1; i < n; j, i = i, i+1 {
if pointOnSegment(p, ring[j], ring[i]) {
return true
}
}
return false
}
func pointOnSegment(p, a, b Point) bool {
cross := (p.Latitude-a.Latitude)*(b.Longitude-a.Longitude) - (p.Longitude-a.Longitude)*(b.Latitude-a.Latitude)
if math.Abs(cross) > epsilon {
return false
}
minLon, maxLon := minMax(a.Longitude, b.Longitude)
minLat, maxLat := minMax(a.Latitude, b.Latitude)
return p.Longitude >= minLon-epsilon &&
p.Longitude <= maxLon+epsilon &&
p.Latitude >= minLat-epsilon &&
p.Latitude <= maxLat+epsilon
}
func minMax(a, b float64) (float64, float64) {
if a < b {
return a, b
}
return b, a
}
func almostEqual(a, b float64) bool {
return math.Abs(a-b) <= epsilon
}

181
internal/geo/point_test.go Normal file
View File

@@ -0,0 +1,181 @@
package geo
import (
"encoding/json"
"strings"
"testing"
)
const squarePolygon = `{
"type": "Polygon",
"coordinates": [[
[-91.0, 38.0],
[-90.0, 38.0],
[-90.0, 39.0],
[-91.0, 39.0],
[-91.0, 38.0]
]]
}`
func TestContainsPointInsideSimplePolygon(t *testing.T) {
got, err := ContainsPoint([]byte(squarePolygon), Point{Longitude: -90.5, Latitude: 38.5})
if err != nil {
t.Fatalf("ContainsPoint() error = %v", err)
}
if !got {
t.Fatalf("ContainsPoint() = false, want true")
}
}
func TestContainsPointAcceptsRawMessage(t *testing.T) {
got, err := ContainsPoint(json.RawMessage(squarePolygon), Point{Longitude: -90.5, Latitude: 38.5})
if err != nil {
t.Fatalf("ContainsPoint() error = %v", err)
}
if !got {
t.Fatalf("ContainsPoint() = false, want true")
}
}
func TestContainsPointOutsideSimplePolygon(t *testing.T) {
got, err := ContainsPoint([]byte(squarePolygon), Point{Longitude: -89.5, Latitude: 38.5})
if err != nil {
t.Fatalf("ContainsPoint() error = %v", err)
}
if got {
t.Fatalf("ContainsPoint() = true, want false")
}
}
func TestContainsPointOnBoundary(t *testing.T) {
got, err := ContainsPoint([]byte(squarePolygon), Point{Longitude: -91.0, Latitude: 38.5})
if err != nil {
t.Fatalf("ContainsPoint() error = %v", err)
}
if !got {
t.Fatalf("ContainsPoint() = false, want true")
}
}
func TestContainsPointInHoleReturnsFalse(t *testing.T) {
const polygonWithHole = `{
"type": "Polygon",
"coordinates": [
[[0,0],[10,0],[10,10],[0,10],[0,0]],
[[4,4],[6,4],[6,6],[4,6],[4,4]]
]
}`
got, err := ContainsPoint([]byte(polygonWithHole), Point{Longitude: 5, Latitude: 5})
if err != nil {
t.Fatalf("ContainsPoint() error = %v", err)
}
if got {
t.Fatalf("ContainsPoint() = true, want false")
}
}
func TestContainsPointOnHoleBoundaryReturnsTrue(t *testing.T) {
const polygonWithHole = `{
"type": "Polygon",
"coordinates": [
[[0,0],[10,0],[10,10],[0,10],[0,0]],
[[4,4],[6,4],[6,6],[4,6],[4,4]]
]
}`
got, err := ContainsPoint([]byte(polygonWithHole), Point{Longitude: 4, Latitude: 5})
if err != nil {
t.Fatalf("ContainsPoint() error = %v", err)
}
if !got {
t.Fatalf("ContainsPoint() = false, want true")
}
}
func TestContainsPointInsideOneMultiPolygonMember(t *testing.T) {
const multiPolygon = `{
"type": "MultiPolygon",
"coordinates": [
[[[0,0],[1,0],[1,1],[0,1],[0,0]]],
[[[10,10],[12,10],[12,12],[10,12],[10,10]]]
]
}`
got, err := ContainsPoint([]byte(multiPolygon), Point{Longitude: 11, Latitude: 11})
if err != nil {
t.Fatalf("ContainsPoint() error = %v", err)
}
if !got {
t.Fatalf("ContainsPoint() = false, want true")
}
}
func TestContainsPointUsesLongitudeLatitudeOrder(t *testing.T) {
const narrowPolygon = `{
"type": "Polygon",
"coordinates": [[
[-91.0, 38.0],
[-90.0, 38.0],
[-90.0, 39.0],
[-91.0, 39.0],
[-91.0, 38.0]
]]
}`
got, err := ContainsPoint([]byte(narrowPolygon), Point{Longitude: -90.5, Latitude: 38.5})
if err != nil {
t.Fatalf("ContainsPoint() error = %v", err)
}
if !got {
t.Fatalf("ContainsPoint() = false, want true")
}
got, err = ContainsPoint([]byte(narrowPolygon), Point{Longitude: 38.5, Latitude: -90.5})
if err != nil {
t.Fatalf("ContainsPoint() reversed error = %v", err)
}
if got {
t.Fatalf("ContainsPoint() with reversed coordinate values = true, want false")
}
}
func TestContainsPointUnsupportedGeometryError(t *testing.T) {
_, err := ContainsPoint([]byte(`{"type":"Point","coordinates":[-90,38]}`), Point{Longitude: -90, Latitude: 38})
if err == nil {
t.Fatalf("ContainsPoint() error = nil, want error")
}
if !strings.Contains(err.Error(), `unsupported geojson geometry type "Point"`) {
t.Fatalf("ContainsPoint() error = %q", err)
}
}
func TestContainsPointInvalidJSONError(t *testing.T) {
_, err := ContainsPoint([]byte(`{"type":"Polygon"`), Point{Longitude: -90, Latitude: 38})
if err == nil {
t.Fatalf("ContainsPoint() error = nil, want error")
}
if !strings.Contains(err.Error(), "decode geojson geometry") {
t.Fatalf("ContainsPoint() error = %q", err)
}
}
func TestContainsPointMalformedCoordinatesError(t *testing.T) {
_, err := ContainsPoint([]byte(`{"type":"Polygon","coordinates":[[[1]]]}`), Point{Longitude: 1, Latitude: 1})
if err == nil {
t.Fatalf("ContainsPoint() error = nil, want error")
}
if !strings.Contains(err.Error(), "need longitude and latitude") {
t.Fatalf("ContainsPoint() error = %q", err)
}
}
func TestContainsPointEmptyRingError(t *testing.T) {
_, err := ContainsPoint([]byte(`{"type":"Polygon","coordinates":[[]]}`), Point{Longitude: 1, Latitude: 1})
if err == nil {
t.Fatalf("ContainsPoint() error = nil, want error")
}
if !strings.Contains(err.Error(), "ring has no points") {
t.Fatalf("ContainsPoint() error = %q", err)
}
}

View File

@@ -7,12 +7,14 @@ import (
"gitea.maximumdirect.net/ejr/weatherfeeder/internal/normalizers/nws"
"gitea.maximumdirect.net/ejr/weatherfeeder/internal/normalizers/openmeteo"
"gitea.maximumdirect.net/ejr/weatherfeeder/internal/normalizers/openweather"
"gitea.maximumdirect.net/ejr/weatherfeeder/internal/normalizers/spc"
)
var builtinRegistrations = []func([]fknormalize.Normalizer) []fknormalize.Normalizer{
nws.Register,
openmeteo.Register,
openweather.Register,
spc.Register,
}
// RegisterBuiltins registers all normalizers shipped with this binary.

View File

@@ -8,6 +8,7 @@ import (
"gitea.maximumdirect.net/ejr/weatherfeeder/internal/normalizers/nws"
"gitea.maximumdirect.net/ejr/weatherfeeder/internal/normalizers/openmeteo"
"gitea.maximumdirect.net/ejr/weatherfeeder/internal/normalizers/openweather"
"gitea.maximumdirect.net/ejr/weatherfeeder/internal/normalizers/spc"
)
func TestRegisterBuiltinsOrder(t *testing.T) {
@@ -25,6 +26,7 @@ func TestRegisterBuiltinsOrder(t *testing.T) {
openmeteo.ObservationNormalizer{},
openmeteo.ForecastNormalizer{},
openweather.ObservationNormalizer{},
spc.ConvectiveOutlookNormalizer{},
}
if len(got) != len(want) {

View File

@@ -5,6 +5,7 @@ import (
"time"
"gitea.maximumdirect.net/ejr/feedkit/event"
"gitea.maximumdirect.net/ejr/weatherfeeder/standards"
)
func TestFinalizeRoundsWeatherPayloadFloats(t *testing.T) {
@@ -14,7 +15,7 @@ func TestFinalizeRoundsWeatherPayloadFloats(t *testing.T) {
in := event.Event{
ID: "evt-1",
Kind: event.Kind("observation"),
Kind: event.Kind(standards.KindObservation),
Source: "source-a",
EmittedAt: time.Date(2026, 3, 28, 12, 0, 0, 0, time.UTC),
Schema: "raw.example.v1",

View File

@@ -18,6 +18,18 @@ import (
// Errors include a small amount of operation context ("extract payload", "decode raw payload").
// Callers typically wrap these with a provider/kind label.
func DecodeJSONPayload[T any](in event.Event) (T, error) {
var zero T
if typed, ok := in.Payload.(T); ok {
return typed, nil
}
if ptr, ok := in.Payload.(*T); ok {
if ptr == nil {
return zero, fmt.Errorf("extract payload: payload pointer is nil")
}
return *ptr, nil
}
return fknormalize.DecodeJSONPayload[T](in)
}

View File

@@ -0,0 +1,39 @@
package common
import (
"testing"
"gitea.maximumdirect.net/ejr/feedkit/event"
)
func TestDecodeJSONPayloadAcceptsTypedPayload(t *testing.T) {
type rawPayload struct {
Value string `json:"value"`
}
got, err := DecodeJSONPayload[rawPayload](event.Event{
Payload: rawPayload{Value: "ok"},
})
if err != nil {
t.Fatalf("DecodeJSONPayload() error = %v", err)
}
if got.Value != "ok" {
t.Fatalf("Value = %q, want ok", got.Value)
}
}
func TestDecodeJSONPayloadAcceptsTypedPointerPayload(t *testing.T) {
type rawPayload struct {
Value string `json:"value"`
}
got, err := DecodeJSONPayload[rawPayload](event.Event{
Payload: &rawPayload{Value: "ok"},
})
if err != nil {
t.Fatalf("DecodeJSONPayload() error = %v", err)
}
if got.Value != "ok" {
t.Fatalf("Value = %q, want ok", got.Value)
}
}

View File

@@ -93,12 +93,8 @@ func buildAlerts(parsed nwsAlertsResponse, fallbackAsOf time.Time) (model.Weathe
sent := nwscommon.ParseTimePtr(p.Sent)
effective := nwscommon.ParseTimePtr(p.Effective)
onset := nwscommon.ParseTimePtr(p.Onset)
// Expires: prefer "expires"; fall back to "ends" if present.
ends := nwscommon.ParseTimePtr(p.Ends)
expires := nwscommon.ParseTimePtr(p.Expires)
if expires == nil {
expires = nwscommon.ParseTimePtr(p.Ends)
}
refs := parseNWSAlertReferences(p.References)
@@ -123,6 +119,7 @@ func buildAlerts(parsed nwsAlertsResponse, fallbackAsOf time.Time) (model.Weathe
Sent: sent,
Effective: effective,
Onset: onset,
Ends: ends,
Expires: expires,
AreaDescription: strings.TrimSpace(p.AreaDesc),

View File

@@ -0,0 +1,136 @@
package nws
import (
"context"
"encoding/json"
"testing"
"time"
"gitea.maximumdirect.net/ejr/feedkit/event"
"gitea.maximumdirect.net/ejr/weatherfeeder/model"
"gitea.maximumdirect.net/ejr/weatherfeeder/standards"
)
func TestAlertsNormalizerMapsEndsSeparatelyFromExpires(t *testing.T) {
raw := []byte(`{
"updated":"2026-06-16T10:00:00+00:00",
"title":"Current watches, warnings, and advisories for St. Louis",
"features":[{
"id":"https://api.weather.gov/alerts/alert-1",
"properties":{
"event":"Flood Warning",
"headline":"Flood Warning issued",
"sent":"2026-06-16T09:55:00+00:00",
"effective":"2026-06-16T10:00:00+00:00",
"onset":"2026-06-16T10:15:00+00:00",
"ends":"2026-06-16T14:00:00+00:00",
"expires":"2026-06-16T11:00:00+00:00"
}
}]
}`)
out, err := AlertsNormalizer{}.Normalize(context.Background(), alertRawEvent(raw))
if err != nil {
t.Fatalf("Normalize() error = %v", err)
}
run := decodeAlertRun(t, out)
if len(run.Alerts) != 1 {
t.Fatalf("expected 1 alert, got %d", len(run.Alerts))
}
alert := run.Alerts[0]
wantEnds := time.Date(2026, 6, 16, 14, 0, 0, 0, time.UTC)
wantExpires := time.Date(2026, 6, 16, 11, 0, 0, 0, time.UTC)
if alert.Ends == nil || !alert.Ends.Equal(wantEnds) {
t.Fatalf("ends = %v, want %s", alert.Ends, wantEnds)
}
if alert.Expires == nil || !alert.Expires.Equal(wantExpires) {
t.Fatalf("expires = %v, want %s", alert.Expires, wantExpires)
}
}
func TestAlertsNormalizerDoesNotFallbackExpiresToEnds(t *testing.T) {
raw := []byte(`{
"updated":"2026-06-16T10:00:00+00:00",
"features":[{
"id":"alert-ends-only",
"properties":{
"event":"Heat Advisory",
"ends":"2026-06-16T22:00:00+00:00"
}
}]
}`)
out, err := AlertsNormalizer{}.Normalize(context.Background(), alertRawEvent(raw))
if err != nil {
t.Fatalf("Normalize() error = %v", err)
}
run := decodeAlertRun(t, out)
alert := run.Alerts[0]
if alert.Ends == nil {
t.Fatal("expected ends to be populated")
}
if alert.Expires != nil {
t.Fatalf("expected expires nil when upstream expires is absent, got %v", alert.Expires)
}
}
func TestAlertsNormalizerIgnoresInvalidEnds(t *testing.T) {
raw := []byte(`{
"updated":"2026-06-16T10:00:00+00:00",
"features":[{
"id":"alert-invalid-ends",
"properties":{
"event":"Special Weather Statement",
"ends":"not-a-time",
"expires":"2026-06-16T11:00:00+00:00"
}
}]
}`)
out, err := AlertsNormalizer{}.Normalize(context.Background(), alertRawEvent(raw))
if err != nil {
t.Fatalf("Normalize() error = %v", err)
}
run := decodeAlertRun(t, out)
alert := run.Alerts[0]
if alert.Ends != nil {
t.Fatalf("expected invalid ends to map nil, got %v", alert.Ends)
}
if alert.Expires == nil {
t.Fatal("expected expires to remain populated")
}
}
func alertRawEvent(raw []byte) event.Event {
emittedAt := time.Date(2026, 6, 16, 10, 5, 0, 0, time.UTC)
return event.Event{
ID: "raw-alerts",
Kind: event.Kind(standards.KindAlert),
Source: "NWSAlerts",
Schema: standards.SchemaRawNWSAlertsV1,
EmittedAt: emittedAt,
Payload: json.RawMessage(raw),
}
}
func decodeAlertRun(t *testing.T, e *event.Event) model.WeatherAlertRun {
t.Helper()
if e == nil {
t.Fatal("expected normalized event")
}
if e.Schema != standards.SchemaWeatherAlertV1 {
t.Fatalf("schema = %q, want %q", e.Schema, standards.SchemaWeatherAlertV1)
}
var run model.WeatherAlertRun
raw, err := json.Marshal(e.Payload)
if err != nil {
t.Fatalf("marshal alert payload: %v", err)
}
if err := json.Unmarshal(raw, &run); err != nil {
t.Fatalf("decode alert payload: %v", err)
}
return run
}

View File

@@ -18,7 +18,7 @@ func TestForecastDiscussionNormalizerProducesCanonicalSchema(t *testing.T) {
out, err := (ForecastDiscussionNormalizer{}).Normalize(nil, event.Event{
ID: "evt-discussion-1",
Kind: event.Kind("forecast_discussion"),
Kind: event.Kind(standards.KindForecastDiscussion),
Source: "nws-discussion-test",
EmittedAt: time.Date(2026, 3, 28, 19, 25, 0, 0, time.UTC),
Schema: standards.SchemaRawNWSForecastDiscussionV1,
@@ -33,7 +33,7 @@ func TestForecastDiscussionNormalizerProducesCanonicalSchema(t *testing.T) {
if out.Schema != standards.SchemaWeatherForecastDiscussionV1 {
t.Fatalf("Schema = %q, want %q", out.Schema, standards.SchemaWeatherForecastDiscussionV1)
}
if out.Kind != event.Kind("forecast_discussion") {
if out.Kind != event.Kind(standards.KindForecastDiscussion) {
t.Fatalf("Kind = %q, want forecast_discussion", out.Kind)
}
@@ -74,7 +74,7 @@ func TestForecastDiscussionNormalizerProducesCanonicalSchema(t *testing.T) {
func TestForecastDiscussionNormalizerRejectsMissingIssueTime(t *testing.T) {
_, err := (ForecastDiscussionNormalizer{}).Normalize(nil, event.Event{
ID: "evt-discussion-bad",
Kind: event.Kind("forecast_discussion"),
Kind: event.Kind(standards.KindForecastDiscussion),
Source: "nws-discussion-test",
EmittedAt: time.Date(2026, 3, 28, 19, 25, 0, 0, time.UTC),
Schema: standards.SchemaRawNWSForecastDiscussionV1,
@@ -93,7 +93,7 @@ func TestForecastDiscussionNormalizerWireShapeHasNoUnexpectedKeys(t *testing.T)
out, err := (ForecastDiscussionNormalizer{}).Normalize(nil, event.Event{
ID: "evt-discussion-2",
Kind: event.Kind("forecast_discussion"),
Kind: event.Kind(standards.KindForecastDiscussion),
Source: "nws-discussion-test",
EmittedAt: time.Date(2026, 3, 28, 19, 25, 0, 0, time.UTC),
Schema: standards.SchemaRawNWSForecastDiscussionV1,

View File

@@ -182,7 +182,7 @@ func TestNormalizeForecastEventBySchemaProducesCanonicalWeatherForecastSchema(t
t.Run(tt.name, func(t *testing.T) {
out, err := normalizeForecastEventBySchema(event.Event{
ID: "evt-1",
Kind: event.Kind("forecast"),
Kind: event.Kind(standards.KindForecast),
Source: "nws-test",
EmittedAt: time.Date(2026, 3, 16, 18, 0, 0, 0, time.UTC),
Schema: tt.schema,

View File

@@ -22,7 +22,7 @@ func TestWeatherStoriesNormalizerProducesCanonicalSchemaAndMapsSample(t *testing
if out.Schema != standards.SchemaWeatherStoryV1 {
t.Fatalf("Schema = %q, want %q", out.Schema, standards.SchemaWeatherStoryV1)
}
if out.Kind != event.Kind("weather_story") {
if out.Kind != event.Kind(standards.KindWeatherStory) {
t.Fatalf("Kind = %q, want weather_story", out.Kind)
}
@@ -118,7 +118,7 @@ func TestWeatherStoriesNormalizerMatch(t *testing.T) {
func weatherStoriesRawEvent(payload string) event.Event {
return event.Event{
ID: "evt-weatherstories-1",
Kind: event.Kind("weather_story"),
Kind: event.Kind(standards.KindWeatherStory),
Source: "nws-weatherstories-test",
EmittedAt: time.Date(2026, 5, 30, 9, 5, 0, 0, time.UTC),
Schema: standards.SchemaRawNWSWeatherStoriesV1,

View File

@@ -0,0 +1,365 @@
package spc
import (
"context"
"encoding/json"
"fmt"
"math"
"regexp"
"sort"
"strings"
"time"
"gitea.maximumdirect.net/ejr/feedkit/event"
"gitea.maximumdirect.net/ejr/weatherfeeder/internal/geo"
normcommon "gitea.maximumdirect.net/ejr/weatherfeeder/internal/normalizers/common"
spcprovider "gitea.maximumdirect.net/ejr/weatherfeeder/internal/providers/spc"
"gitea.maximumdirect.net/ejr/weatherfeeder/model"
"gitea.maximumdirect.net/ejr/weatherfeeder/standards"
)
const (
providerSPC = "spc"
productConvective = "convective"
outlookNormalizer = "spc convective outlook"
outlookTypeUnknown = 99
)
var idTokenRE = regexp.MustCompile(`[^a-z0-9]+`)
// ConvectiveOutlookNormalizer converts:
//
// standards.SchemaRawSPCConvectiveOutlookV1 -> standards.SchemaWeatherOutlookV2
//
// It maps SPC GeoJSON outlook features containing the configured location into
// canonical outlook polygons and adds matching day-level print-page discussions.
type ConvectiveOutlookNormalizer struct{}
func (ConvectiveOutlookNormalizer) Match(e event.Event) bool {
return strings.TrimSpace(e.Schema) == standards.SchemaRawSPCConvectiveOutlookV1
}
func (ConvectiveOutlookNormalizer) Normalize(ctx context.Context, in event.Event) (*event.Event, error) {
_ = ctx
fallbackAsOf := in.EmittedAt.UTC()
if in.EffectiveAt != nil && !in.EffectiveAt.IsZero() {
fallbackAsOf = in.EffectiveAt.UTC()
}
return normcommon.NormalizeJSON(
in,
outlookNormalizer,
standards.SchemaWeatherOutlookV2,
func(parsed spcprovider.RawConvectiveOutlookBundle) (model.WeatherOutlookRun, time.Time, error) {
return buildConvectiveOutlook(parsed, fallbackAsOf)
},
)
}
func buildConvectiveOutlook(bundle spcprovider.RawConvectiveOutlookBundle, fallbackAsOf time.Time) (model.WeatherOutlookRun, time.Time, error) {
if err := validateCoordinates(bundle.Latitude, bundle.Longitude); err != nil {
return model.WeatherOutlookRun{}, time.Time{}, err
}
discussions, latestDiscussionUpdated, err := parseDiscussions(bundle.Discussions)
if err != nil {
return model.WeatherOutlookRun{}, time.Time{}, err
}
products := orderedProducts(bundle.Products)
point := geo.Point{Latitude: bundle.Latitude, Longitude: bundle.Longitude}
outlooks := make([]model.WeatherOutlook, 0)
var latestIssue time.Time
for _, product := range products {
if err := validateProductMetadata(product); err != nil {
return model.WeatherOutlookRun{}, time.Time{}, err
}
if _, ok := discussions[product.Day]; !ok {
return model.WeatherOutlookRun{}, time.Time{}, fmt.Errorf("product %s: discussion for day %d is required", product.Key, product.Day)
}
collection, err := spcprovider.DecodeGeoJSON(product.Body)
if err != nil {
return model.WeatherOutlookRun{}, time.Time{}, fmt.Errorf("product %s: %w", product.Key, err)
}
for i, feature := range collection.Features {
if spcprovider.IsEmptyGeometryCollection(feature.Geometry) {
issuedAt, err := parseRequiredSPCTime(feature.Properties.IssueISO, fmt.Sprintf("product %s feature %d.ISSUE_ISO", product.Key, i))
if err != nil {
return model.WeatherOutlookRun{}, time.Time{}, err
}
if latestIssue.IsZero() || issuedAt.After(latestIssue) {
latestIssue = issuedAt
}
continue
}
outlook, err := mapFeature(product, feature, i, point)
if err != nil {
return model.WeatherOutlookRun{}, time.Time{}, err
}
if latestIssue.IsZero() || outlook.IssuedAt.After(latestIssue) {
latestIssue = outlook.IssuedAt
}
if !outlook.ContainsLocation {
continue
}
outlooks = append(outlooks, outlook)
}
}
runDiscussions, err := buildOutlookDiscussions(outlooks, discussions)
if err != nil {
return model.WeatherOutlookRun{}, time.Time{}, err
}
asOf := latestIssue
if asOf.IsZero() {
asOf = latestDiscussionUpdated
}
if asOf.IsZero() {
asOf = fallbackAsOf.UTC()
}
var issuedAt *time.Time
if !latestIssue.IsZero() {
t := latestIssue.UTC()
issuedAt = &t
}
lat := bundle.Latitude
lon := bundle.Longitude
run := model.WeatherOutlookRun{
LocationID: strings.TrimSpace(bundle.LocationID),
LocationName: strings.TrimSpace(bundle.LocationName),
Latitude: &lat,
Longitude: &lon,
AsOf: asOf.UTC(),
IssuedAt: issuedAt,
Outlooks: outlooks,
Discussions: runDiscussions,
}
return run, run.AsOf, nil
}
type parsedDiscussion struct {
Headline string
Summary string
Discussion string
UpdatedAt *time.Time
}
func parseDiscussions(pages []spcprovider.RawDiscussionPage) (map[int]parsedDiscussion, time.Time, error) {
out := map[int]parsedDiscussion{}
var latestUpdated time.Time
for _, page := range pages {
parsed, err := spcprovider.ParseDiscussionHTML(page.Body)
if err != nil {
return nil, time.Time{}, fmt.Errorf("discussion %s: %w", page.Key, err)
}
day := page.Day
if day == 0 {
if meta, ok := spcprovider.DiscussionProductByKey(page.Key); ok {
day = meta.Day
}
}
if day < 1 || day > 3 {
return nil, time.Time{}, fmt.Errorf("discussion %s: day must be 1, 2, or 3, got %d", page.Key, page.Day)
}
disc := parsedDiscussion{
Headline: strings.TrimSpace(parsed.Headline),
Summary: strings.TrimSpace(parsed.Summary),
Discussion: strings.TrimSpace(parsed.Discussion),
UpdatedAt: parsed.UpdatedAt,
}
out[day] = disc
if parsed.UpdatedAt != nil && (latestUpdated.IsZero() || parsed.UpdatedAt.After(latestUpdated)) {
latestUpdated = parsed.UpdatedAt.UTC()
}
}
return out, latestUpdated, nil
}
func buildOutlookDiscussions(outlooks []model.WeatherOutlook, discussions map[int]parsedDiscussion) ([]model.WeatherOutlookDiscussion, error) {
daysWithOutlooks := map[int]bool{}
for _, outlook := range outlooks {
daysWithOutlooks[outlook.Day] = true
}
days := make([]int, 0, len(daysWithOutlooks))
for day := range daysWithOutlooks {
days = append(days, day)
}
sort.Ints(days)
out := make([]model.WeatherOutlookDiscussion, 0, len(days))
for _, day := range days {
disc, ok := discussions[day]
if !ok {
return nil, fmt.Errorf("discussion for retained day %d is required", day)
}
out = append(out, model.WeatherOutlookDiscussion{
Day: day,
Headline: disc.Headline,
Summary: disc.Summary,
Discussion: disc.Discussion,
UpdatedAt: utcTimePtr(disc.UpdatedAt),
})
}
return out, nil
}
func utcTimePtr(t *time.Time) *time.Time {
if t == nil {
return nil
}
tt := t.UTC()
return &tt
}
func orderedProducts(products []spcprovider.RawOutlookProduct) []spcprovider.RawOutlookProduct {
out := make([]spcprovider.RawOutlookProduct, len(products))
copy(out, products)
sort.SliceStable(out, func(i, j int) bool {
if out[i].Day != out[j].Day {
return out[i].Day < out[j].Day
}
left := outlookTypeOrder(out[i].OutlookType)
right := outlookTypeOrder(out[j].OutlookType)
if left != right {
return left < right
}
return out[i].Key < out[j].Key
})
return out
}
func outlookTypeOrder(outlookType string) int {
switch strings.TrimSpace(outlookType) {
case spcprovider.OutlookTypeCategorical:
return 0
case spcprovider.OutlookTypeTornado:
return 1
case spcprovider.OutlookTypeHail:
return 2
case spcprovider.OutlookTypeWind:
return 3
default:
return outlookTypeUnknown
}
}
func validateProductMetadata(product spcprovider.RawOutlookProduct) error {
if product.Day < 1 || product.Day > 3 {
return fmt.Errorf("product %s: day must be 1, 2, or 3, got %d", product.Key, product.Day)
}
switch strings.TrimSpace(product.OutlookType) {
case spcprovider.OutlookTypeCategorical, spcprovider.OutlookTypeTornado, spcprovider.OutlookTypeHail, spcprovider.OutlookTypeWind:
return nil
default:
return fmt.Errorf("product %s: unsupported outlook type %q", product.Key, product.OutlookType)
}
}
func mapFeature(product spcprovider.RawOutlookProduct, feature spcprovider.GeoJSONFeature, index int, point geo.Point) (model.WeatherOutlook, error) {
fieldPrefix := fmt.Sprintf("product %s feature %d", product.Key, index)
props := feature.Properties
validFrom, err := parseRequiredSPCTime(props.ValidISO, fieldPrefix+".VALID_ISO")
if err != nil {
return model.WeatherOutlook{}, err
}
validTo, err := parseRequiredSPCTime(props.ExpireISO, fieldPrefix+".EXPIRE_ISO")
if err != nil {
return model.WeatherOutlook{}, err
}
issuedAt, err := parseRequiredSPCTime(props.IssueISO, fieldPrefix+".ISSUE_ISO")
if err != nil {
return model.WeatherOutlook{}, err
}
label := strings.TrimSpace(props.Label)
if label == "" {
return model.WeatherOutlook{}, fmt.Errorf("%s.LABEL is required", fieldPrefix)
}
if len(feature.Geometry) == 0 {
return model.WeatherOutlook{}, fmt.Errorf("%s.geometry is required", fieldPrefix)
}
containsLocation, err := geo.ContainsPoint(feature.Geometry, point)
if err != nil {
return model.WeatherOutlook{}, fmt.Errorf("%s.geometry: %w", fieldPrefix, err)
}
geometry := make(json.RawMessage, len(feature.Geometry))
copy(geometry, feature.Geometry)
return model.WeatherOutlook{
ID: outlookID(product.Day, product.OutlookType, label, issuedAt, validFrom, index),
Provider: providerSPC,
Product: productConvective,
Day: product.Day,
OutlookType: strings.TrimSpace(product.OutlookType),
Label: label,
LabelText: strings.TrimSpace(props.Label2),
SeverityRank: props.DN,
ValidFrom: validFrom,
ValidTo: validTo,
IssuedAt: issuedAt,
ExpiresAt: validTo,
Forecaster: strings.TrimSpace(props.Forecaster),
SourceURL: strings.TrimSpace(product.URL),
ImageURL: "",
ContainsLocation: containsLocation,
Geometry: geometry,
}, nil
}
func parseRequiredSPCTime(value, field string) (time.Time, error) {
if strings.TrimSpace(value) == "" {
return time.Time{}, fmt.Errorf("%s is required", field)
}
t, err := spcprovider.ParseISOTimestamp(value)
if err != nil {
return time.Time{}, fmt.Errorf("%s: %w", field, err)
}
return t.UTC(), nil
}
func outlookID(day int, outlookType, label string, issuedAt time.Time, validFrom time.Time, index int) string {
return fmt.Sprintf(
"spc-convective-day%d-%s-%s-%s-%s-%d",
day,
safeIDToken(outlookType),
safeIDToken(label),
issuedAt.UTC().Format(time.RFC3339),
validFrom.UTC().Format(time.RFC3339),
index,
)
}
func safeIDToken(value string) string {
value = strings.ToLower(strings.TrimSpace(value))
value = idTokenRE.ReplaceAllString(value, "-")
value = strings.Trim(value, "-")
if value == "" {
return "unknown"
}
return value
}
func validateCoordinates(latitude, longitude float64) error {
switch {
case math.IsNaN(latitude) || math.IsInf(latitude, 0):
return fmt.Errorf("latitude must be finite")
case math.IsNaN(longitude) || math.IsInf(longitude, 0):
return fmt.Errorf("longitude must be finite")
case latitude < -90 || latitude > 90:
return fmt.Errorf("latitude must be between -90 and 90, got %v", latitude)
case longitude < -180 || longitude > 180:
return fmt.Errorf("longitude must be between -180 and 180, got %v", longitude)
default:
return nil
}
}

View File

@@ -0,0 +1,526 @@
package spc
import (
"encoding/json"
"strings"
"testing"
"time"
"gitea.maximumdirect.net/ejr/feedkit/event"
spcprovider "gitea.maximumdirect.net/ejr/weatherfeeder/internal/providers/spc"
"gitea.maximumdirect.net/ejr/weatherfeeder/model"
"gitea.maximumdirect.net/ejr/weatherfeeder/standards"
)
func TestConvectiveOutlookNormalizerMatch(t *testing.T) {
n := ConvectiveOutlookNormalizer{}
if !n.Match(event.Event{Schema: standards.SchemaRawSPCConvectiveOutlookV1}) {
t.Fatalf("Match(raw SPC outlook) = false, want true")
}
if n.Match(event.Event{Schema: standards.SchemaRawNWSAlertsV1}) {
t.Fatalf("Match(raw NWS alerts) = true, want false")
}
}
func TestConvectiveOutlookNormalizerProducesCanonicalSchemaAndMapsSample(t *testing.T) {
out, err := (ConvectiveOutlookNormalizer{}).Normalize(nil, spcRawEvent(t, spcBundle(t, 38.5, -90.5)))
if err != nil {
t.Fatalf("Normalize() error = %v", err)
}
if out.Schema != standards.SchemaWeatherOutlookV2 {
t.Fatalf("Schema = %q, want %q", out.Schema, standards.SchemaWeatherOutlookV2)
}
if out.Kind != event.Kind(standards.KindOutlook) {
t.Fatalf("Kind = %q, want outlook", out.Kind)
}
run, ok := out.Payload.(model.WeatherOutlookRun)
if !ok {
t.Fatalf("Payload type = %T, want model.WeatherOutlookRun", out.Payload)
}
wantAsOf := time.Date(2026, 6, 11, 19, 45, 0, 0, time.UTC)
if !run.AsOf.Equal(wantAsOf) {
t.Fatalf("AsOf = %s, want %s", run.AsOf, wantAsOf)
}
if run.IssuedAt == nil || !run.IssuedAt.Equal(wantAsOf) {
t.Fatalf("IssuedAt = %v, want %s", run.IssuedAt, wantAsOf)
}
if out.EffectiveAt == nil || !out.EffectiveAt.Equal(wantAsOf) {
t.Fatalf("EffectiveAt = %v, want %s", out.EffectiveAt, wantAsOf)
}
if run.LocationID != "stl" || run.LocationName != "St. Louis, MO" {
t.Fatalf("location metadata = %q/%q", run.LocationID, run.LocationName)
}
if run.Latitude == nil || *run.Latitude != 38.5 || run.Longitude == nil || *run.Longitude != -90.5 {
t.Fatalf("coordinates = %v,%v", run.Latitude, run.Longitude)
}
if len(run.Outlooks) != 4 {
t.Fatalf("Outlooks length = %d, want 4", len(run.Outlooks))
}
assertAllOutlooksContainLocation(t, run.Outlooks)
assertDiscussionDays(t, run.Discussions, 1)
day1Discussion := run.Discussions[0]
if day1Discussion.Headline != "Day 1 Convective Outlook" {
t.Fatalf("day 1 Headline = %q", day1Discussion.Headline)
}
if !strings.Contains(day1Discussion.Summary, "central Plains") {
t.Fatalf("day 1 Summary = %q", day1Discussion.Summary)
}
if !strings.Contains(day1Discussion.Discussion, "...DISCUSSION...") {
t.Fatalf("day 1 Discussion missing product text: %q", day1Discussion.Discussion)
}
if !strings.HasPrefix(day1Discussion.Discussion, "SPC AC 111234") {
t.Fatalf("day 1 Discussion = %q, want SPC product code prefix", day1Discussion.Discussion)
}
got := run.Outlooks[0]
if got.Provider != "spc" || got.Product != "convective" {
t.Fatalf("provider/product = %q/%q", got.Provider, got.Product)
}
if got.Day != 1 || got.OutlookType != spcprovider.OutlookTypeCategorical {
t.Fatalf("day/type = %d/%q", got.Day, got.OutlookType)
}
if got.Label != "SLGT" || got.LabelText != "Slight Risk" {
t.Fatalf("label fields = %q/%q", got.Label, got.LabelText)
}
if got.SeverityRank == nil || *got.SeverityRank != 3 {
t.Fatalf("SeverityRank = %v, want 3", got.SeverityRank)
}
assertTime(t, "ValidFrom", got.ValidFrom, 2026, 6, 11, 13, 0, 0)
assertTime(t, "ValidTo", got.ValidTo, 2026, 6, 12, 12, 0, 0)
assertTime(t, "IssuedAt", got.IssuedAt, 2026, 6, 11, 12, 34, 56)
assertTime(t, "ExpiresAt", got.ExpiresAt, 2026, 6, 12, 12, 0, 0)
if got.Forecaster != "SMITH" {
t.Fatalf("Forecaster = %q, want SMITH", got.Forecaster)
}
if got.SourceURL != "https://example.invalid/day1_categorical.geojson" {
t.Fatalf("SourceURL = %q", got.SourceURL)
}
wantGeometry := `{"type":"Polygon","coordinates":[[[-91.0,38.0],[-90.0,38.0],[-90.0,39.0],[-91.0,39.0],[-91.0,38.0]]]}`
if string(got.Geometry) != wantGeometry {
t.Fatalf("Geometry = %s, want %s", got.Geometry, wantGeometry)
}
if !got.ContainsLocation {
t.Fatalf("ContainsLocation = false, want true")
}
if got.ID != "spc-convective-day1-categorical-slgt-2026-06-11T12:34:56Z-2026-06-11T13:00:00Z-0" {
t.Fatalf("ID = %q", got.ID)
}
}
func TestConvectiveOutlookNormalizerAcceptsTypedSourcePayload(t *testing.T) {
bundle := spcBundle(t, 38.5, -90.5)
in := spcRawEvent(t, bundle)
in.Payload = bundle
out, err := (ConvectiveOutlookNormalizer{}).Normalize(nil, in)
if err != nil {
t.Fatalf("Normalize() error = %v", err)
}
run := out.Payload.(model.WeatherOutlookRun)
if len(run.Outlooks) != 4 {
t.Fatalf("Outlooks length = %d, want 4", len(run.Outlooks))
}
assertAllOutlooksContainLocation(t, run.Outlooks)
assertDiscussionDays(t, run.Discussions, 1)
}
func TestConvectiveOutlookNormalizerEmitsEmptyLocalRunOutsidePolygons(t *testing.T) {
out, err := (ConvectiveOutlookNormalizer{}).Normalize(nil, spcRawEvent(t, spcBundle(t, 0, 0)))
if err != nil {
t.Fatalf("Normalize() error = %v", err)
}
run := out.Payload.(model.WeatherOutlookRun)
if len(run.Outlooks) != 0 {
t.Fatalf("Outlooks length = %d, want 0", len(run.Outlooks))
}
if len(run.Discussions) != 0 {
t.Fatalf("Discussions length = %d, want 0", len(run.Discussions))
}
wantAsOf := time.Date(2026, 6, 11, 19, 45, 0, 0, time.UTC)
if !run.AsOf.Equal(wantAsOf) {
t.Fatalf("AsOf = %s, want latest product issue time %s", run.AsOf, wantAsOf)
}
if run.IssuedAt == nil || !run.IssuedAt.Equal(wantAsOf) {
t.Fatalf("IssuedAt = %v, want latest product issue time %s", run.IssuedAt, wantAsOf)
}
if out.EffectiveAt == nil || !out.EffectiveAt.Equal(run.AsOf) {
t.Fatalf("EffectiveAt = %v, want run AsOf %s", out.EffectiveAt, run.AsOf)
}
}
func TestConvectiveOutlookNormalizerOrdersProductsByDayAndType(t *testing.T) {
bundle := spcBundle(t, 38.5, -90.5)
for i, j := 0, len(bundle.Products)-1; i < j; i, j = i+1, j-1 {
bundle.Products[i], bundle.Products[j] = bundle.Products[j], bundle.Products[i]
}
out, err := (ConvectiveOutlookNormalizer{}).Normalize(nil, spcRawEvent(t, bundle))
if err != nil {
t.Fatalf("Normalize() error = %v", err)
}
run := out.Payload.(model.WeatherOutlookRun)
got := []string{
run.Outlooks[0].OutlookType,
run.Outlooks[1].OutlookType,
run.Outlooks[2].OutlookType,
run.Outlooks[3].OutlookType,
}
want := []string{
spcprovider.OutlookTypeCategorical,
spcprovider.OutlookTypeTornado,
spcprovider.OutlookTypeHail,
spcprovider.OutlookTypeWind,
}
for i := range want {
if got[i] != want[i] || run.Outlooks[i].Day != 1 {
t.Fatalf("outlook[%d] = day %d type %q, want day 1 type %q", i, run.Outlooks[i].Day, got[i], want[i])
}
}
}
func TestConvectiveOutlookNormalizerMapsProbabilisticOutlookTypes(t *testing.T) {
out, err := (ConvectiveOutlookNormalizer{}).Normalize(nil, spcRawEvent(t, spcBundle(t, 38.5, -90.5)))
if err != nil {
t.Fatalf("Normalize() error = %v", err)
}
run := out.Payload.(model.WeatherOutlookRun)
assertAllOutlooksContainLocation(t, run.Outlooks)
for _, outlookType := range []string{
spcprovider.OutlookTypeTornado,
spcprovider.OutlookTypeHail,
spcprovider.OutlookTypeWind,
} {
if findOutlook(run.Outlooks, 1, outlookType) == nil {
t.Fatalf("missing day 1 outlook type %q", outlookType)
}
}
}
func TestConvectiveOutlookNormalizerSkipsEmptyGeometryCollectionPlaceholder(t *testing.T) {
bundle := spcBundle(t, 36, -99)
replaced := false
for i := range bundle.Products {
if bundle.Products[i].Day == 2 && bundle.Products[i].OutlookType == spcprovider.OutlookTypeTornado {
bundle.Products[i].Body = json.RawMessage(emptyGeometryCollectionGeoJSON())
replaced = true
}
}
if !replaced {
t.Fatalf("test setup did not find day 2 tornado product")
}
out, err := (ConvectiveOutlookNormalizer{}).Normalize(nil, spcRawEvent(t, bundle))
if err != nil {
t.Fatalf("Normalize() error = %v", err)
}
run := out.Payload.(model.WeatherOutlookRun)
if len(run.Outlooks) != 3 {
t.Fatalf("Outlooks length = %d, want 3", len(run.Outlooks))
}
assertAllOutlooksContainLocation(t, run.Outlooks)
assertDiscussionDays(t, run.Discussions, 2)
if got := findOutlook(run.Outlooks, 2, spcprovider.OutlookTypeTornado); got != nil {
t.Fatalf("day 2 tornado outlook = %+v, want nil placeholder skipped", *got)
}
wantAsOf := time.Date(2026, 6, 12, 10, 0, 0, 0, time.UTC)
if !run.AsOf.Equal(wantAsOf) {
t.Fatalf("AsOf = %s, want placeholder ISSUE_ISO %s", run.AsOf, wantAsOf)
}
if out.EffectiveAt == nil || !out.EffectiveAt.Equal(wantAsOf) {
t.Fatalf("EffectiveAt = %v, want placeholder ISSUE_ISO %s", out.EffectiveAt, wantAsOf)
}
}
func TestConvectiveOutlookNormalizerIncludesOnlyDayWithContainingPolygons(t *testing.T) {
out, err := (ConvectiveOutlookNormalizer{}).Normalize(nil, spcRawEvent(t, spcBundle(t, 36, -99)))
if err != nil {
t.Fatalf("Normalize() error = %v", err)
}
run := out.Payload.(model.WeatherOutlookRun)
if len(run.Outlooks) == 0 {
t.Fatalf("Outlooks length = 0, want retained day 2 outlooks")
}
assertAllOutlooksContainLocation(t, run.Outlooks)
for i, outlook := range run.Outlooks {
if outlook.Day != 2 {
t.Fatalf("outlook[%d].Day = %d, want 2", i, outlook.Day)
}
}
assertDiscussionDays(t, run.Discussions, 2)
}
func TestConvectiveOutlookNormalizerUsesOneDiscussionForMultipleSameDayOutlooks(t *testing.T) {
out, err := (ConvectiveOutlookNormalizer{}).Normalize(nil, spcRawEvent(t, spcBundle(t, 38.5, -90.5)))
if err != nil {
t.Fatalf("Normalize() error = %v", err)
}
run := out.Payload.(model.WeatherOutlookRun)
if got := countOutlooksByDay(run.Outlooks, 1); got < 2 {
t.Fatalf("day 1 outlook count = %d, want multiple", got)
}
assertAllOutlooksContainLocation(t, run.Outlooks)
assertDiscussionDays(t, run.Discussions, 1)
}
func TestConvectiveOutlookNormalizerPreservesCorrectionMarker(t *testing.T) {
bundle := spcBundle(t, 36, -99)
out, err := (ConvectiveOutlookNormalizer{}).Normalize(nil, spcRawEvent(t, bundle))
if err != nil {
t.Fatalf("Normalize() error = %v", err)
}
run := out.Payload.(model.WeatherOutlookRun)
got := findOutlook(run.Outlooks, 2, spcprovider.OutlookTypeTornado)
if got == nil {
t.Fatalf("missing day 2 tornado outlook")
}
assertDiscussionDays(t, run.Discussions, 2)
if !strings.Contains(run.Discussions[0].Headline, "CORR 1") {
t.Fatalf("day 2 headline = %q, want correction marker", run.Discussions[0].Headline)
}
if !strings.Contains(run.Discussions[0].Discussion, "CORR 1") {
t.Fatalf("day 2 discussion = %q, want correction marker", run.Discussions[0].Discussion)
}
}
func TestConvectiveOutlookNormalizerMissingRSSNormalizes(t *testing.T) {
bundle := spcBundle(t, 0, 0)
bundle.RSS = nil
if _, err := (ConvectiveOutlookNormalizer{}).Normalize(nil, spcRawEvent(t, bundle)); err != nil {
t.Fatalf("Normalize() error = %v", err)
}
}
func TestConvectiveOutlookNormalizerInvalidRequiredTimestampFailsWithContext(t *testing.T) {
bundle := spcBundle(t, 0, 0)
bundle.Products[0].Body = json.RawMessage(strings.Replace(
string(bundle.Products[0].Body),
`"ISSUE_ISO": "2026-06-11T12:34:56Z"`,
`"ISSUE_ISO": "bad"`,
1,
))
_, err := (ConvectiveOutlookNormalizer{}).Normalize(nil, spcRawEvent(t, bundle))
if err == nil {
t.Fatalf("Normalize() error = nil, want error")
}
if !strings.Contains(err.Error(), "product day1_categorical feature 0.ISSUE_ISO") {
t.Fatalf("error = %q, want product and feature context", err)
}
}
func TestConvectiveOutlookNormalizerInvalidGeometryFailsWithContext(t *testing.T) {
bundle := spcBundle(t, 0, 0)
bundle.Products[0].Body = json.RawMessage(strings.Replace(
string(bundle.Products[0].Body),
`"geometry": {`,
`"geometry": {"type":"LineString","coordinates":[[-91,38],[-90,39]]}, "oldGeometry": {`,
1,
))
_, err := (ConvectiveOutlookNormalizer{}).Normalize(nil, spcRawEvent(t, bundle))
if err == nil {
t.Fatalf("Normalize() error = nil, want error")
}
if !strings.Contains(err.Error(), "product day1_categorical feature 0.geometry") {
t.Fatalf("error = %q, want product and feature context", err)
}
}
func TestConvectiveOutlookNormalizerRejectsMissingLabel(t *testing.T) {
bundle := spcBundle(t, 0, 0)
bundle.Products[0].Body = json.RawMessage(strings.Replace(
string(bundle.Products[0].Body),
`"LABEL": "SLGT"`,
`"LABEL": ""`,
1,
))
_, err := (ConvectiveOutlookNormalizer{}).Normalize(nil, spcRawEvent(t, bundle))
if err == nil {
t.Fatalf("Normalize() error = nil, want error")
}
if !strings.Contains(err.Error(), "product day1_categorical feature 0.LABEL") {
t.Fatalf("error = %q, want label context", err)
}
}
func TestConvectiveOutlookNormalizerRejectsMissingDiscussion(t *testing.T) {
bundle := spcBundle(t, 38.5, -90.5)
bundle.Discussions = bundle.Discussions[1:]
_, err := (ConvectiveOutlookNormalizer{}).Normalize(nil, spcRawEvent(t, bundle))
if err == nil {
t.Fatalf("Normalize() error = nil, want error")
}
if !strings.Contains(err.Error(), "product day1_categorical: discussion for day 1 is required") {
t.Fatalf("error = %q, want missing discussion context", err)
}
}
func TestConvectiveOutlookNormalizerOutputJSONShape(t *testing.T) {
out, err := (ConvectiveOutlookNormalizer{}).Normalize(nil, spcRawEvent(t, spcBundle(t, 38.5, -90.5)))
if err != nil {
t.Fatalf("Normalize() error = %v", err)
}
raw, err := json.Marshal(out.Payload)
if err != nil {
t.Fatalf("Marshal(payload) error = %v", err)
}
got := string(raw)
for _, want := range []string{`"asOf"`, `"outlooks"`, `"discussions"`, `"containsLocation"`, `"geometry"`} {
if !strings.Contains(got, want) {
t.Fatalf("payload JSON missing %s: %s", want, got)
}
}
outlookStart := strings.Index(got, `"outlooks"`)
discussionStart := strings.Index(got, `"discussions"`)
if outlookStart == -1 || discussionStart == -1 || discussionStart <= outlookStart {
t.Fatalf("payload JSON has unexpected outlook/discussion order: %s", got)
}
outlookJSON := got[outlookStart:discussionStart]
for _, unwanted := range []string{`"headline"`, `"summary"`, `"discussion"`} {
if strings.Contains(outlookJSON, unwanted) {
t.Fatalf("outlook JSON exposed polygon-level prose key %s: %s", unwanted, got)
}
}
for _, unwanted := range []string{`"products"`, `"fetchedAt"`, `"body"`} {
if strings.Contains(got, unwanted) {
t.Fatalf("payload JSON exposed raw key %s: %s", unwanted, got)
}
}
}
func emptyGeometryCollectionGeoJSON() []byte {
return []byte(`{
"type": "FeatureCollection",
"features": [
{
"type": "Feature",
"properties": {
"VALID_ISO": "2026-06-12T12:00:00Z",
"EXPIRE_ISO": "2026-06-13T12:00:00Z",
"ISSUE_ISO": "2026-06-12T10:00:00Z",
"FORECASTER": "DOE",
"LABEL": "Less Than 2% All Areas",
"LABEL2": "",
"DN": 0
},
"geometry": {
"type": "GeometryCollection",
"geometries": []
}
}
]
}`)
}
func spcRawEvent(t *testing.T, bundle spcprovider.RawConvectiveOutlookBundle) event.Event {
t.Helper()
raw, err := json.Marshal(bundle)
if err != nil {
t.Fatalf("Marshal(bundle) error = %v", err)
}
effectiveAt := time.Date(2026, 6, 11, 19, 45, 0, 0, time.UTC)
return event.Event{
ID: "evt-spc-outlook-1",
Kind: event.Kind(standards.KindOutlook),
Source: "spc-test",
EmittedAt: time.Date(2026, 6, 11, 20, 5, 0, 0, time.UTC),
EffectiveAt: &effectiveAt,
Schema: standards.SchemaRawSPCConvectiveOutlookV1,
Payload: json.RawMessage(raw),
}
}
func spcBundle(t *testing.T, latitude, longitude float64) spcprovider.RawConvectiveOutlookBundle {
t.Helper()
fetchedAt := time.Date(2026, 6, 11, 20, 0, 0, 0, time.UTC)
products := make([]spcprovider.RawOutlookProduct, 0, len(spcprovider.GeoJSONProducts()))
for _, product := range spcprovider.GeoJSONProducts() {
products = append(products, spcprovider.RawOutlookProduct{
Key: product.Key,
Day: product.Day,
OutlookType: product.OutlookType,
URL: "https://example.invalid/" + product.Key + ".geojson",
FetchedAt: fetchedAt,
Body: json.RawMessage(geoJSONFixtureForProduct(t, product.Key)),
})
}
return spcprovider.RawConvectiveOutlookBundle{
LocationID: "stl",
LocationName: "St. Louis, MO",
Latitude: latitude,
Longitude: longitude,
FetchedAt: fetchedAt,
Products: products,
Discussions: []spcprovider.RawDiscussionPage{
{Key: "day1", Day: 1, URL: "https://example.invalid/day1.html", FetchedAt: fetchedAt, Body: string(readSPCTestFixture(t, "day1_prt.html"))},
{Key: "day2", Day: 2, URL: "https://example.invalid/day2.html", FetchedAt: fetchedAt, Body: string(readSPCTestFixture(t, "day2_prt_corr.html"))},
{Key: "day3", Day: 3, URL: "https://example.invalid/day3.html", FetchedAt: fetchedAt, Body: string(readSPCTestFixture(t, "day3_prt.html"))},
},
}
}
func geoJSONFixtureForProduct(t *testing.T, key string) []byte {
t.Helper()
switch {
case strings.HasPrefix(key, "day1_"):
return readSPCTestFixture(t, "day1_cat.geojson")
case strings.HasPrefix(key, "day2_"):
return readSPCTestFixture(t, "day2_torn.geojson")
case strings.HasPrefix(key, "day3_"):
return readSPCTestFixture(t, "day3_cat.geojson")
default:
t.Fatalf("unknown product key %q", key)
return nil
}
}
func findOutlook(outlooks []model.WeatherOutlook, day int, outlookType string) *model.WeatherOutlook {
for i := range outlooks {
if outlooks[i].Day == day && outlooks[i].OutlookType == outlookType {
return &outlooks[i]
}
}
return nil
}
func countOutlooksByDay(outlooks []model.WeatherOutlook, day int) int {
count := 0
for _, outlook := range outlooks {
if outlook.Day == day {
count++
}
}
return count
}
func assertAllOutlooksContainLocation(t *testing.T, outlooks []model.WeatherOutlook) {
t.Helper()
for i, outlook := range outlooks {
if !outlook.ContainsLocation {
t.Fatalf("outlook[%d].ContainsLocation = false, want true", i)
}
}
}
func assertDiscussionDays(t *testing.T, discussions []model.WeatherOutlookDiscussion, want ...int) {
t.Helper()
if len(discussions) != len(want) {
t.Fatalf("Discussions length = %d, want %d", len(discussions), len(want))
}
for i, day := range want {
if discussions[i].Day != day {
t.Fatalf("Discussions[%d].Day = %d, want %d", i, discussions[i].Day, day)
}
}
}
func assertTime(t *testing.T, name string, got time.Time, year int, month time.Month, day int, hour int, minute int, second int) {
t.Helper()
want := time.Date(year, month, day, hour, minute, second, 0, time.UTC)
if !got.Equal(want) {
t.Fatalf("%s = %s, want %s", name, got, want)
}
}

View File

@@ -0,0 +1,17 @@
package spc
import (
"os"
"path/filepath"
"testing"
)
func readSPCTestFixture(t *testing.T, name string) []byte {
t.Helper()
path := filepath.Join("..", "..", "providers", "spc", "testdata", name)
raw, err := os.ReadFile(path)
if err != nil {
t.Fatalf("read fixture %s: %v", path, err)
}
return raw
}

View File

@@ -0,0 +1,14 @@
package spc
import (
fknormalize "gitea.maximumdirect.net/ejr/feedkit/processors/normalize"
)
var builtins = []fknormalize.Normalizer{
ConvectiveOutlookNormalizer{},
}
// Register appends SPC normalizers in stable order.
func Register(in []fknormalize.Normalizer) []fknormalize.Normalizer {
return append(in, builtins...)
}

View File

@@ -0,0 +1,188 @@
package spc
import (
"fmt"
"html"
"regexp"
"strings"
"time"
)
var (
scriptBlockRE = regexp.MustCompile(`(?is)<script\b[^>]*>.*?</script>`)
preBlockRE = regexp.MustCompile(`(?is)<pre\b[^>]*>(.*?)</pre>`)
tagRE = regexp.MustCompile(`(?is)<[^>]+>`)
updatedRE = regexp.MustCompile(`(?im)^\s*Updated:\s*(.+?)\s*$`)
pageUpdatedRE = regexp.MustCompile(`(?i)\bUpdated:\s*((?:\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z)|(?:[A-Z][a-z]{2}\s+[A-Z][a-z]{2}\s+\d{1,2}\s+\d{2}:\d{2}:\d{2}\s+UTC\s+\d{4})|(?:\d{4}\s+UTC\s+[A-Z][a-z]{2}\s+[A-Z][a-z]{2}\s+\d{1,2}\s+\d{4})|(?:\d{4}Z\s+[A-Z][a-z]{2}\s+[A-Z][a-z]{2}\s+\d{1,2}\s+\d{4}))`)
productCodeRE = regexp.MustCompile(`(?i)^SPC\s+AC\s+\d+\s*$`)
sectionRE = regexp.MustCompile(`^\s*\.\.\.[A-Z0-9 /-]+\.{3}\s*$`)
)
// DiscussionText contains parsed text from an SPC print page.
type DiscussionText struct {
ProductTitle string
Headline string
Summary string
Discussion string
UpdatedAt *time.Time
}
// ExtractProductText extracts and cleans the first useful preformatted SPC
// product text block from a print page.
func ExtractProductText(rawHTML string) (string, error) {
matches := preBlockRE.FindAllStringSubmatch(rawHTML, -1)
for _, match := range matches {
if len(match) < 2 {
continue
}
text := cleanHTMLText(match[1])
if strings.TrimSpace(text) != "" {
return text, nil
}
}
return "", fmt.Errorf("no useful pre block found")
}
// ParseDiscussionHTML extracts SPC product text and page-level metadata from a
// print-page HTML document.
func ParseDiscussionHTML(rawHTML string) (DiscussionText, error) {
text, err := ExtractProductText(rawHTML)
if err != nil {
return DiscussionText{}, err
}
parsed := ParseDiscussionText(text)
if updatedAt := ParsePageUpdatedTimestamp(rawHTML); updatedAt != nil {
parsed.UpdatedAt = updatedAt
}
return parsed, nil
}
// ParseDiscussionText extracts common SPC narrative metadata from cleaned
// product text.
func ParseDiscussionText(text string) DiscussionText {
text = trimBlankLines(normalizeNewlines(text))
title := ParseProductTitle(text)
return DiscussionText{
ProductTitle: title,
Headline: title,
Summary: ExtractSummary(text),
Discussion: text,
UpdatedAt: ParseUpdatedTimestamp(text),
}
}
// ParsePageUpdatedTimestamp parses the page-level Updated row from an SPC print
// page. SPC currently places this outside the product <pre> block.
func ParsePageUpdatedTimestamp(rawHTML string) *time.Time {
text := cleanHTMLText(rawHTML)
text = strings.ReplaceAll(text, "\u00a0", " ")
text = strings.Join(strings.Fields(text), " ")
match := pageUpdatedRE.FindStringSubmatch(text)
if len(match) != 2 {
return nil
}
return parseUpdatedValue(match[1])
}
// ParseUpdatedTimestamp parses an SPC print-page Updated line when present.
func ParseUpdatedTimestamp(text string) *time.Time {
match := updatedRE.FindStringSubmatch(normalizeNewlines(text))
if len(match) != 2 {
return nil
}
return parseUpdatedValue(match[1])
}
// ParseProductTitle returns the first non-empty product line from cleaned text.
func ParseProductTitle(text string) string {
for _, line := range strings.Split(normalizeNewlines(text), "\n") {
line = strings.TrimSpace(line)
if line == "" || strings.HasPrefix(line, "Updated:") || productCodeRE.MatchString(line) {
continue
}
return line
}
return ""
}
// ParseHeadline returns the human-facing headline from cleaned text.
func ParseHeadline(text string) string {
return ParseProductTitle(text)
}
// ExtractSummary returns text under the ...SUMMARY... section through the next
// SPC section heading.
func ExtractSummary(text string) string {
lines := strings.Split(normalizeNewlines(text), "\n")
start := -1
for i, line := range lines {
if strings.EqualFold(strings.TrimSpace(line), "...SUMMARY...") {
start = i + 1
break
}
}
if start < 0 {
return ""
}
var out []string
for _, line := range lines[start:] {
if sectionRE.MatchString(line) {
break
}
out = append(out, line)
}
return trimBlankLines(strings.Join(out, "\n"))
}
func cleanHTMLText(raw string) string {
raw = scriptBlockRE.ReplaceAllString(raw, "")
raw = tagRE.ReplaceAllString(raw, "")
raw = html.UnescapeString(raw)
raw = normalizeNewlines(raw)
return trimBlankLines(raw)
}
func normalizeNewlines(text string) string {
text = strings.ReplaceAll(text, "\r\n", "\n")
text = strings.ReplaceAll(text, "\r", "\n")
return text
}
func trimBlankLines(text string) string {
lines := strings.Split(normalizeNewlines(text), "\n")
start := 0
for start < len(lines) && strings.TrimSpace(lines[start]) == "" {
start++
}
end := len(lines)
for end > start && strings.TrimSpace(lines[end-1]) == "" {
end--
}
return strings.Join(lines[start:end], "\n")
}
func parseUpdatedValue(value string) *time.Time {
value = strings.TrimSpace(value)
if value == "" {
return nil
}
if t := parseOptionalISOTimestamp(value); t != nil {
return t
}
for _, layout := range []string{
"Mon Jan 2 15:04:05 UTC 2006",
"1504 UTC Mon Jan 2 2006",
"1504Z Mon Jan 2 2006",
"3:04 PM UTC Mon Jan 2 2006",
time.RFC1123,
time.RFC1123Z,
} {
t, err := time.Parse(layout, value)
if err == nil {
tt := t.UTC()
return &tt
}
}
return nil
}

View File

@@ -0,0 +1,109 @@
package spc
import (
"strings"
"testing"
"time"
)
func TestExtractProductTextCleansPreBlock(t *testing.T) {
raw := string(readTestFile(t, "day1_prt.html"))
got, err := ExtractProductText(raw)
if err != nil {
t.Fatalf("ExtractProductText() error = %v", err)
}
if strings.Contains(got, "<script") || strings.Contains(got, "<pre") {
t.Fatalf("ExtractProductText() retained HTML: %q", got)
}
if strings.Contains(got, "ignore me") {
t.Fatalf("ExtractProductText() retained script content: %q", got)
}
if !strings.Contains(got, "Day 1 Convective Outlook") {
t.Fatalf("ExtractProductText() missing headline: %q", got)
}
if !strings.HasPrefix(got, "SPC AC 111234") {
t.Fatalf("ExtractProductText() = %q, want product code prefix", got)
}
if strings.HasPrefix(got, "\n") || strings.HasSuffix(got, "\n") {
t.Fatalf("ExtractProductText() retained surrounding blank lines: %q", got)
}
}
func TestParseDiscussionHTMLExtractsHeadlineSummaryAndUpdated(t *testing.T) {
got, err := ParseDiscussionHTML(string(readTestFile(t, "day1_prt.html")))
if err != nil {
t.Fatalf("ParseDiscussionHTML() error = %v", err)
}
if got.ProductTitle != "Day 1 Convective Outlook" {
t.Fatalf("ProductTitle = %q", got.ProductTitle)
}
if got.Headline != "Day 1 Convective Outlook" {
t.Fatalf("Headline = %q", got.Headline)
}
wantSummary := "Severe thunderstorms are possible across parts of the central Plains\nand mid Mississippi Valley this afternoon and evening."
if got.Summary != wantSummary {
t.Fatalf("Summary = %q, want %q", got.Summary, wantSummary)
}
if !strings.Contains(got.Discussion, "...DISCUSSION...") {
t.Fatalf("Discussion missing full text: %q", got.Discussion)
}
if !strings.HasPrefix(got.Discussion, "SPC AC 111234") {
t.Fatalf("Discussion = %q, want product code prefix", got.Discussion)
}
wantUpdated := time.Date(2026, 6, 11, 12, 45, 0, 0, time.UTC)
if got.UpdatedAt == nil || !got.UpdatedAt.Equal(wantUpdated) {
t.Fatalf("UpdatedAt = %v, want %s", got.UpdatedAt, wantUpdated)
}
}
func TestParseProductTitleSkipsSPCProductCode(t *testing.T) {
got := ParseProductTitle("SPC AC 101959\nDay 1 Convective Outlook\nNWS Storm Prediction Center Norman OK")
if got != "Day 1 Convective Outlook" {
t.Fatalf("ParseProductTitle() = %q, want Day 1 Convective Outlook", got)
}
}
func TestParseDiscussionTextPreservesCorrectionMarker(t *testing.T) {
got, err := ParseDiscussionHTML(string(readTestFile(t, "day2_prt_corr.html")))
if err != nil {
t.Fatalf("ParseDiscussionHTML() error = %v", err)
}
if !strings.Contains(got.Headline, "CORR 1") {
t.Fatalf("Headline = %q, want correction marker", got.Headline)
}
if !strings.Contains(got.Discussion, "CORR 1") {
t.Fatalf("Discussion = %q, want correction marker", got.Discussion)
}
wantUpdated := time.Date(2026, 6, 11, 17, 30, 0, 0, time.UTC)
if got.UpdatedAt == nil || !got.UpdatedAt.Equal(wantUpdated) {
t.Fatalf("UpdatedAt = %v, want %s", got.UpdatedAt, wantUpdated)
}
}
func TestParseUpdatedTimestampReturnsNilWhenAbsent(t *testing.T) {
text, err := ExtractProductText(string(readTestFile(t, "day2_prt_corr.html")))
if err != nil {
t.Fatalf("ExtractProductText() error = %v", err)
}
if got := ParseUpdatedTimestamp(text); got != nil {
t.Fatalf("ParseUpdatedTimestamp() = %v, want nil", got)
}
}
func TestParseUpdatedTimestampAcceptsSPCUTCFormat(t *testing.T) {
got := ParseUpdatedTimestamp("Updated: 1945 UTC Thu Jun 11 2026")
want := time.Date(2026, 6, 11, 19, 45, 0, 0, time.UTC)
if got == nil || !got.Equal(want) {
t.Fatalf("ParseUpdatedTimestamp() = %v, want %s", got, want)
}
}
func TestParsePageUpdatedTimestampAcceptsLiveSPCShape(t *testing.T) {
got := ParsePageUpdatedTimestamp(string(readTestFile(t, "day3_prt.html")))
want := time.Date(2026, 6, 11, 20, 0, 0, 0, time.UTC)
if got == nil || !got.Equal(want) {
t.Fatalf("ParsePageUpdatedTimestamp() = %v, want %s", got, want)
}
}

View File

@@ -0,0 +1,8 @@
// Package spc contains deterministic helper code for Storm Prediction Center
// products used by sources and normalizers.
//
// Rules:
// - No network I/O here.
// - Keep helpers deterministic and easy to unit test.
// - Preserve upstream payload fragments needed for canonical mapping.
package spc

View File

@@ -0,0 +1,17 @@
package spc
import (
"os"
"path/filepath"
"testing"
)
func readTestFile(t *testing.T, name string) []byte {
t.Helper()
path := filepath.Join("testdata", name)
raw, err := os.ReadFile(path)
if err != nil {
t.Fatalf("read %s: %v", path, err)
}
return raw
}

View File

@@ -0,0 +1,120 @@
package spc
import (
"bytes"
"encoding/json"
"fmt"
"strconv"
"strings"
)
// GeoJSONFeatureCollection is the minimal SPC outlook FeatureCollection shape
// needed by weatherfeeder.
type GeoJSONFeatureCollection struct {
Type string `json:"type"`
Features []GeoJSONFeature `json:"features"`
}
// GeoJSONFeature preserves typed SPC properties and compact raw geometry.
type GeoJSONFeature struct {
Type string `json:"type"`
Properties GeoJSONProperties `json:"properties"`
Geometry json.RawMessage `json:"geometry"`
}
// GeoJSONProperties contains the SPC fields used by canonical mapping.
type GeoJSONProperties struct {
ValidISO string `json:"VALID_ISO"`
ExpireISO string `json:"EXPIRE_ISO"`
IssueISO string `json:"ISSUE_ISO"`
Forecaster string `json:"FORECASTER"`
Label string `json:"LABEL"`
Label2 string `json:"LABEL2"`
DN *int `json:"DN"`
}
type geometryMetadata struct {
Type string `json:"type"`
Geometries []json.RawMessage `json:"geometries"`
}
// DecodeGeoJSON decodes an SPC GeoJSON outlook product and compacts feature
// geometry JSON for stable downstream storage.
func DecodeGeoJSON(raw []byte) (GeoJSONFeatureCollection, error) {
var collection GeoJSONFeatureCollection
if err := json.Unmarshal(raw, &collection); err != nil {
return GeoJSONFeatureCollection{}, fmt.Errorf("decode geojson: %w", err)
}
for i := range collection.Features {
geom, err := compactJSON(collection.Features[i].Geometry)
if err != nil {
return GeoJSONFeatureCollection{}, fmt.Errorf("features[%d].geometry: %w", i, err)
}
collection.Features[i].Geometry = geom
}
return collection, nil
}
// IsEmptyGeometryCollection reports whether raw is SPC's no-polygon placeholder
// geometry shape: a GeometryCollection with no child geometries.
func IsEmptyGeometryCollection(raw json.RawMessage) bool {
var meta geometryMetadata
if err := json.Unmarshal(raw, &meta); err != nil {
return false
}
return meta.Type == "GeometryCollection" && len(meta.Geometries) == 0
}
func (p *GeoJSONProperties) UnmarshalJSON(raw []byte) error {
type alias GeoJSONProperties
var aux struct {
alias
DN any `json:"DN"`
}
if err := json.Unmarshal(raw, &aux); err != nil {
return err
}
*p = GeoJSONProperties(aux.alias)
dn, err := parseSeverityRank(aux.DN)
if err != nil {
return err
}
p.DN = dn
return nil
}
func parseSeverityRank(value any) (*int, error) {
switch v := value.(type) {
case nil:
return nil, nil
case float64:
rank := int(v)
if float64(rank) != v {
return nil, fmt.Errorf("DN must be an integer, got %v", v)
}
return &rank, nil
case string:
v = strings.TrimSpace(v)
if v == "" {
return nil, nil
}
rank, err := strconv.Atoi(v)
if err != nil {
return nil, fmt.Errorf("DN must be an integer, got %q", v)
}
return &rank, nil
default:
return nil, fmt.Errorf("DN must be an integer or string, got %T", value)
}
}
func compactJSON(raw json.RawMessage) (json.RawMessage, error) {
if len(raw) == 0 {
return nil, fmt.Errorf("missing")
}
var buf bytes.Buffer
if err := json.Compact(&buf, raw); err != nil {
return nil, err
}
return json.RawMessage(buf.Bytes()), nil
}

View File

@@ -0,0 +1,115 @@
package spc
import (
"strings"
"testing"
"time"
)
func TestDecodeGeoJSONExposesSPCPropertiesAndCompactGeometry(t *testing.T) {
raw := readTestFile(t, "day1_cat.geojson")
got, err := DecodeGeoJSON(raw)
if err != nil {
t.Fatalf("DecodeGeoJSON() error = %v", err)
}
if got.Type != "FeatureCollection" {
t.Fatalf("Type = %q, want FeatureCollection", got.Type)
}
if len(got.Features) != 1 {
t.Fatalf("Features length = %d, want 1", len(got.Features))
}
feature := got.Features[0]
props := feature.Properties
if props.ValidISO != "2026-06-11T13:00:00Z" {
t.Fatalf("VALID_ISO = %q", props.ValidISO)
}
if props.ExpireISO != "2026-06-12T12:00:00Z" {
t.Fatalf("EXPIRE_ISO = %q", props.ExpireISO)
}
if props.IssueISO != "2026-06-11T12:34:56Z" {
t.Fatalf("ISSUE_ISO = %q", props.IssueISO)
}
if props.Forecaster != "SMITH" {
t.Fatalf("FORECASTER = %q", props.Forecaster)
}
if props.Label != "SLGT" {
t.Fatalf("LABEL = %q", props.Label)
}
if props.Label2 != "Slight Risk" {
t.Fatalf("LABEL2 = %q", props.Label2)
}
if props.DN == nil || *props.DN != 3 {
t.Fatalf("DN = %v, want 3", props.DN)
}
wantGeometry := `{"type":"Polygon","coordinates":[[[-91.0,38.0],[-90.0,38.0],[-90.0,39.0],[-91.0,39.0],[-91.0,38.0]]]}`
if string(feature.Geometry) != wantGeometry {
t.Fatalf("Geometry = %s, want %s", feature.Geometry, wantGeometry)
}
if strings.Contains(string(feature.Geometry), "\n") || strings.Contains(string(feature.Geometry), " ") {
t.Fatalf("Geometry is not compact: %q", feature.Geometry)
}
}
func TestDecodeGeoJSONParsesSeverityRankString(t *testing.T) {
raw := readTestFile(t, "day2_torn.geojson")
got, err := DecodeGeoJSON(raw)
if err != nil {
t.Fatalf("DecodeGeoJSON() error = %v", err)
}
props := got.Features[0].Properties
if props.DN == nil || *props.DN != 5 {
t.Fatalf("DN = %v, want 5", props.DN)
}
}
func TestIsEmptyGeometryCollection(t *testing.T) {
tests := []struct {
name string
raw string
want bool
}{
{
name: "empty geometry collection",
raw: `{"type":"GeometryCollection","geometries":[]}`,
want: true,
},
{
name: "non-empty geometry collection",
raw: `{"type":"GeometryCollection","geometries":[{"type":"Polygon","coordinates":[]} ]}`,
want: false,
},
{
name: "polygon",
raw: `{"type":"Polygon","coordinates":[]}`,
want: false,
},
{
name: "invalid json",
raw: `{`,
want: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := IsEmptyGeometryCollection([]byte(tt.raw)); got != tt.want {
t.Fatalf("IsEmptyGeometryCollection() = %v, want %v", got, tt.want)
}
})
}
}
func TestParseISOTimestampTrimsAndReturnsUTC(t *testing.T) {
got, err := ParseISOTimestamp(" 2026-06-11T12:34:56Z ")
if err != nil {
t.Fatalf("ParseISOTimestamp() error = %v", err)
}
want := time.Date(2026, 6, 11, 12, 34, 56, 0, time.UTC)
if !got.Equal(want) {
t.Fatalf("ParseISOTimestamp() = %s, want %s", got, want)
}
}

View File

@@ -0,0 +1,86 @@
package spc
import "fmt"
const (
OutlookTypeCategorical = "categorical"
OutlookTypeTornado = "tornado"
OutlookTypeHail = "hail"
OutlookTypeWind = "wind"
)
// GeoJSONProduct describes one required SPC convective outlook GeoJSON product.
type GeoJSONProduct struct {
Key string
Day int
OutlookType string
URL string
}
// DiscussionProduct describes one required SPC convective outlook print page.
type DiscussionProduct struct {
Key string
Day int
URL string
}
var geoJSONProducts = []GeoJSONProduct{
{Key: "day1_categorical", Day: 1, OutlookType: OutlookTypeCategorical, URL: "https://www.spc.noaa.gov/products/outlook/day1otlk_cat.nolyr.geojson"},
{Key: "day1_tornado", Day: 1, OutlookType: OutlookTypeTornado, URL: "https://www.spc.noaa.gov/products/outlook/day1otlk_torn.nolyr.geojson"},
{Key: "day1_hail", Day: 1, OutlookType: OutlookTypeHail, URL: "https://www.spc.noaa.gov/products/outlook/day1otlk_hail.nolyr.geojson"},
{Key: "day1_wind", Day: 1, OutlookType: OutlookTypeWind, URL: "https://www.spc.noaa.gov/products/outlook/day1otlk_wind.nolyr.geojson"},
{Key: "day2_categorical", Day: 2, OutlookType: OutlookTypeCategorical, URL: "https://www.spc.noaa.gov/products/outlook/day2otlk_cat.nolyr.geojson"},
{Key: "day2_tornado", Day: 2, OutlookType: OutlookTypeTornado, URL: "https://www.spc.noaa.gov/products/outlook/day2otlk_torn.nolyr.geojson"},
{Key: "day2_hail", Day: 2, OutlookType: OutlookTypeHail, URL: "https://www.spc.noaa.gov/products/outlook/day2otlk_hail.nolyr.geojson"},
{Key: "day2_wind", Day: 2, OutlookType: OutlookTypeWind, URL: "https://www.spc.noaa.gov/products/outlook/day2otlk_wind.nolyr.geojson"},
{Key: "day3_categorical", Day: 3, OutlookType: OutlookTypeCategorical, URL: "https://www.spc.noaa.gov/products/outlook/day3otlk_cat.nolyr.geojson"},
}
var discussionProducts = []DiscussionProduct{
{Key: "day1", Day: 1, URL: "https://www.spc.noaa.gov/products/outlook/day1otlk_prt.html"},
{Key: "day2", Day: 2, URL: "https://www.spc.noaa.gov/products/outlook/day2otlk_prt.html"},
{Key: "day3", Day: 3, URL: "https://www.spc.noaa.gov/products/outlook/day3otlk_prt.html"},
}
// GeoJSONProducts returns the required SPC convective outlook GeoJSON products
// in stable day/type order.
func GeoJSONProducts() []GeoJSONProduct {
out := make([]GeoJSONProduct, len(geoJSONProducts))
copy(out, geoJSONProducts)
return out
}
// DiscussionProducts returns the required SPC convective outlook print pages in
// stable day order.
func DiscussionProducts() []DiscussionProduct {
out := make([]DiscussionProduct, len(discussionProducts))
copy(out, discussionProducts)
return out
}
// GeoJSONProductByKey returns product metadata for a configured product key.
func GeoJSONProductByKey(key string) (GeoJSONProduct, bool) {
for _, product := range geoJSONProducts {
if product.Key == key {
return product, true
}
}
return GeoJSONProduct{}, false
}
// DiscussionProductByKey returns discussion metadata for a configured day key.
func DiscussionProductByKey(key string) (DiscussionProduct, bool) {
for _, product := range discussionProducts {
if product.Key == key {
return product, true
}
}
return DiscussionProduct{}, false
}
func validateProductDay(day int) error {
if day < 1 || day > 3 {
return fmt.Errorf("day must be 1, 2, or 3, got %d", day)
}
return nil
}

View File

@@ -0,0 +1,53 @@
package spc
import "testing"
func TestGeoJSONProductsStableOrder(t *testing.T) {
got := GeoJSONProducts()
if len(got) != 9 {
t.Fatalf("GeoJSONProducts() length = %d, want 9", len(got))
}
wantKeys := []string{
"day1_categorical",
"day1_tornado",
"day1_hail",
"day1_wind",
"day2_categorical",
"day2_tornado",
"day2_hail",
"day2_wind",
"day3_categorical",
}
for i, want := range wantKeys {
if got[i].Key != want {
t.Fatalf("GeoJSONProducts()[%d].Key = %q, want %q", i, got[i].Key, want)
}
if err := validateProductDay(got[i].Day); err != nil {
t.Fatalf("GeoJSONProducts()[%d].Day invalid: %v", i, err)
}
if got[i].URL == "" {
t.Fatalf("GeoJSONProducts()[%d].URL is empty", i)
}
}
}
func TestDiscussionProductsStableOrder(t *testing.T) {
got := DiscussionProducts()
if len(got) != 3 {
t.Fatalf("DiscussionProducts() length = %d, want 3", len(got))
}
wantKeys := []string{"day1", "day2", "day3"}
for i, want := range wantKeys {
if got[i].Key != want {
t.Fatalf("DiscussionProducts()[%d].Key = %q, want %q", i, got[i].Key, want)
}
if got[i].Day != i+1 {
t.Fatalf("DiscussionProducts()[%d].Day = %d, want %d", i, got[i].Day, i+1)
}
if got[i].URL == "" {
t.Fatalf("DiscussionProducts()[%d].URL is empty", i)
}
}
}

View File

@@ -0,0 +1,45 @@
package spc
import (
"encoding/json"
"time"
)
// RawConvectiveOutlookBundle is the provider payload shape for SPC convective
// outlook fetch bundles.
type RawConvectiveOutlookBundle struct {
LocationID string `json:"locationId,omitempty"`
LocationName string `json:"locationName,omitempty"`
Latitude float64 `json:"latitude"`
Longitude float64 `json:"longitude"`
FetchedAt time.Time `json:"fetchedAt"`
Products []RawOutlookProduct `json:"products"`
Discussions []RawDiscussionPage `json:"discussions"`
RSS *RawRSSFeed `json:"rss,omitempty"`
}
// RawOutlookProduct contains one fetched SPC GeoJSON product.
type RawOutlookProduct struct {
Key string `json:"key"`
Day int `json:"day"`
OutlookType string `json:"outlookType"`
URL string `json:"url"`
FetchedAt time.Time `json:"fetchedAt"`
Body json.RawMessage `json:"body"`
}
// RawDiscussionPage contains one fetched SPC print page.
type RawDiscussionPage struct {
Key string `json:"key"`
Day int `json:"day"`
URL string `json:"url"`
FetchedAt time.Time `json:"fetchedAt"`
Body string `json:"body"`
}
// RawRSSFeed contains optional fetched SPC RSS metadata.
type RawRSSFeed struct {
URL string `json:"url"`
FetchedAt time.Time `json:"fetchedAt"`
Body string `json:"body"`
}

View File

@@ -0,0 +1,51 @@
package spc
import (
"encoding/json"
"testing"
"time"
)
func TestRawConvectiveOutlookBundleJSONShape(t *testing.T) {
fetchedAt := time.Date(2026, 6, 11, 20, 0, 0, 0, time.UTC)
bundle := RawConvectiveOutlookBundle{
LocationID: "stl",
LocationName: "St. Louis, MO",
Latitude: 38.6239,
Longitude: -90.3571,
FetchedAt: fetchedAt,
Products: []RawOutlookProduct{{
Key: "day1_categorical",
Day: 1,
OutlookType: OutlookTypeCategorical,
URL: "https://example.invalid/day1.geojson",
FetchedAt: fetchedAt,
Body: json.RawMessage(`{"type":"FeatureCollection","features":[]}`),
}},
Discussions: []RawDiscussionPage{{
Key: "day1",
Day: 1,
URL: "https://example.invalid/day1.html",
FetchedAt: fetchedAt,
Body: "Day 1 Convective Outlook",
}},
}
raw, err := json.Marshal(bundle)
if err != nil {
t.Fatalf("Marshal() error = %v", err)
}
var got map[string]any
if err := json.Unmarshal(raw, &got); err != nil {
t.Fatalf("Unmarshal() error = %v", err)
}
for _, key := range []string{"locationId", "locationName", "latitude", "longitude", "fetchedAt", "products", "discussions"} {
if _, ok := got[key]; !ok {
t.Fatalf("marshaled bundle missing key %q in %s", key, raw)
}
}
if _, ok := got["rss"]; ok {
t.Fatalf("marshaled bundle included empty rss: %s", raw)
}
}

View File

@@ -0,0 +1,81 @@
package spc
import (
"encoding/xml"
"fmt"
"strings"
"time"
)
// RSSFeed is a minimal view of the optional SPC RSS feed.
type RSSFeed struct {
Title string
Link string
Description string
LastBuildDate *time.Time
Items []RSSItem
}
// RSSItem is a minimal view of one optional SPC RSS item.
type RSSItem struct {
Title string
Link string
Description string
PubDate string
GUID string
}
// ParseRSSFeed decodes supplemental SPC RSS metadata.
func ParseRSSFeed(raw string) (RSSFeed, error) {
var doc struct {
Channel struct {
Title string `xml:"title"`
Link string `xml:"link"`
Description string `xml:"description"`
LastBuildDate string `xml:"lastBuildDate"`
Items []struct {
Title string `xml:"title"`
Link string `xml:"link"`
Description string `xml:"description"`
PubDate string `xml:"pubDate"`
GUID string `xml:"guid"`
} `xml:"item"`
} `xml:"channel"`
}
if err := xml.Unmarshal([]byte(raw), &doc); err != nil {
return RSSFeed{}, fmt.Errorf("decode rss: %w", err)
}
feed := RSSFeed{
Title: strings.TrimSpace(doc.Channel.Title),
Link: strings.TrimSpace(doc.Channel.Link),
Description: strings.TrimSpace(doc.Channel.Description),
LastBuildDate: parseRSSDate(doc.Channel.LastBuildDate),
Items: make([]RSSItem, 0, len(doc.Channel.Items)),
}
for _, item := range doc.Channel.Items {
feed.Items = append(feed.Items, RSSItem{
Title: strings.TrimSpace(item.Title),
Link: strings.TrimSpace(item.Link),
Description: strings.TrimSpace(item.Description),
PubDate: strings.TrimSpace(item.PubDate),
GUID: strings.TrimSpace(item.GUID),
})
}
return feed, nil
}
func parseRSSDate(value string) *time.Time {
value = strings.TrimSpace(value)
if value == "" {
return nil
}
for _, layout := range []string{time.RFC1123Z, time.RFC1123} {
t, err := time.Parse(layout, value)
if err == nil {
tt := t.UTC()
return &tt
}
}
return nil
}

View File

@@ -0,0 +1,43 @@
package spc
import (
"testing"
"time"
)
func TestParseRSSFeed(t *testing.T) {
const raw = `<?xml version="1.0"?>
<rss version="2.0">
<channel>
<title>SPC AC RSS</title>
<link>https://www.spc.noaa.gov/products/</link>
<description>SPC products</description>
<lastBuildDate>Thu, 11 Jun 2026 19:00:00 +0000</lastBuildDate>
<item>
<title>Day 1 Convective Outlook</title>
<link>https://www.spc.noaa.gov/products/outlook/day1otlk.html</link>
<description>Outlook text</description>
<pubDate>Thu, 11 Jun 2026 18:55:00 +0000</pubDate>
<guid>day1</guid>
</item>
</channel>
</rss>`
got, err := ParseRSSFeed(raw)
if err != nil {
t.Fatalf("ParseRSSFeed() error = %v", err)
}
if got.Title != "SPC AC RSS" {
t.Fatalf("Title = %q", got.Title)
}
wantBuild := time.Date(2026, 6, 11, 19, 0, 0, 0, time.UTC)
if got.LastBuildDate == nil || !got.LastBuildDate.Equal(wantBuild) {
t.Fatalf("LastBuildDate = %v, want %s", got.LastBuildDate, wantBuild)
}
if len(got.Items) != 1 {
t.Fatalf("Items length = %d, want 1", len(got.Items))
}
if got.Items[0].GUID != "day1" {
t.Fatalf("Item GUID = %q", got.Items[0].GUID)
}
}

View File

@@ -0,0 +1,29 @@
{
"type": "FeatureCollection",
"features": [
{
"type": "Feature",
"properties": {
"VALID_ISO": "2026-06-11T13:00:00Z",
"EXPIRE_ISO": "2026-06-12T12:00:00Z",
"ISSUE_ISO": "2026-06-11T12:34:56Z",
"FORECASTER": "SMITH",
"LABEL": "SLGT",
"LABEL2": "Slight Risk",
"DN": 3
},
"geometry": {
"type": "Polygon",
"coordinates": [
[
[-91.0, 38.0],
[-90.0, 38.0],
[-90.0, 39.0],
[-91.0, 39.0],
[-91.0, 38.0]
]
]
}
}
]
}

View File

@@ -0,0 +1,22 @@
<!doctype html>
<html>
<head><title>Day 1 Convective Outlook</title></head>
<body>
<table>
<tr><td align="center" class="rpttext" nowrap>Updated:&nbsp;Thu Jun 11 12:45:00 UTC 2026&nbsp;(<a href="archive/day1-geojson.zip">geojson</a>)</td></tr>
</table>
<pre>
<script>window.bad = "<b>ignore me</b>";</script>
SPC AC 111234
Day 1 Convective Outlook
NWS Storm Prediction Center Norman OK
...SUMMARY...
Severe thunderstorms are possible across parts of the central Plains
and mid Mississippi Valley this afternoon and evening.
...DISCUSSION...
The primary threats will be damaging wind and large hail.
</pre>
</body>
</html>

View File

@@ -0,0 +1,19 @@
<!doctype html>
<html>
<body>
<table>
<tr><td class="rpttext">Updated:&nbsp;Thu Jun 11 17:30:00 UTC 2026&nbsp;</td></tr>
</table>
<pre>
SPC AC 111730
Day 2 Convective Outlook CORR 1
NWS Storm Prediction Center Norman OK
...SUMMARY...
Scattered severe thunderstorms remain possible across the southern Plains.
...DISCUSSION...
Corrected outlook text remains otherwise unchanged.
</pre>
</body>
</html>

View File

@@ -0,0 +1,29 @@
{
"type": "FeatureCollection",
"features": [
{
"type": "Feature",
"properties": {
"VALID_ISO": "2026-06-12T12:00:00Z",
"EXPIRE_ISO": "2026-06-13T12:00:00Z",
"ISSUE_ISO": "2026-06-11T17:30:00Z",
"FORECASTER": "DOE",
"LABEL": "5",
"LABEL2": "5% Tornado",
"DN": "5"
},
"geometry": {
"type": "Polygon",
"coordinates": [
[
[-100.0, 35.0],
[-98.0, 35.0],
[-98.0, 37.0],
[-100.0, 37.0],
[-100.0, 35.0]
]
]
}
}
]
}

View File

@@ -0,0 +1,31 @@
{
"type": "FeatureCollection",
"features": [
{
"type": "Feature",
"properties": {
"VALID_ISO": "2026-06-13T12:00:00Z",
"EXPIRE_ISO": "2026-06-14T12:00:00Z",
"ISSUE_ISO": "2026-06-11T19:45:00Z",
"FORECASTER": "LEE",
"LABEL": "MRGL",
"LABEL2": "Marginal Risk",
"DN": 2
},
"geometry": {
"type": "MultiPolygon",
"coordinates": [
[
[
[-103.0, 34.0],
[-101.0, 34.0],
[-101.0, 36.0],
[-103.0, 36.0],
[-103.0, 34.0]
]
]
]
}
}
]
}

View File

@@ -0,0 +1,19 @@
<!doctype html>
<html>
<body>
<table>
<tr><td class="rpttext">Updated:&nbsp;Thu Jun 11 20:00:00 UTC 2026&nbsp;</td></tr>
</table>
<pre>
SPC AC 112000
Day 3 Convective Outlook
NWS Storm Prediction Center Norman OK
...SUMMARY...
A corridor of strong to severe storms may develop near a frontal zone.
...DISCUSSION...
Confidence remains moderate for organized storms.
</pre>
</body>
</html>

View File

@@ -0,0 +1,24 @@
package spc
import (
"strings"
"time"
)
// ParseISOTimestamp parses SPC ISO timestamps from GeoJSON properties.
func ParseISOTimestamp(value string) (time.Time, error) {
return time.Parse(time.RFC3339, strings.TrimSpace(value))
}
func parseOptionalISOTimestamp(value string) *time.Time {
value = strings.TrimSpace(value)
if value == "" {
return nil
}
t, err := ParseISOTimestamp(value)
if err != nil {
return nil
}
tt := t.UTC()
return &tt
}

View File

@@ -11,6 +11,7 @@
// - weather.forecast_discussion.v1 -> model.WeatherForecastDiscussion
// - weather.weather_story.v1 -> model.WeatherStoryRun
// - weather.alert.v1 -> model.WeatherAlertRun
// - weather.outlook.v2 -> model.WeatherOutlookRun
//
// Parent/child relationships:
// - observations.event_id -> observation_present_weather.event_id
@@ -19,9 +20,11 @@
// - weather_story_runs.event_id -> weather_stories.run_event_id
// - alert_runs.event_id -> alerts.run_event_id
// - alerts.(run_event_id, alert_index) -> alert_references.(run_event_id, alert_index)
// - outlook_runs.event_id -> outlooks.run_event_id
// - outlook_runs.event_id -> outlook_discussions.run_event_id
//
// Dedupe and retention behavior:
// - Parent primary keys (event_id): observations, forecasts, alert_runs.
// - Parent primary keys (event_id): observations, forecasts, alert_runs, outlook_runs.
// - Child primary keys use positional indexes to preserve payload order.
// - Prune columns:
// - observations.observed_at
@@ -35,11 +38,14 @@
// - alert_runs.as_of
// - alerts.as_of
// - alert_references.as_of
// - outlook_runs.as_of
// - outlooks.as_of
// - outlook_discussions.as_of
//
// Envelope field mapping (shared parent columns)
//
// These columns exist on parent tables such as observations, forecasts,
// forecast_discussions, weather_story_runs, and alert_runs:
// forecast_discussions, weather_story_runs, alert_runs, and outlook_runs:
// - event_id TEXT -> event.id
// - event_kind TEXT -> event.kind
// - event_source TEXT -> event.source
@@ -192,6 +198,7 @@
// - sent TIMESTAMPTZ NULL -> payload.alerts[i].sent
// - effective TIMESTAMPTZ NULL -> payload.alerts[i].effective
// - onset TIMESTAMPTZ NULL -> payload.alerts[i].onset
// - ends TIMESTAMPTZ NULL -> payload.alerts[i].ends
// - expires TIMESTAMPTZ NULL -> payload.alerts[i].expires
// - area_description TEXT NULL -> payload.alerts[i].areaDescription
// - sender_name TEXT NULL -> payload.alerts[i].senderName
@@ -208,6 +215,57 @@
// - sender TEXT NULL -> payload.alerts[i].references[j].sender
// - sent TIMESTAMPTZ NULL -> payload.alerts[i].references[j].sent
//
// 10. outlook_runs (PK: event_id)
//
// - event_id TEXT -> event.id
// - event_kind TEXT -> event.kind
// - event_source TEXT -> event.source
// - event_schema TEXT -> event.schema
// - event_emitted_at TIMESTAMPTZ -> event.emitted_at
// - event_effective_at TIMESTAMPTZ NULL -> event.effective_at
// - location_id TEXT NULL -> payload.locationId
// - location_name TEXT NULL -> payload.locationName
// - latitude DOUBLE PRECISION NULL -> payload.latitude
// - longitude DOUBLE PRECISION NULL -> payload.longitude
// - as_of TIMESTAMPTZ -> payload.asOf
// - issued_at TIMESTAMPTZ NULL -> payload.issuedAt
// - outlook_count INTEGER -> len(payload.outlooks)
// - discussion_count INTEGER -> len(payload.discussions)
//
// 11. outlooks (PK: run_event_id, outlook_index)
//
// - run_event_id TEXT -> outlook_runs.event_id / payload.outlooks[i]
// - outlook_index INTEGER -> i (array position in payload.outlooks)
// - as_of TIMESTAMPTZ -> payload.asOf (copied from parent)
// - outlook_id TEXT -> payload.outlooks[i].id
// - provider TEXT -> payload.outlooks[i].provider
// - product TEXT -> payload.outlooks[i].product
// - day INTEGER -> payload.outlooks[i].day
// - outlook_type TEXT -> payload.outlooks[i].outlookType
// - label TEXT -> payload.outlooks[i].label
// - label_text TEXT NULL -> payload.outlooks[i].labelText
// - severity_rank INTEGER NULL -> payload.outlooks[i].severityRank
// - valid_from TIMESTAMPTZ -> payload.outlooks[i].validFrom
// - valid_to TIMESTAMPTZ -> payload.outlooks[i].validTo
// - issued_at TIMESTAMPTZ -> payload.outlooks[i].issuedAt
// - expires_at TIMESTAMPTZ -> payload.outlooks[i].expiresAt
// - forecaster TEXT NULL -> payload.outlooks[i].forecaster
// - source_url TEXT NULL -> payload.outlooks[i].sourceUrl
// - image_url TEXT NULL -> payload.outlooks[i].imageUrl
// - contains_location BOOLEAN -> payload.outlooks[i].containsLocation
// - geometry_json TEXT -> compact JSON payload.outlooks[i].geometry
//
// 12. outlook_discussions (PK: run_event_id, discussion_index)
//
// - run_event_id TEXT -> outlook_runs.event_id / payload.discussions[i]
// - discussion_index INTEGER -> i (array position in payload.discussions)
// - as_of TIMESTAMPTZ -> payload.asOf (copied from parent)
// - day INTEGER -> payload.discussions[i].day
// - headline TEXT NULL -> payload.discussions[i].headline
// - summary TEXT NULL -> payload.discussions[i].summary
// - discussion TEXT NULL -> payload.discussions[i].discussion
// - updated_at TIMESTAMPTZ NULL -> payload.discussions[i].updatedAt
//
// Reconstructing canonical JSON payloads
//
// - WeatherObservation:
@@ -226,4 +284,9 @@
// read one row from alert_runs, join alerts by run_event_id ordered by
// alert_index, then join alert_references by (run_event_id, alert_index)
// ordered by reference_index to rebuild references per alert.
//
// - WeatherOutlookRun:
// read one row from outlook_runs, join outlooks by run_event_id ordered by
// outlook_index to rebuild outlooks, then join outlook_discussions by
// run_event_id ordered by discussion_index to rebuild discussions.
package postgres

View File

@@ -0,0 +1,33 @@
package postgres
import (
"os"
"strings"
"testing"
)
func TestDocumentedOutlookDiscussionStorage(t *testing.T) {
for _, path := range []string{
"../../../docs/integrations/postgres.md",
"../../../docs/internal/postgres-sink.md",
} {
t.Run(path, func(t *testing.T) {
raw, err := os.ReadFile(path)
if err != nil {
t.Fatalf("ReadFile(%s) error = %v", path, err)
}
doc := string(raw)
for _, want := range []string{
tableOutlookDiscussions,
"discussion_count",
"discussion_index",
"weather.outlook.v2",
} {
if !strings.Contains(doc, want) {
t.Fatalf("%s missing %q", path, want)
}
}
})
}
}

View File

@@ -1,6 +1,7 @@
package postgres
import (
"bytes"
"context"
"encoding/json"
"fmt"
@@ -26,6 +27,8 @@ func mapPostgresEvent(_ context.Context, e fkevent.Event) ([]fksinks.PostgresWri
return mapWeatherStoryEvent(e)
case standards.SchemaWeatherAlertV1:
return mapAlertEvent(e)
case standards.SchemaWeatherOutlookV2:
return mapOutlookEvent(e)
default:
return nil, nil
}
@@ -45,13 +48,7 @@ func mapObservationEvent(e fkevent.Event) ([]fksinks.PostgresWrite, error) {
writes = append(writes, fksinks.PostgresWrite{
Table: tableObservations,
Values: map[string]any{
"event_id": e.ID,
"event_kind": string(e.Kind),
"event_source": e.Source,
"event_schema": e.Schema,
"event_emitted_at": e.EmittedAt.UTC(),
"event_effective_at": nullableTime(e.EffectiveAt),
Values: parentEventValues(e, map[string]any{
"station_id": nullableString(obs.StationID),
"station_name": nullableString(obs.StationName),
"observed_at": observedAt,
@@ -67,7 +64,7 @@ func mapObservationEvent(e fkevent.Event) ([]fksinks.PostgresWrite, error) {
"visibility_meters": nullableFloat64(obs.VisibilityMeters),
"relative_humidity_percent": nullableFloat64(obs.RelativeHumidityPercent),
"apparent_temperature_c": nullableFloat64(obs.ApparentTemperatureC),
},
}),
})
for i, pw := range obs.PresentWeather {
@@ -106,23 +103,17 @@ func mapForecastEvent(e fkevent.Event) ([]fksinks.PostgresWrite, error) {
writes = append(writes, fksinks.PostgresWrite{
Table: tableForecasts,
Values: map[string]any{
"event_id": e.ID,
"event_kind": string(e.Kind),
"event_source": e.Source,
"event_schema": e.Schema,
"event_emitted_at": e.EmittedAt.UTC(),
"event_effective_at": nullableTime(e.EffectiveAt),
"location_id": nullableString(run.LocationID),
"location_name": nullableString(run.LocationName),
"issued_at": issuedAt,
"updated_at": nullableTime(run.UpdatedAt),
"product": string(run.Product),
"latitude": nullableFloat64(run.Latitude),
"longitude": nullableFloat64(run.Longitude),
"elevation_meters": nullableFloat64(run.ElevationMeters),
"period_count": len(run.Periods),
},
Values: parentEventValues(e, map[string]any{
"location_id": nullableString(run.LocationID),
"location_name": nullableString(run.LocationName),
"issued_at": issuedAt,
"updated_at": nullableTime(run.UpdatedAt),
"product": string(run.Product),
"latitude": nullableFloat64(run.Latitude),
"longitude": nullableFloat64(run.Longitude),
"elevation_meters": nullableFloat64(run.ElevationMeters),
"period_count": len(run.Periods),
}),
})
for i, p := range run.Periods {
@@ -183,13 +174,7 @@ func mapForecastDiscussionEvent(e fkevent.Event) ([]fksinks.PostgresWrite, error
writes := make([]fksinks.PostgresWrite, 0, 1+len(run.KeyMessages))
writes = append(writes, fksinks.PostgresWrite{
Table: tableForecastDiscussions,
Values: map[string]any{
"event_id": e.ID,
"event_kind": string(e.Kind),
"event_source": e.Source,
"event_schema": e.Schema,
"event_emitted_at": e.EmittedAt.UTC(),
"event_effective_at": nullableTime(e.EffectiveAt),
Values: parentEventValues(e, map[string]any{
"office_id": nullableString(run.OfficeID),
"office_name": nullableString(run.OfficeName),
"issued_at": issuedAt,
@@ -202,7 +187,7 @@ func mapForecastDiscussionEvent(e fkevent.Event) ([]fksinks.PostgresWrite, error
"long_term_issued_at": longTermIssuedAt,
"long_term_text": longTermText,
"key_message_count": len(run.KeyMessages),
},
}),
})
for i, msg := range run.KeyMessages {
@@ -233,17 +218,11 @@ func mapWeatherStoryEvent(e fkevent.Event) ([]fksinks.PostgresWrite, error) {
writes := make([]fksinks.PostgresWrite, 0, 1+len(run.Stories))
writes = append(writes, fksinks.PostgresWrite{
Table: tableWeatherStoryRuns,
Values: map[string]any{
"event_id": e.ID,
"event_kind": string(e.Kind),
"event_source": e.Source,
"event_schema": e.Schema,
"event_emitted_at": e.EmittedAt.UTC(),
"event_effective_at": nullableTime(e.EffectiveAt),
"office_id": nullableString(run.OfficeID),
"as_of": asOf,
"story_count": len(run.Stories),
},
Values: parentEventValues(e, map[string]any{
"office_id": nullableString(run.OfficeID),
"as_of": asOf,
"story_count": len(run.Stories),
}),
})
for i, story := range run.Stories {
@@ -287,20 +266,14 @@ func mapAlertEvent(e fkevent.Event) ([]fksinks.PostgresWrite, error) {
writes = append(writes, fksinks.PostgresWrite{
Table: tableAlertRuns,
Values: map[string]any{
"event_id": e.ID,
"event_kind": string(e.Kind),
"event_source": e.Source,
"event_schema": e.Schema,
"event_emitted_at": e.EmittedAt.UTC(),
"event_effective_at": nullableTime(e.EffectiveAt),
"location_id": nullableString(run.LocationID),
"location_name": nullableString(run.LocationName),
"as_of": asOf,
"latitude": nullableFloat64(run.Latitude),
"longitude": nullableFloat64(run.Longitude),
"alert_count": len(run.Alerts),
},
Values: parentEventValues(e, map[string]any{
"location_id": nullableString(run.LocationID),
"location_name": nullableString(run.LocationName),
"as_of": asOf,
"latitude": nullableFloat64(run.Latitude),
"longitude": nullableFloat64(run.Longitude),
"alert_count": len(run.Alerts),
}),
})
for i, a := range run.Alerts {
@@ -329,6 +302,7 @@ func mapAlertEvent(e fkevent.Event) ([]fksinks.PostgresWrite, error) {
"sent": nullableTime(a.Sent),
"effective": nullableTime(a.Effective),
"onset": nullableTime(a.Onset),
"ends": nullableTime(a.Ends),
"expires": nullableTime(a.Expires),
"area_description": nullableString(a.AreaDescription),
"sender_name": nullableString(a.SenderName),
@@ -356,6 +330,143 @@ func mapAlertEvent(e fkevent.Event) ([]fksinks.PostgresWrite, error) {
return writes, nil
}
func mapOutlookEvent(e fkevent.Event) ([]fksinks.PostgresWrite, error) {
run, err := decodePayload[model.WeatherOutlookRun](e.Payload)
if err != nil {
return nil, fmt.Errorf("decode outlook payload: %w", err)
}
if run.AsOf.IsZero() {
return nil, fmt.Errorf("decode outlook payload: asOf is required")
}
asOf := run.AsOf.UTC()
if err := validateOutlookDiscussions(run.Discussions); err != nil {
return nil, err
}
writes := make([]fksinks.PostgresWrite, 0, 1+len(run.Outlooks)+len(run.Discussions))
writes = append(writes, fksinks.PostgresWrite{
Table: tableOutlookRuns,
Values: parentEventValues(e, map[string]any{
"location_id": nullableString(run.LocationID),
"location_name": nullableString(run.LocationName),
"latitude": nullableFloat64(run.Latitude),
"longitude": nullableFloat64(run.Longitude),
"as_of": asOf,
"issued_at": nullableTime(run.IssuedAt),
"outlook_count": len(run.Outlooks),
"discussion_count": len(run.Discussions),
}),
})
for i, outlook := range run.Outlooks {
if err := validateOutlook(outlook, i); err != nil {
return nil, err
}
geometryJSON, err := requiredCompactJSONText(outlook.Geometry)
if err != nil {
return nil, fmt.Errorf("decode outlook payload: outlooks[%d].geometry: %w", i, err)
}
writes = append(writes, fksinks.PostgresWrite{
Table: tableOutlooks,
Values: map[string]any{
"run_event_id": e.ID,
"outlook_index": i,
"as_of": asOf,
"outlook_id": outlook.ID,
"provider": outlook.Provider,
"product": outlook.Product,
"day": outlook.Day,
"outlook_type": outlook.OutlookType,
"label": outlook.Label,
"label_text": nullableString(outlook.LabelText),
"severity_rank": nullableInt(outlook.SeverityRank),
"valid_from": outlook.ValidFrom.UTC(),
"valid_to": outlook.ValidTo.UTC(),
"issued_at": outlook.IssuedAt.UTC(),
"expires_at": outlook.ExpiresAt.UTC(),
"forecaster": nullableString(outlook.Forecaster),
"source_url": nullableString(outlook.SourceURL),
"image_url": nullableString(outlook.ImageURL),
"contains_location": outlook.ContainsLocation,
"geometry_json": geometryJSON,
},
})
}
for i, discussion := range run.Discussions {
writes = append(writes, fksinks.PostgresWrite{
Table: tableOutlookDiscussions,
Values: map[string]any{
"run_event_id": e.ID,
"discussion_index": i,
"as_of": asOf,
"day": discussion.Day,
"headline": nullableString(discussion.Headline),
"summary": nullableString(discussion.Summary),
"discussion": nullableString(discussion.Discussion),
"updated_at": nullableTime(discussion.UpdatedAt),
},
})
}
return writes, nil
}
func validateOutlook(outlook model.WeatherOutlook, index int) error {
if strings.TrimSpace(outlook.ID) == "" {
return fmt.Errorf("decode outlook payload: outlooks[%d].id is required", index)
}
if strings.TrimSpace(outlook.Provider) == "" {
return fmt.Errorf("decode outlook payload: outlooks[%d].provider is required", index)
}
if strings.TrimSpace(outlook.Product) == "" {
return fmt.Errorf("decode outlook payload: outlooks[%d].product is required", index)
}
if outlook.Day == 0 {
return fmt.Errorf("decode outlook payload: outlooks[%d].day is required", index)
}
if strings.TrimSpace(outlook.OutlookType) == "" {
return fmt.Errorf("decode outlook payload: outlooks[%d].outlookType is required", index)
}
if strings.TrimSpace(outlook.Label) == "" {
return fmt.Errorf("decode outlook payload: outlooks[%d].label is required", index)
}
if outlook.ValidFrom.IsZero() || outlook.ValidTo.IsZero() {
return fmt.Errorf("decode outlook payload: outlooks[%d] validFrom/validTo are required", index)
}
if outlook.IssuedAt.IsZero() || outlook.ExpiresAt.IsZero() {
return fmt.Errorf("decode outlook payload: outlooks[%d] issuedAt/expiresAt are required", index)
}
if len(outlook.Geometry) == 0 {
return fmt.Errorf("decode outlook payload: outlooks[%d].geometry is required", index)
}
if !outlook.ContainsLocation {
return fmt.Errorf("decode outlook payload: outlooks[%d].containsLocation must be true", index)
}
return nil
}
func validateOutlookDiscussions(discussions []model.WeatherOutlookDiscussion) error {
seenDays := map[int]int{}
for i, discussion := range discussions {
if discussion.Day < 1 || discussion.Day > 3 {
return fmt.Errorf("decode outlook payload: discussions[%d].day must be 1, 2, or 3", i)
}
if strings.TrimSpace(discussion.Headline) == "" &&
strings.TrimSpace(discussion.Summary) == "" &&
strings.TrimSpace(discussion.Discussion) == "" {
return fmt.Errorf("decode outlook payload: discussions[%d] headline, summary, or discussion is required", i)
}
if first, ok := seenDays[discussion.Day]; ok {
return fmt.Errorf("decode outlook payload: discussions[%d].day duplicates discussions[%d].day %d", i, first, discussion.Day)
}
seenDays[discussion.Day] = i
}
return nil
}
func decodePayload[T any](payload any) (T, error) {
var out T
if payload == nil {
@@ -382,6 +493,21 @@ func decodePayload[T any](payload any) (T, error) {
return out, nil
}
func parentEventValues(e fkevent.Event, values map[string]any) map[string]any {
out := map[string]any{
"event_id": e.ID,
"event_kind": string(e.Kind),
"event_source": e.Source,
"event_schema": e.Schema,
"event_emitted_at": e.EmittedAt.UTC(),
"event_effective_at": nullableTime(e.EffectiveAt),
}
for k, v := range values {
out[k] = v
}
return out
}
func nullableDiscussionSection(section *model.WeatherForecastDiscussionSection) (any, any, any) {
if section == nil {
return nil, nil, nil
@@ -418,6 +544,13 @@ func nullableBool(v *bool) any {
return *v
}
func nullableInt(v *int) any {
if v == nil {
return nil
}
return *v
}
func nullableTime(v *time.Time) any {
if v == nil || v.IsZero() {
return nil
@@ -445,3 +578,19 @@ func compactJSONText(v any) (any, error) {
}
return string(b), nil
}
func requiredCompactJSONText(v any) (string, error) {
compact, err := compactJSONText(v)
if err != nil {
return "", err
}
s, ok := compact.(string)
if !ok || strings.TrimSpace(s) == "" || strings.TrimSpace(s) == "null" {
return "", fmt.Errorf("is required")
}
var buf bytes.Buffer
if err := json.Compact(&buf, []byte(s)); err != nil {
return "", err
}
return buf.String(), nil
}

View File

@@ -27,7 +27,7 @@ func TestMapPostgresEventObservationStructPayload(t *testing.T) {
PresentWeather: []model.PresentWeather{{Raw: map[string]any{"a": 1, "b": "x"}}},
}
writes, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherObservationV1, "observation", obs))
writes, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherObservationV1, standards.KindObservation, obs))
if err != nil {
t.Fatalf("mapPostgresEvent() error = %v", err)
}
@@ -74,7 +74,7 @@ func TestMapPostgresEventForecastStructPayload(t *testing.T) {
},
}
writes, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherForecastV1, "forecast", run))
writes, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherForecastV1, standards.KindForecast, run))
if err != nil {
t.Fatalf("mapPostgresEvent() error = %v", err)
}
@@ -103,6 +103,8 @@ func TestMapPostgresEventForecastStructPayload(t *testing.T) {
func TestMapPostgresEventAlertStructPayload(t *testing.T) {
sent := time.Date(2026, 3, 16, 17, 0, 0, 0, time.UTC)
ends := time.Date(2026, 3, 16, 20, 0, 0, 0, time.UTC)
expires := time.Date(2026, 3, 16, 18, 30, 0, 0, time.UTC)
run := model.WeatherAlertRun{
AsOf: time.Date(2026, 3, 16, 18, 0, 0, 0, time.UTC),
Alerts: []model.WeatherAlert{
@@ -110,6 +112,8 @@ func TestMapPostgresEventAlertStructPayload(t *testing.T) {
ID: "urn:alert:1",
Headline: "Winter Weather Advisory",
Severity: "Moderate",
Ends: &ends,
Expires: &expires,
References: []model.AlertReference{
{ID: "urn:ref:1", Sent: &sent},
{Identifier: "ref-two"},
@@ -122,7 +126,7 @@ func TestMapPostgresEventAlertStructPayload(t *testing.T) {
},
}
writes, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherAlertV1, "alert", run))
writes, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherAlertV1, standards.KindAlert, run))
if err != nil {
t.Fatalf("mapPostgresEvent() error = %v", err)
}
@@ -145,6 +149,20 @@ func TestMapPostgresEventAlertStructPayload(t *testing.T) {
if got := firstAlert.Values["reference_count"]; got != 2 {
t.Fatalf("alerts reference_count = %#v, want 2", got)
}
if got := firstAlert.Values["ends"]; got != ends {
t.Fatalf("alerts ends = %#v, want %#v", got, ends)
}
if got := firstAlert.Values["expires"]; got != expires {
t.Fatalf("alerts expires = %#v, want %#v", got, expires)
}
alertWrites := writesForTable(writes, tableAlerts)
if len(alertWrites) != 2 {
t.Fatalf("alert writes len = %d, want 2", len(alertWrites))
}
if got := alertWrites[1].Values["ends"]; got != nil {
t.Fatalf("second alert ends = %#v, want nil", got)
}
assertAllWritesIncludeAllColumns(t, writes)
}
@@ -163,7 +181,7 @@ func TestMapPostgresEventForecastDiscussionStructPayload(t *testing.T) {
LongTerm: &model.WeatherForecastDiscussionSection{Text: "Long term text"},
}
writes, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherForecastDiscussionV1, "forecast_discussion", run))
writes, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherForecastDiscussionV1, standards.KindForecastDiscussion, run))
if err != nil {
t.Fatalf("mapPostgresEvent() error = %v", err)
}
@@ -213,7 +231,7 @@ func TestMapPostgresEventWeatherStoryStructPayload(t *testing.T) {
},
}
writes, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherStoryV1, "weather_story", run))
writes, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherStoryV1, standards.KindWeatherStory, run))
if err != nil {
t.Fatalf("mapPostgresEvent() error = %v", err)
}
@@ -239,8 +257,331 @@ func TestMapPostgresEventWeatherStoryStructPayload(t *testing.T) {
assertAllWritesIncludeAllColumns(t, writes)
}
func TestMapPostgresEventOutlookStructPayload(t *testing.T) {
lat := 38.6239
lon := -90.3571
issuedAt := time.Date(2026, 6, 11, 19, 45, 0, 0, time.FixedZone("UTC-5", -5*60*60))
updatedAt := time.Date(2026, 6, 11, 21, 15, 0, 0, time.FixedZone("UTC-5", -5*60*60))
severity := 3
run := model.WeatherOutlookRun{
LocationID: "stl",
LocationName: "St. Louis, MO",
Latitude: &lat,
Longitude: &lon,
AsOf: time.Date(2026, 6, 12, 0, 45, 0, 0, time.UTC),
IssuedAt: &issuedAt,
Outlooks: []model.WeatherOutlook{
{
ID: "outlook-1",
Provider: "spc",
Product: "convective",
Day: 1,
OutlookType: "categorical",
Label: "SLGT",
LabelText: "Slight Risk",
SeverityRank: &severity,
ValidFrom: time.Date(2026, 6, 11, 13, 0, 0, 0, time.FixedZone("UTC-5", -5*60*60)),
ValidTo: time.Date(2026, 6, 12, 12, 0, 0, 0, time.UTC),
IssuedAt: issuedAt,
ExpiresAt: time.Date(2026, 6, 12, 12, 0, 0, 0, time.UTC),
Forecaster: "SMITH",
SourceURL: "https://example.invalid/day1.geojson",
ContainsLocation: true,
Geometry: json.RawMessage(`{ "type" : "Polygon", "coordinates" : [ [ [ -91.0, 38.0 ], [ -90.0, 38.0 ], [ -90.0, 39.0 ], [ -91.0, 39.0 ], [ -91.0, 38.0 ] ] ] }`),
},
{
ID: "outlook-2",
Provider: "spc",
Product: "convective",
Day: 1,
OutlookType: "wind",
Label: "15",
ValidFrom: time.Date(2026, 6, 11, 13, 0, 0, 0, time.UTC),
ValidTo: time.Date(2026, 6, 12, 12, 0, 0, 0, time.UTC),
IssuedAt: time.Date(2026, 6, 11, 19, 45, 0, 0, time.UTC),
ExpiresAt: time.Date(2026, 6, 12, 12, 0, 0, 0, time.UTC),
ContainsLocation: true,
Geometry: json.RawMessage(`{"type":"Polygon","coordinates":[[[-100,35],[-98,35],[-98,37],[-100,37],[-100,35]]]}`),
},
},
Discussions: []model.WeatherOutlookDiscussion{
{
Day: 1,
Headline: "Day 1 Convective Outlook",
Summary: "Severe thunderstorms are possible.",
Discussion: "Full discussion text.",
UpdatedAt: &updatedAt,
},
},
}
writes, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherOutlookV2, standards.KindOutlook, run))
if err != nil {
t.Fatalf("mapPostgresEvent() error = %v", err)
}
if len(writes) != 4 {
t.Fatalf("mapPostgresEvent() writes len = %d, want 4", len(writes))
}
if writes[0].Table != tableOutlookRuns {
t.Fatalf("writes[0].Table = %q, want %q", writes[0].Table, tableOutlookRuns)
}
if got := writes[0].Values["outlook_count"]; got != 2 {
t.Fatalf("outlook_runs outlook_count = %#v, want 2", got)
}
if got := writes[0].Values["discussion_count"]; got != 1 {
t.Fatalf("outlook_runs discussion_count = %#v, want 1", got)
}
if got := writes[0].Values["issued_at"]; got != issuedAt.UTC() {
t.Fatalf("outlook_runs issued_at = %#v, want UTC %s", got, issuedAt.UTC())
}
if writes[1].Table != tableOutlooks || writes[2].Table != tableOutlooks {
t.Fatalf("outlook writes not in expected order")
}
if got := writes[1].Values["outlook_index"]; got != 0 {
t.Fatalf("first outlook index = %#v, want 0", got)
}
if got := writes[1].Values["outlook_id"]; got != "outlook-1" {
t.Fatalf("first outlook_id = %#v, want outlook-1", got)
}
if got := writes[1].Values["provider"]; got != "spc" {
t.Fatalf("first provider = %#v, want spc", got)
}
if got := writes[1].Values["valid_from"]; got != run.Outlooks[0].ValidFrom.UTC() {
t.Fatalf("first valid_from = %#v, want UTC %s", got, run.Outlooks[0].ValidFrom.UTC())
}
if got := writes[1].Values["geometry_json"]; got != `{"type":"Polygon","coordinates":[[[-91.0,38.0],[-90.0,38.0],[-90.0,39.0],[-91.0,39.0],[-91.0,38.0]]]}` {
t.Fatalf("first geometry_json = %#v", got)
}
if got := writes[2].Values["contains_location"]; got != true {
t.Fatalf("second contains_location = %#v, want true", got)
}
if writes[3].Table != tableOutlookDiscussions {
t.Fatalf("writes[3].Table = %q, want %q", writes[3].Table, tableOutlookDiscussions)
}
if got := writes[3].Values["discussion_index"]; got != 0 {
t.Fatalf("discussion_index = %#v, want 0", got)
}
if got := writes[3].Values["as_of"]; got != run.AsOf.UTC() {
t.Fatalf("discussion as_of = %#v, want %s", got, run.AsOf.UTC())
}
if got := writes[3].Values["day"]; got != 1 {
t.Fatalf("discussion day = %#v, want 1", got)
}
if got := writes[3].Values["headline"]; got != "Day 1 Convective Outlook" {
t.Fatalf("discussion headline = %#v", got)
}
if got := writes[3].Values["summary"]; got != "Severe thunderstorms are possible." {
t.Fatalf("discussion summary = %#v", got)
}
if got := writes[3].Values["discussion"]; got != "Full discussion text." {
t.Fatalf("discussion text = %#v", got)
}
if got := writes[3].Values["updated_at"]; got != updatedAt.UTC() {
t.Fatalf("discussion updated_at = %#v, want UTC %s", got, updatedAt.UTC())
}
assertAllWritesIncludeAllColumns(t, writes)
}
func TestMapPostgresEventOutlookEmptyLocalRun(t *testing.T) {
run := model.WeatherOutlookRun{
AsOf: time.Date(2026, 6, 11, 19, 45, 0, 0, time.UTC),
}
writes, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherOutlookV2, standards.KindOutlook, run))
if err != nil {
t.Fatalf("mapPostgresEvent() error = %v", err)
}
if len(writes) != 1 {
t.Fatalf("mapPostgresEvent() writes len = %d, want 1", len(writes))
}
if writes[0].Table != tableOutlookRuns {
t.Fatalf("writes[0].Table = %q, want %q", writes[0].Table, tableOutlookRuns)
}
if got := writes[0].Values["outlook_count"]; got != 0 {
t.Fatalf("outlook_runs outlook_count = %#v, want 0", got)
}
if got := writes[0].Values["discussion_count"]; got != 0 {
t.Fatalf("outlook_runs discussion_count = %#v, want 0", got)
}
assertAllWritesIncludeAllColumns(t, writes)
}
func TestMapPostgresEventOutlookRejectsMissingAsOf(t *testing.T) {
_, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherOutlookV2, standards.KindOutlook, model.WeatherOutlookRun{}))
if err == nil {
t.Fatalf("mapPostgresEvent() error = nil, want missing asOf error")
}
if !strings.Contains(err.Error(), "asOf is required") {
t.Fatalf("error = %q, want asOf context", err)
}
}
func TestMapPostgresEventOutlookRejectsMissingIDAndProvider(t *testing.T) {
base := model.WeatherOutlook{
ID: "outlook-1",
Provider: "spc",
Product: "convective",
Day: 1,
OutlookType: "categorical",
Label: "SLGT",
ValidFrom: time.Date(2026, 6, 11, 13, 0, 0, 0, time.UTC),
ValidTo: time.Date(2026, 6, 12, 12, 0, 0, 0, time.UTC),
IssuedAt: time.Date(2026, 6, 11, 19, 45, 0, 0, time.UTC),
ExpiresAt: time.Date(2026, 6, 12, 12, 0, 0, 0, time.UTC),
ContainsLocation: true,
Geometry: json.RawMessage(`{"type":"Polygon","coordinates":[[[-91,38],[-90,38],[-90,39],[-91,39],[-91,38]]]}`),
}
tests := []struct {
name string
mutate func(*model.WeatherOutlook)
wantErr string
}{
{
name: "missing id",
mutate: func(outlook *model.WeatherOutlook) { outlook.ID = "" },
wantErr: "outlooks[0].id is required",
},
{
name: "missing provider",
mutate: func(outlook *model.WeatherOutlook) { outlook.Provider = "" },
wantErr: "outlooks[0].provider is required",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
outlook := base
tt.mutate(&outlook)
run := model.WeatherOutlookRun{
AsOf: time.Date(2026, 6, 11, 19, 45, 0, 0, time.UTC),
Outlooks: []model.WeatherOutlook{outlook},
}
_, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherOutlookV2, standards.KindOutlook, run))
if err == nil {
t.Fatalf("mapPostgresEvent() error = nil, want %q", tt.wantErr)
}
if !strings.Contains(err.Error(), tt.wantErr) {
t.Fatalf("error = %q, want %q", err, tt.wantErr)
}
})
}
}
func TestMapPostgresEventOutlookRejectsMissingRequiredTimes(t *testing.T) {
run := model.WeatherOutlookRun{
AsOf: time.Date(2026, 6, 11, 19, 45, 0, 0, time.UTC),
Outlooks: []model.WeatherOutlook{{
ID: "outlook-1",
Provider: "spc",
Product: "convective",
Day: 1,
OutlookType: "categorical",
Label: "SLGT",
ContainsLocation: true,
Geometry: json.RawMessage(`{"type":"Polygon","coordinates":[[[-91,38],[-90,38],[-90,39],[-91,39],[-91,38]]]}`),
}},
}
_, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherOutlookV2, standards.KindOutlook, run))
if err == nil {
t.Fatalf("mapPostgresEvent() error = nil, want missing time error")
}
if !strings.Contains(err.Error(), "outlooks[0] validFrom/validTo are required") {
t.Fatalf("error = %q, want outlook time context", err)
}
}
func TestMapPostgresEventOutlookRejectsEmptyGeometry(t *testing.T) {
run := model.WeatherOutlookRun{
AsOf: time.Date(2026, 6, 11, 19, 45, 0, 0, time.UTC),
Outlooks: []model.WeatherOutlook{{
ID: "outlook-1",
Provider: "spc",
Product: "convective",
Day: 1,
OutlookType: "categorical",
Label: "SLGT",
ValidFrom: time.Date(2026, 6, 11, 13, 0, 0, 0, time.UTC),
ValidTo: time.Date(2026, 6, 12, 12, 0, 0, 0, time.UTC),
IssuedAt: time.Date(2026, 6, 11, 19, 45, 0, 0, time.UTC),
ExpiresAt: time.Date(2026, 6, 12, 12, 0, 0, 0, time.UTC),
ContainsLocation: true,
}},
}
_, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherOutlookV2, standards.KindOutlook, run))
if err == nil {
t.Fatalf("mapPostgresEvent() error = nil, want geometry error")
}
if !strings.Contains(err.Error(), "outlooks[0].geometry is required") {
t.Fatalf("error = %q, want geometry context", err)
}
}
func TestMapPostgresEventOutlookRejectsDuplicateDiscussionDay(t *testing.T) {
run := model.WeatherOutlookRun{
AsOf: time.Date(2026, 6, 11, 19, 45, 0, 0, time.UTC),
Discussions: []model.WeatherOutlookDiscussion{
{Day: 1, Discussion: "First day one discussion."},
{Day: 1, Discussion: "Duplicate day one discussion."},
},
}
_, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherOutlookV2, standards.KindOutlook, run))
if err == nil {
t.Fatalf("mapPostgresEvent() error = nil, want duplicate discussion day error")
}
if !strings.Contains(err.Error(), "discussions[1].day duplicates discussions[0].day 1") {
t.Fatalf("error = %q, want duplicate discussion day context", err)
}
}
func TestMapPostgresEventOutlookRejectsInvalidDiscussionDay(t *testing.T) {
run := model.WeatherOutlookRun{
AsOf: time.Date(2026, 6, 11, 19, 45, 0, 0, time.UTC),
Discussions: []model.WeatherOutlookDiscussion{{Day: 4, Discussion: "Invalid day."}},
}
_, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherOutlookV2, standards.KindOutlook, run))
if err == nil {
t.Fatalf("mapPostgresEvent() error = nil, want invalid discussion day error")
}
if !strings.Contains(err.Error(), "discussions[0].day must be 1, 2, or 3") {
t.Fatalf("error = %q, want invalid discussion day context", err)
}
}
func TestMapPostgresEventOutlookRejectsEmptyDiscussionContent(t *testing.T) {
run := model.WeatherOutlookRun{
AsOf: time.Date(2026, 6, 11, 19, 45, 0, 0, time.UTC),
Discussions: []model.WeatherOutlookDiscussion{{Day: 1}},
}
_, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherOutlookV2, standards.KindOutlook, run))
if err == nil {
t.Fatalf("mapPostgresEvent() error = nil, want empty discussion content error")
}
if !strings.Contains(err.Error(), "discussions[0] headline, summary, or discussion is required") {
t.Fatalf("error = %q, want empty discussion content context", err)
}
}
func TestMapPostgresEventOutlookRejectsContainsLocationFalse(t *testing.T) {
run := model.WeatherOutlookRun{
AsOf: time.Date(2026, 6, 11, 19, 45, 0, 0, time.UTC),
Outlooks: []model.WeatherOutlook{validTestOutlook()},
}
run.Outlooks[0].ContainsLocation = false
_, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherOutlookV2, standards.KindOutlook, run))
if err == nil {
t.Fatalf("mapPostgresEvent() error = nil, want containsLocation error")
}
if !strings.Contains(err.Error(), "outlooks[0].containsLocation must be true") {
t.Fatalf("error = %q, want containsLocation context", err)
}
}
func TestMapPostgresEventWeatherStoryRejectsMissingAsOf(t *testing.T) {
_, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherStoryV1, "weather_story", model.WeatherStoryRun{}))
_, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherStoryV1, standards.KindWeatherStory, model.WeatherStoryRun{}))
if err == nil {
t.Fatalf("mapPostgresEvent() error = nil, want missing asOf error")
}
@@ -254,7 +595,7 @@ func TestMapPostgresEventWeatherStoryRejectsMissingStoryTimes(t *testing.T) {
AsOf: time.Date(2026, 5, 30, 9, 0, 34, 0, time.UTC),
Stories: []model.WeatherStory{{Title: "missing times"}},
}
_, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherStoryV1, "weather_story", run))
_, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherStoryV1, standards.KindWeatherStory, run))
if err == nil {
t.Fatalf("mapPostgresEvent() error = nil, want missing story times error")
}
@@ -284,7 +625,7 @@ func TestMapPostgresEventMapPayload(t *testing.T) {
t.Fatalf("json.Unmarshal() error = %v", err)
}
writes, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherForecastV1, "forecast", payload))
writes, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherForecastV1, standards.KindForecast, payload))
if err != nil {
t.Fatalf("mapPostgresEvent() error = %v", err)
}
@@ -299,7 +640,18 @@ func TestMapPostgresEventMapPayload(t *testing.T) {
}
func TestMapPostgresEventUnknownSchemaNoOp(t *testing.T) {
writes, err := mapPostgresEvent(context.Background(), testEvent("weather.unknown.v1", "observation", map[string]any{"x": 1}))
writes, err := mapPostgresEvent(context.Background(), testEvent("weather.unknown.v1", standards.KindObservation, map[string]any{"x": 1}))
if err != nil {
t.Fatalf("mapPostgresEvent() error = %v", err)
}
if len(writes) != 0 {
t.Fatalf("mapPostgresEvent() writes len = %d, want 0", len(writes))
}
}
func TestMapPostgresEventLegacyOutlookSchemaNoOp(t *testing.T) {
run := model.WeatherOutlookRun{AsOf: time.Date(2026, 6, 11, 19, 45, 0, 0, time.UTC)}
writes, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherOutlookV1, standards.KindOutlook, run))
if err != nil {
t.Fatalf("mapPostgresEvent() error = %v", err)
}
@@ -309,7 +661,7 @@ func TestMapPostgresEventUnknownSchemaNoOp(t *testing.T) {
}
func TestMapPostgresEventMalformedPayload(t *testing.T) {
_, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherForecastV1, "forecast", "bad"))
_, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherForecastV1, standards.KindForecast, "bad"))
if err == nil {
t.Fatalf("mapPostgresEvent() expected error for malformed payload")
}
@@ -319,7 +671,7 @@ func TestMapPostgresEventMalformedPayload(t *testing.T) {
}
func TestMapPostgresEventForecastDiscussionMalformedPayload(t *testing.T) {
_, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherForecastDiscussionV1, "forecast_discussion", "bad"))
_, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherForecastDiscussionV1, standards.KindForecastDiscussion, "bad"))
if err == nil {
t.Fatalf("mapPostgresEvent() expected error for malformed payload")
}
@@ -328,6 +680,56 @@ func TestMapPostgresEventForecastDiscussionMalformedPayload(t *testing.T) {
}
}
func TestParentEventValuesAddsEnvelopeAndPreservesProductValues(t *testing.T) {
emittedAt := time.Date(2026, 3, 16, 13, 31, 0, 0, time.FixedZone("CDT", -5*60*60))
effectiveAt := time.Date(2026, 3, 16, 13, 30, 0, 0, time.FixedZone("CDT", -5*60*60))
event := fkevent.Event{
ID: "evt-envelope",
Kind: fkevent.Kind(standards.KindForecast),
Source: "test-source",
Schema: standards.SchemaWeatherForecastV1,
EmittedAt: emittedAt,
EffectiveAt: &effectiveAt,
}
got := parentEventValues(event, map[string]any{"product_col": "product-value"})
assertParentEnvelopeValues(t, got, event)
if got["product_col"] != "product-value" {
t.Fatalf("product_col = %#v, want product-value", got["product_col"])
}
}
func TestParentEventValuesNullEffectiveAt(t *testing.T) {
base := fkevent.Event{
ID: "evt-envelope",
Kind: fkevent.Kind(standards.KindObservation),
Source: "test-source",
Schema: standards.SchemaWeatherObservationV1,
EmittedAt: time.Date(2026, 3, 16, 18, 31, 0, 0, time.UTC),
}
for _, tt := range []struct {
name string
mut func(*fkevent.Event)
}{
{name: "nil", mut: func(*fkevent.Event) {}},
{name: "zero", mut: func(event *fkevent.Event) {
zero := time.Time{}
event.EffectiveAt = &zero
}},
} {
t.Run(tt.name, func(t *testing.T) {
event := base
tt.mut(&event)
got := parentEventValues(event, nil)
if got["event_effective_at"] != nil {
t.Fatalf("event_effective_at = %#v, want nil", got["event_effective_at"])
}
})
}
}
func testEvent(schema string, kind fkevent.Kind, payload any) fkevent.Event {
effectiveAt := time.Date(2026, 3, 16, 18, 30, 0, 0, time.UTC)
return fkevent.Event{
@@ -341,6 +743,30 @@ func testEvent(schema string, kind fkevent.Kind, payload any) fkevent.Event {
}
}
func assertParentEnvelopeValues(t *testing.T, values map[string]any, event fkevent.Event) {
t.Helper()
if got := values["event_id"]; got != event.ID {
t.Fatalf("event_id = %#v, want %q", got, event.ID)
}
if got := values["event_kind"]; got != string(event.Kind) {
t.Fatalf("event_kind = %#v, want %q", got, event.Kind)
}
if got := values["event_source"]; got != event.Source {
t.Fatalf("event_source = %#v, want %q", got, event.Source)
}
if got := values["event_schema"]; got != event.Schema {
t.Fatalf("event_schema = %#v, want %q", got, event.Schema)
}
if got := values["event_emitted_at"]; got != event.EmittedAt.UTC() {
t.Fatalf("event_emitted_at = %#v, want %s", got, event.EmittedAt.UTC())
}
wantEffective := nullableTime(event.EffectiveAt)
if got := values["event_effective_at"]; got != wantEffective {
t.Fatalf("event_effective_at = %#v, want %#v", got, wantEffective)
}
}
func firstWriteForTable(writes []fksinks.PostgresWrite, table string) (fksinks.PostgresWrite, bool) {
for _, w := range writes {
if w.Table == table {
@@ -350,6 +776,16 @@ func firstWriteForTable(writes []fksinks.PostgresWrite, table string) (fksinks.P
return fksinks.PostgresWrite{}, false
}
func writesForTable(writes []fksinks.PostgresWrite, table string) []fksinks.PostgresWrite {
out := make([]fksinks.PostgresWrite, 0)
for _, w := range writes {
if w.Table == table {
out = append(out, w)
}
}
return out
}
func assertAllWritesIncludeAllColumns(t *testing.T, writes []fksinks.PostgresWrite) {
t.Helper()
colCounts := tableColumnCounts()
@@ -373,6 +809,23 @@ func tableColumnCounts() map[string]int {
return m
}
func validTestOutlook() model.WeatherOutlook {
return model.WeatherOutlook{
ID: "outlook-1",
Provider: "spc",
Product: "convective",
Day: 1,
OutlookType: "categorical",
Label: "SLGT",
ValidFrom: time.Date(2026, 6, 11, 13, 0, 0, 0, time.UTC),
ValidTo: time.Date(2026, 6, 12, 12, 0, 0, 0, time.UTC),
IssuedAt: time.Date(2026, 6, 11, 19, 45, 0, 0, time.UTC),
ExpiresAt: time.Date(2026, 6, 12, 12, 0, 0, 0, time.UTC),
ContainsLocation: true,
Geometry: json.RawMessage(`{"type":"Polygon","coordinates":[[[-91,38],[-90,38],[-90,39],[-91,39],[-91,38]]]}`),
}
}
func wmoCodePtr(v model.WMOCode) *model.WMOCode {
out := v
return &out

View File

@@ -16,6 +16,9 @@ const (
tableAlertRuns = "alert_runs"
tableAlerts = "alerts"
tableAlertReferences = "alert_references"
tableOutlookRuns = "outlook_runs"
tableOutlooks = "outlooks"
tableOutlookDiscussions = "outlook_discussions"
)
// PostgresSchema returns weatherfeeder's Postgres schema definition.
@@ -24,13 +27,7 @@ func PostgresSchema() fksinks.PostgresSchema {
Tables: []fksinks.PostgresTable{
{
Name: tableObservations,
Columns: []fksinks.PostgresColumn{
{Name: "event_id", Type: "TEXT", Nullable: false},
{Name: "event_kind", Type: "TEXT", Nullable: false},
{Name: "event_source", Type: "TEXT", Nullable: false},
{Name: "event_schema", Type: "TEXT", Nullable: false},
{Name: "event_emitted_at", Type: "TIMESTAMPTZ", Nullable: false},
{Name: "event_effective_at", Type: "TIMESTAMPTZ", Nullable: true},
Columns: parentEnvelopeColumns([]fksinks.PostgresColumn{
{Name: "station_id", Type: "TEXT", Nullable: true},
{Name: "station_name", Type: "TEXT", Nullable: true},
{Name: "observed_at", Type: "TIMESTAMPTZ", Nullable: false},
@@ -46,7 +43,7 @@ func PostgresSchema() fksinks.PostgresSchema {
{Name: "visibility_meters", Type: "DOUBLE PRECISION", Nullable: true},
{Name: "relative_humidity_percent", Type: "DOUBLE PRECISION", Nullable: true},
{Name: "apparent_temperature_c", Type: "DOUBLE PRECISION", Nullable: true},
},
}...),
PrimaryKey: []string{"event_id"},
PruneColumn: "observed_at",
Indexes: []fksinks.PostgresIndex{
@@ -71,13 +68,7 @@ func PostgresSchema() fksinks.PostgresSchema {
},
{
Name: tableForecasts,
Columns: []fksinks.PostgresColumn{
{Name: "event_id", Type: "TEXT", Nullable: false},
{Name: "event_kind", Type: "TEXT", Nullable: false},
{Name: "event_source", Type: "TEXT", Nullable: false},
{Name: "event_schema", Type: "TEXT", Nullable: false},
{Name: "event_emitted_at", Type: "TIMESTAMPTZ", Nullable: false},
{Name: "event_effective_at", Type: "TIMESTAMPTZ", Nullable: true},
Columns: parentEnvelopeColumns([]fksinks.PostgresColumn{
{Name: "location_id", Type: "TEXT", Nullable: true},
{Name: "location_name", Type: "TEXT", Nullable: true},
{Name: "issued_at", Type: "TIMESTAMPTZ", Nullable: false},
@@ -87,7 +78,7 @@ func PostgresSchema() fksinks.PostgresSchema {
{Name: "longitude", Type: "DOUBLE PRECISION", Nullable: true},
{Name: "elevation_meters", Type: "DOUBLE PRECISION", Nullable: true},
{Name: "period_count", Type: "INTEGER", Nullable: false},
},
}...),
PrimaryKey: []string{"event_id"},
PruneColumn: "issued_at",
Indexes: []fksinks.PostgresIndex{
@@ -135,13 +126,7 @@ func PostgresSchema() fksinks.PostgresSchema {
},
{
Name: tableForecastDiscussions,
Columns: []fksinks.PostgresColumn{
{Name: "event_id", Type: "TEXT", Nullable: false},
{Name: "event_kind", Type: "TEXT", Nullable: false},
{Name: "event_source", Type: "TEXT", Nullable: false},
{Name: "event_schema", Type: "TEXT", Nullable: false},
{Name: "event_emitted_at", Type: "TIMESTAMPTZ", Nullable: false},
{Name: "event_effective_at", Type: "TIMESTAMPTZ", Nullable: true},
Columns: parentEnvelopeColumns([]fksinks.PostgresColumn{
{Name: "office_id", Type: "TEXT", Nullable: true},
{Name: "office_name", Type: "TEXT", Nullable: true},
{Name: "issued_at", Type: "TIMESTAMPTZ", Nullable: false},
@@ -154,7 +139,7 @@ func PostgresSchema() fksinks.PostgresSchema {
{Name: "long_term_issued_at", Type: "TIMESTAMPTZ", Nullable: true},
{Name: "long_term_text", Type: "TEXT", Nullable: true},
{Name: "key_message_count", Type: "INTEGER", Nullable: false},
},
}...),
PrimaryKey: []string{"event_id"},
PruneColumn: "issued_at",
Indexes: []fksinks.PostgresIndex{
@@ -178,17 +163,11 @@ func PostgresSchema() fksinks.PostgresSchema {
},
{
Name: tableWeatherStoryRuns,
Columns: []fksinks.PostgresColumn{
{Name: "event_id", Type: "TEXT", Nullable: false},
{Name: "event_kind", Type: "TEXT", Nullable: false},
{Name: "event_source", Type: "TEXT", Nullable: false},
{Name: "event_schema", Type: "TEXT", Nullable: false},
{Name: "event_emitted_at", Type: "TIMESTAMPTZ", Nullable: false},
{Name: "event_effective_at", Type: "TIMESTAMPTZ", Nullable: true},
Columns: parentEnvelopeColumns([]fksinks.PostgresColumn{
{Name: "office_id", Type: "TEXT", Nullable: true},
{Name: "as_of", Type: "TIMESTAMPTZ", Nullable: false},
{Name: "story_count", Type: "INTEGER", Nullable: false},
},
}...),
PrimaryKey: []string{"event_id"},
PruneColumn: "as_of",
Indexes: []fksinks.PostgresIndex{
@@ -223,20 +202,14 @@ func PostgresSchema() fksinks.PostgresSchema {
},
{
Name: tableAlertRuns,
Columns: []fksinks.PostgresColumn{
{Name: "event_id", Type: "TEXT", Nullable: false},
{Name: "event_kind", Type: "TEXT", Nullable: false},
{Name: "event_source", Type: "TEXT", Nullable: false},
{Name: "event_schema", Type: "TEXT", Nullable: false},
{Name: "event_emitted_at", Type: "TIMESTAMPTZ", Nullable: false},
{Name: "event_effective_at", Type: "TIMESTAMPTZ", Nullable: true},
Columns: parentEnvelopeColumns([]fksinks.PostgresColumn{
{Name: "location_id", Type: "TEXT", Nullable: true},
{Name: "location_name", Type: "TEXT", Nullable: true},
{Name: "as_of", Type: "TIMESTAMPTZ", Nullable: false},
{Name: "latitude", Type: "DOUBLE PRECISION", Nullable: true},
{Name: "longitude", Type: "DOUBLE PRECISION", Nullable: true},
{Name: "alert_count", Type: "INTEGER", Nullable: false},
},
}...),
PrimaryKey: []string{"event_id"},
PruneColumn: "as_of",
Indexes: []fksinks.PostgresIndex{
@@ -265,6 +238,7 @@ func PostgresSchema() fksinks.PostgresSchema {
{Name: "sent", Type: "TIMESTAMPTZ", Nullable: true},
{Name: "effective", Type: "TIMESTAMPTZ", Nullable: true},
{Name: "onset", Type: "TIMESTAMPTZ", Nullable: true},
{Name: "ends", Type: "TIMESTAMPTZ", Nullable: true},
{Name: "expires", Type: "TIMESTAMPTZ", Nullable: true},
{Name: "area_description", Type: "TEXT", Nullable: true},
{Name: "sender_name", Type: "TEXT", Nullable: true},
@@ -297,7 +271,89 @@ func PostgresSchema() fksinks.PostgresSchema {
{Name: "idx_wf_alert_refs_sent", Columns: []string{"sent"}},
},
},
{
Name: tableOutlookRuns,
Columns: parentEnvelopeColumns([]fksinks.PostgresColumn{
{Name: "location_id", Type: "TEXT", Nullable: true},
{Name: "location_name", Type: "TEXT", Nullable: true},
{Name: "latitude", Type: "DOUBLE PRECISION", Nullable: true},
{Name: "longitude", Type: "DOUBLE PRECISION", Nullable: true},
{Name: "as_of", Type: "TIMESTAMPTZ", Nullable: false},
{Name: "issued_at", Type: "TIMESTAMPTZ", Nullable: true},
{Name: "outlook_count", Type: "INTEGER", Nullable: false},
{Name: "discussion_count", Type: "INTEGER", Nullable: false},
}...),
PrimaryKey: []string{"event_id"},
PruneColumn: "as_of",
Indexes: []fksinks.PostgresIndex{
{Name: "idx_wf_outlook_run_location_as_of", Columns: []string{"location_id", "as_of"}},
{Name: "idx_wf_outlook_run_as_of", Columns: []string{"as_of"}},
},
},
{
Name: tableOutlooks,
Columns: []fksinks.PostgresColumn{
{Name: "run_event_id", Type: "TEXT REFERENCES outlook_runs(event_id) ON DELETE CASCADE", Nullable: false},
{Name: "outlook_index", Type: "INTEGER", Nullable: false},
{Name: "as_of", Type: "TIMESTAMPTZ", Nullable: false},
{Name: "outlook_id", Type: "TEXT", Nullable: false},
{Name: "provider", Type: "TEXT", Nullable: false},
{Name: "product", Type: "TEXT", Nullable: false},
{Name: "day", Type: "INTEGER", Nullable: false},
{Name: "outlook_type", Type: "TEXT", Nullable: false},
{Name: "label", Type: "TEXT", Nullable: false},
{Name: "label_text", Type: "TEXT", Nullable: true},
{Name: "severity_rank", Type: "INTEGER", Nullable: true},
{Name: "valid_from", Type: "TIMESTAMPTZ", Nullable: false},
{Name: "valid_to", Type: "TIMESTAMPTZ", Nullable: false},
{Name: "issued_at", Type: "TIMESTAMPTZ", Nullable: false},
{Name: "expires_at", Type: "TIMESTAMPTZ", Nullable: false},
{Name: "forecaster", Type: "TEXT", Nullable: true},
{Name: "source_url", Type: "TEXT", Nullable: true},
{Name: "image_url", Type: "TEXT", Nullable: true},
{Name: "contains_location", Type: "BOOLEAN", Nullable: false},
{Name: "geometry_json", Type: "TEXT", Nullable: false},
},
PrimaryKey: []string{"run_event_id", "outlook_index"},
PruneColumn: "as_of",
Indexes: []fksinks.PostgresIndex{
{Name: "idx_wf_outlooks_contains_valid", Columns: []string{"contains_location", "valid_from", "valid_to"}},
{Name: "idx_wf_outlooks_day_type_label", Columns: []string{"day", "outlook_type", "label"}},
{Name: "idx_wf_outlooks_valid", Columns: []string{"valid_from", "valid_to"}},
},
},
{
Name: tableOutlookDiscussions,
Columns: []fksinks.PostgresColumn{
{Name: "run_event_id", Type: "TEXT REFERENCES outlook_runs(event_id) ON DELETE CASCADE", Nullable: false},
{Name: "discussion_index", Type: "INTEGER", Nullable: false},
{Name: "as_of", Type: "TIMESTAMPTZ", Nullable: false},
{Name: "day", Type: "INTEGER", Nullable: false},
{Name: "headline", Type: "TEXT", Nullable: true},
{Name: "summary", Type: "TEXT", Nullable: true},
{Name: "discussion", Type: "TEXT", Nullable: true},
{Name: "updated_at", Type: "TIMESTAMPTZ", Nullable: true},
},
PrimaryKey: []string{"run_event_id", "discussion_index"},
PruneColumn: "as_of",
Indexes: []fksinks.PostgresIndex{
{Name: "idx_wf_outlook_discussions_day_as_of", Columns: []string{"day", "as_of"}},
{Name: "idx_wf_outlook_discussions_run_day", Columns: []string{"run_event_id", "day"}, Unique: true},
},
},
},
MapEvent: mapPostgresEvent,
}
}
func parentEnvelopeColumns(extra ...fksinks.PostgresColumn) []fksinks.PostgresColumn {
columns := []fksinks.PostgresColumn{
{Name: "event_id", Type: "TEXT", Nullable: false},
{Name: "event_kind", Type: "TEXT", Nullable: false},
{Name: "event_source", Type: "TEXT", Nullable: false},
{Name: "event_schema", Type: "TEXT", Nullable: false},
{Name: "event_emitted_at", Type: "TIMESTAMPTZ", Nullable: false},
{Name: "event_effective_at", Type: "TIMESTAMPTZ", Nullable: true},
}
return append(columns, extra...)
}

View File

@@ -1,6 +1,12 @@
package postgres
import "testing"
import (
"reflect"
"strings"
"testing"
fksinks "gitea.maximumdirect.net/ejr/feedkit/sinks"
)
func TestWeatherPostgresSchemaShape(t *testing.T) {
s := PostgresSchema()
@@ -20,6 +26,9 @@ func TestWeatherPostgresSchemaShape(t *testing.T) {
tableAlertRuns: true,
tableAlerts: true,
tableAlertReferences: true,
tableOutlookRuns: true,
tableOutlooks: true,
tableOutlookDiscussions: true,
}
if len(s.Tables) != len(wantTables) {
@@ -43,6 +52,54 @@ func TestWeatherPostgresSchemaShape(t *testing.T) {
}
}
func TestWeatherPostgresSchemaIncludesOutlookTables(t *testing.T) {
runColumns := columnsForTable(t, tableOutlookRuns)
for _, col := range []string{"event_id", "event_kind", "event_source", "event_schema", "event_emitted_at", "event_effective_at", "location_id", "location_name", "latitude", "longitude", "as_of", "issued_at", "outlook_count", "discussion_count"} {
if !runColumns[col] {
t.Fatalf("%s missing %s column", tableOutlookRuns, col)
}
}
assertTablePrimaryKey(t, tableOutlookRuns, []string{"event_id"})
assertTableIndex(t, tableOutlookRuns, "idx_wf_outlook_run_location_as_of", []string{"location_id", "as_of"})
assertTableIndex(t, tableOutlookRuns, "idx_wf_outlook_run_as_of", []string{"as_of"})
outlookColumns := columnsForTable(t, tableOutlooks)
for _, col := range []string{"run_event_id", "outlook_index", "as_of", "outlook_id", "provider", "product", "day", "outlook_type", "label", "label_text", "severity_rank", "valid_from", "valid_to", "issued_at", "expires_at", "forecaster", "source_url", "image_url", "contains_location", "geometry_json"} {
if !outlookColumns[col] {
t.Fatalf("%s missing %s column", tableOutlooks, col)
}
}
for _, col := range []string{"headline", "summary", "discussion"} {
if outlookColumns[col] {
t.Fatalf("%s still includes legacy %s column", tableOutlooks, col)
}
}
assertTablePrimaryKey(t, tableOutlooks, []string{"run_event_id", "outlook_index"})
assertTableIndex(t, tableOutlooks, "idx_wf_outlooks_contains_valid", []string{"contains_location", "valid_from", "valid_to"})
assertTableIndex(t, tableOutlooks, "idx_wf_outlooks_day_type_label", []string{"day", "outlook_type", "label"})
assertTableIndex(t, tableOutlooks, "idx_wf_outlooks_valid", []string{"valid_from", "valid_to"})
discussionColumns := columnsForTable(t, tableOutlookDiscussions)
for _, col := range []string{"run_event_id", "discussion_index", "as_of", "day", "headline", "summary", "discussion", "updated_at"} {
if !discussionColumns[col] {
t.Fatalf("%s missing %s column", tableOutlookDiscussions, col)
}
}
assertTablePrimaryKey(t, tableOutlookDiscussions, []string{"run_event_id", "discussion_index"})
assertTablePruneColumn(t, tableOutlookDiscussions, "as_of")
assertTableIndex(t, tableOutlookDiscussions, "idx_wf_outlook_discussions_day_as_of", []string{"day", "as_of"})
assertTableUniqueIndex(t, tableOutlookDiscussions, "idx_wf_outlook_discussions_run_day", []string{"run_event_id", "day"})
}
func TestWeatherPostgresSchemaIncludesAlertEndsColumn(t *testing.T) {
alertColumns := columnsForTable(t, tableAlerts)
for _, col := range []string{"run_event_id", "alert_index", "as_of", "alert_id", "onset", "ends", "expires"} {
if !alertColumns[col] {
t.Fatalf("%s missing %s column", tableAlerts, col)
}
}
}
func TestWeatherPostgresSchemaIncludesWeatherStoryColumns(t *testing.T) {
runColumns := columnsForTable(t, tableWeatherStoryRuns)
if !runColumns["as_of"] {
@@ -60,20 +117,102 @@ func TestWeatherPostgresSchemaIncludesWeatherStoryColumns(t *testing.T) {
}
}
func columnsForTable(t *testing.T, table string) map[string]bool {
func TestWeatherPostgresSchemaParentTablesStartWithEnvelopeColumns(t *testing.T) {
for _, table := range []string{
tableObservations,
tableForecasts,
tableForecastDiscussions,
tableWeatherStoryRuns,
tableAlertRuns,
tableOutlookRuns,
} {
t.Run(table, func(t *testing.T) {
columns := orderedColumnsForTable(t, table)
want := parentEnvelopeColumns()
if len(columns) < len(want) {
t.Fatalf("%s has %d columns, want at least %d", table, len(columns), len(want))
}
if !reflect.DeepEqual(columns[:len(want)], want) {
t.Fatalf("%s envelope prefix = %#v, want %#v", table, columns[:len(want)], want)
}
})
}
}
func assertTablePrimaryKey(t *testing.T, table string, want []string) {
t.Helper()
tbl := tableByName(t, table)
if strings.Join(tbl.PrimaryKey, ",") != strings.Join(want, ",") {
t.Fatalf("%s primary key = %#v, want %#v", table, tbl.PrimaryKey, want)
}
}
func assertTablePruneColumn(t *testing.T, table string, want string) {
t.Helper()
tbl := tableByName(t, table)
if tbl.PruneColumn != want {
t.Fatalf("%s prune column = %q, want %q", table, tbl.PruneColumn, want)
}
}
func assertTableIndex(t *testing.T, table string, name string, want []string) {
t.Helper()
assertTableIndexWithUnique(t, table, name, want, false)
}
func assertTableUniqueIndex(t *testing.T, table string, name string, want []string) {
t.Helper()
assertTableIndexWithUnique(t, table, name, want, true)
}
func assertTableIndexWithUnique(t *testing.T, table string, name string, want []string, unique bool) {
t.Helper()
tbl := tableByName(t, table)
for _, idx := range tbl.Indexes {
if idx.Name == name {
if strings.Join(idx.Columns, ",") != strings.Join(want, ",") {
t.Fatalf("%s index %s columns = %#v, want %#v", table, name, idx.Columns, want)
}
if idx.Unique != unique {
t.Fatalf("%s index %s unique = %v, want %v", table, name, idx.Unique, unique)
}
return
}
}
t.Fatalf("%s missing index %s", table, name)
}
func tableByName(t *testing.T, table string) fksinks.PostgresTable {
t.Helper()
for _, tbl := range PostgresSchema().Tables {
if tbl.Name == table {
return tbl
}
}
t.Fatalf("missing table %q", table)
return fksinks.PostgresTable{}
}
func orderedColumnsForTable(t *testing.T, table string) []fksinks.PostgresColumn {
t.Helper()
schema := PostgresSchema()
for _, tbl := range schema.Tables {
if tbl.Name != table {
continue
if tbl.Name == table {
return tbl.Columns
}
cols := make(map[string]bool, len(tbl.Columns))
for _, col := range tbl.Columns {
cols[col.Name] = true
}
return cols
}
t.Fatalf("missing table %q", table)
return nil
}
func columnsForTable(t *testing.T, table string) map[string]bool {
t.Helper()
ordered := orderedColumnsForTable(t, table)
cols := make(map[string]bool, len(ordered))
for _, col := range ordered {
cols[col.Name] = true
}
return cols
}

View File

@@ -4,6 +4,7 @@ import (
"gitea.maximumdirect.net/ejr/weatherfeeder/internal/sources/nws"
"gitea.maximumdirect.net/ejr/weatherfeeder/internal/sources/openmeteo"
"gitea.maximumdirect.net/ejr/weatherfeeder/internal/sources/openweather"
"gitea.maximumdirect.net/ejr/weatherfeeder/internal/sources/spc"
"gitea.maximumdirect.net/ejr/feedkit/config"
fksource "gitea.maximumdirect.net/ejr/feedkit/sources"
@@ -15,19 +16,22 @@ type pollDriverRegistration struct {
}
var pollDriverRegistrations = []pollDriverRegistration{
{driver: "nws_observation", factory: func(cfg config.SourceConfig) (fksource.PollSource, error) { return nws.NewObservationSource(cfg) }},
{driver: "nws_alerts", factory: func(cfg config.SourceConfig) (fksource.PollSource, error) { return nws.NewAlertsSource(cfg) }},
{driver: "nws_forecast_hourly", factory: func(cfg config.SourceConfig) (fksource.PollSource, error) { return nws.NewHourlyForecastSource(cfg) }},
{driver: "nws_forecast_narrative", factory: func(cfg config.SourceConfig) (fksource.PollSource, error) { return nws.NewNarrativeForecastSource(cfg) }},
{driver: "nws_forecast_discussion", factory: func(cfg config.SourceConfig) (fksource.PollSource, error) {
{driver: nws.DriverObservation, factory: func(cfg config.SourceConfig) (fksource.PollSource, error) { return nws.NewObservationSource(cfg) }},
{driver: nws.DriverAlerts, factory: func(cfg config.SourceConfig) (fksource.PollSource, error) { return nws.NewAlertsSource(cfg) }},
{driver: nws.DriverForecastHourly, factory: func(cfg config.SourceConfig) (fksource.PollSource, error) { return nws.NewHourlyForecastSource(cfg) }},
{driver: nws.DriverForecastNarrative, factory: func(cfg config.SourceConfig) (fksource.PollSource, error) { return nws.NewNarrativeForecastSource(cfg) }},
{driver: nws.DriverForecastDiscussion, factory: func(cfg config.SourceConfig) (fksource.PollSource, error) {
return nws.NewForecastDiscussionSource(cfg)
}},
{driver: "nws_weatherstories", factory: func(cfg config.SourceConfig) (fksource.PollSource, error) { return nws.NewWeatherStoriesSource(cfg) }},
{driver: "openmeteo_observation", factory: func(cfg config.SourceConfig) (fksource.PollSource, error) { return openmeteo.NewObservationSource(cfg) }},
{driver: "openmeteo_forecast", factory: func(cfg config.SourceConfig) (fksource.PollSource, error) { return openmeteo.NewForecastSource(cfg) }},
{driver: "openweather_observation", factory: func(cfg config.SourceConfig) (fksource.PollSource, error) {
{driver: nws.DriverWeatherStories, factory: func(cfg config.SourceConfig) (fksource.PollSource, error) { return nws.NewWeatherStoriesSource(cfg) }},
{driver: openmeteo.DriverObservation, factory: func(cfg config.SourceConfig) (fksource.PollSource, error) { return openmeteo.NewObservationSource(cfg) }},
{driver: openmeteo.DriverForecast, factory: func(cfg config.SourceConfig) (fksource.PollSource, error) { return openmeteo.NewForecastSource(cfg) }},
{driver: openweather.DriverObservation, factory: func(cfg config.SourceConfig) (fksource.PollSource, error) {
return openweather.NewObservationSource(cfg)
}},
{driver: spc.DriverConvectiveOutlook, factory: func(cfg config.SourceConfig) (fksource.PollSource, error) {
return spc.NewConvectiveOutlookSource(cfg)
}},
}
// RegisterBuiltins registers the source drivers that ship with this binary.

View File

@@ -6,57 +6,29 @@ import (
"gitea.maximumdirect.net/ejr/feedkit/config"
fksource "gitea.maximumdirect.net/ejr/feedkit/sources"
"gitea.maximumdirect.net/ejr/weatherfeeder/internal/sources/openweather"
"gitea.maximumdirect.net/ejr/weatherfeeder/internal/sources/spc"
)
func TestRegisterBuiltinsRegistersNWSHourlyForecastDriver(t *testing.T) {
func TestRegisterBuiltinsRegistersCurrentPollDrivers(t *testing.T) {
reg := fksource.NewRegistry()
RegisterBuiltins(reg)
in, err := reg.BuildInput(sourceConfigForDriver("nws_forecast_hourly"))
if err != nil {
t.Fatalf("BuildInput(nws_forecast_hourly) error = %v", err)
if len(pollDriverRegistrations) == 0 {
t.Fatalf("pollDriverRegistrations is empty")
}
if _, ok := in.(fksource.PollSource); !ok {
t.Fatalf("BuildInput(nws_forecast_hourly) type = %T, want PollSource", in)
}
}
func TestRegisterBuiltinsRegistersNWSNarrativeForecastDriver(t *testing.T) {
reg := fksource.NewRegistry()
RegisterBuiltins(reg)
in, err := reg.BuildInput(sourceConfigForDriver("nws_forecast_narrative"))
if err != nil {
t.Fatalf("BuildInput(nws_forecast_narrative) error = %v", err)
}
if _, ok := in.(fksource.PollSource); !ok {
t.Fatalf("BuildInput(nws_forecast_narrative) type = %T, want PollSource", in)
}
}
func TestRegisterBuiltinsRegistersNWSForecastDiscussionDriver(t *testing.T) {
reg := fksource.NewRegistry()
RegisterBuiltins(reg)
in, err := reg.BuildInput(sourceConfigForDriver("nws_forecast_discussion"))
if err != nil {
t.Fatalf("BuildInput(nws_forecast_discussion) error = %v", err)
}
if _, ok := in.(fksource.PollSource); !ok {
t.Fatalf("BuildInput(nws_forecast_discussion) type = %T, want PollSource", in)
}
}
func TestRegisterBuiltinsRegistersNWSWeatherStoriesDriver(t *testing.T) {
reg := fksource.NewRegistry()
RegisterBuiltins(reg)
in, err := reg.BuildInput(sourceConfigForDriver("nws_weatherstories"))
if err != nil {
t.Fatalf("BuildInput(nws_weatherstories) error = %v", err)
}
if _, ok := in.(fksource.PollSource); !ok {
t.Fatalf("BuildInput(nws_weatherstories) type = %T, want PollSource", in)
for _, tt := range pollDriverRegistrations {
tt := tt
t.Run(tt.driver, func(t *testing.T) {
in, err := reg.BuildInput(sourceConfigForDriver(tt.driver))
if err != nil {
t.Fatalf("BuildInput(%q) error = %v", tt.driver, err)
}
if _, ok := in.(fksource.PollSource); !ok {
t.Fatalf("BuildInput(%q) type = %T, want PollSource", tt.driver, in)
}
})
}
}
@@ -73,45 +45,23 @@ func TestRegisterBuiltinsDoesNotRegisterLegacyNWSForecastDriver(t *testing.T) {
}
}
func TestRegisterBuiltinsRegistersAllCurrentDrivers(t *testing.T) {
reg := fksource.NewRegistry()
RegisterBuiltins(reg)
drivers := []string{
"nws_observation",
"nws_alerts",
"nws_forecast_hourly",
"nws_forecast_narrative",
"nws_forecast_discussion",
"nws_weatherstories",
"openmeteo_observation",
"openmeteo_forecast",
"openweather_observation",
}
for _, driver := range drivers {
in, err := reg.BuildInput(sourceConfigForDriver(driver))
if err != nil {
t.Fatalf("BuildInput(%s) error = %v", driver, err)
}
if _, ok := in.(fksource.PollSource); !ok {
t.Fatalf("BuildInput(%s) type = %T, want PollSource", driver, in)
}
}
}
func sourceConfigForDriver(driver string) config.SourceConfig {
url := "https://example.invalid"
if driver == "openweather_observation" {
if driver == openweather.DriverObservation {
url = "https://example.invalid?units=metric"
}
params := map[string]any{
"url": url,
"user_agent": "test-agent",
}
if driver == spc.DriverConvectiveOutlook {
params["latitude"] = 38.6239
params["longitude"] = -90.3571
}
return config.SourceConfig{
Name: "test-source",
Driver: driver,
Mode: config.SourceModePoll,
Params: map[string]any{
"url": url,
"user_agent": "test-agent",
},
Params: params,
}
}

View File

@@ -0,0 +1,32 @@
package sources
import (
"os"
"strings"
"testing"
)
func TestDocumentedRegisteredSourceDrivers(t *testing.T) {
docs := map[string]string{
"docs/config.md": readDoc(t, "../../docs/config.md"),
"docs/internal/sources.md": readDoc(t, "../../docs/internal/sources.md"),
}
for _, reg := range pollDriverRegistrations {
for path, doc := range docs {
if !strings.Contains(doc, reg.driver) {
t.Fatalf("%s missing source driver %q", path, reg.driver)
}
}
}
}
func readDoc(t *testing.T, path string) string {
t.Helper()
raw, err := os.ReadFile(path)
if err != nil {
t.Fatalf("ReadFile(%s) error = %v", path, err)
}
return string(raw)
}

View File

@@ -0,0 +1,58 @@
package httpconfig
import (
"fmt"
"strings"
"time"
"gitea.maximumdirect.net/ejr/feedkit/config"
"gitea.maximumdirect.net/ejr/feedkit/transport"
)
// Settings contains common HTTP client config for sources that fetch multiple documents.
type Settings struct {
Name string
UserAgent string
Timeout time.Duration
BodyLimitBytes int64
}
func Parse(driver string, cfg config.SourceConfig) (Settings, error) {
name := strings.TrimSpace(cfg.Name)
if name == "" {
return Settings{}, fmt.Errorf("%s: name is required", driver)
}
if cfg.Params == nil {
return Settings{}, fmt.Errorf("%s %q: params are required", driver, name)
}
userAgent, ok := cfg.ParamString("user_agent", "userAgent")
if !ok {
return Settings{}, fmt.Errorf("%s %q: params.user_agent is required", driver, name)
}
timeout := transport.DefaultHTTPTimeout
if _, exists := cfg.Params["http_timeout"]; exists {
var ok bool
timeout, ok = cfg.ParamDuration("http_timeout")
if !ok || timeout <= 0 {
return Settings{}, fmt.Errorf("source %q: params.http_timeout must be a positive duration", name)
}
}
bodyLimit := transport.DefaultHTTPResponseBodyLimitBytes
if _, exists := cfg.Params["http_response_body_limit_bytes"]; exists {
rawLimit, ok := cfg.ParamInt("http_response_body_limit_bytes")
if !ok || rawLimit <= 0 {
return Settings{}, fmt.Errorf("source %q: params.http_response_body_limit_bytes must be a positive integer", name)
}
bodyLimit = int64(rawLimit)
}
return Settings{
Name: name,
UserAgent: userAgent,
Timeout: timeout,
BodyLimitBytes: bodyLimit,
}, nil
}

View File

@@ -0,0 +1,115 @@
package httpconfig
import (
"strings"
"testing"
"time"
"gitea.maximumdirect.net/ejr/feedkit/config"
"gitea.maximumdirect.net/ejr/feedkit/transport"
)
func TestParseUsesRequiredValuesAndDefaults(t *testing.T) {
got, err := Parse("test_driver", config.SourceConfig{
Name: " test-source ",
Params: map[string]any{
"user_agent": "test-agent",
},
})
if err != nil {
t.Fatalf("Parse() error = %v", err)
}
if got.Name != "test-source" {
t.Fatalf("Name = %q, want test-source", got.Name)
}
if got.UserAgent != "test-agent" {
t.Fatalf("UserAgent = %q, want test-agent", got.UserAgent)
}
if got.Timeout != transport.DefaultHTTPTimeout {
t.Fatalf("Timeout = %s, want %s", got.Timeout, transport.DefaultHTTPTimeout)
}
if got.BodyLimitBytes != transport.DefaultHTTPResponseBodyLimitBytes {
t.Fatalf("BodyLimitBytes = %d, want %d", got.BodyLimitBytes, transport.DefaultHTTPResponseBodyLimitBytes)
}
}
func TestParseUsesAliasesAndOverrides(t *testing.T) {
got, err := Parse("test_driver", config.SourceConfig{
Name: "test-source",
Params: map[string]any{
"userAgent": "test-agent",
"http_timeout": "2s",
"http_response_body_limit_bytes": 2048,
},
})
if err != nil {
t.Fatalf("Parse() error = %v", err)
}
if got.UserAgent != "test-agent" {
t.Fatalf("UserAgent = %q, want test-agent", got.UserAgent)
}
if got.Timeout != 2*time.Second {
t.Fatalf("Timeout = %s, want 2s", got.Timeout)
}
if got.BodyLimitBytes != 2048 {
t.Fatalf("BodyLimitBytes = %d, want 2048", got.BodyLimitBytes)
}
}
func TestParseRejectsInvalidConfig(t *testing.T) {
tests := []struct {
name string
cfg config.SourceConfig
wantErr string
}{
{
name: "missing name",
cfg: config.SourceConfig{Params: map[string]any{"user_agent": "test-agent"}},
wantErr: "test_driver: name is required",
},
{
name: "missing params",
cfg: config.SourceConfig{Name: "test-source"},
wantErr: `test_driver "test-source": params are required`,
},
{
name: "missing user agent",
cfg: config.SourceConfig{Name: "test-source", Params: map[string]any{}},
wantErr: `test_driver "test-source": params.user_agent is required`,
},
{
name: "invalid timeout",
cfg: config.SourceConfig{
Name: "test-source",
Params: map[string]any{
"user_agent": "test-agent",
"http_timeout": "0s",
},
},
wantErr: `source "test-source": params.http_timeout must be a positive duration`,
},
{
name: "invalid body limit",
cfg: config.SourceConfig{
Name: "test-source",
Params: map[string]any{
"user_agent": "test-agent",
"http_response_body_limit_bytes": 0,
},
},
wantErr: `source "test-source": params.http_response_body_limit_bytes must be a positive integer`,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
_, err := Parse("test_driver", tt.cfg)
if err == nil {
t.Fatalf("Parse() error = nil, want %q", tt.wantErr)
}
if !strings.Contains(err.Error(), tt.wantErr) {
t.Fatalf("Parse() error = %q, want %q", err, tt.wantErr)
}
})
}
}

View File

@@ -26,10 +26,8 @@ type AlertsSource struct {
}
func NewAlertsSource(cfg config.SourceConfig) (*AlertsSource, error) {
const driver = "nws_alerts"
// NWS alerts responses are GeoJSON-ish; allow fallback to plain JSON as well.
hs, err := fksources.NewHTTPSource(driver, cfg, "application/geo+json, application/json")
hs, err := fksources.NewHTTPSource(DriverAlerts, cfg, "application/geo+json, application/json")
if err != nil {
return nil, err
}
@@ -40,7 +38,7 @@ func NewAlertsSource(cfg config.SourceConfig) (*AlertsSource, error) {
func (s *AlertsSource) Name() string { return s.http.Name }
// Kinds is used for routing/policy.
func (s *AlertsSource) Kinds() []event.Kind { return []event.Kind{event.Kind("alert")} }
func (s *AlertsSource) Kinds() []event.Kind { return []event.Kind{event.Kind(standards.KindAlert)} }
func (s *AlertsSource) Poll(ctx context.Context) ([]event.Event, error) {
raw, meta, changed, err := s.fetchRaw(ctx)
@@ -71,7 +69,7 @@ func (s *AlertsSource) Poll(ctx context.Context) ([]event.Event, error) {
eventID := fksources.DefaultEventID("", s.http.Name, effectiveAt, emittedAt)
return fksources.SingleEvent(
event.Kind("alert"),
event.Kind(standards.KindAlert),
s.http.Name,
standards.SchemaRawNWSAlertsV1,
eventID,

View File

@@ -0,0 +1,11 @@
package nws
// Source driver strings registered by weatherfeeder for NWS sources.
const (
DriverObservation = "nws_observation"
DriverAlerts = "nws_alerts"
DriverForecastHourly = "nws_forecast_hourly"
DriverForecastNarrative = "nws_forecast_narrative"
DriverForecastDiscussion = "nws_forecast_discussion"
DriverWeatherStories = "nws_weatherstories"
)

View File

@@ -10,6 +10,7 @@ import (
"gitea.maximumdirect.net/ejr/feedkit/event"
fksources "gitea.maximumdirect.net/ejr/feedkit/sources"
nwscommon "gitea.maximumdirect.net/ejr/weatherfeeder/internal/providers/nws"
"gitea.maximumdirect.net/ejr/weatherfeeder/standards"
)
const nwsForecastAccept = "application/geo+json, application/json"
@@ -44,7 +45,9 @@ func newForecastSource(cfg config.SourceConfig, driver, rawSchema string) (*fore
func (s *forecastSource) Name() string { return s.http.Name }
func (s *forecastSource) Kinds() []event.Kind { return []event.Kind{event.Kind("forecast")} }
func (s *forecastSource) Kinds() []event.Kind {
return []event.Kind{event.Kind(standards.KindForecast)}
}
func (s *forecastSource) Poll(ctx context.Context) ([]event.Event, error) {
raw, meta, changed, err := s.fetchRaw(ctx)
@@ -69,7 +72,7 @@ func (s *forecastSource) Poll(ctx context.Context) ([]event.Event, error) {
eventID := fksources.DefaultEventID("", s.http.Name, effectiveAt, emittedAt)
return fksources.SingleEvent(
event.Kind("forecast"),
event.Kind(standards.KindForecast),
s.http.Name,
s.rawSchema,
eventID,

View File

@@ -20,9 +20,7 @@ type ForecastDiscussionSource struct {
}
func NewForecastDiscussionSource(cfg config.SourceConfig) (*ForecastDiscussionSource, error) {
const driver = "nws_forecast_discussion"
hs, err := fksources.NewHTTPSource(driver, cfg, "text/html, application/xhtml+xml")
hs, err := fksources.NewHTTPSource(DriverForecastDiscussion, cfg, "text/html, application/xhtml+xml")
if err != nil {
return nil, err
}
@@ -33,7 +31,7 @@ func NewForecastDiscussionSource(cfg config.SourceConfig) (*ForecastDiscussionSo
func (s *ForecastDiscussionSource) Name() string { return s.http.Name }
func (s *ForecastDiscussionSource) Kinds() []event.Kind {
return []event.Kind{event.Kind("forecast_discussion")}
return []event.Kind{event.Kind(standards.KindForecastDiscussion)}
}
func (s *ForecastDiscussionSource) Poll(ctx context.Context) ([]event.Event, error) {
@@ -57,7 +55,7 @@ func (s *ForecastDiscussionSource) Poll(ctx context.Context) ([]event.Event, err
eventID := fksources.DefaultEventID("", s.http.Name, effectiveAt, emittedAt)
return fksources.SingleEvent(
event.Kind("forecast_discussion"),
event.Kind(standards.KindForecastDiscussion),
s.http.Name,
standards.SchemaRawNWSForecastDiscussionV1,
eventID,

View File

@@ -27,7 +27,7 @@ func TestForecastDiscussionSourcePollEmitsExpectedEvent(t *testing.T) {
if err != nil {
t.Fatalf("NewForecastDiscussionSource() error = %v", err)
}
if got := src.Kinds(); len(got) != 1 || got[0] != event.Kind("forecast_discussion") {
if got := src.Kinds(); len(got) != 1 || got[0] != event.Kind(standards.KindForecastDiscussion) {
t.Fatalf("Kinds() = %#v, want [forecast_discussion]", got)
}
@@ -40,7 +40,7 @@ func TestForecastDiscussionSourcePollEmitsExpectedEvent(t *testing.T) {
}
got := events[0]
if got.Kind != event.Kind("forecast_discussion") {
if got.Kind != event.Kind(standards.KindForecastDiscussion) {
t.Fatalf("Kind = %q, want forecast_discussion", got.Kind)
}
if got.Schema != standards.SchemaRawNWSForecastDiscussionV1 {
@@ -117,7 +117,7 @@ func TestForecastDiscussionSourcePollRejectsInvalidHTML(t *testing.T) {
func forecastDiscussionSourceConfig(url string) config.SourceConfig {
return config.SourceConfig{
Name: "test-forecast-discussion-source",
Driver: "nws_forecast_discussion",
Driver: DriverForecastDiscussion,
Mode: config.SourceModePoll,
Params: map[string]any{
"url": url,

View File

@@ -18,8 +18,7 @@ type HourlyForecastSource struct {
}
func NewHourlyForecastSource(cfg config.SourceConfig) (*HourlyForecastSource, error) {
const driver = "nws_forecast_hourly"
src, err := newForecastSource(cfg, driver, standards.SchemaRawNWSHourlyForecastV1)
src, err := newForecastSource(cfg, DriverForecastHourly, standards.SchemaRawNWSHourlyForecastV1)
if err != nil {
return nil, err
}

View File

@@ -18,8 +18,7 @@ type NarrativeForecastSource struct {
}
func NewNarrativeForecastSource(cfg config.SourceConfig) (*NarrativeForecastSource, error) {
const driver = "nws_forecast_narrative"
src, err := newForecastSource(cfg, driver, standards.SchemaRawNWSNarrativeForecastV1)
src, err := newForecastSource(cfg, DriverForecastNarrative, standards.SchemaRawNWSNarrativeForecastV1)
if err != nil {
return nil, err
}

View File

@@ -26,7 +26,7 @@ func TestForecastSourcesEmitExpectedSchemaAndPreferGeneratedAt(t *testing.T) {
}{
{
name: "hourly",
driver: "nws_forecast_hourly",
driver: DriverForecastHourly,
wantSchema: standards.SchemaRawNWSHourlyForecastV1,
newSource: func(cfg config.SourceConfig) (forecastPoller, error) {
return NewHourlyForecastSource(cfg)
@@ -34,7 +34,7 @@ func TestForecastSourcesEmitExpectedSchemaAndPreferGeneratedAt(t *testing.T) {
},
{
name: "narrative",
driver: "nws_forecast_narrative",
driver: DriverForecastNarrative,
wantSchema: standards.SchemaRawNWSNarrativeForecastV1,
newSource: func(cfg config.SourceConfig) (forecastPoller, error) {
return NewNarrativeForecastSource(cfg)
@@ -55,7 +55,7 @@ func TestForecastSourcesEmitExpectedSchemaAndPreferGeneratedAt(t *testing.T) {
}
if ks, ok := src.(interface{ Kinds() []event.Kind }); !ok {
t.Fatalf("source does not implement Kinds()")
} else if gotKinds := ks.Kinds(); len(gotKinds) != 1 || gotKinds[0] != event.Kind("forecast") {
} else if gotKinds := ks.Kinds(); len(gotKinds) != 1 || gotKinds[0] != event.Kind(standards.KindForecast) {
t.Fatalf("Kinds() = %#v, want [forecast]", gotKinds)
}
@@ -69,7 +69,7 @@ func TestForecastSourcesEmitExpectedSchemaAndPreferGeneratedAt(t *testing.T) {
if got[0].Schema != tt.wantSchema {
t.Fatalf("Poll() schema = %q, want %q", got[0].Schema, tt.wantSchema)
}
if got[0].Kind != event.Kind("forecast") {
if got[0].Kind != event.Kind(standards.KindForecast) {
t.Fatalf("Poll() kind = %q, want forecast", got[0].Kind)
}
@@ -117,7 +117,7 @@ func TestForecastSourcePollEffectiveAtFallbackOrder(t *testing.T) {
}))
defer srv.Close()
src, err := NewHourlyForecastSource(forecastSourceConfig("nws_forecast_hourly", srv.URL))
src, err := NewHourlyForecastSource(forecastSourceConfig(DriverForecastHourly, srv.URL))
if err != nil {
t.Fatalf("NewHourlyForecastSource() error = %v", err)
}
@@ -148,7 +148,7 @@ func TestForecastSourcePollMetadataDecodeFailureStillEmitsRawEvent(t *testing.T)
}))
defer srv.Close()
src, err := NewNarrativeForecastSource(forecastSourceConfig("nws_forecast_narrative", srv.URL))
src, err := NewNarrativeForecastSource(forecastSourceConfig(DriverForecastNarrative, srv.URL))
if err != nil {
t.Fatalf("NewNarrativeForecastSource() error = %v", err)
}

View File

@@ -20,9 +20,7 @@ type ObservationSource struct {
}
func NewObservationSource(cfg config.SourceConfig) (*ObservationSource, error) {
const driver = "nws_observation"
hs, err := fksources.NewHTTPSource(driver, cfg, "application/geo+json, application/json")
hs, err := fksources.NewHTTPSource(DriverObservation, cfg, "application/geo+json, application/json")
if err != nil {
return nil, err
}
@@ -32,7 +30,9 @@ func NewObservationSource(cfg config.SourceConfig) (*ObservationSource, error) {
func (s *ObservationSource) Name() string { return s.http.Name }
func (s *ObservationSource) Kinds() []event.Kind { return []event.Kind{event.Kind("observation")} }
func (s *ObservationSource) Kinds() []event.Kind {
return []event.Kind{event.Kind(standards.KindObservation)}
}
func (s *ObservationSource) Poll(ctx context.Context) ([]event.Event, error) {
raw, meta, changed, err := s.fetchRaw(ctx)
@@ -54,7 +54,7 @@ func (s *ObservationSource) Poll(ctx context.Context) ([]event.Event, error) {
eventID := fksources.DefaultEventID(meta.ID, s.http.Name, effectiveAt, emittedAt)
return fksources.SingleEvent(
event.Kind("observation"),
event.Kind(standards.KindObservation),
s.http.Name,
standards.SchemaRawNWSObservationV1,
eventID,

View File

@@ -8,6 +8,7 @@ import (
"gitea.maximumdirect.net/ejr/feedkit/config"
"gitea.maximumdirect.net/ejr/feedkit/event"
"gitea.maximumdirect.net/ejr/weatherfeeder/standards"
)
func TestObservationSourcePollReturnsNoEventsOn304(t *testing.T) {
@@ -31,7 +32,7 @@ func TestObservationSourcePollReturnsNoEventsOn304(t *testing.T) {
src, err := NewObservationSource(config.SourceConfig{
Name: "NWSObservationTest",
Driver: "nws_observation",
Driver: DriverObservation,
Mode: config.SourceModePoll,
Params: map[string]any{
"url": srv.URL,
@@ -41,7 +42,7 @@ func TestObservationSourcePollReturnsNoEventsOn304(t *testing.T) {
if err != nil {
t.Fatalf("NewObservationSource() error = %v", err)
}
if got := src.Kinds(); len(got) != 1 || got[0] != event.Kind("observation") {
if got := src.Kinds(); len(got) != 1 || got[0] != event.Kind(standards.KindObservation) {
t.Fatalf("Kinds() = %#v, want [observation]", got)
}
@@ -52,7 +53,7 @@ func TestObservationSourcePollReturnsNoEventsOn304(t *testing.T) {
if len(first) != 1 {
t.Fatalf("first Poll() len = %d, want 1", len(first))
}
if first[0].Kind != event.Kind("observation") {
if first[0].Kind != event.Kind(standards.KindObservation) {
t.Fatalf("first Poll() kind = %q", first[0].Kind)
}

View File

@@ -22,9 +22,7 @@ type WeatherStoriesSource struct {
}
func NewWeatherStoriesSource(cfg config.SourceConfig) (*WeatherStoriesSource, error) {
const driver = "nws_weatherstories"
hs, err := fksources.NewHTTPSource(driver, cfg, "application/geo+json, application/json")
hs, err := fksources.NewHTTPSource(DriverWeatherStories, cfg, "application/geo+json, application/json")
if err != nil {
return nil, err
}
@@ -35,7 +33,7 @@ func NewWeatherStoriesSource(cfg config.SourceConfig) (*WeatherStoriesSource, er
func (s *WeatherStoriesSource) Name() string { return s.http.Name }
func (s *WeatherStoriesSource) Kinds() []event.Kind {
return []event.Kind{event.Kind("weather_story")}
return []event.Kind{event.Kind(standards.KindWeatherStory)}
}
func (s *WeatherStoriesSource) Poll(ctx context.Context) ([]event.Event, error) {
@@ -61,7 +59,7 @@ func (s *WeatherStoriesSource) Poll(ctx context.Context) ([]event.Event, error)
eventID := fksources.DefaultEventID("", s.http.Name, effectiveAt, emittedAt)
return fksources.SingleEvent(
event.Kind("weather_story"),
event.Kind(standards.KindWeatherStory),
s.http.Name,
standards.SchemaRawNWSWeatherStoriesV1,
eventID,

View File

@@ -28,7 +28,7 @@ func TestWeatherStoriesSourcePollEmitsExpectedEventAndPrefersLatestUpdateTime(t
if err != nil {
t.Fatalf("NewWeatherStoriesSource() error = %v", err)
}
if got := src.Kinds(); len(got) != 1 || got[0] != event.Kind("weather_story") {
if got := src.Kinds(); len(got) != 1 || got[0] != event.Kind(standards.KindWeatherStory) {
t.Fatalf("Kinds() = %#v, want [weather_story]", got)
}
@@ -41,7 +41,7 @@ func TestWeatherStoriesSourcePollEmitsExpectedEventAndPrefersLatestUpdateTime(t
}
got := events[0]
if got.Kind != event.Kind("weather_story") {
if got.Kind != event.Kind(standards.KindWeatherStory) {
t.Fatalf("Kind = %q, want weather_story", got.Kind)
}
if got.Schema != standards.SchemaRawNWSWeatherStoriesV1 {
@@ -148,7 +148,7 @@ func TestWeatherStoriesSourcePollMetadataDecodeFailureStillEmitsRawEvent(t *test
func weatherStoriesSourceConfig(url string) config.SourceConfig {
return config.SourceConfig{
Name: "test-weatherstories-source",
Driver: "nws_weatherstories",
Driver: DriverWeatherStories,
Mode: config.SourceModePoll,
Params: map[string]any{
"url": url,

View File

@@ -0,0 +1,7 @@
package openmeteo
// Source driver strings registered by weatherfeeder for Open-Meteo sources.
const (
DriverObservation = "openmeteo_observation"
DriverForecast = "openmeteo_forecast"
)

View File

@@ -19,9 +19,7 @@ type ForecastSource struct {
}
func NewForecastSource(cfg config.SourceConfig) (*ForecastSource, error) {
const driver = "openmeteo_forecast"
hs, err := fksources.NewHTTPSource(driver, cfg, "application/json")
hs, err := fksources.NewHTTPSource(DriverForecast, cfg, "application/json")
if err != nil {
return nil, err
}
@@ -31,7 +29,9 @@ func NewForecastSource(cfg config.SourceConfig) (*ForecastSource, error) {
func (s *ForecastSource) Name() string { return s.http.Name }
func (s *ForecastSource) Kinds() []event.Kind { return []event.Kind{event.Kind("forecast")} }
func (s *ForecastSource) Kinds() []event.Kind {
return []event.Kind{event.Kind(standards.KindForecast)}
}
func (s *ForecastSource) Poll(ctx context.Context) ([]event.Event, error) {
raw, meta, changed, err := s.fetchRaw(ctx)
@@ -55,7 +55,7 @@ func (s *ForecastSource) Poll(ctx context.Context) ([]event.Event, error) {
eventID := fksources.DefaultEventID("", s.http.Name, effectiveAt, emittedAt)
return fksources.SingleEvent(
event.Kind("forecast"),
event.Kind(standards.KindForecast),
s.http.Name,
standards.SchemaRawOpenMeteoHourlyForecastV1,
eventID,

View File

@@ -19,9 +19,7 @@ type ObservationSource struct {
}
func NewObservationSource(cfg config.SourceConfig) (*ObservationSource, error) {
const driver = "openmeteo_observation"
hs, err := fksources.NewHTTPSource(driver, cfg, "application/json")
hs, err := fksources.NewHTTPSource(DriverObservation, cfg, "application/json")
if err != nil {
return nil, err
}
@@ -31,7 +29,9 @@ func NewObservationSource(cfg config.SourceConfig) (*ObservationSource, error) {
func (s *ObservationSource) Name() string { return s.http.Name }
func (s *ObservationSource) Kinds() []event.Kind { return []event.Kind{event.Kind("observation")} }
func (s *ObservationSource) Kinds() []event.Kind {
return []event.Kind{event.Kind(standards.KindObservation)}
}
func (s *ObservationSource) Poll(ctx context.Context) ([]event.Event, error) {
raw, meta, changed, err := s.fetchRaw(ctx)
@@ -52,7 +52,7 @@ func (s *ObservationSource) Poll(ctx context.Context) ([]event.Event, error) {
eventID := fksources.DefaultEventID("", s.http.Name, effectiveAt, emittedAt)
return fksources.SingleEvent(
event.Kind("observation"),
event.Kind(standards.KindObservation),
s.http.Name,
standards.SchemaRawOpenMeteoCurrentV1,
eventID,

View File

@@ -5,12 +5,13 @@ import (
"gitea.maximumdirect.net/ejr/feedkit/config"
"gitea.maximumdirect.net/ejr/feedkit/event"
"gitea.maximumdirect.net/ejr/weatherfeeder/standards"
)
func TestObservationSourceAdvertisesKinds(t *testing.T) {
src, err := NewObservationSource(config.SourceConfig{
Name: "openmeteo-observation-test",
Driver: "openmeteo_observation",
Driver: DriverObservation,
Mode: config.SourceModePoll,
Params: map[string]any{
"url": "https://example.invalid",
@@ -20,7 +21,7 @@ func TestObservationSourceAdvertisesKinds(t *testing.T) {
if err != nil {
t.Fatalf("NewObservationSource() error = %v", err)
}
if got := src.Kinds(); len(got) != 1 || got[0] != event.Kind("observation") {
if got := src.Kinds(); len(got) != 1 || got[0] != event.Kind(standards.KindObservation) {
t.Fatalf("Kinds() = %#v, want [observation]", got)
}
}
@@ -28,7 +29,7 @@ func TestObservationSourceAdvertisesKinds(t *testing.T) {
func TestForecastSourceAdvertisesKinds(t *testing.T) {
src, err := NewForecastSource(config.SourceConfig{
Name: "openmeteo-forecast-test",
Driver: "openmeteo_forecast",
Driver: DriverForecast,
Mode: config.SourceModePoll,
Params: map[string]any{
"url": "https://example.invalid",
@@ -38,7 +39,7 @@ func TestForecastSourceAdvertisesKinds(t *testing.T) {
if err != nil {
t.Fatalf("NewForecastSource() error = %v", err)
}
if got := src.Kinds(); len(got) != 1 || got[0] != event.Kind("forecast") {
if got := src.Kinds(); len(got) != 1 || got[0] != event.Kind(standards.KindForecast) {
t.Fatalf("Kinds() = %#v, want [forecast]", got)
}
}

View File

@@ -0,0 +1,6 @@
package openweather
// Source driver strings registered by weatherfeeder for OpenWeather sources.
const (
DriverObservation = "openweather_observation"
)

View File

@@ -19,9 +19,7 @@ type ObservationSource struct {
}
func NewObservationSource(cfg config.SourceConfig) (*ObservationSource, error) {
const driver = "openweather_observation"
hs, err := fksources.NewHTTPSource(driver, cfg, "application/json")
hs, err := fksources.NewHTTPSource(DriverObservation, cfg, "application/json")
if err != nil {
return nil, err
}
@@ -35,7 +33,9 @@ func NewObservationSource(cfg config.SourceConfig) (*ObservationSource, error) {
func (s *ObservationSource) Name() string { return s.http.Name }
func (s *ObservationSource) Kinds() []event.Kind { return []event.Kind{event.Kind("observation")} }
func (s *ObservationSource) Kinds() []event.Kind {
return []event.Kind{event.Kind(standards.KindObservation)}
}
func (s *ObservationSource) Poll(ctx context.Context) ([]event.Event, error) {
if err := owcommon.RequireMetricUnits(s.http.URL); err != nil {
@@ -60,7 +60,7 @@ func (s *ObservationSource) Poll(ctx context.Context) ([]event.Event, error) {
eventID := fksources.DefaultEventID("", s.http.Name, effectiveAt, emittedAt)
return fksources.SingleEvent(
event.Kind("observation"),
event.Kind(standards.KindObservation),
s.http.Name,
standards.SchemaRawOpenWeatherCurrentV1,
eventID,

View File

@@ -5,12 +5,13 @@ import (
"gitea.maximumdirect.net/ejr/feedkit/config"
"gitea.maximumdirect.net/ejr/feedkit/event"
"gitea.maximumdirect.net/ejr/weatherfeeder/standards"
)
func TestObservationSourceAdvertisesKinds(t *testing.T) {
src, err := NewObservationSource(config.SourceConfig{
Name: "openweather-observation-test",
Driver: "openweather_observation",
Driver: DriverObservation,
Mode: config.SourceModePoll,
Params: map[string]any{
"url": "https://example.invalid?units=metric",
@@ -20,7 +21,7 @@ func TestObservationSourceAdvertisesKinds(t *testing.T) {
if err != nil {
t.Fatalf("NewObservationSource() error = %v", err)
}
if got := src.Kinds(); len(got) != 1 || got[0] != event.Kind("observation") {
if got := src.Kinds(); len(got) != 1 || got[0] != event.Kind(standards.KindObservation) {
t.Fatalf("Kinds() = %#v, want [observation]", got)
}
}

View File

@@ -0,0 +1,380 @@
package spc
import (
"context"
"crypto/sha256"
"encoding/json"
"fmt"
"math"
"net/http"
"strconv"
"strings"
"time"
"gitea.maximumdirect.net/ejr/feedkit/config"
"gitea.maximumdirect.net/ejr/feedkit/event"
fksources "gitea.maximumdirect.net/ejr/feedkit/sources"
"gitea.maximumdirect.net/ejr/feedkit/transport"
spcprovider "gitea.maximumdirect.net/ejr/weatherfeeder/internal/providers/spc"
"gitea.maximumdirect.net/ejr/weatherfeeder/internal/sources/internal/httpconfig"
"gitea.maximumdirect.net/ejr/weatherfeeder/standards"
)
const (
acceptGeoJSON = "application/geo+json, application/json"
acceptDiscussion = "text/html, application/xhtml+xml"
acceptRSS = "application/rss+xml, application/xml, text/xml"
)
type fetchProduct struct {
Key string
Day int
OutlookType string
URL string
Accept string
}
// ConvectiveOutlookSource polls SPC Day 1-3 convective outlook products and
// emits one raw outlook bundle event.
type ConvectiveOutlookSource struct {
name string
userAgent string
locationID string
locationName string
latitude float64
longitude float64
client *http.Client
bodyLimit int64
geoJSONProducts []fetchProduct
discussions []fetchProduct
rssURL string
lastHash [sha256.Size]byte
hasHash bool
}
func NewConvectiveOutlookSource(cfg config.SourceConfig) (*ConvectiveOutlookSource, error) {
httpSettings, err := httpconfig.Parse(DriverConvectiveOutlook, cfg)
if err != nil {
return nil, err
}
latitude, err := requireFloatParam(cfg, "latitude")
if err != nil {
return nil, fmt.Errorf("%s %q: %w", DriverConvectiveOutlook, httpSettings.Name, err)
}
longitude, err := requireFloatParam(cfg, "longitude")
if err != nil {
return nil, fmt.Errorf("%s %q: %w", DriverConvectiveOutlook, httpSettings.Name, err)
}
geoJSONProducts, err := configuredGeoJSONProducts(cfg)
if err != nil {
return nil, fmt.Errorf("%s %q: %w", DriverConvectiveOutlook, httpSettings.Name, err)
}
discussions, err := configuredDiscussionProducts(cfg)
if err != nil {
return nil, fmt.Errorf("%s %q: %w", DriverConvectiveOutlook, httpSettings.Name, err)
}
rssURL := ""
if rawRSSURL, ok := cfg.ParamString("rss_url", "rssURL"); ok {
rssURL = rawRSSURL
}
locationID, _ := cfg.ParamString("location_id", "locationID")
locationName, _ := cfg.ParamString("location_name", "locationName")
return &ConvectiveOutlookSource{
name: httpSettings.Name,
userAgent: httpSettings.UserAgent,
locationID: locationID,
locationName: locationName,
latitude: latitude,
longitude: longitude,
client: transport.NewHTTPClient(httpSettings.Timeout),
bodyLimit: httpSettings.BodyLimitBytes,
geoJSONProducts: geoJSONProducts,
discussions: discussions,
rssURL: rssURL,
}, nil
}
func (s *ConvectiveOutlookSource) Name() string { return s.name }
func (s *ConvectiveOutlookSource) Kinds() []event.Kind {
return []event.Kind{event.Kind(standards.KindOutlook)}
}
func (s *ConvectiveOutlookSource) Poll(ctx context.Context) ([]event.Event, error) {
fetchedAt := time.Now().UTC()
bundle := spcprovider.RawConvectiveOutlookBundle{
LocationID: s.locationID,
LocationName: s.locationName,
Latitude: s.latitude,
Longitude: s.longitude,
FetchedAt: fetchedAt,
Products: make([]spcprovider.RawOutlookProduct, 0, len(s.geoJSONProducts)),
Discussions: make([]spcprovider.RawDiscussionPage, 0, len(s.discussions)),
}
hash := sha256.New()
var latestIssue time.Time
for _, product := range s.geoJSONProducts {
body, err := s.fetch(ctx, product.URL, product.Accept)
if err != nil {
return nil, fmt.Errorf("fetch geojson %s: %w", product.Key, err)
}
addHashPart(hash, product.Key, product.URL, body)
bundle.Products = append(bundle.Products, spcprovider.RawOutlookProduct{
Key: product.Key,
Day: product.Day,
OutlookType: product.OutlookType,
URL: product.URL,
FetchedAt: fetchedAt,
Body: json.RawMessage(body),
})
if t := latestIssueTime(body); !t.IsZero() && (latestIssue.IsZero() || t.After(latestIssue)) {
latestIssue = t
}
}
var latestUpdated time.Time
for _, product := range s.discussions {
body, err := s.fetch(ctx, product.URL, product.Accept)
if err != nil {
return nil, fmt.Errorf("fetch discussion %s: %w", product.Key, err)
}
addHashPart(hash, product.Key, product.URL, body)
bodyText := string(body)
bundle.Discussions = append(bundle.Discussions, spcprovider.RawDiscussionPage{
Key: product.Key,
Day: product.Day,
URL: product.URL,
FetchedAt: fetchedAt,
Body: bodyText,
})
if t := discussionUpdatedTime(bodyText); !t.IsZero() && (latestUpdated.IsZero() || t.After(latestUpdated)) {
latestUpdated = t
}
}
var rssBuild time.Time
if s.rssURL != "" {
body, err := s.fetch(ctx, s.rssURL, acceptRSS)
if err != nil {
return nil, fmt.Errorf("fetch rss: %w", err)
}
addHashPart(hash, "rss", s.rssURL, body)
bodyText := string(body)
bundle.RSS = &spcprovider.RawRSSFeed{
URL: s.rssURL,
FetchedAt: fetchedAt,
Body: bodyText,
}
if feed, err := spcprovider.ParseRSSFeed(bodyText); err == nil && feed.LastBuildDate != nil {
rssBuild = feed.LastBuildDate.UTC()
}
}
var currentHash [sha256.Size]byte
copy(currentHash[:], hash.Sum(nil))
if s.hasHash && currentHash == s.lastHash {
return nil, nil
}
s.lastHash = currentHash
s.hasHash = true
effectiveAt := chooseEffectiveTime(latestIssue, latestUpdated, rssBuild, fetchedAt)
emittedAt := time.Now().UTC()
eventID := fksources.DefaultEventID("", s.name, &effectiveAt, emittedAt)
return fksources.SingleEvent(
event.Kind(standards.KindOutlook),
s.name,
standards.SchemaRawSPCConvectiveOutlookV1,
eventID,
emittedAt,
&effectiveAt,
bundle,
)
}
func (s *ConvectiveOutlookSource) fetch(ctx context.Context, url, accept string) ([]byte, error) {
return transport.FetchBodyWithLimit(ctx, s.client, url, s.userAgent, accept, s.bodyLimit)
}
func configuredGeoJSONProducts(cfg config.SourceConfig) ([]fetchProduct, error) {
overrides, err := optionalStringMap(cfg, "geojson_urls")
if err != nil {
return nil, err
}
out := make([]fetchProduct, 0, len(spcprovider.GeoJSONProducts()))
for _, product := range spcprovider.GeoJSONProducts() {
url := product.URL
if override := strings.TrimSpace(overrides[product.Key]); override != "" {
url = override
}
out = append(out, fetchProduct{
Key: product.Key,
Day: product.Day,
OutlookType: product.OutlookType,
URL: url,
Accept: acceptGeoJSON,
})
}
return out, nil
}
func configuredDiscussionProducts(cfg config.SourceConfig) ([]fetchProduct, error) {
overrides, err := optionalStringMap(cfg, "discussion_urls")
if err != nil {
return nil, err
}
out := make([]fetchProduct, 0, len(spcprovider.DiscussionProducts()))
for _, product := range spcprovider.DiscussionProducts() {
url := product.URL
if override := strings.TrimSpace(overrides[product.Key]); override != "" {
url = override
}
out = append(out, fetchProduct{
Key: product.Key,
Day: product.Day,
URL: url,
Accept: acceptDiscussion,
})
}
return out, nil
}
func optionalStringMap(cfg config.SourceConfig, key string) (map[string]string, error) {
raw, ok := cfg.Params[key]
if !ok || raw == nil {
return map[string]string{}, nil
}
out := map[string]string{}
switch typed := raw.(type) {
case map[string]string:
for k, v := range typed {
if strings.TrimSpace(k) != "" && strings.TrimSpace(v) != "" {
out[strings.TrimSpace(k)] = strings.TrimSpace(v)
}
}
case map[string]any:
for k, v := range typed {
s, ok := v.(string)
if !ok {
return nil, fmt.Errorf("params.%s[%q] must be a string", key, k)
}
if strings.TrimSpace(k) != "" && strings.TrimSpace(s) != "" {
out[strings.TrimSpace(k)] = strings.TrimSpace(s)
}
}
default:
return nil, fmt.Errorf("params.%s must be a string map", key)
}
return out, nil
}
func requireFloatParam(cfg config.SourceConfig, key string) (float64, error) {
raw, ok := cfg.Params[key]
if !ok || raw == nil {
return 0, fmt.Errorf("params.%s is required", key)
}
v, ok := numberFromAny(raw)
if !ok {
return 0, fmt.Errorf("params.%s must be a number", key)
}
if math.IsNaN(v) || math.IsInf(v, 0) {
return 0, fmt.Errorf("params.%s must be finite", key)
}
return v, nil
}
func numberFromAny(raw any) (float64, bool) {
switch v := raw.(type) {
case float64:
return v, true
case float32:
return float64(v), true
case int:
return float64(v), true
case int8:
return float64(v), true
case int16:
return float64(v), true
case int32:
return float64(v), true
case int64:
return float64(v), true
case uint:
return float64(v), true
case uint8:
return float64(v), true
case uint16:
return float64(v), true
case uint32:
return float64(v), true
case uint64:
return float64(v), true
case string:
parsed, err := strconv.ParseFloat(strings.TrimSpace(v), 64)
return parsed, err == nil
default:
return 0, false
}
}
func latestIssueTime(raw []byte) time.Time {
collection, err := spcprovider.DecodeGeoJSON(raw)
if err != nil {
return time.Time{}
}
var latest time.Time
for _, feature := range collection.Features {
t, err := spcprovider.ParseISOTimestamp(feature.Properties.IssueISO)
if err != nil {
continue
}
t = t.UTC()
if latest.IsZero() || t.After(latest) {
latest = t
}
}
return latest
}
func discussionUpdatedTime(rawHTML string) time.Time {
t := spcprovider.ParsePageUpdatedTimestamp(rawHTML)
if t == nil {
return time.Time{}
}
return t.UTC()
}
func chooseEffectiveTime(issue time.Time, updated time.Time, rss time.Time, fetched time.Time) time.Time {
switch {
case !issue.IsZero():
return issue.UTC()
case !updated.IsZero():
return updated.UTC()
case !rss.IsZero():
return rss.UTC()
default:
return fetched.UTC()
}
}
func addHashPart(hash interface{ Write([]byte) (int, error) }, key, url string, body []byte) {
_, _ = hash.Write([]byte(key))
_, _ = hash.Write([]byte{0})
_, _ = hash.Write([]byte(url))
_, _ = hash.Write([]byte{0})
_, _ = hash.Write(body)
_, _ = hash.Write([]byte{0})
}

View File

@@ -0,0 +1,347 @@
package spc
import (
"encoding/json"
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
"gitea.maximumdirect.net/ejr/feedkit/config"
"gitea.maximumdirect.net/ejr/feedkit/event"
spcprovider "gitea.maximumdirect.net/ejr/weatherfeeder/internal/providers/spc"
"gitea.maximumdirect.net/ejr/weatherfeeder/standards"
)
func TestConvectiveOutlookSourceKinds(t *testing.T) {
src, err := NewConvectiveOutlookSource(convectiveOutlookConfig(map[string]any{}))
if err != nil {
t.Fatalf("NewConvectiveOutlookSource() error = %v", err)
}
got := src.Kinds()
if len(got) != 1 || got[0] != event.Kind(standards.KindOutlook) {
t.Fatalf("Kinds() = %#v, want [outlook]", got)
}
}
func TestConvectiveOutlookSourceRequiresLatitudeAndLongitude(t *testing.T) {
for _, key := range []string{"latitude", "longitude"} {
cfg := convectiveOutlookConfig(map[string]any{})
delete(cfg.Params, key)
_, err := NewConvectiveOutlookSource(cfg)
if err == nil {
t.Fatalf("NewConvectiveOutlookSource() without %s error = nil, want error", key)
}
if !strings.Contains(err.Error(), "params."+key+" is required") {
t.Fatalf("error = %q, want missing %s", err, key)
}
}
}
func TestConvectiveOutlookSourcePollEmitsRawBundle(t *testing.T) {
srv := newSPCTestServer(t, spcServerOptions{})
src, err := NewConvectiveOutlookSource(convectiveOutlookConfig(serverOverrideParams(srv.URL, false)))
if err != nil {
t.Fatalf("NewConvectiveOutlookSource() error = %v", err)
}
events, err := src.Poll(t.Context())
if err != nil {
t.Fatalf("Poll() error = %v", err)
}
if len(events) != 1 {
t.Fatalf("Poll() returned %d events, want 1", len(events))
}
got := events[0]
if got.Kind != event.Kind(standards.KindOutlook) {
t.Fatalf("Kind = %q, want outlook", got.Kind)
}
if got.Schema != standards.SchemaRawSPCConvectiveOutlookV1 {
t.Fatalf("Schema = %q, want %q", got.Schema, standards.SchemaRawSPCConvectiveOutlookV1)
}
wantEffective := time.Date(2026, 6, 11, 19, 45, 0, 0, time.UTC)
if got.EffectiveAt == nil || !got.EffectiveAt.Equal(wantEffective) {
t.Fatalf("EffectiveAt = %v, want %s", got.EffectiveAt, wantEffective)
}
bundle, ok := got.Payload.(spcprovider.RawConvectiveOutlookBundle)
if !ok {
t.Fatalf("Payload type = %T, want RawConvectiveOutlookBundle", got.Payload)
}
if bundle.LocationID != "stl" || bundle.LocationName != "St. Louis, MO" {
t.Fatalf("location metadata = %q/%q", bundle.LocationID, bundle.LocationName)
}
if bundle.Latitude != 38.6239 || bundle.Longitude != -90.3571 {
t.Fatalf("coordinates = %v,%v", bundle.Latitude, bundle.Longitude)
}
if len(bundle.Products) != 9 {
t.Fatalf("Products length = %d, want 9", len(bundle.Products))
}
if len(bundle.Discussions) != 3 {
t.Fatalf("Discussions length = %d, want 3", len(bundle.Discussions))
}
if bundle.RSS != nil {
t.Fatalf("RSS = %#v, want nil", bundle.RSS)
}
}
func TestConvectiveOutlookSourceEffectiveAtFallsBackToDiscussionUpdated(t *testing.T) {
srv := newSPCTestServer(t, spcServerOptions{blankIssueISO: true})
src, err := NewConvectiveOutlookSource(convectiveOutlookConfig(serverOverrideParams(srv.URL, false)))
if err != nil {
t.Fatalf("NewConvectiveOutlookSource() error = %v", err)
}
events, err := src.Poll(t.Context())
if err != nil {
t.Fatalf("Poll() error = %v", err)
}
want := time.Date(2026, 6, 11, 20, 0, 0, 0, time.UTC)
if events[0].EffectiveAt == nil || !events[0].EffectiveAt.Equal(want) {
t.Fatalf("EffectiveAt = %v, want %s", events[0].EffectiveAt, want)
}
}
func TestConvectiveOutlookSourceIncludesRSSOnlyWhenConfigured(t *testing.T) {
srv := newSPCTestServer(t, spcServerOptions{})
withoutRSS, err := NewConvectiveOutlookSource(convectiveOutlookConfig(serverOverrideParams(srv.URL, false)))
if err != nil {
t.Fatalf("NewConvectiveOutlookSource(without RSS) error = %v", err)
}
events, err := withoutRSS.Poll(t.Context())
if err != nil {
t.Fatalf("Poll(without RSS) error = %v", err)
}
if events[0].Payload.(spcprovider.RawConvectiveOutlookBundle).RSS != nil {
t.Fatalf("RSS present without rss_url")
}
withRSS, err := NewConvectiveOutlookSource(convectiveOutlookConfig(serverOverrideParams(srv.URL, true)))
if err != nil {
t.Fatalf("NewConvectiveOutlookSource(with RSS) error = %v", err)
}
events, err = withRSS.Poll(t.Context())
if err != nil {
t.Fatalf("Poll(with RSS) error = %v", err)
}
if events[0].Payload.(spcprovider.RawConvectiveOutlookBundle).RSS == nil {
t.Fatalf("RSS missing with rss_url")
}
}
func TestConvectiveOutlookSourceUnchangedResponseEmitsNoEvents(t *testing.T) {
srv := newSPCTestServer(t, spcServerOptions{})
src, err := NewConvectiveOutlookSource(convectiveOutlookConfig(serverOverrideParams(srv.URL, false)))
if err != nil {
t.Fatalf("NewConvectiveOutlookSource() error = %v", err)
}
events, err := src.Poll(t.Context())
if err != nil {
t.Fatalf("first Poll() error = %v", err)
}
if len(events) != 1 {
t.Fatalf("first Poll() events = %d, want 1", len(events))
}
events, err = src.Poll(t.Context())
if err != nil {
t.Fatalf("second Poll() error = %v", err)
}
if len(events) != 0 {
t.Fatalf("second Poll() events = %d, want 0", len(events))
}
}
func TestConvectiveOutlookSourceGeoJSONFailureReturnsError(t *testing.T) {
srv := newSPCTestServer(t, spcServerOptions{failGeoJSONKey: "day2_wind"})
src, err := NewConvectiveOutlookSource(convectiveOutlookConfig(serverOverrideParams(srv.URL, false)))
if err != nil {
t.Fatalf("NewConvectiveOutlookSource() error = %v", err)
}
events, err := src.Poll(t.Context())
if err == nil {
t.Fatalf("Poll() error = nil, want error")
}
if len(events) != 0 {
t.Fatalf("Poll() events = %d, want 0", len(events))
}
if !strings.Contains(err.Error(), "fetch geojson day2_wind") {
t.Fatalf("error = %q", err)
}
}
func TestConvectiveOutlookSourceDiscussionFailureReturnsError(t *testing.T) {
srv := newSPCTestServer(t, spcServerOptions{failDiscussionKey: "day2"})
src, err := NewConvectiveOutlookSource(convectiveOutlookConfig(serverOverrideParams(srv.URL, false)))
if err != nil {
t.Fatalf("NewConvectiveOutlookSource() error = %v", err)
}
events, err := src.Poll(t.Context())
if err == nil {
t.Fatalf("Poll() error = nil, want error")
}
if len(events) != 0 {
t.Fatalf("Poll() events = %d, want 0", len(events))
}
if !strings.Contains(err.Error(), "fetch discussion day2") {
t.Fatalf("error = %q", err)
}
}
func TestConvectiveOutlookSourcePayloadJSONShape(t *testing.T) {
srv := newSPCTestServer(t, spcServerOptions{})
src, err := NewConvectiveOutlookSource(convectiveOutlookConfig(serverOverrideParams(srv.URL, false)))
if err != nil {
t.Fatalf("NewConvectiveOutlookSource() error = %v", err)
}
events, err := src.Poll(t.Context())
if err != nil {
t.Fatalf("Poll() error = %v", err)
}
raw, err := json.Marshal(events[0].Payload)
if err != nil {
t.Fatalf("Marshal(payload) error = %v", err)
}
if !strings.Contains(string(raw), `"products"`) || !strings.Contains(string(raw), `"discussions"`) {
t.Fatalf("payload JSON missing raw bundle fields: %s", raw)
}
}
type spcServerOptions struct {
blankIssueISO bool
failGeoJSONKey string
failDiscussionKey string
}
func newSPCTestServer(t *testing.T, opts spcServerOptions) *httptest.Server {
t.Helper()
mux := http.NewServeMux()
for _, product := range spcprovider.GeoJSONProducts() {
product := product
mux.HandleFunc("/geojson/"+product.Key, func(w http.ResponseWriter, r *http.Request) {
if r.Header.Get("Accept") != acceptGeoJSON {
t.Errorf("geojson Accept = %q, want %q", r.Header.Get("Accept"), acceptGeoJSON)
}
if product.Key == opts.failGeoJSONKey {
http.Error(w, "failed", http.StatusInternalServerError)
return
}
w.Header().Set("Content-Type", "application/geo+json")
_, _ = w.Write(geoJSONFixtureForProduct(t, product.Key, opts.blankIssueISO))
})
}
for _, product := range spcprovider.DiscussionProducts() {
product := product
mux.HandleFunc("/discussion/"+product.Key, func(w http.ResponseWriter, r *http.Request) {
if r.Header.Get("Accept") != acceptDiscussion {
t.Errorf("discussion Accept = %q, want %q", r.Header.Get("Accept"), acceptDiscussion)
}
if product.Key == opts.failDiscussionKey {
http.Error(w, "failed", http.StatusInternalServerError)
return
}
w.Header().Set("Content-Type", "text/html")
_, _ = w.Write(discussionFixtureForProduct(t, product.Key))
})
}
mux.HandleFunc("/rss", func(w http.ResponseWriter, r *http.Request) {
if r.Header.Get("Accept") != acceptRSS {
t.Errorf("rss Accept = %q, want %q", r.Header.Get("Accept"), acceptRSS)
}
w.Header().Set("Content-Type", "application/rss+xml")
_, _ = w.Write([]byte(testRSS))
})
return httptest.NewServer(mux)
}
func convectiveOutlookConfig(extra map[string]any) config.SourceConfig {
params := map[string]any{
"latitude": 38.6239,
"longitude": -90.3571,
"location_id": "stl",
"location_name": "St. Louis, MO",
"user_agent": "test-agent",
}
for k, v := range extra {
params[k] = v
}
return config.SourceConfig{
Name: "spc-test",
Driver: DriverConvectiveOutlook,
Mode: config.SourceModePoll,
Params: params,
}
}
func serverOverrideParams(baseURL string, includeRSS bool) map[string]any {
geoJSONURLs := map[string]any{}
for _, product := range spcprovider.GeoJSONProducts() {
geoJSONURLs[product.Key] = baseURL + "/geojson/" + product.Key
}
discussionURLs := map[string]any{}
for _, product := range spcprovider.DiscussionProducts() {
discussionURLs[product.Key] = baseURL + "/discussion/" + product.Key
}
out := map[string]any{
"geojson_urls": geoJSONURLs,
"discussion_urls": discussionURLs,
}
if includeRSS {
out["rss_url"] = baseURL + "/rss"
}
return out
}
func geoJSONFixtureForProduct(t *testing.T, key string, blankIssueISO bool) []byte {
t.Helper()
var name string
switch {
case strings.HasPrefix(key, "day1_"):
name = "day1_cat.geojson"
case strings.HasPrefix(key, "day2_"):
name = "day2_torn.geojson"
case strings.HasPrefix(key, "day3_"):
name = "day3_cat.geojson"
default:
t.Fatalf("unknown product key %q", key)
}
raw := readSPCTestFixture(t, name)
if blankIssueISO {
raw = []byte(strings.ReplaceAll(string(raw), `"ISSUE_ISO": "2026-06-11T12:34:56Z"`, `"ISSUE_ISO": ""`))
raw = []byte(strings.ReplaceAll(string(raw), `"ISSUE_ISO": "2026-06-11T17:30:00Z"`, `"ISSUE_ISO": ""`))
raw = []byte(strings.ReplaceAll(string(raw), `"ISSUE_ISO": "2026-06-11T19:45:00Z"`, `"ISSUE_ISO": ""`))
}
return raw
}
func discussionFixtureForProduct(t *testing.T, key string) []byte {
t.Helper()
switch key {
case "day1":
return readSPCTestFixture(t, "day1_prt.html")
case "day2":
return readSPCTestFixture(t, "day2_prt_corr.html")
case "day3":
return readSPCTestFixture(t, "day3_prt.html")
default:
t.Fatalf("unknown discussion key %q", key)
return nil
}
}
const testRSS = `<?xml version="1.0"?>
<rss version="2.0">
<channel>
<title>SPC AC RSS</title>
<lastBuildDate>Thu, 11 Jun 2026 21:00:00 +0000</lastBuildDate>
</channel>
</rss>`

View File

@@ -0,0 +1,6 @@
package spc
// Source driver strings registered by weatherfeeder for SPC sources.
const (
DriverConvectiveOutlook = "spc_convective_outlook"
)

View File

@@ -0,0 +1,17 @@
package spc
import (
"os"
"path/filepath"
"testing"
)
func readSPCTestFixture(t *testing.T, name string) []byte {
t.Helper()
path := filepath.Join("..", "..", "providers", "spc", "testdata", name)
raw, err := os.ReadFile(path)
if err != nil {
t.Fatalf("read fixture %s: %v", path, err)
}
return raw
}

View File

@@ -55,9 +55,11 @@ type WeatherAlert struct {
Instruction string `json:"instruction,omitempty"`
// Timing (all optional; provider-dependent).
// Onset and Ends describe the alert period. Expires is provider expiration metadata.
Sent *time.Time `json:"sent,omitempty"`
Effective *time.Time `json:"effective,omitempty"`
Onset *time.Time `json:"onset,omitempty"`
Ends *time.Time `json:"ends,omitempty"`
Expires *time.Time `json:"expires,omitempty"`
// Scope / affected area.

Some files were not shown because too many files have changed in this diff Show More