43 Commits

Author SHA1 Message Date
9b88be4dd2 Handle additional NWS forecast discussion formats
All checks were successful
ci/woodpecker/manual/build-image Pipeline was successful
2026-08-03 01:14:47 +00:00
456a46e01b Mark NWS forecast discussion resilience implemented 2026-08-03 00:24:26 +00:00
3740c779eb Add NWS forecast discussion fixture coverage 2026-08-03 00:22:51 +00:00
6943a5ebc9 Harden NWS key message parsing 2026-08-03 00:19:48 +00:00
9a09454621 Recognize NWS key points sections 2026-08-03 00:15:52 +00:00
b8c6708439 Support parenthesized NWS discussion headings 2026-08-03 00:13:58 +00:00
a62cb87b78 Mark NWS forecast discussion resilience implemented 2026-08-02 23:02:43 +00:00
0b5eaf46f4 Add cross-office NWS forecast discussion coverage 2026-08-02 23:01:11 +00:00
882059014c Normalize NWS forecast discussion preambles 2026-08-02 22:58:48 +00:00
0a3e52d0e5 Scan NWS forecast discussion blocks once 2026-08-02 22:56:03 +00:00
2b19a121fa Generalize NWS forecast discussion heading parsing 2026-08-02 22:53:00 +00:00
29c65971eb Mark NWS forecast discussion heading variants implemented 2026-08-02 20:54:12 +00:00
3ecf4c5b7f Add NWS forecast discussion heading regressions 2026-08-02 20:53:30 +00:00
f402e27542 Centralize NWS forecast discussion section headings 2026-08-02 20:50:51 +00:00
f720b6cdc0 Plan support for NWS AFD section heading variants 2026-08-02 20:47:24 +00:00
50215d2105 Add a new field to the alert schema to fix a mismatch between the prior schema and the upstream NWS API
All checks were successful
ci/woodpecker/push/build-image Pipeline was successful
2026-06-16 19:57:48 -05:00
74411e3f54 Clean up completed roadmap and documentation for the new outlook schema
All checks were successful
ci/woodpecker/push/build-image Pipeline was successful
2026-06-12 08:30:41 -05:00
4358a7cdce Verify outlook schema release readiness
Some checks failed
ci/woodpecker/push/build-image Pipeline failed
2026-06-12 04:40:33 +00:00
dcea5261ab Document current outlook schema behavior 2026-06-12 04:38:44 +00:00
97141c7a9b Add outlook schema transition guide 2026-06-12 04:32:52 +00:00
2e2d36024e Align normalizer docs with outlook v2 2026-06-12 04:31:15 +00:00
4d2cddf801 Update Postgres outlook storage for v2 2026-06-12 04:29:00 +00:00
21a35a5205 Emit location-filtered SPC outlook v2 runs 2026-06-12 04:24:12 +00:00
435d1ade07 Add outlook v2 model contract 2026-06-12 04:19:56 +00:00
819ac24aed Added an implementation plan for updates and revisions to the outlook code
All checks were successful
ci/woodpecker/push/build-image Pipeline was successful
2026-06-11 23:15:00 -05:00
2c472449e8 Fix a bug in the GeoJSON handling when there are no active polygons
All checks were successful
ci/woodpecker/push/build-image Pipeline was successful
2026-06-11 22:21:59 -05:00
5d7f604a2c Implement remaining cleanup items prior to the next release
All checks were successful
ci/woodpecker/push/build-image Pipeline was successful
2026-06-11 10:17:49 -05:00
8041f99782 Clean and update documentation
All checks were successful
ci/woodpecker/push/build-image Pipeline was successful
2026-06-11 10:00:15 -05:00
c417c892d9 Enhance JSON payload decoding to accept both typed and pointer payloads, and add corresponding tests
All checks were successful
ci/woodpecker/push/build-image Pipeline was successful
2026-06-10 22:11:38 -05:00
481215c5db Update .dockerignore to include docs files required by tests
All checks were successful
ci/woodpecker/push/build-image Pipeline was successful
2026-06-10 22:03:39 -05:00
5d94d3f32d Remove invalid SPC URLs for day 3 tornado/wind/hail risk
Some checks failed
ci/woodpecker/push/build-image Pipeline failed
2026-06-10 21:56:11 -05:00
f8f1b8d4a5 Update documentation
Some checks failed
ci/woodpecker/push/build-image Pipeline failed
2026-06-10 21:46:33 -05:00
06d5973746 Clean up stale internal literals
Some checks failed
ci/woodpecker/push/build-image Pipeline was canceled
2026-06-11 02:23:42 +00:00
8045b27173 Move SPC provider fixture helper 2026-06-11 02:20:25 +00:00
985468c1b9 Add documentation identifier consistency tests 2026-06-11 02:18:15 +00:00
6a0b30b7c7 Centralize Postgres event envelope mapping 2026-06-11 02:15:52 +00:00
86ce4eb68c Share HTTP config parsing for multi-document sources 2026-06-11 02:13:38 +00:00
33541a71fc Table-drive source registry tests 2026-06-11 02:10:13 +00:00
b7277e0c02 Centralize weather event and driver identifiers 2026-06-11 02:08:36 +00:00
dec05821bf Add a staged cleanup roadmap to address the code quality audit
All checks were successful
ci/woodpecker/push/build-image Pipeline was successful
2026-06-10 21:02:07 -05:00
1a9f462fbf Audit code quality and deduplication opportunities 2026-06-10 20:04:00 -05:00
a990da957b Finalize SPC outlook feature addition and clean up implemented roadmap documentation
All checks were successful
ci/woodpecker/push/build-image Pipeline was successful
2026-06-10 19:54:39 -05:00
a4cd63ca4e Update the SPC outlook implementation plan to identify remaining gaps and corrections
All checks were successful
ci/woodpecker/push/build-image Pipeline was successful
2026-06-10 19:40:46 -05:00
94 changed files with 4974 additions and 1701 deletions

View File

@@ -1,7 +1,8 @@
.git
.gitignore
**/*.md
!docs/*.md
!docs/**/*.md
dist/
tmp/
.DS_Store

View File

@@ -29,8 +29,10 @@ current working directory.
- [Operations guide](docs/operations.md)
- [Troubleshooting guide](docs/troubleshooting.md)
- [Example configs](examples/)
- [Go consumer guide](docs/consumers/api.md)
- [Event wire contract](docs/integrations/events.md)
- [Postgres table contract](docs/integrations/postgres.md)
- [Feedkit integration notes](docs/integrations/feedkit.md)
- [NWS integration notes](docs/integrations/nws.md)
- [SPC integration notes](docs/integrations/spc.md)
- [Open-Meteo integration notes](docs/integrations/openmeteo.md)

View File

@@ -20,6 +20,7 @@ import (
wfnormalizers "gitea.maximumdirect.net/ejr/weatherfeeder/internal/normalizers"
wfsources "gitea.maximumdirect.net/ejr/weatherfeeder/internal/sources"
"gitea.maximumdirect.net/ejr/weatherfeeder/standards"
)
type testInput struct {
@@ -36,10 +37,10 @@ type testKindsSource struct {
func (s testKindsSource) Kinds() []fkevent.Kind { return s.kinds }
func TestValidateSourceExpectedKindsSubsetAllowed(t *testing.T) {
sc := config.SourceConfig{Kinds: []string{"observation"}}
sc := config.SourceConfig{Kinds: []string{standards.KindObservation}}
in := testKindsSource{
testInput: testInput{name: "test"},
kinds: []fkevent.Kind{"observation", "forecast"},
kinds: []fkevent.Kind{fkevent.Kind(standards.KindObservation), fkevent.Kind(standards.KindForecast)},
}
if err := fksources.ValidateExpectedKinds(sc, in); err != nil {
@@ -48,10 +49,10 @@ func TestValidateSourceExpectedKindsSubsetAllowed(t *testing.T) {
}
func TestValidateSourceExpectedKindsMismatchFails(t *testing.T) {
sc := config.SourceConfig{Kinds: []string{"alert"}}
sc := config.SourceConfig{Kinds: []string{standards.KindAlert}}
in := testKindsSource{
testInput: testInput{name: "test"},
kinds: []fkevent.Kind{"observation", "forecast"},
kinds: []fkevent.Kind{fkevent.Kind(standards.KindObservation), fkevent.Kind(standards.KindForecast)},
}
err := fksources.ValidateExpectedKinds(sc, in)
@@ -64,7 +65,7 @@ func TestValidateSourceExpectedKindsMismatchFails(t *testing.T) {
}
func TestValidateSourceExpectedKindsNoMetadataSkipsCheck(t *testing.T) {
sc := config.SourceConfig{Kinds: []string{"alert"}}
sc := config.SourceConfig{Kinds: []string{standards.KindAlert}}
in := testInput{name: "test"}
if err := fksources.ValidateExpectedKinds(sc, in); err != nil {
@@ -111,6 +112,10 @@ func TestMaintainedConfigExamplesLoad(t *testing.T) {
func assertConfigSourcesBuildSchedulerJobs(t *testing.T, cfg *config.Config) {
t.Helper()
if len(cfg.Sources) == 0 {
t.Fatalf("config has no sources")
}
reg := fksources.NewRegistry()
wfsources.RegisterBuiltins(reg)
@@ -158,7 +163,7 @@ func TestNormalizeNoMatchPassThrough(t *testing.T) {
pl := &fkpipeline.Pipeline{Processors: chain}
in := fkevent.Event{
ID: "evt-no-match",
Kind: fkevent.Kind("observation"),
Kind: fkevent.Kind(standards.KindObservation),
Source: "test",
EmittedAt: time.Now().UTC(),
Schema: "raw.weatherfeeder.unknown.v1",
@@ -188,7 +193,7 @@ func TestDedupeDropsSecondEventWithSameID(t *testing.T) {
pl := &fkpipeline.Pipeline{Processors: chain}
in := fkevent.Event{
ID: "evt-dedupe-1",
Kind: fkevent.Kind("observation"),
Kind: fkevent.Kind(standards.KindObservation),
Source: "test",
EmittedAt: time.Now().UTC(),
Schema: "raw.weatherfeeder.unknown.v1",

View File

@@ -147,7 +147,7 @@ URL omits it or sets another unit system.
## SPC Convective Outlook Params
`spc_convective_outlook` fetches the twelve required Day 1-3 GeoJSON outlook
`spc_convective_outlook` fetches the nine required Day 1-3 GeoJSON outlook
products and the three required Day 1-3 print pages as one atomic bundle.
| Param | Required | Description |
@@ -165,8 +165,8 @@ products and the three required Day 1-3 print pages as one atomic bundle.
GeoJSON product keys are `day1_categorical`, `day1_tornado`, `day1_hail`,
`day1_wind`, `day2_categorical`, `day2_tornado`, `day2_hail`, `day2_wind`,
`day3_categorical`, `day3_tornado`, `day3_hail`, and `day3_wind`.
Discussion keys are `day1`, `day2`, and `day3`.
and `day3_categorical`. SPC does not provide Day 3 tornado, hail, or wind
GeoJSON products. Discussion keys are `day1`, `day2`, and `day3`.
```yaml
sources:

92
docs/consumers/api.md Normal file
View File

@@ -0,0 +1,92 @@
# Consumer API Guide
## Purpose
This guide is for developers and LLM coding agents integrating `weatherfeeder`
from another Go codebase.
`weatherfeeder` is primarily a daemon, not an SDK. Its public integration
surface is intentionally narrow:
- `model`: canonical weather payload structs.
- `standards`: schema strings, event kind strings, and shared WMO constants.
- JSON event output from stdout and NATS sinks.
- Postgres tables written by the optional Postgres sink.
Packages under `internal/` are implementation details and are not public
integration surfaces.
## Recommended Workflow
Consumers should switch on the event `schema` value and decode `payload` into
the matching `model` type.
Minimal example:
```go
package consumer
import (
"encoding/json"
"fmt"
"gitea.maximumdirect.net/ejr/weatherfeeder/model"
"gitea.maximumdirect.net/ejr/weatherfeeder/standards"
)
type Event struct {
ID string `json:"id"`
Kind string `json:"kind"`
Schema string `json:"schema"`
Payload json.RawMessage `json:"payload"`
}
func Decode(payload []byte) (any, error) {
var evt Event
if err := json.Unmarshal(payload, &evt); err != nil {
return nil, err
}
switch evt.Schema {
case standards.SchemaWeatherObservationV1:
var out model.WeatherObservation
return &out, json.Unmarshal(evt.Payload, &out)
case standards.SchemaWeatherForecastV1:
var out model.WeatherForecastRun
return &out, json.Unmarshal(evt.Payload, &out)
case standards.SchemaWeatherForecastDiscussionV1:
var out model.WeatherForecastDiscussion
return &out, json.Unmarshal(evt.Payload, &out)
case standards.SchemaWeatherStoryV1:
var out model.WeatherStoryRun
return &out, json.Unmarshal(evt.Payload, &out)
case standards.SchemaWeatherAlertV1:
var out model.WeatherAlertRun
return &out, json.Unmarshal(evt.Payload, &out)
case standards.SchemaWeatherOutlookV2:
var out model.WeatherOutlookRun
return &out, json.Unmarshal(evt.Payload, &out)
default:
return nil, fmt.Errorf("unsupported weatherfeeder schema %q", evt.Schema)
}
}
```
## Consumer Responsibilities
- Treat event IDs as opaque.
- Treat absent `omitempty` fields as unknown, not zero.
- Prefer schema constants from `standards` over string literals in Go code.
- Expect canonical numeric measurements to use metric units.
- Expect canonical timestamps from normalizers to be UTC unless a field-specific
contract says otherwise.
- Handle additive fields within the same schema version.
- Do not import `internal/...` packages.
## Canonical References
- Public payload package: [`pkg-model.md`](pkg-model.md).
- Public constants package: [`pkg-standards.md`](pkg-standards.md).
- JSON event wire contract: [`../integrations/events.md`](../integrations/events.md).
- Postgres table contract: [`../integrations/postgres.md`](../integrations/postgres.md).
- Runtime and adapter architecture: [`../policy/architecture.md`](../policy/architecture.md).

View File

@@ -0,0 +1,64 @@
# Package `model`
## Import Path
```go
import "gitea.maximumdirect.net/ejr/weatherfeeder/model"
```
## Purpose
Package `model` defines `weatherfeeder`'s canonical weather payload structs.
These structs are emitted as the `payload` of canonical `weather.*` events and
are also the domain types consumed by downstream applications such as
`weatherapi`.
The JSON field tags on these structs are part of the wire contract. For the full
field-by-field JSON contract, use the [event wire contract](../integrations/events.md).
## Payload Types
Current canonical schema families map to these public types:
| Schema | Primary type |
|---|---|
| `weather.observation.v1` | `WeatherObservation` |
| `weather.forecast.v1` | `WeatherForecastRun` |
| `weather.forecast_discussion.v1` | `WeatherForecastDiscussion` |
| `weather.weather_story.v1` | `WeatherStoryRun` |
| `weather.alert.v1` | `WeatherAlertRun` |
| `weather.outlook.v2` | `WeatherOutlookRun` |
Related child types include:
- `WeatherObservationPresentWeather`
- `WeatherForecastPeriod`
- `WeatherForecastDiscussionSection`
- `WeatherStory`
- `WeatherAlert`
- `WeatherAlertReference`
- `WeatherOutlook`
- `WeatherOutlookDiscussion`
- `WMOCode`
## Wire And Compatibility Rules
- JSON tags define canonical payload field names.
- Pointer fields and fields tagged `omitempty` are optional on the wire.
- Missing optional fields mean unknown or not applicable.
- Canonical measurements use metric units.
- Canonical timestamps are `time.Time` values encoded by Go's JSON encoder.
- Normalized canonical timestamps are UTC unless a field-specific contract says
otherwise.
- Additive fields are compatible within a schema version.
- Removing, renaming, or changing the meaning of a field requires a new schema
identifier.
## Boundaries
`model` should not depend on source adapters, sinks, SQL column names, provider
HTTP shapes, or runtime configuration.
Consumers should not rely on packages under `internal/...`. Use `model` with
schema constants from [`standards`](pkg-standards.md) and the JSON contract in
[`docs/integrations/events.md`](../integrations/events.md).

View File

@@ -0,0 +1,96 @@
# Package `standards`
## Import Path
```go
import "gitea.maximumdirect.net/ejr/weatherfeeder/standards"
```
## Purpose
Package `standards` defines stable identifiers and shared weather constants used
by `weatherfeeder` producers and Go consumers.
Use this package when switching on event schemas, comparing event kinds, or
working with canonical WMO condition codes.
## Event Kind Constants
Current event kind constants are:
| Constant | Value |
|---|---|
| `KindObservation` | `observation` |
| `KindForecast` | `forecast` |
| `KindForecastDiscussion` | `forecast_discussion` |
| `KindWeatherStory` | `weather_story` |
| `KindAlert` | `alert` |
| `KindOutlook` | `outlook` |
These are plain string constants. Convert them at adapter boundaries when using
feedkit's `event.Kind` type.
## Canonical Schema Constants
Canonical schemas emitted after normalization:
| Constant | Value |
|---|---|
| `SchemaWeatherObservationV1` | `weather.observation.v1` |
| `SchemaWeatherForecastV1` | `weather.forecast.v1` |
| `SchemaWeatherForecastDiscussionV1` | `weather.forecast_discussion.v1` |
| `SchemaWeatherStoryV1` | `weather.weather_story.v1` |
| `SchemaWeatherAlertV1` | `weather.alert.v1` |
| `SchemaWeatherOutlookV2` | `weather.outlook.v2` |
Historical canonical schema constant:
| Constant | Value |
|---|---|
| `SchemaWeatherOutlookV1` | `weather.outlook.v1` |
## Raw Schema Constants
Raw source schemas emitted by current registered sources:
| Constant | Value |
|---|---|
| `SchemaRawNWSObservationV1` | `raw.nws.observation.v1` |
| `SchemaRawOpenMeteoCurrentV1` | `raw.openmeteo.current.v1` |
| `SchemaRawOpenWeatherCurrentV1` | `raw.openweather.current.v1` |
| `SchemaRawNWSHourlyForecastV1` | `raw.nws.hourly.forecast.v1` |
| `SchemaRawNWSNarrativeForecastV1` | `raw.nws.narrative.forecast.v1` |
| `SchemaRawNWSForecastDiscussionV1` | `raw.nws.forecast_discussion.v1` |
| `SchemaRawNWSWeatherStoriesV1` | `raw.nws.weatherstories.v1` |
| `SchemaRawOpenMeteoHourlyForecastV1` | `raw.openmeteo.hourly.forecast.v1` |
| `SchemaRawNWSAlertsV1` | `raw.nws.alerts.v1` |
| `SchemaRawSPCConvectiveOutlookV1` | `raw.spc.convective_outlook.v1` |
Additional raw schema constant:
| Constant | Value |
|---|---|
| `SchemaRawOpenWeatherHourlyForecastV1` | `raw.openweather.hourly.forecast.v1` |
`SchemaRawOpenWeatherHourlyForecastV1` exists in code, but no current registered
source emits it. Consumers should not expect that raw schema unless a later
registered source documents it as part of the current event contract.
## WMO Constants And Text
`standards` also defines the canonical `WMOCode` vocabulary and text helpers
used by normalized observations and forecasts.
Consumer guidance:
- Treat `WMOUnknown` as unknown condition data.
- Observation `conditionCode` is required in the current event contract.
- Forecast period `conditionCode` is optional because some forecast products do
not provide a meaningful WMO condition.
- Prefer WMO constants and helper functions from this package instead of
duplicating code tables in consumers.
## Boundaries
`standards` is provider-agnostic. Provider-specific parsing belongs in
`weatherfeeder` internals, not in this package and not in consumers.

View File

@@ -37,7 +37,7 @@ Canonical schemas emitted after normalization:
| `forecast_discussion` | `weather.forecast_discussion.v1` | `WeatherForecastDiscussion` |
| `weather_story` | `weather.weather_story.v1` | `WeatherStoryRun` |
| `alert` | `weather.alert.v1` | `WeatherAlertRun` |
| `outlook` | `weather.outlook.v1` | `WeatherOutlookRun` |
| `outlook` | `weather.outlook.v2` | `WeatherOutlookRun` |
Raw upstream schemas emitted by current sources:
@@ -211,8 +211,9 @@ Payload type: `WeatherAlertRun`.
| `instruction` | string | no | Alert instruction. |
| `sent` | timestamp | no | Provider sent time. |
| `effective` | timestamp | no | Effective time. |
| `onset` | timestamp | no | Onset time. |
| `expires` | timestamp | no | Expiration time. |
| `onset` | timestamp | no | Alert period start. |
| `ends` | timestamp | no | Alert period end. |
| `expires` | timestamp | no | Provider expiration metadata; not necessarily the alert period end. |
| `areaDescription` | string | no | Affected area description. |
| `senderName` | string | no | Provider sender name. |
| `references` | array | no | Related alerts. |
@@ -220,13 +221,15 @@ Payload type: `WeatherAlertRun`.
`references[]` entries contain optional `id`, `identifier`, `sender`, and
`sent` fields.
## `weather.outlook.v1`
## `weather.outlook.v2`
Payload type: `WeatherOutlookRun`.
The current producer is the SPC convective outlook normalizer. It emits Day 1-3
convective outlook polygons for categorical, tornado, hail, and wind products.
All timestamps are UTC.
convective outlook snapshots for categorical, tornado, hail, and wind products
that apply to the configured forecast point. Raw SPC bundles remain complete;
canonical outlook payloads are filtered to local polygons. All timestamps are
UTC.
| Field | Type | Required | Notes |
|---|---|:---:|---|
@@ -236,7 +239,8 @@ All timestamps are UTC.
| `longitude` | number | no | Configured point longitude in decimal degrees. |
| `asOf` | timestamp | yes | Snapshot time. For SPC, this is the latest outlook issue time when available. |
| `issuedAt` | timestamp | no | Latest issue time across outlook features when any feature exists. |
| `outlooks` | array | yes | Ordered outlook polygons. |
| `outlooks` | array | yes | Ordered outlook polygons containing the configured point. |
| `discussions` | array | yes | Run-level day discussions for retained outlook days. |
`outlooks[]` entries:
@@ -255,19 +259,53 @@ All timestamps are UTC.
| `issuedAt` | timestamp | yes | Feature issue time. |
| `expiresAt` | timestamp | yes | Expiration time; currently equal to `validTo`. |
| `forecaster` | string | no | SPC forecaster text, when present. |
| `headline` | string | no | Matching Day 1-3 print-page product title. |
| `summary` | string | no | Text from the print-page `...SUMMARY...` section. |
| `discussion` | string | no | Cleaned full print-page product text. |
| `sourceUrl` | string | no | GeoJSON product URL for this outlook feature. |
| `imageUrl` | string | no | Reserved for provider image URLs; currently empty. |
| `containsLocation` | boolean | yes | Whether the configured point is inside or on the boundary of the polygon. |
| `containsLocation` | boolean | yes | Always `true` for emitted v2 outlooks. |
| `geometry` | object | yes | Compact GeoJSON `Polygon` or `MultiPolygon` geometry. |
`geometry` preserves the SPC feature geometry as compact GeoJSON using
`[longitude, latitude]` coordinate order. `containsLocation` is computed with
that geometry and the configured source `latitude`/`longitude`; boundary points
count as contained. All outlook polygons are emitted, including polygons that do
not contain the configured point.
count as contained. Polygons that do not contain the configured point are not
included in canonical v2 payloads.
`discussions[]` entries:
| Field | Type | Required | Notes |
|---|---|:---:|---|
| `day` | integer | yes | SPC outlook day, currently `1`, `2`, or `3`. |
| `headline` | string | no | Matching Day 1-3 print-page product title. |
| `summary` | string | no | Text from the print-page `...SUMMARY...` section. |
| `discussion` | string | no | Cleaned full print-page product text. |
| `updatedAt` | timestamp | no | Print-page update time, when present. |
When no SPC polygons apply locally, the run is still emitted with `outlooks: []`
and `discussions: []`. Discussions are included only for days represented by at
least one retained outlook, and multiple retained outlook types for the same day
share one discussion entry.
### SPC Outlook Supersession
Consumers should prefer latest-run semantics for current conditions: read the
latest `WeatherOutlookRun` for the configured location and use its `outlooks`
and `discussions` arrays together.
Historical SQL consumers that collapse older rows should identify superseded
outlooks by `provider`, `product`, `outlookType`, `validFrom`, and `validTo`,
then keep rows with the greatest `issuedAt`. `day` and `label` are not identity
fields. When multiple retained polygons share that latest `issuedAt`, preserve
the full group.
## Legacy `weather.outlook.v1`
`weather.outlook.v1` is a historical canonical schema retained as a standards
constant for older data and consumers. Current SPC normalization emits
`weather.outlook.v2`.
The v1 payload used `WeatherOutlookRun` and placed `headline`, `summary`, and
`discussion` on each `outlooks[]` polygon. It also represented the pre-v2 SPC
canonical behavior, where national polygons were preserved in canonical output.
## Compact Example

View File

@@ -0,0 +1,106 @@
# Feedkit Integration
## Purpose
This document describes the feedkit runtime behavior that `weatherfeeder`
currently depends on. It is for maintainers and LLM coding agents changing
runtime wiring, config behavior, source construction, processing, routing, or
sink behavior.
Weather-domain behavior belongs in `weatherfeeder`. Generic daemon mechanics
belong to feedkit.
## Current Dependency
`weatherfeeder` imports feedkit as its daemon framework dependency. The exact
module version is declared in `go.mod`.
Feedkit provides:
- YAML config loading and validation.
- Source, processor, and sink registries.
- HTTP source helper behavior.
- Scheduler polling.
- Normalize and dedupe processors.
- Route compilation and sink dispatch.
- Built-in stdout, NATS, and Postgres sink mechanics.
## Config Contract
`cmd/weatherfeeder` calls feedkit config loading for `config.yml` in the current
working directory.
Implemented behavior relied on by weatherfeeder docs and tests:
- Top-level config contains `sources`, `sinks`, and optional `routes`.
- Config struct fields are decoded strictly, so misspelled struct fields fail
startup.
- Driver-specific `params` maps are decoded generically and validated by the
source or sink constructor that consumes them.
- Source `kinds` can be validated against a source's advertised `Kinds()`.
## Source And HTTP Contract
Most weatherfeeder sources use feedkit's single-document HTTP source helper for:
- request construction;
- `User-Agent` and `Accept` headers;
- optional conditional GET validators;
- response body size limits;
- context-aware HTTP work;
- unchanged `304 Not Modified` responses that emit no events.
The SPC convective outlook source fetches multiple documents itself, but it uses
feedkit transport helpers for HTTP clients and response body limits.
## Scheduler And Processing Contract
Weatherfeeder builds feedkit scheduler jobs from source configs. Current source
drivers are polling drivers and use the configured `every` interval.
Events flow through a feedkit pipeline in this order:
1. normalize processor;
2. dedupe processor.
The normalize processor is configured with `RequireMatch=false`, so unmatched
schemas pass through unchanged. Weatherfeeder registers its built-in normalizers
and owns the provider-to-canonical mapping.
The dedupe processor stores a bounded in-memory set of recent event IDs. The
bound is configured in `cmd/weatherfeeder`.
## Dispatch And Sink Contract
Feedkit compiles routes from config and dispatches processed events to matching
sinks. If `routes` is omitted, every configured sink receives every event kind.
Feedkit owns sink fanout mechanics, per-sink workers, queueing, context-aware
shutdown, and sink error logging. Weatherfeeder owns the event kinds and schemas
that make routes meaningful.
Built-in feedkit sinks used by weatherfeeder:
- `stdout`: validates and writes JSON events to stdout.
- `nats`: publishes JSON events to a configured subject.
- generic `postgres` sink factory: opens the database, ensures tables/indexes,
runs transactions, inserts mapped rows, and prunes when configured.
Weatherfeeder supplies its Postgres table schema and event mapper to feedkit's
Postgres sink factory. The table contract is documented in
[`postgres.md`](postgres.md).
## Boundaries
Do not move weather-domain policy into feedkit. Weatherfeeder owns:
- provider source drivers;
- raw and canonical schema constants;
- event kind meaning;
- canonical payload structs;
- normalizers;
- Postgres table shape and row mapping.
Do not duplicate generic feedkit mechanics in weatherfeeder unless there is a
narrow weather-specific reason. Runtime composition details are documented in
[`../internal/runtime.md`](../internal/runtime.md).

View File

@@ -44,6 +44,9 @@ normalizer uses fields under `properties` such as `stationId`, `stationName`,
`nws_alerts` expects an alerts FeatureCollection. The normalizer uses the
collection `updated` timestamp, `title`, each feature ID, alert classification
fields, narrative fields, timing fields, sender fields, and references.
`properties.onset` and `properties.ends` map to the canonical alert period
start and end. `properties.expires` maps only to canonical `expires` provider
metadata and is not treated as the alert period end.
`nws_forecast_hourly` and `nws_forecast_narrative` expect gridpoint forecast
GeoJSON with `properties.generatedAt`, `properties.updateTime`, elevation,
@@ -98,8 +101,9 @@ unset. Forecast temperatures are converted to Celsius when NWS supplies
Fahrenheit, and wind speed strings are converted to kilometers per hour.
Alert timing fields are parsed best-effort. Invalid per-alert timestamps are
left unset rather than failing the whole alert run. Missing alert IDs are
synthesized from the run snapshot time and array position.
left unset rather than failing the whole alert run. NWS `ends` is preserved
separately from `expires`; `expires` does not fall back to `ends`. Missing alert
IDs are synthesized from the run snapshot time and array position.
Forecast discussion parsing requires an issue time. Weather story entries require
start time, end time, and update time.

View File

@@ -23,7 +23,7 @@ Events are mapped only for canonical weather schemas:
- `weather.forecast_discussion.v1`
- `weather.weather_story.v1`
- `weather.alert.v1`
- `weather.outlook.v1`
- `weather.outlook.v2`
Unsupported schemas produce no writes for this sink. Mapped events are inserted
transactionally. Inserts use ordinary `INSERT`; duplicate primary keys fail the
@@ -59,6 +59,7 @@ Parent tables store the feed event envelope:
| `alert_references` | `run_event_id`, `alert_index`, `reference_index` | `as_of` |
| `outlook_runs` | `event_id` | `as_of` |
| `outlooks` | `run_event_id`, `outlook_index` | `as_of` |
| `outlook_discussions` | `run_event_id`, `discussion_index` | `as_of` |
## Table Contract
@@ -352,6 +353,7 @@ Indexes:
| `sent` | `TIMESTAMPTZ` | yes | `payload.alerts[].sent` |
| `effective` | `TIMESTAMPTZ` | yes | `payload.alerts[].effective` |
| `onset` | `TIMESTAMPTZ` | yes | `payload.alerts[].onset` |
| `ends` | `TIMESTAMPTZ` | yes | `payload.alerts[].ends` |
| `expires` | `TIMESTAMPTZ` | yes | `payload.alerts[].expires` |
| `area_description` | `TEXT` | yes | `payload.alerts[].areaDescription` |
| `sender_name` | `TEXT` | yes | `payload.alerts[].senderName` |
@@ -408,6 +410,7 @@ Indexes:
| `as_of` | `TIMESTAMPTZ` | no | `payload.asOf` |
| `issued_at` | `TIMESTAMPTZ` | yes | `payload.issuedAt` |
| `outlook_count` | `INTEGER` | no | `len(payload.outlooks)` |
| `discussion_count` | `INTEGER` | no | `len(payload.discussions)` |
### `outlooks`
@@ -429,6 +432,8 @@ Indexes:
| `run_event_id` | `TEXT REFERENCES outlook_runs(event_id) ON DELETE CASCADE` | no | Parent event ID. |
| `outlook_index` | `INTEGER` | no | `payload.outlooks[]` index. |
| `as_of` | `TIMESTAMPTZ` | no | Parent `payload.asOf` |
| `outlook_id` | `TEXT` | no | `payload.outlooks[].id` |
| `provider` | `TEXT` | no | `payload.outlooks[].provider` |
| `product` | `TEXT` | no | `payload.outlooks[].product` |
| `day` | `INTEGER` | no | `payload.outlooks[].day` |
| `outlook_type` | `TEXT` | no | `payload.outlooks[].outlookType` |
@@ -440,14 +445,36 @@ Indexes:
| `issued_at` | `TIMESTAMPTZ` | no | `payload.outlooks[].issuedAt` |
| `expires_at` | `TIMESTAMPTZ` | no | `payload.outlooks[].expiresAt` |
| `forecaster` | `TEXT` | yes | `payload.outlooks[].forecaster` |
| `headline` | `TEXT` | yes | `payload.outlooks[].headline` |
| `summary` | `TEXT` | yes | `payload.outlooks[].summary` |
| `discussion` | `TEXT` | yes | `payload.outlooks[].discussion` |
| `source_url` | `TEXT` | yes | `payload.outlooks[].sourceUrl` |
| `image_url` | `TEXT` | yes | `payload.outlooks[].imageUrl` |
| `contains_location` | `BOOLEAN` | no | `payload.outlooks[].containsLocation` |
| `geometry_json` | `TEXT` | no | Compact JSON from `payload.outlooks[].geometry` |
### `outlook_discussions`
Primary key: `run_event_id`, `discussion_index`
Prune column: `as_of`
Foreign key: `run_event_id` references `outlook_runs(event_id)` with cascade
delete.
Indexes:
- `idx_wf_outlook_discussions_day_as_of` on `day`, `as_of`
- unique `idx_wf_outlook_discussions_run_day` on `run_event_id`, `day`
| Column | Type | Null | Source |
|---|---|:---:|---|
| `run_event_id` | `TEXT REFERENCES outlook_runs(event_id) ON DELETE CASCADE` | no | Parent event ID. |
| `discussion_index` | `INTEGER` | no | `payload.discussions[]` index. |
| `as_of` | `TIMESTAMPTZ` | no | Parent `payload.asOf` |
| `day` | `INTEGER` | no | `payload.discussions[].day` |
| `headline` | `TEXT` | yes | `payload.discussions[].headline` |
| `summary` | `TEXT` | yes | `payload.discussions[].summary` |
| `discussion` | `TEXT` | yes | `payload.discussions[].discussion` |
| `updated_at` | `TIMESTAMPTZ` | yes | `payload.discussions[].updatedAt` |
## Retention
When sink param `prune` is set, every successful write transaction deletes rows
@@ -470,5 +497,6 @@ binary does not provide CLI commands for them.
- `WeatherAlertRun`: read `alert_runs`, join `alerts` by `run_event_id` ordered
by `alert_index`, then join `alert_references` by `run_event_id` and
`alert_index` ordered by `reference_index`.
- `WeatherOutlookRun`: read `outlook_runs`, then join `outlooks` by
`run_event_id` ordered by `outlook_index`.
- `WeatherOutlookRun`: read `outlook_runs`, join `outlooks` by `run_event_id`
ordered by `outlook_index`, then join `outlook_discussions` by
`run_event_id` ordered by `discussion_index`.

View File

@@ -13,7 +13,7 @@ events are documented in [event wire contract](events.md).
| Driver | Kind | Raw schema | Canonical schema |
| --- | --- | --- | --- |
| `spc_convective_outlook` | `outlook` | `raw.spc.convective_outlook.v1` | `weather.outlook.v1` |
| `spc_convective_outlook` | `outlook` | `raw.spc.convective_outlook.v1` | `weather.outlook.v2` |
## Config Requirements
@@ -39,11 +39,11 @@ current Day 1-3 SPC product URLs.
## Upstream Products Used
The source fetches twelve required GeoJSON products every poll:
The source fetches nine required GeoJSON products every poll:
- Day 1 categorical, tornado, hail, and wind
- Day 2 categorical, tornado, hail, and wind
- Day 3 categorical, tornado, hail, and wind
- Day 3 categorical
It also fetches three required print pages:
@@ -52,8 +52,8 @@ It also fetches three required print pages:
- Day 3 convective outlook print page
GeoJSON products are authoritative for outlook polygons, valid windows, issue
times, labels, and severity rank. Print pages are authoritative for headline,
summary, and discussion text.
times, labels, and severity rank. Print pages are authoritative for run-level
day discussion headline, summary, and discussion text.
## Accept Headers
@@ -99,23 +99,33 @@ Raw source `effective_at` prefers:
4. fetch time.
The normalizer sets canonical `asOf` and normalized event `effective_at` from
the latest valid outlook feature `issuedAt`, with fallback to print-page update
time and then the incoming event envelope.
the latest valid GeoJSON issue time across the complete raw bundle, including
empty no-risk placeholders, with fallback to print-page update time and then the
incoming event envelope.
## Mapping Notes
Each GeoJSON feature becomes one canonical outlook. Products are ordered by day,
then categorical, tornado, hail, and wind. Feature order is preserved within
each product.
The raw source fetches and envelopes the complete SPC bundle. The normalizer
decodes every configured GeoJSON product, skips empty no-risk
`GeometryCollection` placeholders, and emits canonical outlooks only when the
configured point is inside or on the boundary of a real feature. Products are
ordered by day, then categorical, tornado, hail, and wind. Retained feature order
is preserved within each product.
The normalizer computes `containsLocation` with the configured latitude and
longitude against compact GeoJSON `Polygon` or `MultiPolygon` geometry.
Coordinates use GeoJSON order, `[longitude, latitude]`, and boundary points
count as contained.
All outlook polygons are preserved, including polygons that do not contain the
configured point. Matching print-page headline, summary, and discussion text is
attached to every outlook for the same day.
Canonical outlook runs are emitted even when no polygons apply locally. In that
case the payload contains empty `outlooks` and `discussions` arrays.
Print-page prose is represented as run-level day discussions. Discussions are
included only for days represented by at least one retained outlook. Multiple
retained outlook types for the same day share one discussion entry.
For downstream current-state and historical supersession guidance, see the
[event wire contract](events.md#spc-outlook-supersession).
## Failure Behavior

View File

@@ -27,7 +27,7 @@ Current mappings:
| `raw.nws.forecast_discussion.v1` | `weather.forecast_discussion.v1` |
| `raw.nws.weatherstories.v1` | `weather.weather_story.v1` |
| `raw.nws.alerts.v1` | `weather.alert.v1` |
| `raw.spc.convective_outlook.v1` | `weather.outlook.v1` |
| `raw.spc.convective_outlook.v1` | `weather.outlook.v2` |
## Boundaries
@@ -59,13 +59,13 @@ Weatherfeeder registers normalizers in a stable order:
The current normalizers avoid ambiguous matches by using schema equality.
The SPC outlook normalizer decodes the raw multi-document bundle, maps each
GeoJSON feature to a canonical outlook, and enriches all outlooks for a day with
the matching print-page headline, summary, and discussion. It preserves compact
The SPC outlook normalizer decodes the raw multi-document bundle, maps
location-containing GeoJSON features to canonical outlooks, and adds one
run-level print-page discussion per retained outlook day. It preserves compact
GeoJSON feature geometry and computes `containsLocation` with
`internal/geo.ContainsPoint` using the source-configured point. Boundary points
count as contained, and all polygons are preserved whether or not they contain
the point.
count as contained. Polygons that do not contain the point are omitted from the
canonical run.
## State

View File

@@ -19,11 +19,16 @@ Inputs are canonical feed events. The mapper currently handles these schemas:
- `weather.forecast_discussion.v1`
- `weather.weather_story.v1`
- `weather.alert.v1`
- `weather.outlook.v1`
- `weather.outlook.v2`
Outputs are feedkit `PostgresWrite` values for weatherfeeder-owned tables.
Unsupported schemas produce no writes and no error.
Outlook runs are written to `outlook_runs`, retained local polygons are written
to `outlooks`, and run-level day discussions are written to
`outlook_discussions`. The parent run row stores `outlook_count` and
`discussion_count`.
## Boundaries
- Weatherfeeder owns table definitions in `schema.go`.
@@ -82,6 +87,7 @@ Child rows use positional indexes to preserve canonical array order:
- `alert_index`
- `reference_index`
- `outlook_index`
- `discussion_index`
Required canonical fields are validated before writes are returned:
@@ -94,7 +100,10 @@ Required canonical fields are validated before writes are returned:
- alert runs require `asOf`, and each alert requires `id`;
- outlook runs require `asOf`, and each outlook requires `id`, `provider`,
`product`, `day`, `outlookType`, `label`, `validFrom`, `validTo`, `issuedAt`,
`expiresAt`, and `geometry`.
`expiresAt`, `containsLocation: true`, and `geometry`;
- outlook discussions require day `1`, `2`, or `3`; at least one of
`headline`, `summary`, or `discussion`; and no duplicate discussion day in the
same run.
Nullable canonical values are converted to SQL nulls by mapper helpers.
Observation present-weather raw values and outlook geometry values are stored as

View File

@@ -68,6 +68,21 @@ Runtime composition uses feedkit for:
Weatherfeeder registers its own source drivers and its Postgres schema mapper.
Responsibility split:
| Runtime concern | Owner |
| --- | --- |
| Config loading and generic validation | feedkit |
| Source, processor, and sink registries | feedkit mechanics; weatherfeeder registrations |
| Source polling and stream supervision | feedkit scheduler |
| Raw weather data fetching | weatherfeeder source adapters |
| Normalizer execution order and pass-through behavior | feedkit normalize processor |
| Weather raw-to-canonical mapping | weatherfeeder normalizers |
| Dedupe mechanics | feedkit dedupe processor |
| Route compilation and sink fanout | feedkit dispatch |
| Weather Postgres table shape and row mapping | weatherfeeder Postgres adapter |
| Postgres connection, DDL, inserts, transactions, and pruning | feedkit Postgres sink |
## State
Weatherfeeder-owned runtime state is in process:

View File

@@ -82,7 +82,7 @@ document bodies as the previous successful poll.
Every event passes through normalization first and dedupe second.
Normalizers match raw source schemas and produce canonical `weather.*.v1`
Normalizers match raw source schemas and produce versioned canonical `weather.*`
payloads. If an event has no matching normalizer, the normalize processor passes
it through unchanged.

View File

@@ -23,10 +23,32 @@ The implemented runtime flow is:
Canonical payload structs live in `model`. Schema identifiers and cross-provider wire conventions live in `standards`. Source adapters live under `internal/sources`. Normalizers live under `internal/normalizers`. Provider-specific parsing helpers shared by sources and normalizers live under `internal/providers`. Sink-specific persistence mapping lives under `internal/sinks`.
## Architecture Style
`weatherfeeder` uses a pragmatic ports-and-adapters architecture rather than a
formal framework. Provider APIs, config loading, scheduling, dispatch, and sinks
sit outside the weather domain model and normalization rules.
The implementation style is:
- Pipeline-oriented: events flow from source polling through normalization,
dedupe, routing, and sink fanout.
- Schema-routed: normalizers select raw payloads by explicit schema strings, not
source names or configured routes.
- Provider-isolated: NWS, Open-Meteo, OpenWeather, and SPC quirks stay in
provider-specific source, provider-helper, and normalizer packages.
- Registry-based: built-in source drivers, normalizers, processors, and sinks
are assembled explicitly through registries instead of dynamic plugin loading.
- Adapter-clean: persistence and external-system details stay behind source and
sink adapters, not in `model` or normalizers.
- Direct Go: prefer small package-level constructors and straightforward code
over broad abstractions.
## Core Design Principles
- Hexagonal boundaries: provider APIs, config loading, scheduling, dispatch, and sinks are external mechanisms around the weather domain model and normalization logic.
- Raw-to-canonical flow: sources should fetch and envelope raw provider payloads; normalizers should own provider-to-canonical mapping.
- Location-focused canonical data: canonical weather events represent data relevant to the configured forecast location or configured provider object; `weatherfeeder` is not a national provider-data archive.
- Schema-based routing: normalizers match on event schema, not source name or event kind.
- Composable registries: source drivers, normalizers, processors, and sinks are assembled explicitly through registries.
- Bounded concurrency: scheduling and sink fanout are concurrent, but the application should keep queues, goroutine ownership, logging, and cancellation behavior visible.
@@ -57,6 +79,23 @@ Tests and examples:
- The sample `cmd/weatherfeeder/config.yml` is executable test input and is load-tested.
- Tests should keep exercising package contracts directly rather than relying only on full-daemon execution.
## Feedkit Boundary
`feedkit` provides reusable daemon infrastructure. `weatherfeeder` provides the
weather-domain adapters, models, schemas, and mapping policy.
| Area | Feedkit owns | Weatherfeeder owns |
| --- | --- | --- |
| Config | Generic YAML shape: sources, sinks, routes, modes, cadence, and params. | Driver-specific config rules such as NWS `user_agent`, OpenWeather `units=metric`, and SPC coordinates. |
| Events | Domain-agnostic event envelope: ID, kind, source, emitted/effective times, schema, and payload. | Event kind meaning, schema strings, and canonical weather payloads. |
| Sources | Source interfaces, registry, expected-kind validation, HTTP helper, and default event ID helper. | NWS/Open-Meteo/OpenWeather/SPC source drivers and raw schema emission. |
| Processing | Processor registry, normalize processor, dedupe processor, and pipeline execution. | Weather normalizers and schema-specific raw-to-canonical mapping. |
| Dispatch | Route compilation and sink fanout mechanics. | Which weather event kinds are configured and meaningful. |
| Sinks | Generic stdout, NATS, and Postgres sink mechanics. | Weather-specific Postgres schema and canonical event-to-row mapping. |
Do not move weather-domain policy into `feedkit`, and do not duplicate generic
daemon mechanics in `weatherfeeder` when feedkit already provides the boundary.
## Modules Or Processing Steps
The implemented processing steps are source polling, normalization, dedupe, and sink dispatch.

View File

@@ -24,7 +24,8 @@ docs, not here.
normalizers.
- `internal/sinks/postgres/`: weatherfeeder-owned Postgres schema and canonical
event mapper.
- `docs/`: current behavior, policies, integration contracts, and roadmap files.
- `docs/`: current behavior, consumer guides, integration contracts, policies,
and roadmap files.
- `examples/`: maintained, copyable configuration examples.
## Build And Test
@@ -70,6 +71,32 @@ Use fixtures, local test servers, and package-level tests.
- Prefer explicit registries and small package-level constructors over hidden
global behavior.
## Architecture-Preserving Changes
When changing `weatherfeeder`, preserve the split between feedkit
infrastructure and weather-domain behavior.
Do:
- keep generic scheduling, dispatch, processor, config, and sink mechanics in
feedkit;
- keep weather provider rules in source adapters, provider helpers, and
normalizers;
- keep canonical weather payloads in `model` and schema/wire identifiers in
`standards`;
- keep Postgres table and row mapping under `internal/sinks/postgres`;
- use explicit registries for built-in sources and normalizers.
Do not:
- move provider parsing, WMO mapping, or canonical weather policy into
`cmd/weatherfeeder`;
- move weather-specific constants, schemas, or validation rules into feedkit;
- put database column metadata or sink-specific tags on canonical model structs;
- replace explicit registries with dynamic plugin loading;
- introduce broad abstractions when a small provider-specific helper preserves
clarity.
## Dependency Policy
Prefer the Go standard library unless a dependency materially improves
@@ -183,6 +210,7 @@ When behavior changes, update the canonical docs in the same change:
- CLI behavior: `docs/cli.md`;
- operations and recovery: `docs/operations.md`;
- troubleshooting: `docs/troubleshooting.md`;
- public Go package consumption: `docs/consumers/`;
- external contracts: `docs/integrations/`;
- internal component behavior: `docs/internal/`;
- copyable configs: `examples/`.

View File

@@ -2,12 +2,13 @@
## Purpose
Project documentation must help four audiences:
Project documentation must help five audiences:
1. users who need to run the application;
2. administrators/operators who need to configure and operate it;
3. developers who need to understand and change it safely;
4. LLM coding agents that need clear scope, boundaries, and invariants.
4. LLM coding agents that need clear scope, boundaries, and invariants;
5. developers and LLM coding agents integrating this project from another codebase.
Docs should be accurate, concise, task-oriented, and organized by audience. Prefer links to canonical docs over repetition.
@@ -42,11 +43,14 @@ Canonical homes:
- project purpose and quickstart: `README.md`
- development principles: `docs/policy/architecture.md`
- public HTTP API reference: `docs/api.md`
- configuration reference: `docs/config.md`
- CLI reference: `docs/cli.md`
- operations and recovery: `docs/operations.md`
- troubleshooting: `docs/troubleshooting.md`
- public API/package consumer guidance: `docs/consumers/`
- implemented internals: `docs/internal/`
- external protocol, service, and file-format contracts: `docs/integrations/`
- future work: `docs/roadmap/`
- contributor workflow: `docs/policy/development.md`
- copyable examples: `examples/`
@@ -106,7 +110,7 @@ Recommended:
- `examples/`
- `docs/policy/development.md`
### Modular, staged, service-oriented, or orchestration application
### Modular, service-oriented, or orchestration application
Required:
- `docs/cli.md`, if CLI-based
@@ -119,6 +123,31 @@ Recommended:
- `docs/troubleshooting.md`
- validated examples under `examples/`
### Public HTTP API service
Required:
- `docs/api.md`
- `docs/cli.md`, if CLI-based
- `docs/config.md`, if config-driven
- `docs/operations.md`
- `docs/internal/`
- `docs/policy/development.md`
Recommended:
- `docs/troubleshooting.md`
- `docs/consumers/`, for task-oriented client integration guides
- `docs/integrations/`, for upstream/downstream service contracts
- validated examples under `examples/`
### Project with public packages or consumer APIs
Required:
- `docs/consumers/api.md`
- one `docs/consumers/pkg-<name>.md` file per public package, if public packages exist
Recommended:
- copyable consumer examples under `examples/`, if practical
## Required Documents
### README.md
@@ -161,6 +190,32 @@ It should include:
For small projects, this file may be brief. It may simply state that the project is intentionally narrow, monolithic, and dependency-light.
### docs/api.md
**Audience:** external HTTP API consumers, developers, LLM coding agents integrating by HTTP
Required for projects whose primary public interface is HTTP.
`docs/api.md` is the canonical public HTTP API contract. It should be normative for external consumers and should not be duplicated by README, operations docs, consumer guides, or integration docs.
It should include:
1. base URL conventions;
2. authentication and authorization behavior, if implemented;
3. response envelope;
4. supported media types and content negotiation behavior;
5. shared query parameters;
6. endpoint reference grouped by route family;
7. request parameters and validation rules;
8. response fields, units, nullability, and optionality;
9. error response shape and status codes;
10. pagination, caching, rate-limit, idempotency, and retry behavior, if implemented;
11. compact request and response examples.
It must document only implemented endpoints and behavior. Planned endpoints, proposed fields, future filters, and experimental response shapes belong only under `docs/roadmap/`.
For HTTP API projects, `docs/consumers/` may provide task-oriented client integration guides, but those guides should link to `docs/api.md` for the authoritative endpoint contract.
### docs/policy/development.md
**Audience:** developers, LLM coding agents
@@ -175,7 +230,7 @@ It should include:
- dependency policy;
- how to add config fields;
- how to add CLI flags;
- how to add stages/modules/adapters, if applicable;
- how to add modules or adapters, if applicable;
- how to update examples;
- documentation update expectations.
@@ -216,7 +271,7 @@ Explain when commands are useful, not just their syntax.
**Audience:** administrators, operators
Required for applications that maintain state, support resume behavior, run multiple stages, write durable artifacts, use remote storage, or require recovery procedures.
Required for applications that maintain state, support resume behavior, run multi-step workflows, write durable artifacts, use remote storage, or require recovery procedures.
It should cover:
@@ -244,11 +299,40 @@ Each entry should include:
- safe fix;
- relevant links.
### docs/consumers/
**Audience:** developers and LLM coding agents integrating this project from another codebase
Required for projects with public packages, SDKs, client APIs, plugin APIs, or other application-facing integration surfaces.
This directory describes how an external codebase should consume the project's public API. It should be task-oriented and copyable where useful. It is not the place for internal implementation details or operator procedures.
For projects whose public API is HTTP, `docs/consumers/` is not required, and it should not duplicate the endpoint reference in `docs/api.md`. If present, it may provide practical integration workflows, client-specific examples, or migration notes that link back to `docs/api.md`.
`docs/consumers/api.md` should provide the consumer-facing overview and primary implementation workflow. It should include:
1. intended consumer audience and use cases;
2. required inputs supplied by operators or deployment configuration;
3. recommended public package or API workflow;
4. minimal copyable example;
5. consumer responsibilities and boundaries;
6. retry, idempotency, or status behavior, if applicable;
7. links to package-specific docs and canonical integration contracts.
Package-specific docs should be named `pkg-<name>.md` and should include:
1. import path;
2. intended use cases;
3. primary types and functions needed by consumers;
4. minimal examples;
5. validation, error, retry, and boundary behavior;
6. links to canonical file-format or wire-protocol contracts.
### docs/internal/
**Audience:** developers, LLM coding agents
Required for modular, staged, service-oriented, or orchestration projects.
Required for modular, service-oriented, or orchestration projects.
This directory describes implemented internal components. It is not the roadmap.
@@ -289,7 +373,9 @@ Roadmap docs should not be confused with current behavior.
Required for projects that depend on external CLIs, APIs, services, protocols, or file formats where the integration contract is important to maintain.
This directory contains concise, versioned reference notes for external integration contracts. It should document only the parts of the external system that this project actually uses.
This directory contains concise, versioned reference notes for external integration contracts. It should document only the parts of the external system that this project actually uses or exposes.
For public HTTP API services, `docs/integrations/` should document upstream, downstream, storage, protocol, or runtime contracts that the service depends on or bridges. It should not become a second copy of the public HTTP endpoint reference; that belongs in `docs/api.md`.
Use one file per integration where useful.
@@ -346,8 +432,10 @@ Before merging documentation changes, verify:
- README is concise and orientation-focused.
- `docs/policy/architecture.md` describes development principles.
- `docs/api.md` is the canonical HTTP contract for HTTP API services.
- Future work appears only under `docs/roadmap/`.
- User-facing docs avoid unnecessary internals.
- Consumer-facing docs explain public APIs without duplicating HTTP endpoint or integration contracts.
- Developer-facing docs preserve boundaries and invariants.
- Config examples match the schema.
- CLI examples match real commands and flags.

99
docs/roadmap/future.md Normal file
View File

@@ -0,0 +1,99 @@
# Future Work
## Purpose
This document is the catch-all roadmap for planned, deferred, aspirational, experimental, or unimplemented weatherfeeder work. Current behavior belongs in the canonical docs outside `docs/roadmap/`.
## NWS AFD Parsing Resilience
The current parser handles the concrete RAH, LWX, and MFR variants that
motivated these ideas. Future work should keep those extension points
maintainable as additional evidence appears.
### Generalize Wrapper-Scoped Embedded Sections
The scanner currently permits undotted nested headings only inside `PREV
DISCUSSION`. If other wrapper identities are observed, replace the single
wrapper check with a small explicit provider-local registry and add a fixture
for each wrapper family. Do not make the leading dot globally optional: wrapper
scope is the safeguard against classifying uppercase prose as a section.
### Extend Conservative Preamble Classification
Leading key-message metadata currently supports validated `Issued at`, `Updated
at`, and `As of <clock> <weekday>...` forms. Add future wording variants as
small, ordered classifiers with strict label boundaries and value grammars.
Every addition should include collision tests proving that similar message prose
and malformed metadata remain canonical content.
### Keep List-Marker Recognition Extensible
The marker parser currently supports hyphens, asterisks, `N)`, `N.`, `(N)`, and
composite forms such as `- (N)`. If new decorators appear, evolve the helper
toward an explicit marker grammar or typed classification result rather than a
broad punctuation heuristic. Preserve positive-number and whitespace-boundary
checks so ordinary prose is not stripped.
### Maintain a Cross-Office Fixture Corpus
The compact RAH, LWX, and current MFR fixtures seed regression coverage for the
observed layouts. Future parser changes should add concise, deterministic HTML
fixtures for materially distinct office formats and exercise them through both
the provider parser and normalizer. Fixture comments should identify the format
family and state that edited prose is not an archived product; tests must remain
offline and assert both intended extraction and adjacent-section isolation.
## SPC Convective Outlook Follow-Ups
### Weatherapi Outlook Endpoints
Expose persisted SPC convective outlooks through `weatherapi` after the weatherfeeder storage contract is stable.
Likely endpoints:
- `GET /outlooks/convective`
- `GET /outlooks/convective/active`
- `GET /outlooks/convective/location`
Recommended behavior:
- Return the latest outlook run by default.
- Support active outlook filtering by current time and `containsLocation=true`.
- Consider optional query filters for `day`, `outlookType`, and `containsLocation`.
- Preserve canonical outlook geometry for downstream display and audit use.
### SPC Day 4-8 Outlooks
Add SPC Day 4-8 convective outlook support as a schema-compatible extension only after Day 1-3 operation is proven.
Notes:
- Day 4-8 products have different semantics from Day 1-3 categorical/tornado/hail/wind products.
- Avoid forcing Day 4-8 assumptions into the current Day 1-3 model until the source shapes and consumer needs are reviewed.
- Prefer reusing `weather.outlook.v2` if the fields remain accurate; otherwise write a separate roadmap before changing the canonical contract.
### Degraded SPC Bundle Mode
Evaluate whether the SPC source should support degraded partial bundles when one required upstream product fails.
Current behavior should remain atomic: if a required GeoJSON or print-page fetch fails, emit no event for that poll.
Future degraded mode would need a clear contract for:
- distinguishing "no risk polygon" from "product missing";
- exposing per-product fetch errors without leaking raw provider internals into canonical events;
- deciding whether downstream sinks and APIs should store or serve partial snapshots.
### Richer SPC Page Assets And Tables
Evaluate whether to parse additional SPC print-page metadata beyond the current discussion text.
Possible additions:
- archive GeoJSON/shapefile/KML links;
- image URLs;
- page risk tables;
- city tables;
- richer discussion section metadata.
Keep GeoJSON products authoritative for polygons, validity windows, and point matching unless a future roadmap explicitly changes that contract.

View File

@@ -1,697 +0,0 @@
# SPC Convective Outlook Implementation Plan
## Purpose
Implement `weatherfeeder` support for Storm Prediction Center Day 1-3 convective outlooks described in [`docs/roadmap/spc.md`](spc.md). This plan is written for an LLM coding agent and should be followed stage by stage.
This is a planning document only. The implementation must preserve the existing weatherfeeder architecture: sources emit raw provider events, normalizers map raw payloads into canonical model types, and sinks persist canonical schemas.
## Decisions
The following choices are fixed for this implementation:
- Scope is Day 1-3 SPC convective outlooks only.
- Day 4-8 outlooks are out of scope.
- GeoJSON files are authoritative for polygons, validity windows, issue times, outlook labels, and severity ranking.
- Day 1-3 print pages are authoritative for discussion text.
- RSS is optional supplemental metadata only and must not be required for correctness.
- Do not fetch RSS by default. Include RSS only when an optional `rss_url` source param is configured.
- A poll is atomic for required products. If any configured GeoJSON or print-page URL fails or returns a non-2xx response, return an error and emit no event.
- Use compact GeoJSON geometry in the canonical payload for auditability and downstream display.
- Use standard-library-first parsing. Do not add an HTML parsing dependency unless string-based extraction proves unmaintainable during implementation.
- Keep all new planned behavior inside `weatherfeeder`; do not make `weatherapi` changes in this pass.
## Public Contract
Add schema constants in `standards/schema.go`:
- `SchemaRawSPCConvectiveOutlookV1 = "raw.spc.convective_outlook.v1"`
- `SchemaWeatherOutlookV1 = "weather.outlook.v1"`
Add source driver:
- `spc_convective_outlook`
Add event kind:
- `outlook`
Add canonical model types:
- `model.WeatherOutlookRun`
- `model.WeatherOutlook`
Canonical run fields:
```go
type WeatherOutlookRun struct {
LocationID string `json:"locationId,omitempty"`
LocationName string `json:"locationName,omitempty"`
Latitude *float64 `json:"latitude,omitempty"`
Longitude *float64 `json:"longitude,omitempty"`
AsOf time.Time `json:"asOf"`
IssuedAt *time.Time `json:"issuedAt,omitempty"`
Outlooks []WeatherOutlook `json:"outlooks"`
}
```
Canonical outlook fields:
```go
type WeatherOutlook struct {
ID string `json:"id"`
Provider string `json:"provider"`
Product string `json:"product"`
Day int `json:"day"`
OutlookType string `json:"outlookType"`
Label string `json:"label"`
LabelText string `json:"labelText,omitempty"`
SeverityRank *int `json:"severityRank,omitempty"`
ValidFrom time.Time `json:"validFrom"`
ValidTo time.Time `json:"validTo"`
IssuedAt time.Time `json:"issuedAt"`
ExpiresAt time.Time `json:"expiresAt"`
Forecaster string `json:"forecaster,omitempty"`
Headline string `json:"headline,omitempty"`
Summary string `json:"summary,omitempty"`
Discussion string `json:"discussion,omitempty"`
SourceURL string `json:"sourceUrl,omitempty"`
ImageURL string `json:"imageUrl,omitempty"`
ContainsLocation bool `json:"containsLocation"`
Geometry json.RawMessage `json:"geometry"`
}
```
Required canonical fields:
- Run: `asOf`, `outlooks`.
- Outlook: `id`, `provider`, `product`, `day`, `outlookType`, `label`, `validFrom`, `validTo`, `issuedAt`, `expiresAt`, `containsLocation`, `geometry`.
Canonical values:
- `provider` is `spc`.
- `product` is `convective`.
- `outlookType` is one of `categorical`, `tornado`, `hail`, `wind`.
- `day` is one of `1`, `2`, `3`.
## Source Inputs
Default required GeoJSON products:
- `https://www.spc.noaa.gov/products/outlook/day1otlk_cat.nolyr.geojson`
- `https://www.spc.noaa.gov/products/outlook/day1otlk_torn.nolyr.geojson`
- `https://www.spc.noaa.gov/products/outlook/day1otlk_hail.nolyr.geojson`
- `https://www.spc.noaa.gov/products/outlook/day1otlk_wind.nolyr.geojson`
- `https://www.spc.noaa.gov/products/outlook/day2otlk_cat.nolyr.geojson`
- `https://www.spc.noaa.gov/products/outlook/day2otlk_torn.nolyr.geojson`
- `https://www.spc.noaa.gov/products/outlook/day2otlk_hail.nolyr.geojson`
- `https://www.spc.noaa.gov/products/outlook/day2otlk_wind.nolyr.geojson`
- `https://www.spc.noaa.gov/products/outlook/day3otlk_cat.nolyr.geojson`
- `https://www.spc.noaa.gov/products/outlook/day3otlk_torn.nolyr.geojson`
- `https://www.spc.noaa.gov/products/outlook/day3otlk_hail.nolyr.geojson`
- `https://www.spc.noaa.gov/products/outlook/day3otlk_wind.nolyr.geojson`
Default required print-page products:
- `https://www.spc.noaa.gov/products/outlook/day1otlk_prt.html`
- `https://www.spc.noaa.gov/products/outlook/day2otlk_prt.html`
- `https://www.spc.noaa.gov/products/outlook/day3otlk_prt.html`
Optional RSS product:
- `https://www.spc.noaa.gov/products/spcacrss.xml`
Recommended config shape:
```yaml
- name: SPCConvectiveOutlookSTL
mode: poll
kinds: ["outlook"]
driver: spc_convective_outlook
every: 30m
params:
latitude: 38.6239
longitude: -90.3571
location_id: "stl"
location_name: "St. Louis, MO"
user_agent: "HomeOps (eric@maximumdirect.net)"
```
Optional source params:
- `geojson_urls`: map from product key to URL, used by tests and future upstream changes.
- `discussion_urls`: map from day key to URL, used by tests and future upstream changes.
- `rss_url`: string; when non-empty, fetch RSS as supplemental metadata.
Product keys for `geojson_urls`:
- `day1_categorical`, `day1_tornado`, `day1_hail`, `day1_wind`
- `day2_categorical`, `day2_tornado`, `day2_hail`, `day2_wind`
- `day3_categorical`, `day3_tornado`, `day3_hail`, `day3_wind`
Discussion keys for `discussion_urls`:
- `day1`, `day2`, `day3`
## Raw Bundle Shape
Create a provider raw bundle type under `internal/providers/spc` or `internal/normalizers/spc` and use it consistently between source tests and normalizer tests. Prefer `internal/providers/spc` if source metadata extraction and normalizer parsing share helpers.
Suggested raw payload shape:
```go
type RawConvectiveOutlookBundle struct {
LocationID string `json:"locationId,omitempty"`
LocationName string `json:"locationName,omitempty"`
Latitude float64 `json:"latitude"`
Longitude float64 `json:"longitude"`
FetchedAt time.Time `json:"fetchedAt"`
Products []RawOutlookProduct `json:"products"`
Discussions []RawDiscussionPage `json:"discussions"`
RSS *RawRSSFeed `json:"rss,omitempty"`
}
```
```go
type RawOutlookProduct struct {
Key string `json:"key"`
Day int `json:"day"`
OutlookType string `json:"outlookType"`
URL string `json:"url"`
FetchedAt time.Time `json:"fetchedAt"`
Body json.RawMessage `json:"body"`
}
```
```go
type RawDiscussionPage struct {
Key string `json:"key"`
Day int `json:"day"`
URL string `json:"url"`
FetchedAt time.Time `json:"fetchedAt"`
Body string `json:"body"`
}
```
```go
type RawRSSFeed struct {
URL string `json:"url"`
FetchedAt time.Time `json:"fetchedAt"`
Body string `json:"body"`
}
```
Do not put parsed canonical fields into the raw bundle except configured metadata and product keys needed to identify each fetched upstream document. Source-level timestamp parsing is allowed only for event `effectiveAt` selection.
## Stage 1: Provider Helpers And Fixtures
Goal: add deterministic SPC parsing primitives and test fixtures before wiring the source or normalizer.
Files to add:
- `internal/providers/spc/doc.go`
- `internal/providers/spc/time.go`
- `internal/providers/spc/product.go`
- `internal/providers/spc/geojson.go`
- `internal/providers/spc/discussion.go`
- `internal/providers/spc/rss.go`, only if optional RSS parsing is implemented
- `internal/providers/spc/testdata/day1_cat.geojson`
- `internal/providers/spc/testdata/day2_torn.geojson`
- `internal/providers/spc/testdata/day3_wind.geojson`
- `internal/providers/spc/testdata/day1_prt.html`
- `internal/providers/spc/testdata/day2_prt_corr.html`
- `internal/providers/spc/testdata/day3_prt.html`
Provider helper behavior:
- Define stable product metadata for the 12 required GeoJSON products.
- Define stable discussion metadata for the 3 required print-page products.
- Parse SPC ISO timestamps from GeoJSON properties using `time.Parse(time.RFC3339, value)` after trimming whitespace.
- Decode enough GeoJSON to expose feature properties and raw geometry without owning canonical mapping.
- Preserve raw geometry as compact JSON bytes for later canonical use.
- Extract print-page product text from the first useful `<pre>` block.
- Strip embedded `<script>` blocks and remaining tags from extracted `<pre>` content.
- Use `html.UnescapeString`, normalize CRLF to LF, and trim surrounding blank lines.
- Parse print-page `Updated:` timestamps when present.
- Parse discussion headline/product title from product text.
- Parse `...SUMMARY...` content through the next section heading.
- Preserve full product text as discussion text after cleanup.
- Preserve correction markers such as `CORR 1` in headline or metadata when present.
Dependency rule:
- Start with `strings`, `regexp`, `encoding/json`, `encoding/xml`, and `html` from the standard library.
- Do not add `golang.org/x/net/html` or another HTML parser unless tests show the string extraction is too brittle.
Tests:
- `go test ./internal/providers/spc`
- Product metadata contains exactly 12 GeoJSON products in day/type order.
- Discussion metadata contains exactly 3 print pages in day order.
- GeoJSON fixture decode exposes `VALID_ISO`, `EXPIRE_ISO`, `ISSUE_ISO`, `FORECASTER`, `LABEL`, `LABEL2`, `DN`, and geometry.
- Print-page fixture extraction returns product text without scripts or tags.
- Day 2 correction fixture preserves `CORR 1`.
- Summary extraction returns only the summary paragraph content.
- Updated timestamp parser returns UTC time when present and nil when absent.
Stage completion criteria:
- Provider helper tests pass.
- No source, normalizer, model, standards, sink, config, or docs current-behavior files are changed in this stage unless needed for package compilation.
## Stage 2: Geometry Helper
Goal: implement point-in-polygon support independent of SPC parsing.
Files to add:
- `internal/geo/geojson.go`
- `internal/geo/point.go`
- `internal/geo/point_test.go`
Behavior:
- Support GeoJSON `Polygon` and `MultiPolygon` only.
- Accept geometry as `json.RawMessage` or `[]byte`.
- Interpret GeoJSON coordinate order as `[longitude, latitude]`.
- Treat the first ring as the exterior ring.
- Treat subsequent rings as holes.
- Count boundary points as inside.
- Return a clear error for unsupported geometry types, malformed coordinates, empty rings, or invalid JSON.
- Use planar ray casting. This is sufficient for operational point-in-polygon checks at SPC polygon scale.
Tests:
- Point inside simple polygon returns true.
- Point outside simple polygon returns false.
- Point on polygon boundary returns true.
- Point in a hole returns false.
- Point inside one `MultiPolygon` member returns true.
- Longitude/latitude ordering regression test fails if coordinates are reversed.
- Unsupported geometry returns a useful error.
Verification:
```sh
go test ./internal/geo
```
Stage completion criteria:
- Geometry helper has no dependency on SPC, source, normalizer, or sink packages.
## Stage 3: Source Driver And Raw Schema
Goal: emit raw SPC outlook bundles from a new poll source.
Files to update:
- `standards/schema.go`
- `internal/sources/builtins.go`
- `internal/sources/builtins_test.go`
Files to add:
- `internal/sources/spc/convective_outlook.go`
- `internal/sources/spc/convective_outlook_test.go`
Source constructor:
- Export `NewConvectiveOutlookSource(cfg config.SourceConfig) (*ConvectiveOutlookSource, error)`.
- Register driver `spc_convective_outlook` in `internal/sources/builtins.go`.
- Validate required params `latitude` and `longitude`.
- Accept optional params `location_id`, `location_name`, `geojson_urls`, `discussion_urls`, and `rss_url`.
- Require `user_agent` when the existing HTTP source conventions require it.
- Advertise `Kinds() []event.Kind{event.Kind("outlook")}`.
Fetch behavior:
- Fetch all required GeoJSON URLs and print-page URLs every poll cycle.
- Fetch optional RSS only when `rss_url` is configured and non-empty.
- Use Accept `application/geo+json, application/json` for GeoJSON requests.
- Use Accept `text/html, application/xhtml+xml` for print pages.
- Use Accept `application/rss+xml, application/xml, text/xml` for RSS.
- Respect `context.Context` on all requests.
- Do not emit partial bundles.
- If the complete raw bundle is unchanged from the previous successful poll, return no events. Implement source-local change detection by hashing the combined fetched bodies if feedkit HTTP conditional helpers do not directly support multi-document polling.
- Build the raw bundle with fetched bodies and metadata.
- Emit one raw event with kind `outlook` and schema `standards.SchemaRawSPCConvectiveOutlookV1`.
- Use `fksources.DefaultEventID("", sourceName, effectiveAt, emittedAt)`.
Effective time policy:
- Prefer latest valid `ISSUE_ISO` across all GeoJSON features.
- Fallback to latest valid print-page `Updated:` timestamp.
- Fallback to RSS `lastBuildDate` if RSS was fetched and parseable.
- Fallback to fetch time if no provider timestamp is parseable.
- Keep fetch-time fallback as the last source effective-time option so downstream consumers still receive a stable effective time when provider metadata is absent.
Tests:
- Builtin registry builds `spc_convective_outlook` as a `PollSource`.
- `Kinds()` returns `outlook`.
- Constructor rejects missing `latitude` or `longitude`.
- Poll emits one raw event with schema `raw.spc.convective_outlook.v1`.
- Raw bundle includes 12 products and 3 discussions when defaults are overridden to test-server URLs.
- Poll effectiveAt prefers latest GeoJSON `ISSUE_ISO`.
- Poll effectiveAt falls back to print-page `Updated:` when GeoJSON issue times are absent.
- Poll includes RSS only when `rss_url` is configured.
- Unchanged response emits no events on a second poll.
- A configured GeoJSON fetch failure returns error and emits no event.
- A configured print-page fetch failure returns error and emits no event.
- Tests use `httptest.Server`; do not call live SPC services.
Verification:
```sh
go test ./internal/providers/spc ./internal/sources
```
Stage completion criteria:
- Source emits raw bundles only.
- No canonical model, normalizer, or sink mapping is added in this stage except schema constants required for compilation.
## Stage 4: Canonical Model And Normalizer
Goal: convert raw SPC bundles into `weather.outlook.v1` events.
Files to update:
- `standards/schema.go`
- `internal/normalizers/builtins_test.go`
Files to add:
- `model/outlook.go`
- `internal/normalizers/spc/convective_outlook.go`
- `internal/normalizers/spc/register.go`
- `internal/normalizers/spc/convective_outlook_test.go`
Files to update for registration:
- `internal/normalizers/builtins.go`
Normalizer behavior:
- Add `SPCConvectiveOutlookNormalizer`.
- Match only `standards.SchemaRawSPCConvectiveOutlookV1`.
- Decode `RawConvectiveOutlookBundle`.
- Build one `model.WeatherOutlookRun` per raw bundle.
- Map each GeoJSON feature to one `model.WeatherOutlook`.
- Preserve feature order within each product.
- Order products by day then outlook type in this order: `categorical`, `tornado`, `hail`, `wind`.
- Compute `containsLocation` with `internal/geo` using the configured bundle latitude and longitude.
- Store compact feature geometry JSON in `WeatherOutlook.Geometry`.
- Set output schema to `standards.SchemaWeatherOutlookV1`.
- Set output effectiveAt to run `AsOf`.
- Use `internal/normalizers/common.Finalize` or existing normalizer helper patterns so envelope handling remains consistent.
GeoJSON property mapping:
- `VALID_ISO` maps to `validFrom` and is required.
- `EXPIRE_ISO` maps to `validTo` and `expiresAt` and is required.
- `ISSUE_ISO` maps to `issuedAt` and is required.
- `FORECASTER` maps to `forecaster` and is optional.
- `LABEL` maps to `label` and is required.
- `LABEL2` maps to `labelText` and is optional.
- `DN` maps to `severityRank` and is optional.
Derived mapping:
- `provider` is `spc`.
- `product` is `convective`.
- `day` and `outlookType` come from raw product metadata, not from brittle URL parsing when product metadata is available.
- `id` is deterministic: join day, outlook type, normalized label, issuedAt UTC in RFC3339 format, validFrom UTC in RFC3339 format, and product-local feature index. Use a stable ASCII-safe format.
- `sourceUrl` is the GeoJSON product URL for geometry/probability features.
- `imageUrl` is empty for v1.
Discussion enrichment:
- Parse each print-page discussion with `internal/providers/spc` helpers.
- Attach Day 1 print-page headline, summary, and discussion to Day 1 outlooks, Day 2 to Day 2 outlooks, and Day 3 to Day 3 outlooks.
- Use the same discussion text for categorical, tornado, hail, and wind outlooks for the same day.
- If a print page is fetched but discussion extraction fails, normalization should fail because print pages are required v1 inputs.
- RSS metadata must not be required for discussion enrichment.
Run timestamp policy:
- `AsOf` is latest valid `issuedAt` across all outlook features.
- Fallback to latest print-page `Updated:` timestamp.
- Fallback to input event `EffectiveAt`.
- Fallback to input event `EmittedAt`.
- `IssuedAt` is latest valid `issuedAt` across all outlook features when any feature exists; otherwise nil.
Error behavior:
- Fail normalization if required GeoJSON timestamps are missing or unparseable.
- Fail normalization if required `LABEL` is empty.
- Fail normalization if required geometry is missing or invalid.
- Fail normalization if latitude or longitude is missing or invalid in the raw bundle.
- Include product key and feature index in errors.
Tests:
- Normalizer matches only `raw.spc.convective_outlook.v1`.
- Builtin normalizer order includes the SPC normalizer after existing provider normalizers unless a specific order is needed.
- Canonical schema is `weather.outlook.v1`.
- Categorical fixture maps expected day, type, label, label text, severity rank, valid times, issue time, forecaster, source URL, and geometry.
- Probabilistic fixtures for tornado, hail, and wind map expected `outlookType` values.
- `containsLocation` is true for a fixture point inside a polygon.
- `containsLocation` is false for a fixture point outside a polygon.
- Print-page text maps to headline, summary, and discussion.
- Day 2 correction marker `CORR 1` is preserved in headline or discussion metadata.
- Missing optional RSS still normalizes successfully.
- Invalid required timestamp fails with product key and feature index context.
- Invalid geometry fails with product key and feature index context.
- Output JSON uses the intended field names and does not expose raw bundle internals.
Verification:
```sh
go test ./model ./internal/geo ./internal/providers/spc ./internal/normalizers ./internal/normalizers/spc
```
Stage completion criteria:
- A raw SPC bundle normalizes into canonical `model.WeatherOutlookRun`.
- No Postgres schema or current-behavior docs are updated in this stage unless the canonical schema documentation is intentionally updated with implementation.
## Stage 5: Postgres Sink
Goal: persist canonical outlook events through the weatherfeeder Postgres sink.
Files to update:
- `internal/sinks/postgres/schema.go`
- `internal/sinks/postgres/map.go`
- `internal/sinks/postgres/schema_test.go`
- `internal/sinks/postgres/map_test.go`
- `internal/sinks/postgres/doc.go`
Tables:
- `outlook_runs`
- `outlooks`
Add table constants:
- `tableOutlookRuns = "outlook_runs"`
- `tableOutlooks = "outlooks"`
`outlook_runs` columns:
- `event_id TEXT NOT NULL`
- `event_kind TEXT NOT NULL`
- `event_source TEXT NOT NULL`
- `event_schema TEXT NOT NULL`
- `event_emitted_at TIMESTAMPTZ NOT NULL`
- `event_effective_at TIMESTAMPTZ NULL`
- `location_id TEXT NULL`
- `location_name TEXT NULL`
- `latitude DOUBLE PRECISION NULL`
- `longitude DOUBLE PRECISION NULL`
- `as_of TIMESTAMPTZ NOT NULL`
- `issued_at TIMESTAMPTZ NULL`
- `outlook_count INTEGER NOT NULL`
`outlook_runs` keys and indexes:
- Primary key: `event_id`
- Prune column: `as_of`
- Index `idx_wf_outlook_run_location_as_of` on `location_id, as_of`
- Index `idx_wf_outlook_run_as_of` on `as_of`
`outlooks` columns:
- `run_event_id TEXT REFERENCES outlook_runs(event_id) ON DELETE CASCADE NOT NULL`
- `outlook_index INTEGER NOT NULL`
- `as_of TIMESTAMPTZ NOT NULL`
- `product TEXT NOT NULL`
- `day INTEGER NOT NULL`
- `outlook_type TEXT NOT NULL`
- `label TEXT NOT NULL`
- `label_text TEXT NULL`
- `severity_rank INTEGER NULL`
- `valid_from TIMESTAMPTZ NOT NULL`
- `valid_to TIMESTAMPTZ NOT NULL`
- `issued_at TIMESTAMPTZ NOT NULL`
- `expires_at TIMESTAMPTZ NOT NULL`
- `forecaster TEXT NULL`
- `headline TEXT NULL`
- `summary TEXT NULL`
- `discussion TEXT NULL`
- `source_url TEXT NULL`
- `image_url TEXT NULL`
- `contains_location BOOLEAN NOT NULL`
- `geometry_json TEXT NOT NULL`
`outlooks` keys and indexes:
- Primary key: `run_event_id, outlook_index`
- Prune column: `as_of`
- Index `idx_wf_outlooks_contains_valid` on `contains_location, valid_from, valid_to`
- Index `idx_wf_outlooks_day_type_label` on `day, outlook_type, label`
- Index `idx_wf_outlooks_valid` on `valid_from, valid_to`
Mapper behavior:
- Extend `mapPostgresEvent` for `standards.SchemaWeatherOutlookV1`.
- Decode `model.WeatherOutlookRun`.
- Require run `AsOf`.
- Map envelope columns exactly like existing parent run tables.
- Store all times as UTC.
- Write one `outlook_runs` row and one `outlooks` row per outlook.
- Use `outlook_index` as the zero-based slice index.
- Require outlook `ID`, `Provider`, `Product`, `Day`, `OutlookType`, `Label`, `ValidFrom`, `ValidTo`, `IssuedAt`, `ExpiresAt`, and non-empty `Geometry`.
- Store compact geometry JSON text in `geometry_json` using the existing compact JSON helper or a similar helper.
- Preserve all outlook polygons, not only those containing the configured point.
Tests:
- Schema includes `outlook_runs` and `outlooks`.
- Schema includes required columns and indexes.
- Mapper writes one run row plus one row per outlook.
- Mapper stores UTC times.
- Mapper stores compact geometry JSON.
- Mapper preserves `contains_location` false as false, not nil.
- Mapper rejects missing run `asOf`.
- Mapper rejects missing required outlook times.
- Mapper rejects empty geometry.
- Mapper ignores unrelated schemas unchanged.
Verification:
```sh
go test ./internal/sinks/postgres
```
Stage completion criteria:
- Postgres schema and mapper tests pass.
- No weatherapi read path is added in this stage.
## Stage 6: Config, Runtime Wiring, And Current-Behavior Docs
Goal: expose the completed source/normalizer/sink behavior in maintained configuration and documentation after implementation exists.
Files to update:
- `cmd/weatherfeeder/config.yml`
- `cmd/weatherfeeder/main_test.go`, if config load expectations need updates
- `docs/config.md`
- `docs/integrations/events.md`
- `docs/integrations/postgres.md`
- `docs/internal/sources.md`
- `docs/internal/normalizers.md`
- `docs/internal/postgres-sink.md`
- `README.md`
Config changes:
- Add sample source `SPCConvectiveOutlookSTL` with driver `spc_convective_outlook`, kind `outlook`, `every: 30m`, latitude, longitude, location metadata, and `user_agent`.
- Add `outlook` route examples for stdout, NATS, and Postgres where the existing config style includes kind lists.
- Keep optional RSS disabled in sample config unless there is a specific operator reason to include it.
Docs changes:
- Update current-behavior docs only after the code for that behavior exists.
- `docs/config.md` should document required and optional source params.
- `docs/integrations/events.md` should document `weather.outlook.v1`, including field definitions, required fields, optional fields, geometry semantics, and `containsLocation` semantics.
- `docs/integrations/postgres.md` should document `outlook_runs` and `outlooks`.
- Internal docs should explain provider boundaries, print-page discussion parsing, and point-in-polygon behavior where useful for maintainers.
- `README.md` should only briefly list SPC convective outlook support and link to canonical docs.
- Do not document weatherapi endpoints in weatherfeeder current-behavior docs.
Tests:
- Existing config load tests pass.
- Add config test coverage if the sample config is expected to build scheduler jobs for the new source.
- Documentation examples should use the real driver name and event kind.
Verification:
```sh
go test ./cmd/weatherfeeder ./internal/sources ./internal/normalizers ./internal/sinks/postgres
```
Stage completion criteria:
- Sample config remains loadable.
- Current-behavior docs match implemented code.
- No roadmap-only claims leak into non-roadmap docs beyond the implemented behavior.
## Stage 7: Full Verification And Cleanup
Goal: validate the complete feature and remove implementation-only rough edges.
Run focused tests:
```sh
go test ./internal/providers/spc ./internal/geo ./internal/sources ./internal/normalizers/... ./internal/sinks/postgres ./model ./cmd/weatherfeeder
```
Run full tests:
```sh
go test ./...
```
Manual review checklist:
- `standards/schema.go` contains raw and canonical SPC schema constants.
- `model/outlook.go` uses stable JSON tags and no provider-specific names except canonical strings.
- Source registry includes `spc_convective_outlook`.
- Normalizer registry includes the SPC normalizer.
- Source tests do not use live SPC services.
- Normalizer tests use fixtures and cover discussion parsing, geometry, and timestamp failures.
- Postgres tests cover schema shape and mapper validation.
- Docs follow `docs/policy/documentation.md` and use canonical homes.
- No weatherapi files are changed.
- No broad dependency was added without clear justification.
Acceptance criteria:
- A configured SPC source emits one raw bundled `outlook` event per changed complete product snapshot.
- Normalization produces `weather.outlook.v1` events.
- All Day 1-3 categorical, tornado, hail, and wind products are represented.
- Each outlook includes `containsLocation` for the configured latitude and longitude.
- Each outlook preserves compact GeoJSON geometry.
- Day 1-3 print-page discussion text is preserved in canonical headline, summary, and discussion fields where parseable.
- RSS metadata is optional and supplemental only.
- Postgres sink persists outlook runs and outlook rows.
- Sample config and current-behavior docs describe the implemented kind, driver, schema, and storage contract.
## Suggested Prompt Boundaries
This feature is too broad for a single safe implementation prompt. Use these implementation prompts in order:
1. Implement Stage 1 provider helpers and fixtures only.
2. Implement Stage 2 geometry helper only.
3. Implement Stage 3 source driver and raw schema only.
4. Implement Stage 4 canonical model and normalizer only.
5. Implement Stage 5 Postgres sink mapping only.
6. Implement Stage 6 config and current-behavior documentation only.
7. Run Stage 7 verification and perform targeted fixes only.
Each prompt should run its stage-specific tests before moving on. Do not proceed to the next stage with failing tests unless the failure is unrelated and explicitly documented.

View File

@@ -0,0 +1,87 @@
# SPC Outlook Postgres Schema Transition
## Purpose
This document describes how to reset existing Postgres outlook tables from the
`weather.outlook.v1` storage shape to the `weather.outlook.v2` compatible
storage shape.
Updated `weatherfeeder` versions create outlook tables with run-level
discussion storage. Existing databases that already contain the old outlook
table family need a manual reset because the Postgres sink creates tables with
`CREATE TABLE IF NOT EXISTS`.
## Scope
This reset drops only the outlook table family and lets updated `weatherfeeder`
recreate it:
- `outlook_discussions`
- `outlooks`
- `outlook_runs`
Other weather tables are not affected.
## Warning
These commands delete stored SPC outlook history. Existing `weather.outlook.v1`
outlook rows are intentionally removed. Downstream readers should be updated
intentionally for the new outlook shape.
## Deployment Order
1. Stop `weatherfeeder`.
2. Drop the existing outlook tables.
3. Deploy updated `weatherfeeder`.
4. Start `weatherfeeder` so the Postgres sink recreates the new outlook tables.
5. Deploy updated downstream consumers such as `weatherapi`.
## Reset SQL
```sql
DROP TABLE IF EXISTS outlook_discussions;
DROP TABLE IF EXISTS outlooks;
DROP TABLE IF EXISTS outlook_runs;
```
## Verification SQL
Before or after the updated daemon starts, this query shows which outlook tables
exist:
```sql
SELECT table_name
FROM information_schema.tables
WHERE table_name IN ('outlook_runs', 'outlooks', 'outlook_discussions')
ORDER BY table_name;
```
After the updated daemon has started and recreated the tables, verify the new
run column:
```sql
SELECT column_name, is_nullable, data_type
FROM information_schema.columns
WHERE table_name = 'outlook_runs'
AND column_name = 'discussion_count';
```
Verify the discussion table indexes:
```sql
SELECT indexname
FROM pg_indexes
WHERE tablename = 'outlook_discussions'
ORDER BY indexname;
```
Verify that legacy polygon-level prose columns are gone from `outlooks`:
```sql
SELECT column_name
FROM information_schema.columns
WHERE table_name = 'outlooks'
AND column_name IN ('headline', 'summary', 'discussion');
```
The final query should return zero rows.

View File

@@ -1,349 +0,0 @@
# SPC Convective Outlook Support Roadmap
## Summary
Add `weatherfeeder` support for Storm Prediction Center convective outlooks as a new canonical outlook domain. The feature should poll SPC GeoJSON outlook products, enrich them with Day 1-3 print-page discussion text, compute whether the configured forecast point is inside each active outlook polygon, normalize the result into a provider-independent schema, and persist it through existing sinks.
This is a new domain, not an extension of `weather.alert.v1`. SPC outlooks describe probabilistic/categorical risk areas over a validity window; alerts describe active actionable hazard messages.
## Public Contract
Add schema constants:
- Raw schema: `raw.spc.convective_outlook.v1`
- Canonical schema: `weather.outlook.v1`
Add source driver:
- `spc_convective_outlook`
Add event kind:
- `outlook`
Add canonical model types:
- `model.WeatherOutlookRun`
- `model.WeatherOutlook`
Recommended canonical run fields:
- `locationId`, `locationName`
- `latitude`, `longitude`
- `asOf`
- `issuedAt`
- `outlooks`
Recommended canonical outlook fields:
- `id`
- `provider`
- `product`
- `day`
- `outlookType`
- `label`
- `labelText`
- `severityRank`
- `validFrom`
- `validTo`
- `issuedAt`
- `expiresAt`
- `forecaster`
- `headline`
- `summary`
- `discussion`
- `sourceUrl`
- `imageUrl`
- `containsLocation`
- `geometry`
Contract defaults:
- `product` should be `convective`.
- `outlookType` should be one of `categorical`, `tornado`, `hail`, `wind`.
- `day` should be `1`, `2`, or `3` for this first implementation.
- `containsLocation` is computed against configured forecast coordinates.
- `geometry` should preserve compact GeoJSON geometry for auditability and future API use.
- Day 4-8 outlooks are out of scope for this first implementation. They have different product semantics and should be added in a later schema-compatible extension only after Day 1-3 support is proven.
## Source Scope
The source should fetch a bundle of SPC products in one poll cycle and emit one raw event containing fetched GeoJSON products, fetched Day 1-3 print pages, optional RSS metadata, configured point metadata, and per-product fetch metadata. The source may decode minimal timestamps for event identity/effective time, but provider-to-canonical mapping belongs in the normalizer.
Poll these GeoJSON URLs:
- `https://www.spc.noaa.gov/products/outlook/day1otlk_cat.nolyr.geojson`
- `https://www.spc.noaa.gov/products/outlook/day1otlk_torn.nolyr.geojson`
- `https://www.spc.noaa.gov/products/outlook/day1otlk_hail.nolyr.geojson`
- `https://www.spc.noaa.gov/products/outlook/day1otlk_wind.nolyr.geojson`
- `https://www.spc.noaa.gov/products/outlook/day2otlk_cat.nolyr.geojson`
- `https://www.spc.noaa.gov/products/outlook/day2otlk_torn.nolyr.geojson`
- `https://www.spc.noaa.gov/products/outlook/day2otlk_hail.nolyr.geojson`
- `https://www.spc.noaa.gov/products/outlook/day2otlk_wind.nolyr.geojson`
- `https://www.spc.noaa.gov/products/outlook/day3otlk_cat.nolyr.geojson`
- `https://www.spc.noaa.gov/products/outlook/day3otlk_torn.nolyr.geojson`
- `https://www.spc.noaa.gov/products/outlook/day3otlk_hail.nolyr.geojson`
- `https://www.spc.noaa.gov/products/outlook/day3otlk_wind.nolyr.geojson`
Implementation note: the initial candidate list duplicated Day 3 wind and omitted Day 2 wind. Use the corrected 12-product matrix above.
Poll these print-page discussion URLs:
- `https://www.spc.noaa.gov/products/outlook/day1otlk_prt.html`
- `https://www.spc.noaa.gov/products/outlook/day2otlk_prt.html`
- `https://www.spc.noaa.gov/products/outlook/day3otlk_prt.html`
Print-page usage:
- Use print pages as the authoritative source for Day 1-3 outlook discussion text.
- Preserve source URL, page title/header where available, page-level `Updated:` timestamp where parseable, and the raw HTML or extracted `<pre>` product block in the raw bundle.
- Extract canonical `headline`, `summary`, `discussion`, and discussion metadata in the normalizer or shared SPC provider helper.
- Treat the GeoJSON files, not print pages, as authoritative for polygons and validity windows.
- Ignore page risk tables and image assets for v1 unless they are needed to support discussion parsing.
Optionally poll:
- `https://www.spc.noaa.gov/products/spcacrss.xml`
RSS usage:
- Use RSS as supplemental discovery or health metadata, not as the primary discussion or geometry source.
- Preserve item title, link, description text, pubDate, and guid where useful.
- Do not depend on RSS completeness for Day 1-3 discussion text. The RSS feed can omit current outlook products, including a current Day 2 discussion.
- Do not attempt to parse polygons from RSS HTML.
Polling cadence:
- Use a default cadence of `30m`, not daily/twice-daily. SPC current outlook files can update several times per day, and a 30-minute poll with HTTP caching is low cost and less likely to miss updates.
- Rely on ETag/Last-Modified handling from the HTTP source layer where available.
- Keep the cadence configurable via `every`.
Recommended config shape:
```yaml
- name: SPCConvectiveOutlookSTL
mode: poll
kinds: ["outlook"]
driver: spc_convective_outlook
every: 30m
params:
latitude: 38.6239
longitude: -90.3571
location_id: "stl"
location_name: "St. Louis, MO"
user_agent: "HomeOps (eric@maximumdirect.net)"
```
The source should own default SPC URLs, with optional params to override GeoJSON URLs, print-page URLs, and the RSS URL for tests or future provider changes.
## Implementation Stages
### Stage 1: Raw Source and Schema
- Add schema constants.
- Add `internal/sources/spc` package.
- Implement `spc_convective_outlook` as a poll source.
- Fetch all configured GeoJSON products and Day 1-3 print pages during a poll cycle.
- Fetch RSS only when configured or retained as an optional supplemental source.
- Emit one raw event with kind `outlook` and schema `raw.spc.convective_outlook.v1`.
- Use the latest valid `ISSUE_ISO`, print-page `Updated:` timestamp, RSS `lastBuildDate`, or fetch time for `effectiveAt`, in that order.
- Preserve partial fetch errors as source errors for the first implementation; do not emit incomplete outlook bundles unless a later explicit degraded-mode policy is added.
- Keep source tests deterministic with local test servers and fixtures; do not call live SPC services from unit tests.
Tests:
- source driver builds as a `PollSource`
- source advertises kind `outlook`
- source emits one raw bundled event
- source chooses expected effective time
- source uses configured location metadata
- source fails clearly on missing latitude/longitude
- source fails clearly when a configured GeoJSON or print-page fetch fails
### Stage 2: GeoJSON and Spatial Matching
- Add an internal geometry helper package, for example `internal/geo`.
- Support GeoJSON `Polygon` and `MultiPolygon`.
- Implement point-in-polygon with ring support:
- first ring is the exterior
- subsequent rings are holes
- boundary counts as inside
- GeoJSON coordinate order is `[longitude, latitude]`
- Use planar ray casting for this operational point-in-polygon check.
Tests:
- point inside polygon
- point outside polygon
- point on polygon boundary
- point inside a hole is outside
- point inside one multipolygon member is inside
- longitude/latitude order regression test
### Stage 3: Normalizer and Canonical Mapping
- Add `model.WeatherOutlookRun` and `model.WeatherOutlook`.
- Add `SPCConvectiveOutlookNormalizer`.
- Match only `raw.spc.convective_outlook.v1`.
- Decode the raw source bundle.
- Map each GeoJSON feature to one `WeatherOutlook`.
- Preserve SPC feature order within each product, then order products by day and type.
- Map properties:
- `VALID_ISO` -> `validFrom`
- `EXPIRE_ISO` -> `validTo` and `expiresAt`
- `ISSUE_ISO` -> `issuedAt`
- `FORECASTER` -> `forecaster`
- `LABEL` -> `label`
- `LABEL2` -> `labelText`
- `DN` -> `severityRank`
- Derive:
- `day` from product key or URL
- `outlookType` from product key or URL
- `id` from day, type, label, issuedAt, validFrom, and feature index
- `containsLocation` from configured point and GeoJSON geometry
- Enrich narrative fields from the matching Day 1-3 print page:
- parse the main product text from the `<pre>` block
- strip embedded tags and scripts before text parsing
- preserve line breaks in discussion text
- parse page/title metadata, product title, issued line, valid line, correction markers such as `CORR 1`, forecaster, summary, and full discussion where practical
- use print-page text for canonical `headline`, `summary`, and `discussion`
- Use RSS only as optional supplemental metadata when present; missing or incomplete RSS must not prevent GeoJSON plus print-page normalization.
- Set run `asOf` to the latest valid `issuedAt` across features, falling back to print-page `Updated:` timestamps, RSS `lastBuildDate`, then input event time.
- Set run `issuedAt` to the latest valid `issuedAt` across features.
- Set output event `effectiveAt` to run `asOf`.
Tests:
- normalizer routes only raw SPC schema
- categorical and probabilistic products map expected fields
- `containsLocation` is true for a known point inside a fixture polygon
- `containsLocation` is false outside
- print-page discussion text maps to canonical headline, summary, and discussion
- correction markers such as `CORR 1` are preserved where present
- missing optional RSS still permits GeoJSON and print-page normalization
- malformed required GeoJSON timestamps fail with useful context
- canonical JSON shape does not expose raw-provider-only bundle internals
### Stage 4: Postgres Sink
Add tables:
- `outlook_runs`
- `outlooks`
Suggested `outlook_runs` columns:
- event envelope columns
- `location_id`
- `location_name`
- `latitude`
- `longitude`
- `as_of`
- `issued_at`
- `outlook_count`
Suggested `outlooks` columns:
- `run_event_id`
- `outlook_index`
- `as_of`
- `product`
- `day`
- `outlook_type`
- `label`
- `label_text`
- `severity_rank`
- `valid_from`
- `valid_to`
- `issued_at`
- `expires_at`
- `forecaster`
- `headline`
- `summary`
- `discussion`
- `source_url`
- `image_url`
- `contains_location`
- `geometry_json`
Suggested indexes:
- `outlook_runs(location_id, as_of)`
- `outlooks(contains_location, valid_from, valid_to)`
- `outlooks(day, outlook_type, label)`
- `outlooks(valid_from, valid_to)`
Mapping rules:
- Store `geometry` as compact GeoJSON text in `geometry_json`.
- Require run `asOf`.
- Require outlook `validFrom`, `validTo`, `issuedAt`, `day`, `outlookType`, and `label`.
- Preserve all outlook polygons, not only polygons containing the configured point.
Tests:
- schema includes outlook tables and indexes
- mapper writes one run row plus one row per outlook
- mapper stores compact geometry JSON
- mapper rejects missing required run/outlook fields
### Stage 5: Config and Documentation
- Update sample config with `SPCConvectiveOutlookSTL`.
- Add `outlook` to route examples.
- Update current-behavior docs in the implementation change, not before behavior exists.
- Update `docs/config.md` with source params.
- Update `docs/integrations/events.md` with `weather.outlook.v1`.
- Update `docs/integrations/postgres.md` with outlook table contract.
- Update relevant internal docs under `docs/internal/` if source, normalizer, or sink behavior needs maintainer context.
- Update `README.md` only with brief implemented-provider capability and links to canonical docs.
Docs should clearly state:
- Print pages are used for Day 1-3 narrative/discussion text.
- RSS, if retained, is supplemental and not required for complete discussion coverage.
- GeoJSON is used for polygons, validity windows, and point matching.
- `containsLocation` is computed by weatherfeeder at ingestion time.
- Geometry is stored for downstream audit/display.
- Day 4-8 outlooks are not included in the initial implementation.
### Stage 6: Weatherapi Follow-Up
Do not include weatherapi changes in the first weatherfeeder implementation unless explicitly requested.
Likely future weatherapi endpoints:
- `GET /outlooks/convective`
- `GET /outlooks/convective/active`
- `GET /outlooks/convective/location`
Recommended API behavior:
- latest run by default
- active outlooks filtered by current time and `containsLocation=true`
- optional filter query params for `day`, `outlookType`, and `containsLocation`
## Verification Commands
Run focused tests:
```sh
go test ./internal/sources ./internal/normalizers/... ./internal/sinks/postgres ./model
```
Run full weatherfeeder tests:
```sh
go test ./...
```
## Acceptance Criteria
- A configured SPC source emits raw outlook bundles.
- Normalization produces `weather.outlook.v1` events.
- Each outlook indicates whether the configured forecast point is inside its polygon.
- All current Day 1-3 categorical/tornado/hail/wind products are represented.
- Day 1-3 print-page discussion text is preserved where available.
- RSS metadata is optional and supplemental if retained.
- Postgres sink persists outlook runs and outlook rows.
- Sample config and public docs describe the new kind, driver, schema, and storage contract.

View File

@@ -5,6 +5,7 @@ import (
"time"
"gitea.maximumdirect.net/ejr/feedkit/event"
"gitea.maximumdirect.net/ejr/weatherfeeder/standards"
)
func TestFinalizeRoundsWeatherPayloadFloats(t *testing.T) {
@@ -14,7 +15,7 @@ func TestFinalizeRoundsWeatherPayloadFloats(t *testing.T) {
in := event.Event{
ID: "evt-1",
Kind: event.Kind("observation"),
Kind: event.Kind(standards.KindObservation),
Source: "source-a",
EmittedAt: time.Date(2026, 3, 28, 12, 0, 0, 0, time.UTC),
Schema: "raw.example.v1",

View File

@@ -18,6 +18,18 @@ import (
// Errors include a small amount of operation context ("extract payload", "decode raw payload").
// Callers typically wrap these with a provider/kind label.
func DecodeJSONPayload[T any](in event.Event) (T, error) {
var zero T
if typed, ok := in.Payload.(T); ok {
return typed, nil
}
if ptr, ok := in.Payload.(*T); ok {
if ptr == nil {
return zero, fmt.Errorf("extract payload: payload pointer is nil")
}
return *ptr, nil
}
return fknormalize.DecodeJSONPayload[T](in)
}

View File

@@ -0,0 +1,39 @@
package common
import (
"testing"
"gitea.maximumdirect.net/ejr/feedkit/event"
)
func TestDecodeJSONPayloadAcceptsTypedPayload(t *testing.T) {
type rawPayload struct {
Value string `json:"value"`
}
got, err := DecodeJSONPayload[rawPayload](event.Event{
Payload: rawPayload{Value: "ok"},
})
if err != nil {
t.Fatalf("DecodeJSONPayload() error = %v", err)
}
if got.Value != "ok" {
t.Fatalf("Value = %q, want ok", got.Value)
}
}
func TestDecodeJSONPayloadAcceptsTypedPointerPayload(t *testing.T) {
type rawPayload struct {
Value string `json:"value"`
}
got, err := DecodeJSONPayload[rawPayload](event.Event{
Payload: &rawPayload{Value: "ok"},
})
if err != nil {
t.Fatalf("DecodeJSONPayload() error = %v", err)
}
if got.Value != "ok" {
t.Fatalf("Value = %q, want ok", got.Value)
}
}

View File

@@ -93,12 +93,8 @@ func buildAlerts(parsed nwsAlertsResponse, fallbackAsOf time.Time) (model.Weathe
sent := nwscommon.ParseTimePtr(p.Sent)
effective := nwscommon.ParseTimePtr(p.Effective)
onset := nwscommon.ParseTimePtr(p.Onset)
// Expires: prefer "expires"; fall back to "ends" if present.
ends := nwscommon.ParseTimePtr(p.Ends)
expires := nwscommon.ParseTimePtr(p.Expires)
if expires == nil {
expires = nwscommon.ParseTimePtr(p.Ends)
}
refs := parseNWSAlertReferences(p.References)
@@ -123,6 +119,7 @@ func buildAlerts(parsed nwsAlertsResponse, fallbackAsOf time.Time) (model.Weathe
Sent: sent,
Effective: effective,
Onset: onset,
Ends: ends,
Expires: expires,
AreaDescription: strings.TrimSpace(p.AreaDesc),

View File

@@ -0,0 +1,136 @@
package nws
import (
"context"
"encoding/json"
"testing"
"time"
"gitea.maximumdirect.net/ejr/feedkit/event"
"gitea.maximumdirect.net/ejr/weatherfeeder/model"
"gitea.maximumdirect.net/ejr/weatherfeeder/standards"
)
func TestAlertsNormalizerMapsEndsSeparatelyFromExpires(t *testing.T) {
raw := []byte(`{
"updated":"2026-06-16T10:00:00+00:00",
"title":"Current watches, warnings, and advisories for St. Louis",
"features":[{
"id":"https://api.weather.gov/alerts/alert-1",
"properties":{
"event":"Flood Warning",
"headline":"Flood Warning issued",
"sent":"2026-06-16T09:55:00+00:00",
"effective":"2026-06-16T10:00:00+00:00",
"onset":"2026-06-16T10:15:00+00:00",
"ends":"2026-06-16T14:00:00+00:00",
"expires":"2026-06-16T11:00:00+00:00"
}
}]
}`)
out, err := AlertsNormalizer{}.Normalize(context.Background(), alertRawEvent(raw))
if err != nil {
t.Fatalf("Normalize() error = %v", err)
}
run := decodeAlertRun(t, out)
if len(run.Alerts) != 1 {
t.Fatalf("expected 1 alert, got %d", len(run.Alerts))
}
alert := run.Alerts[0]
wantEnds := time.Date(2026, 6, 16, 14, 0, 0, 0, time.UTC)
wantExpires := time.Date(2026, 6, 16, 11, 0, 0, 0, time.UTC)
if alert.Ends == nil || !alert.Ends.Equal(wantEnds) {
t.Fatalf("ends = %v, want %s", alert.Ends, wantEnds)
}
if alert.Expires == nil || !alert.Expires.Equal(wantExpires) {
t.Fatalf("expires = %v, want %s", alert.Expires, wantExpires)
}
}
func TestAlertsNormalizerDoesNotFallbackExpiresToEnds(t *testing.T) {
raw := []byte(`{
"updated":"2026-06-16T10:00:00+00:00",
"features":[{
"id":"alert-ends-only",
"properties":{
"event":"Heat Advisory",
"ends":"2026-06-16T22:00:00+00:00"
}
}]
}`)
out, err := AlertsNormalizer{}.Normalize(context.Background(), alertRawEvent(raw))
if err != nil {
t.Fatalf("Normalize() error = %v", err)
}
run := decodeAlertRun(t, out)
alert := run.Alerts[0]
if alert.Ends == nil {
t.Fatal("expected ends to be populated")
}
if alert.Expires != nil {
t.Fatalf("expected expires nil when upstream expires is absent, got %v", alert.Expires)
}
}
func TestAlertsNormalizerIgnoresInvalidEnds(t *testing.T) {
raw := []byte(`{
"updated":"2026-06-16T10:00:00+00:00",
"features":[{
"id":"alert-invalid-ends",
"properties":{
"event":"Special Weather Statement",
"ends":"not-a-time",
"expires":"2026-06-16T11:00:00+00:00"
}
}]
}`)
out, err := AlertsNormalizer{}.Normalize(context.Background(), alertRawEvent(raw))
if err != nil {
t.Fatalf("Normalize() error = %v", err)
}
run := decodeAlertRun(t, out)
alert := run.Alerts[0]
if alert.Ends != nil {
t.Fatalf("expected invalid ends to map nil, got %v", alert.Ends)
}
if alert.Expires == nil {
t.Fatal("expected expires to remain populated")
}
}
func alertRawEvent(raw []byte) event.Event {
emittedAt := time.Date(2026, 6, 16, 10, 5, 0, 0, time.UTC)
return event.Event{
ID: "raw-alerts",
Kind: event.Kind(standards.KindAlert),
Source: "NWSAlerts",
Schema: standards.SchemaRawNWSAlertsV1,
EmittedAt: emittedAt,
Payload: json.RawMessage(raw),
}
}
func decodeAlertRun(t *testing.T, e *event.Event) model.WeatherAlertRun {
t.Helper()
if e == nil {
t.Fatal("expected normalized event")
}
if e.Schema != standards.SchemaWeatherAlertV1 {
t.Fatalf("schema = %q, want %q", e.Schema, standards.SchemaWeatherAlertV1)
}
var run model.WeatherAlertRun
raw, err := json.Marshal(e.Payload)
if err != nil {
t.Fatalf("marshal alert payload: %v", err)
}
if err := json.Unmarshal(raw, &run); err != nil {
t.Fatalf("decode alert payload: %v", err)
}
return run
}

View File

@@ -4,6 +4,7 @@ import (
"encoding/json"
"os"
"path/filepath"
"reflect"
"strings"
"testing"
"time"
@@ -18,7 +19,7 @@ func TestForecastDiscussionNormalizerProducesCanonicalSchema(t *testing.T) {
out, err := (ForecastDiscussionNormalizer{}).Normalize(nil, event.Event{
ID: "evt-discussion-1",
Kind: event.Kind("forecast_discussion"),
Kind: event.Kind(standards.KindForecastDiscussion),
Source: "nws-discussion-test",
EmittedAt: time.Date(2026, 3, 28, 19, 25, 0, 0, time.UTC),
Schema: standards.SchemaRawNWSForecastDiscussionV1,
@@ -33,7 +34,7 @@ func TestForecastDiscussionNormalizerProducesCanonicalSchema(t *testing.T) {
if out.Schema != standards.SchemaWeatherForecastDiscussionV1 {
t.Fatalf("Schema = %q, want %q", out.Schema, standards.SchemaWeatherForecastDiscussionV1)
}
if out.Kind != event.Kind("forecast_discussion") {
if out.Kind != event.Kind(standards.KindForecastDiscussion) {
t.Fatalf("Kind = %q, want forecast_discussion", out.Kind)
}
@@ -71,10 +72,281 @@ func TestForecastDiscussionNormalizerProducesCanonicalSchema(t *testing.T) {
}
}
func TestForecastDiscussionNormalizerSupportsMixedHeadingFormats(t *testing.T) {
out, err := (ForecastDiscussionNormalizer{}).Normalize(nil, event.Event{
ID: "evt-discussion-mixed-format",
Kind: event.Kind(standards.KindForecastDiscussion),
Source: "nws-discussion-test",
EmittedAt: time.Date(2026, 3, 28, 19, 25, 0, 0, time.UTC),
Schema: standards.SchemaRawNWSForecastDiscussionV1,
Payload: loadMixedFormatForecastDiscussionSampleHTML(t),
})
if err != nil {
t.Fatalf("Normalize() error = %v", err)
}
if out == nil {
t.Fatalf("Normalize() returned nil output")
}
if out.Kind != event.Kind(standards.KindForecastDiscussion) {
t.Fatalf("Kind = %q, want forecast_discussion", out.Kind)
}
if out.Schema != standards.SchemaWeatherForecastDiscussionV1 {
t.Fatalf("Schema = %q, want %q", out.Schema, standards.SchemaWeatherForecastDiscussionV1)
}
wantEffectiveAt := time.Date(2026, 3, 28, 19, 24, 0, 0, time.UTC)
if out.EffectiveAt == nil || !out.EffectiveAt.Equal(wantEffectiveAt) {
t.Fatalf("EffectiveAt = %v, want %s", out.EffectiveAt, wantEffectiveAt.Format(time.RFC3339))
}
payload, ok := out.Payload.(model.WeatherForecastDiscussion)
if !ok {
t.Fatalf("Payload type = %T, want model.WeatherForecastDiscussion", out.Payload)
}
if payload.ShortTerm == nil || payload.LongTerm == nil {
t.Fatalf("ShortTerm=%v LongTerm=%v, want both populated", payload.ShortTerm, payload.LongTerm)
}
if payload.ShortTerm.Qualifier != "Through Late Sunday Night" {
t.Fatalf("ShortTerm.Qualifier = %q", payload.ShortTerm.Qualifier)
}
if !strings.Contains(payload.ShortTerm.Text, "After a chilly morning") {
t.Fatalf("ShortTerm.Text missing expected prose: %q", payload.ShortTerm.Text)
}
if payload.LongTerm.Qualifier != "Monday through Next Saturday" {
t.Fatalf("LongTerm.Qualifier = %q", payload.LongTerm.Qualifier)
}
if !strings.Contains(payload.LongTerm.Text, "The peak of the warmth arrives Monday and Tuesday") {
t.Fatalf("LongTerm.Text missing expected prose: %q", payload.LongTerm.Text)
}
if strings.Contains(payload.LongTerm.Text, "AVIATION") || strings.Contains(payload.LongTerm.Text, "VFR conditions are expected") {
t.Fatalf("LongTerm.Text includes aviation content: %q", payload.LongTerm.Text)
}
b, err := json.Marshal(out.Payload)
if err != nil {
t.Fatalf("json.Marshal(payload) error = %v", err)
}
var fields map[string]any
if err := json.Unmarshal(b, &fields); err != nil {
t.Fatalf("json.Unmarshal(payload) error = %v", err)
}
for _, key := range []string{"aviation", "sections"} {
if _, ok := fields[key]; ok {
t.Fatalf("unexpected key %q in canonical payload", key)
}
}
}
func TestForecastDiscussionNormalizerSupportsCrossOfficeLayout(t *testing.T) {
in := event.Event{
ID: "evt-discussion-bou",
Kind: event.Kind(standards.KindForecastDiscussion),
Source: "nws-discussion-bou-test",
EmittedAt: time.Date(2026, 4, 7, 19, 1, 0, 0, time.UTC),
Schema: standards.SchemaRawNWSForecastDiscussionV1,
Payload: loadForecastDiscussionBOUSampleHTML(t),
}
out, err := (ForecastDiscussionNormalizer{}).Normalize(nil, in)
if err != nil {
t.Fatalf("Normalize() error = %v", err)
}
if out == nil {
t.Fatalf("Normalize() returned nil output")
}
if out.ID != in.ID || out.Source != in.Source || !out.EmittedAt.Equal(in.EmittedAt) {
t.Fatalf("envelope = %#v, want ID/source/emittedAt from input", out)
}
if out.Kind != event.Kind(standards.KindForecastDiscussion) {
t.Fatalf("Kind = %q, want forecast_discussion", out.Kind)
}
if out.Schema != standards.SchemaWeatherForecastDiscussionV1 {
t.Fatalf("Schema = %q, want %q", out.Schema, standards.SchemaWeatherForecastDiscussionV1)
}
wantEffectiveAt := time.Date(2026, 4, 7, 19, 0, 0, 0, time.UTC)
if out.EffectiveAt == nil || !out.EffectiveAt.Equal(wantEffectiveAt) {
t.Fatalf("EffectiveAt = %v, want %s", out.EffectiveAt, wantEffectiveAt.Format(time.RFC3339))
}
payload, ok := out.Payload.(model.WeatherForecastDiscussion)
if !ok {
t.Fatalf("Payload type = %T, want model.WeatherForecastDiscussion", out.Payload)
}
if payload.OfficeID != "BOU" || payload.OfficeName != "National Weather Service Denver CO" {
t.Fatalf("OfficeID=%q OfficeName=%q", payload.OfficeID, payload.OfficeName)
}
wantMessages := []string{
"Strong winds are expected along the Front Range this evening.",
"Cooler temperatures arrive on Wednesday.",
}
if len(payload.KeyMessages) != len(wantMessages) {
t.Fatalf("KeyMessages = %#v, want %#v", payload.KeyMessages, wantMessages)
}
for i := range wantMessages {
if payload.KeyMessages[i] != wantMessages[i] {
t.Fatalf("KeyMessages[%d] = %q, want %q", i, payload.KeyMessages[i], wantMessages[i])
}
}
if payload.ShortTerm == nil || payload.LongTerm == nil {
t.Fatalf("ShortTerm=%v LongTerm=%v, want both populated", payload.ShortTerm, payload.LongTerm)
}
if payload.ShortTerm.Qualifier != "(Tonight through Wednesday)" || payload.ShortTerm.Text != "Gusty west winds will continue through the evening before decreasing overnight." {
t.Fatalf("ShortTerm = %#v", payload.ShortTerm)
}
if payload.LongTerm.Qualifier != "(Thursday through Saturday)" || payload.LongTerm.Text != "Warmer and drier conditions return Thursday, followed by a chance of showers Friday." {
t.Fatalf("LongTerm = %#v", payload.LongTerm)
}
if payload.ShortTerm.IssuedAt == nil || payload.LongTerm.IssuedAt == nil ||
!payload.ShortTerm.IssuedAt.Equal(wantEffectiveAt) || !payload.LongTerm.IssuedAt.Equal(wantEffectiveAt) {
t.Fatalf("section issue times = short %v long %v, want %s", payload.ShortTerm.IssuedAt, payload.LongTerm.IssuedAt, wantEffectiveAt.Format(time.RFC3339))
}
b, err := json.Marshal(out.Payload)
if err != nil {
t.Fatalf("json.Marshal(payload) error = %v", err)
}
var fields map[string]any
if err := json.Unmarshal(b, &fields); err != nil {
t.Fatalf("json.Unmarshal(payload) error = %v", err)
}
for _, key := range []string{"aviation", "discussion", "sections"} {
if _, ok := fields[key]; ok {
t.Fatalf("unexpected key %q in canonical payload", key)
}
}
}
func TestForecastDiscussionNormalizerSupportsCrossOfficeKeyMessageFixtures(t *testing.T) {
tests := []struct {
name string
filename string
id string
source string
emittedAt time.Time
effectiveAt time.Time
messages []string
}{
{
name: "numbered key messages",
filename: "forecast_discussion_bgm_numbered_sample.html",
id: "evt-discussion-bgm",
source: "nws-discussion-bgm-test",
emittedAt: time.Date(2026, 4, 10, 17, 31, 0, 0, time.UTC),
effectiveAt: time.Date(2026, 4, 10, 17, 30, 0, 0, time.UTC),
messages: []string{
"Periods of rain are expected through Saturday, with locally heavier amounts possible.",
"Cooler temperatures return late this weekend.",
},
},
{
name: "key points alias",
filename: "forecast_discussion_mfr_key_points_sample.html",
id: "evt-discussion-mfr",
source: "nws-discussion-mfr-test",
emittedAt: time.Date(2026, 4, 10, 19, 46, 0, 0, time.UTC),
effectiveAt: time.Date(2026, 4, 10, 19, 45, 0, 0, time.UTC),
messages: []string{
"Gusty winds will develop over exposed ridges, especially during the afternoon.",
"Inland valleys remain dry through Saturday.",
},
},
{
name: "as of preamble",
filename: "forecast_discussion_rah_as_of_sample.html",
id: "evt-discussion-rah",
source: "nws-discussion-rah-test",
emittedAt: time.Date(2026, 8, 2, 16, 36, 0, 0, time.UTC),
effectiveAt: time.Date(2026, 8, 2, 16, 35, 0, 0, time.UTC),
messages: []string{
"Scattered storms may produce locally heavy rain this afternoon.",
"Drier weather arrives Monday.",
},
},
{
name: "parenthesized numeric markers",
filename: "forecast_discussion_lwx_parenthesized_number_sample.html",
id: "evt-discussion-lwx",
source: "nws-discussion-lwx-test",
emittedAt: time.Date(2026, 8, 2, 18, 1, 0, 0, time.UTC),
effectiveAt: time.Date(2026, 8, 2, 18, 0, 0, 0, time.UTC),
messages: []string{
"Thunderstorms remain possible near the Blue Ridge this evening.",
"Seasonably warm conditions continue Monday.",
},
},
{
name: "embedded key messages in previous discussion",
filename: "forecast_discussion_mfr_prev_discussion_sample.html",
id: "evt-discussion-mfr-previous",
source: "nws-discussion-mfr-previous-test",
emittedAt: time.Date(2026, 8, 2, 22, 20, 0, 0, time.UTC),
effectiveAt: time.Date(2026, 8, 2, 22, 19, 0, 0, time.UTC),
messages: []string{
"Heat returns to inland valleys Monday.",
"Gusty afternoon winds develop east of the Cascades.",
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
in := event.Event{
ID: tt.id,
Kind: event.Kind(standards.KindForecastDiscussion),
Source: tt.source,
EmittedAt: tt.emittedAt,
Schema: standards.SchemaRawNWSForecastDiscussionV1,
Payload: loadForecastDiscussionFixtureHTML(t, tt.filename),
}
out, err := (ForecastDiscussionNormalizer{}).Normalize(nil, in)
if err != nil {
t.Fatalf("Normalize() error = %v", err)
}
if out == nil {
t.Fatalf("Normalize() returned nil output")
}
if out.ID != in.ID || out.Source != in.Source || !out.EmittedAt.Equal(in.EmittedAt) {
t.Fatalf("envelope = %#v, want ID/source/emittedAt from input", out)
}
if out.Kind != event.Kind(standards.KindForecastDiscussion) {
t.Fatalf("Kind = %q, want forecast_discussion", out.Kind)
}
if out.Schema != standards.SchemaWeatherForecastDiscussionV1 {
t.Fatalf("Schema = %q, want %q", out.Schema, standards.SchemaWeatherForecastDiscussionV1)
}
if out.EffectiveAt == nil || !out.EffectiveAt.Equal(tt.effectiveAt) {
t.Fatalf("EffectiveAt = %v, want %s", out.EffectiveAt, tt.effectiveAt.Format(time.RFC3339))
}
payload, ok := out.Payload.(model.WeatherForecastDiscussion)
if !ok {
t.Fatalf("Payload type = %T, want model.WeatherForecastDiscussion", out.Payload)
}
if !reflect.DeepEqual(payload.KeyMessages, tt.messages) {
t.Fatalf("KeyMessages = %#v, want %#v", payload.KeyMessages, tt.messages)
}
b, err := json.Marshal(payload)
if err != nil {
t.Fatalf("json.Marshal(payload) error = %v", err)
}
var fields map[string]any
if err := json.Unmarshal(b, &fields); err != nil {
t.Fatalf("json.Unmarshal(payload) error = %v", err)
}
for _, key := range []string{"nearTerm", "discussion", "aviation", "sections"} {
if _, ok := fields[key]; ok {
t.Fatalf("unexpected key %q in canonical payload", key)
}
}
})
}
}
func TestForecastDiscussionNormalizerRejectsMissingIssueTime(t *testing.T) {
_, err := (ForecastDiscussionNormalizer{}).Normalize(nil, event.Event{
ID: "evt-discussion-bad",
Kind: event.Kind("forecast_discussion"),
Kind: event.Kind(standards.KindForecastDiscussion),
Source: "nws-discussion-test",
EmittedAt: time.Date(2026, 3, 28, 19, 25, 0, 0, time.UTC),
Schema: standards.SchemaRawNWSForecastDiscussionV1,
@@ -93,7 +365,7 @@ func TestForecastDiscussionNormalizerWireShapeHasNoUnexpectedKeys(t *testing.T)
out, err := (ForecastDiscussionNormalizer{}).Normalize(nil, event.Event{
ID: "evt-discussion-2",
Kind: event.Kind("forecast_discussion"),
Kind: event.Kind(standards.KindForecastDiscussion),
Source: "nws-discussion-test",
EmittedAt: time.Date(2026, 3, 28, 19, 25, 0, 0, time.UTC),
Schema: standards.SchemaRawNWSForecastDiscussionV1,
@@ -128,3 +400,56 @@ func loadForecastDiscussionSampleHTML(t *testing.T) string {
}
return string(b)
}
func loadForecastDiscussionBOUSampleHTML(t *testing.T) string {
t.Helper()
path := filepath.Join("..", "..", "providers", "nws", "testdata", "forecast_discussion_bou_sample.html")
b, err := os.ReadFile(path)
if err != nil {
t.Fatalf("os.ReadFile(%q) error = %v", path, err)
}
return string(b)
}
func loadForecastDiscussionFixtureHTML(t *testing.T, filename string) string {
t.Helper()
path := filepath.Join("..", "..", "providers", "nws", "testdata", filename)
b, err := os.ReadFile(path)
if err != nil {
t.Fatalf("os.ReadFile(%q) error = %v", path, err)
}
return string(b)
}
func loadMixedFormatForecastDiscussionSampleHTML(t *testing.T) string {
t.Helper()
raw := loadForecastDiscussionSampleHTML(t)
replacements := []struct {
original string
replacement string
}{
{
original: ".SHORT TERM... (Through Late Sunday Night)",
replacement: ".SHORT TERM /Through Late Sunday Night/...",
},
{
original: ".LONG TERM... (Monday through Next Saturday)",
replacement: ".LONG TERM /Monday through Next Saturday/...",
},
{
original: ".AVIATION... (For the 18z TAFs through 18z Sunday Afternoon)",
replacement: ".AVIATION /For the 18z TAFs through 18z Sunday Afternoon/...",
},
}
for _, replacement := range replacements {
if !strings.Contains(raw, replacement.original) {
t.Fatalf("fixture missing heading %q", replacement.original)
}
raw = strings.Replace(raw, replacement.original, replacement.replacement, 1)
}
return raw
}

View File

@@ -182,7 +182,7 @@ func TestNormalizeForecastEventBySchemaProducesCanonicalWeatherForecastSchema(t
t.Run(tt.name, func(t *testing.T) {
out, err := normalizeForecastEventBySchema(event.Event{
ID: "evt-1",
Kind: event.Kind("forecast"),
Kind: event.Kind(standards.KindForecast),
Source: "nws-test",
EmittedAt: time.Date(2026, 3, 16, 18, 0, 0, 0, time.UTC),
Schema: tt.schema,

View File

@@ -22,7 +22,7 @@ func TestWeatherStoriesNormalizerProducesCanonicalSchemaAndMapsSample(t *testing
if out.Schema != standards.SchemaWeatherStoryV1 {
t.Fatalf("Schema = %q, want %q", out.Schema, standards.SchemaWeatherStoryV1)
}
if out.Kind != event.Kind("weather_story") {
if out.Kind != event.Kind(standards.KindWeatherStory) {
t.Fatalf("Kind = %q, want weather_story", out.Kind)
}
@@ -118,7 +118,7 @@ func TestWeatherStoriesNormalizerMatch(t *testing.T) {
func weatherStoriesRawEvent(payload string) event.Event {
return event.Event{
ID: "evt-weatherstories-1",
Kind: event.Kind("weather_story"),
Kind: event.Kind(standards.KindWeatherStory),
Source: "nws-weatherstories-test",
EmittedAt: time.Date(2026, 5, 30, 9, 5, 0, 0, time.UTC),
Schema: standards.SchemaRawNWSWeatherStoriesV1,

View File

@@ -22,7 +22,6 @@ const (
providerSPC = "spc"
productConvective = "convective"
outlookNormalizer = "spc convective outlook"
outlookKind = "outlook"
outlookTypeUnknown = 99
)
@@ -30,10 +29,10 @@ var idTokenRE = regexp.MustCompile(`[^a-z0-9]+`)
// ConvectiveOutlookNormalizer converts:
//
// standards.SchemaRawSPCConvectiveOutlookV1 -> standards.SchemaWeatherOutlookV1
// standards.SchemaRawSPCConvectiveOutlookV1 -> standards.SchemaWeatherOutlookV2
//
// It maps SPC GeoJSON outlook features into canonical outlook polygons and
// enriches each day with the matching required print-page discussion.
// It maps SPC GeoJSON outlook features containing the configured location into
// canonical outlook polygons and adds matching day-level print-page discussions.
type ConvectiveOutlookNormalizer struct{}
func (ConvectiveOutlookNormalizer) Match(e event.Event) bool {
@@ -51,7 +50,7 @@ func (ConvectiveOutlookNormalizer) Normalize(ctx context.Context, in event.Event
return normcommon.NormalizeJSON(
in,
outlookNormalizer,
standards.SchemaWeatherOutlookV1,
standards.SchemaWeatherOutlookV2,
func(parsed spcprovider.RawConvectiveOutlookBundle) (model.WeatherOutlookRun, time.Time, error) {
return buildConvectiveOutlook(parsed, fallbackAsOf)
},
@@ -77,8 +76,7 @@ func buildConvectiveOutlook(bundle spcprovider.RawConvectiveOutlookBundle, fallb
if err := validateProductMetadata(product); err != nil {
return model.WeatherOutlookRun{}, time.Time{}, err
}
discussion, ok := discussions[product.Day]
if !ok {
if _, ok := discussions[product.Day]; !ok {
return model.WeatherOutlookRun{}, time.Time{}, fmt.Errorf("product %s: discussion for day %d is required", product.Key, product.Day)
}
@@ -88,17 +86,36 @@ func buildConvectiveOutlook(bundle spcprovider.RawConvectiveOutlookBundle, fallb
}
for i, feature := range collection.Features {
outlook, err := mapFeature(product, feature, i, point, discussion)
if spcprovider.IsEmptyGeometryCollection(feature.Geometry) {
issuedAt, err := parseRequiredSPCTime(feature.Properties.IssueISO, fmt.Sprintf("product %s feature %d.ISSUE_ISO", product.Key, i))
if err != nil {
return model.WeatherOutlookRun{}, time.Time{}, err
}
if latestIssue.IsZero() || issuedAt.After(latestIssue) {
latestIssue = issuedAt
}
continue
}
outlook, err := mapFeature(product, feature, i, point)
if err != nil {
return model.WeatherOutlookRun{}, time.Time{}, err
}
if latestIssue.IsZero() || outlook.IssuedAt.After(latestIssue) {
latestIssue = outlook.IssuedAt
}
if !outlook.ContainsLocation {
continue
}
outlooks = append(outlooks, outlook)
}
}
runDiscussions, err := buildOutlookDiscussions(outlooks, discussions)
if err != nil {
return model.WeatherOutlookRun{}, time.Time{}, err
}
asOf := latestIssue
if asOf.IsZero() {
asOf = latestDiscussionUpdated
@@ -123,6 +140,7 @@ func buildConvectiveOutlook(bundle spcprovider.RawConvectiveOutlookBundle, fallb
AsOf: asOf.UTC(),
IssuedAt: issuedAt,
Outlooks: outlooks,
Discussions: runDiscussions,
}
return run, run.AsOf, nil
}
@@ -138,11 +156,10 @@ func parseDiscussions(pages []spcprovider.RawDiscussionPage) (map[int]parsedDisc
out := map[int]parsedDiscussion{}
var latestUpdated time.Time
for _, page := range pages {
text, err := spcprovider.ExtractProductText(page.Body)
parsed, err := spcprovider.ParseDiscussionHTML(page.Body)
if err != nil {
return nil, time.Time{}, fmt.Errorf("discussion %s: %w", page.Key, err)
}
parsed := spcprovider.ParseDiscussionText(text)
day := page.Day
if day == 0 {
if meta, ok := spcprovider.DiscussionProductByKey(page.Key); ok {
@@ -166,6 +183,43 @@ func parseDiscussions(pages []spcprovider.RawDiscussionPage) (map[int]parsedDisc
return out, latestUpdated, nil
}
func buildOutlookDiscussions(outlooks []model.WeatherOutlook, discussions map[int]parsedDiscussion) ([]model.WeatherOutlookDiscussion, error) {
daysWithOutlooks := map[int]bool{}
for _, outlook := range outlooks {
daysWithOutlooks[outlook.Day] = true
}
days := make([]int, 0, len(daysWithOutlooks))
for day := range daysWithOutlooks {
days = append(days, day)
}
sort.Ints(days)
out := make([]model.WeatherOutlookDiscussion, 0, len(days))
for _, day := range days {
disc, ok := discussions[day]
if !ok {
return nil, fmt.Errorf("discussion for retained day %d is required", day)
}
out = append(out, model.WeatherOutlookDiscussion{
Day: day,
Headline: disc.Headline,
Summary: disc.Summary,
Discussion: disc.Discussion,
UpdatedAt: utcTimePtr(disc.UpdatedAt),
})
}
return out, nil
}
func utcTimePtr(t *time.Time) *time.Time {
if t == nil {
return nil
}
tt := t.UTC()
return &tt
}
func orderedProducts(products []spcprovider.RawOutlookProduct) []spcprovider.RawOutlookProduct {
out := make([]spcprovider.RawOutlookProduct, len(products))
copy(out, products)
@@ -210,7 +264,7 @@ func validateProductMetadata(product spcprovider.RawOutlookProduct) error {
}
}
func mapFeature(product spcprovider.RawOutlookProduct, feature spcprovider.GeoJSONFeature, index int, point geo.Point, discussion parsedDiscussion) (model.WeatherOutlook, error) {
func mapFeature(product spcprovider.RawOutlookProduct, feature spcprovider.GeoJSONFeature, index int, point geo.Point) (model.WeatherOutlook, error) {
fieldPrefix := fmt.Sprintf("product %s feature %d", product.Key, index)
props := feature.Properties
@@ -255,9 +309,6 @@ func mapFeature(product spcprovider.RawOutlookProduct, feature spcprovider.GeoJS
IssuedAt: issuedAt,
ExpiresAt: validTo,
Forecaster: strings.TrimSpace(props.Forecaster),
Headline: discussion.Headline,
Summary: discussion.Summary,
Discussion: discussion.Discussion,
SourceURL: strings.TrimSpace(product.URL),
ImageURL: "",
ContainsLocation: containsLocation,

View File

@@ -2,8 +2,6 @@ package spc
import (
"encoding/json"
"os"
"path/filepath"
"strings"
"testing"
"time"
@@ -29,10 +27,10 @@ func TestConvectiveOutlookNormalizerProducesCanonicalSchemaAndMapsSample(t *test
if err != nil {
t.Fatalf("Normalize() error = %v", err)
}
if out.Schema != standards.SchemaWeatherOutlookV1 {
t.Fatalf("Schema = %q, want %q", out.Schema, standards.SchemaWeatherOutlookV1)
if out.Schema != standards.SchemaWeatherOutlookV2 {
t.Fatalf("Schema = %q, want %q", out.Schema, standards.SchemaWeatherOutlookV2)
}
if out.Kind != event.Kind("outlook") {
if out.Kind != event.Kind(standards.KindOutlook) {
t.Fatalf("Kind = %q, want outlook", out.Kind)
}
@@ -56,8 +54,23 @@ func TestConvectiveOutlookNormalizerProducesCanonicalSchemaAndMapsSample(t *test
if run.Latitude == nil || *run.Latitude != 38.5 || run.Longitude == nil || *run.Longitude != -90.5 {
t.Fatalf("coordinates = %v,%v", run.Latitude, run.Longitude)
}
if len(run.Outlooks) != 12 {
t.Fatalf("Outlooks length = %d, want 12", len(run.Outlooks))
if len(run.Outlooks) != 4 {
t.Fatalf("Outlooks length = %d, want 4", len(run.Outlooks))
}
assertAllOutlooksContainLocation(t, run.Outlooks)
assertDiscussionDays(t, run.Discussions, 1)
day1Discussion := run.Discussions[0]
if day1Discussion.Headline != "Day 1 Convective Outlook" {
t.Fatalf("day 1 Headline = %q", day1Discussion.Headline)
}
if !strings.Contains(day1Discussion.Summary, "central Plains") {
t.Fatalf("day 1 Summary = %q", day1Discussion.Summary)
}
if !strings.Contains(day1Discussion.Discussion, "...DISCUSSION...") {
t.Fatalf("day 1 Discussion missing product text: %q", day1Discussion.Discussion)
}
if !strings.HasPrefix(day1Discussion.Discussion, "SPC AC 111234") {
t.Fatalf("day 1 Discussion = %q, want SPC product code prefix", day1Discussion.Discussion)
}
got := run.Outlooks[0]
@@ -90,22 +103,54 @@ func TestConvectiveOutlookNormalizerProducesCanonicalSchemaAndMapsSample(t *test
if !got.ContainsLocation {
t.Fatalf("ContainsLocation = false, want true")
}
if got.Headline != "Day 1 Convective Outlook" {
t.Fatalf("Headline = %q", got.Headline)
}
if !strings.Contains(got.Summary, "central Plains") {
t.Fatalf("Summary = %q", got.Summary)
}
if !strings.Contains(got.Discussion, "...DISCUSSION...") {
t.Fatalf("Discussion missing product text: %q", got.Discussion)
}
if got.ID != "spc-convective-day1-categorical-slgt-2026-06-11T12:34:56Z-2026-06-11T13:00:00Z-0" {
t.Fatalf("ID = %q", got.ID)
}
}
func TestConvectiveOutlookNormalizerAcceptsTypedSourcePayload(t *testing.T) {
bundle := spcBundle(t, 38.5, -90.5)
in := spcRawEvent(t, bundle)
in.Payload = bundle
out, err := (ConvectiveOutlookNormalizer{}).Normalize(nil, in)
if err != nil {
t.Fatalf("Normalize() error = %v", err)
}
run := out.Payload.(model.WeatherOutlookRun)
if len(run.Outlooks) != 4 {
t.Fatalf("Outlooks length = %d, want 4", len(run.Outlooks))
}
assertAllOutlooksContainLocation(t, run.Outlooks)
assertDiscussionDays(t, run.Discussions, 1)
}
func TestConvectiveOutlookNormalizerEmitsEmptyLocalRunOutsidePolygons(t *testing.T) {
out, err := (ConvectiveOutlookNormalizer{}).Normalize(nil, spcRawEvent(t, spcBundle(t, 0, 0)))
if err != nil {
t.Fatalf("Normalize() error = %v", err)
}
run := out.Payload.(model.WeatherOutlookRun)
if len(run.Outlooks) != 0 {
t.Fatalf("Outlooks length = %d, want 0", len(run.Outlooks))
}
if len(run.Discussions) != 0 {
t.Fatalf("Discussions length = %d, want 0", len(run.Discussions))
}
wantAsOf := time.Date(2026, 6, 11, 19, 45, 0, 0, time.UTC)
if !run.AsOf.Equal(wantAsOf) {
t.Fatalf("AsOf = %s, want latest product issue time %s", run.AsOf, wantAsOf)
}
if run.IssuedAt == nil || !run.IssuedAt.Equal(wantAsOf) {
t.Fatalf("IssuedAt = %v, want latest product issue time %s", run.IssuedAt, wantAsOf)
}
if out.EffectiveAt == nil || !out.EffectiveAt.Equal(run.AsOf) {
t.Fatalf("EffectiveAt = %v, want run AsOf %s", out.EffectiveAt, run.AsOf)
}
}
func TestConvectiveOutlookNormalizerOrdersProductsByDayAndType(t *testing.T) {
bundle := spcBundle(t, 0, 0)
bundle := spcBundle(t, 38.5, -90.5)
for i, j := 0, len(bundle.Products)-1; i < j; i, j = i+1, j-1 {
bundle.Products[i], bundle.Products[j] = bundle.Products[j], bundle.Products[i]
}
@@ -135,11 +180,12 @@ func TestConvectiveOutlookNormalizerOrdersProductsByDayAndType(t *testing.T) {
}
func TestConvectiveOutlookNormalizerMapsProbabilisticOutlookTypes(t *testing.T) {
out, err := (ConvectiveOutlookNormalizer{}).Normalize(nil, spcRawEvent(t, spcBundle(t, 0, 0)))
out, err := (ConvectiveOutlookNormalizer{}).Normalize(nil, spcRawEvent(t, spcBundle(t, 38.5, -90.5)))
if err != nil {
t.Fatalf("Normalize() error = %v", err)
}
run := out.Payload.(model.WeatherOutlookRun)
assertAllOutlooksContainLocation(t, run.Outlooks)
for _, outlookType := range []string{
spcprovider.OutlookTypeTornado,
spcprovider.OutlookTypeHail,
@@ -151,19 +197,75 @@ func TestConvectiveOutlookNormalizerMapsProbabilisticOutlookTypes(t *testing.T)
}
}
func TestConvectiveOutlookNormalizerContainsLocationFalseOutsidePolygon(t *testing.T) {
out, err := (ConvectiveOutlookNormalizer{}).Normalize(nil, spcRawEvent(t, spcBundle(t, 0, 0)))
func TestConvectiveOutlookNormalizerSkipsEmptyGeometryCollectionPlaceholder(t *testing.T) {
bundle := spcBundle(t, 36, -99)
replaced := false
for i := range bundle.Products {
if bundle.Products[i].Day == 2 && bundle.Products[i].OutlookType == spcprovider.OutlookTypeTornado {
bundle.Products[i].Body = json.RawMessage(emptyGeometryCollectionGeoJSON())
replaced = true
}
}
if !replaced {
t.Fatalf("test setup did not find day 2 tornado product")
}
out, err := (ConvectiveOutlookNormalizer{}).Normalize(nil, spcRawEvent(t, bundle))
if err != nil {
t.Fatalf("Normalize() error = %v", err)
}
run := out.Payload.(model.WeatherOutlookRun)
if run.Outlooks[0].ContainsLocation {
t.Fatalf("ContainsLocation = true, want false")
if len(run.Outlooks) != 3 {
t.Fatalf("Outlooks length = %d, want 3", len(run.Outlooks))
}
assertAllOutlooksContainLocation(t, run.Outlooks)
assertDiscussionDays(t, run.Discussions, 2)
if got := findOutlook(run.Outlooks, 2, spcprovider.OutlookTypeTornado); got != nil {
t.Fatalf("day 2 tornado outlook = %+v, want nil placeholder skipped", *got)
}
wantAsOf := time.Date(2026, 6, 12, 10, 0, 0, 0, time.UTC)
if !run.AsOf.Equal(wantAsOf) {
t.Fatalf("AsOf = %s, want placeholder ISSUE_ISO %s", run.AsOf, wantAsOf)
}
if out.EffectiveAt == nil || !out.EffectiveAt.Equal(wantAsOf) {
t.Fatalf("EffectiveAt = %v, want placeholder ISSUE_ISO %s", out.EffectiveAt, wantAsOf)
}
}
func TestConvectiveOutlookNormalizerIncludesOnlyDayWithContainingPolygons(t *testing.T) {
out, err := (ConvectiveOutlookNormalizer{}).Normalize(nil, spcRawEvent(t, spcBundle(t, 36, -99)))
if err != nil {
t.Fatalf("Normalize() error = %v", err)
}
run := out.Payload.(model.WeatherOutlookRun)
if len(run.Outlooks) == 0 {
t.Fatalf("Outlooks length = 0, want retained day 2 outlooks")
}
assertAllOutlooksContainLocation(t, run.Outlooks)
for i, outlook := range run.Outlooks {
if outlook.Day != 2 {
t.Fatalf("outlook[%d].Day = %d, want 2", i, outlook.Day)
}
}
assertDiscussionDays(t, run.Discussions, 2)
}
func TestConvectiveOutlookNormalizerUsesOneDiscussionForMultipleSameDayOutlooks(t *testing.T) {
out, err := (ConvectiveOutlookNormalizer{}).Normalize(nil, spcRawEvent(t, spcBundle(t, 38.5, -90.5)))
if err != nil {
t.Fatalf("Normalize() error = %v", err)
}
run := out.Payload.(model.WeatherOutlookRun)
if got := countOutlooksByDay(run.Outlooks, 1); got < 2 {
t.Fatalf("day 1 outlook count = %d, want multiple", got)
}
assertAllOutlooksContainLocation(t, run.Outlooks)
assertDiscussionDays(t, run.Discussions, 1)
}
func TestConvectiveOutlookNormalizerPreservesCorrectionMarker(t *testing.T) {
out, err := (ConvectiveOutlookNormalizer{}).Normalize(nil, spcRawEvent(t, spcBundle(t, 0, 0)))
bundle := spcBundle(t, 36, -99)
out, err := (ConvectiveOutlookNormalizer{}).Normalize(nil, spcRawEvent(t, bundle))
if err != nil {
t.Fatalf("Normalize() error = %v", err)
}
@@ -172,11 +274,12 @@ func TestConvectiveOutlookNormalizerPreservesCorrectionMarker(t *testing.T) {
if got == nil {
t.Fatalf("missing day 2 tornado outlook")
}
if !strings.Contains(got.Headline, "CORR 1") {
t.Fatalf("Headline = %q, want correction marker", got.Headline)
assertDiscussionDays(t, run.Discussions, 2)
if !strings.Contains(run.Discussions[0].Headline, "CORR 1") {
t.Fatalf("day 2 headline = %q, want correction marker", run.Discussions[0].Headline)
}
if !strings.Contains(got.Discussion, "CORR 1") {
t.Fatalf("Discussion = %q, want correction marker", got.Discussion)
if !strings.Contains(run.Discussions[0].Discussion, "CORR 1") {
t.Fatalf("day 2 discussion = %q, want correction marker", run.Discussions[0].Discussion)
}
}
@@ -242,6 +345,19 @@ func TestConvectiveOutlookNormalizerRejectsMissingLabel(t *testing.T) {
}
}
func TestConvectiveOutlookNormalizerRejectsMissingDiscussion(t *testing.T) {
bundle := spcBundle(t, 38.5, -90.5)
bundle.Discussions = bundle.Discussions[1:]
_, err := (ConvectiveOutlookNormalizer{}).Normalize(nil, spcRawEvent(t, bundle))
if err == nil {
t.Fatalf("Normalize() error = nil, want error")
}
if !strings.Contains(err.Error(), "product day1_categorical: discussion for day 1 is required") {
t.Fatalf("error = %q, want missing discussion context", err)
}
}
func TestConvectiveOutlookNormalizerOutputJSONShape(t *testing.T) {
out, err := (ConvectiveOutlookNormalizer{}).Normalize(nil, spcRawEvent(t, spcBundle(t, 38.5, -90.5)))
if err != nil {
@@ -252,18 +368,53 @@ func TestConvectiveOutlookNormalizerOutputJSONShape(t *testing.T) {
t.Fatalf("Marshal(payload) error = %v", err)
}
got := string(raw)
for _, want := range []string{`"asOf"`, `"outlooks"`, `"containsLocation"`, `"geometry"`} {
for _, want := range []string{`"asOf"`, `"outlooks"`, `"discussions"`, `"containsLocation"`, `"geometry"`} {
if !strings.Contains(got, want) {
t.Fatalf("payload JSON missing %s: %s", want, got)
}
}
for _, unwanted := range []string{`"products"`, `"discussions"`, `"fetchedAt"`, `"body"`} {
outlookStart := strings.Index(got, `"outlooks"`)
discussionStart := strings.Index(got, `"discussions"`)
if outlookStart == -1 || discussionStart == -1 || discussionStart <= outlookStart {
t.Fatalf("payload JSON has unexpected outlook/discussion order: %s", got)
}
outlookJSON := got[outlookStart:discussionStart]
for _, unwanted := range []string{`"headline"`, `"summary"`, `"discussion"`} {
if strings.Contains(outlookJSON, unwanted) {
t.Fatalf("outlook JSON exposed polygon-level prose key %s: %s", unwanted, got)
}
}
for _, unwanted := range []string{`"products"`, `"fetchedAt"`, `"body"`} {
if strings.Contains(got, unwanted) {
t.Fatalf("payload JSON exposed raw key %s: %s", unwanted, got)
}
}
}
func emptyGeometryCollectionGeoJSON() []byte {
return []byte(`{
"type": "FeatureCollection",
"features": [
{
"type": "Feature",
"properties": {
"VALID_ISO": "2026-06-12T12:00:00Z",
"EXPIRE_ISO": "2026-06-13T12:00:00Z",
"ISSUE_ISO": "2026-06-12T10:00:00Z",
"FORECASTER": "DOE",
"LABEL": "Less Than 2% All Areas",
"LABEL2": "",
"DN": 0
},
"geometry": {
"type": "GeometryCollection",
"geometries": []
}
}
]
}`)
}
func spcRawEvent(t *testing.T, bundle spcprovider.RawConvectiveOutlookBundle) event.Event {
t.Helper()
raw, err := json.Marshal(bundle)
@@ -273,7 +424,7 @@ func spcRawEvent(t *testing.T, bundle spcprovider.RawConvectiveOutlookBundle) ev
effectiveAt := time.Date(2026, 6, 11, 19, 45, 0, 0, time.UTC)
return event.Event{
ID: "evt-spc-outlook-1",
Kind: event.Kind("outlook"),
Kind: event.Kind(standards.KindOutlook),
Source: "spc-test",
EmittedAt: time.Date(2026, 6, 11, 20, 5, 0, 0, time.UTC),
EffectiveAt: &effectiveAt,
@@ -319,23 +470,13 @@ func geoJSONFixtureForProduct(t *testing.T, key string) []byte {
case strings.HasPrefix(key, "day2_"):
return readSPCTestFixture(t, "day2_torn.geojson")
case strings.HasPrefix(key, "day3_"):
return readSPCTestFixture(t, "day3_wind.geojson")
return readSPCTestFixture(t, "day3_cat.geojson")
default:
t.Fatalf("unknown product key %q", key)
return nil
}
}
func readSPCTestFixture(t *testing.T, name string) []byte {
t.Helper()
path := filepath.Join("..", "..", "providers", "spc", "testdata", name)
raw, err := os.ReadFile(path)
if err != nil {
t.Fatalf("read fixture %s: %v", path, err)
}
return raw
}
func findOutlook(outlooks []model.WeatherOutlook, day int, outlookType string) *model.WeatherOutlook {
for i := range outlooks {
if outlooks[i].Day == day && outlooks[i].OutlookType == outlookType {
@@ -345,6 +486,37 @@ func findOutlook(outlooks []model.WeatherOutlook, day int, outlookType string) *
return nil
}
func countOutlooksByDay(outlooks []model.WeatherOutlook, day int) int {
count := 0
for _, outlook := range outlooks {
if outlook.Day == day {
count++
}
}
return count
}
func assertAllOutlooksContainLocation(t *testing.T, outlooks []model.WeatherOutlook) {
t.Helper()
for i, outlook := range outlooks {
if !outlook.ContainsLocation {
t.Fatalf("outlook[%d].ContainsLocation = false, want true", i)
}
}
}
func assertDiscussionDays(t *testing.T, discussions []model.WeatherOutlookDiscussion, want ...int) {
t.Helper()
if len(discussions) != len(want) {
t.Fatalf("Discussions length = %d, want %d", len(discussions), len(want))
}
for i, day := range want {
if discussions[i].Day != day {
t.Fatalf("Discussions[%d].Day = %d, want %d", i, discussions[i].Day, day)
}
}
}
func assertTime(t *testing.T, name string, got time.Time, year int, month time.Month, day int, hour int, minute int, second int) {
t.Helper()
want := time.Date(year, month, day, hour, minute, second, 0, time.UTC)

View File

@@ -0,0 +1,17 @@
package spc
import (
"os"
"path/filepath"
"testing"
)
func readSPCTestFixture(t *testing.T, name string) []byte {
t.Helper()
path := filepath.Join("..", "..", "providers", "spc", "testdata", name)
raw, err := os.ReadFile(path)
if err != nil {
t.Fatalf("read fixture %s: %v", path, err)
}
return raw
}

View File

@@ -27,11 +27,34 @@ type ForecastDiscussionSection struct {
Text string
}
type forecastDiscussionSectionRole uint8
const (
forecastDiscussionSectionRoleKeyMessages forecastDiscussionSectionRole = iota
forecastDiscussionSectionRoleShortTerm
forecastDiscussionSectionRoleLongTerm
)
type forecastDiscussionSectionHeading struct {
section string
qualifier string
}
type forecastDiscussionSectionBlock struct {
heading forecastDiscussionSectionHeading
body []string
}
var (
forecastDiscussionHeaderRE = regexp.MustCompile(`^\.(KEY MESSAGES|SHORT TERM|LONG TERM|AVIATION)\.\.\.(.*)$`)
forecastDiscussionAFDRE = regexp.MustCompile(`^AFD([A-Z]{3})$`)
forecastDiscussionWMORE = regexp.MustCompile(`\bK([A-Z]{3})\b`)
forecastDiscussionSigRE = regexp.MustCompile(`^[A-Z]{2,6}$`)
forecastDiscussionSectionRoles = map[string]forecastDiscussionSectionRole{
"KEY MESSAGES": forecastDiscussionSectionRoleKeyMessages,
"KEY POINTS": forecastDiscussionSectionRoleKeyMessages,
"SHORT TERM": forecastDiscussionSectionRoleShortTerm,
"LONG TERM": forecastDiscussionSectionRoleLongTerm,
}
forecastDiscussionAFDRE = regexp.MustCompile(`^AFD([A-Z]{3})$`)
forecastDiscussionWMORE = regexp.MustCompile(`\bK([A-Z]{3})\b`)
forecastDiscussionSigRE = regexp.MustCompile(`^[A-Z]{2,6}$`)
)
func ParseForecastDiscussionHTML(raw string) (ForecastDiscussion, error) {
@@ -99,22 +122,30 @@ func ParseForecastDiscussionText(text string) (ForecastDiscussion, error) {
IssuedAt: issuedAt.UTC(),
}
if block, ok := extractForecastDiscussionSection(lines, "KEY MESSAGES"); ok {
out.KeyMessages = parseForecastDiscussionKeyMessages(block)
}
if block, ok := extractForecastDiscussionSection(lines, "SHORT TERM"); ok {
section, err := parseForecastDiscussionTextSection(block)
if err != nil {
return ForecastDiscussion{}, fmt.Errorf("parse SHORT TERM: %w", err)
seenRoles := make(map[forecastDiscussionSectionRole]bool, len(forecastDiscussionSectionRoles))
for _, block := range parseForecastDiscussionSectionBlocks(lines) {
role, ok := forecastDiscussionSectionRoles[block.heading.section]
if !ok || seenRoles[role] {
continue
}
out.ShortTerm = &section
}
if block, ok := extractForecastDiscussionSection(lines, "LONG TERM"); ok {
section, err := parseForecastDiscussionTextSection(block)
if err != nil {
return ForecastDiscussion{}, fmt.Errorf("parse LONG TERM: %w", err)
seenRoles[role] = true
switch role {
case forecastDiscussionSectionRoleKeyMessages:
out.KeyMessages = parseForecastDiscussionKeyMessages(block.body)
case forecastDiscussionSectionRoleShortTerm:
section, err := parseForecastDiscussionTextSection(block)
if err != nil {
return ForecastDiscussion{}, fmt.Errorf("parse %s: %w", block.heading.section, err)
}
out.ShortTerm = &section
case forecastDiscussionSectionRoleLongTerm:
section, err := parseForecastDiscussionTextSection(block)
if err != nil {
return ForecastDiscussion{}, fmt.Errorf("parse %s: %w", block.heading.section, err)
}
out.LongTerm = &section
}
out.LongTerm = &section
}
return out, nil
@@ -285,8 +316,9 @@ func parseForecastDiscussionHeader(lines []string) (string, time.Time, error) {
func parseForecastDiscussionIssueTime(line string) (time.Time, error) {
line = strings.TrimSpace(line)
line = strings.TrimPrefix(line, "Issued at ")
line = strings.TrimSpace(line)
if isForecastDiscussionIssuedAtLine(line) {
line = strings.TrimSpace(line[len("Issued at"):])
}
parts := strings.Fields(line)
if len(parts) != 7 {
@@ -386,40 +418,217 @@ func forecastDiscussionLocation(abbrev string) (*time.Location, error) {
return time.FixedZone(abbr, offset), nil
}
func extractForecastDiscussionSection(lines []string, section string) ([]string, bool) {
target := "." + section + "..."
for i, raw := range lines {
func parseForecastDiscussionSectionHeading(line string) (forecastDiscussionSectionHeading, bool) {
line = strings.TrimSpace(line)
if len(line) < 2 || line[0] != '.' {
return forecastDiscussionSectionHeading{}, false
}
if strings.HasSuffix(line, "/...") {
return parseForecastDiscussionSlashQualifiedHeading(line)
}
if strings.HasSuffix(line, "...") {
if heading, ok := parseForecastDiscussionParenthesizedTerminalHeading(line); ok {
return heading, true
}
}
return parseForecastDiscussionEllipsisHeading(line)
}
func parseForecastDiscussionSlashQualifiedHeading(line string) (forecastDiscussionSectionHeading, bool) {
content := strings.TrimSuffix(line[1:], "/...")
separator := -1
for i := 1; i < len(content); i++ {
if content[i] == '/' && isForecastDiscussionHorizontalWhitespace(content[i-1]) {
separator = i
break
}
}
if separator < 0 {
return forecastDiscussionSectionHeading{}, false
}
section, ok := normalizeForecastDiscussionSectionIdentity(content[:separator])
if !ok {
return forecastDiscussionSectionHeading{}, false
}
qualifier := strings.TrimSpace(content[separator+1:])
if qualifier == "" {
return forecastDiscussionSectionHeading{}, false
}
return forecastDiscussionSectionHeading{section: section, qualifier: qualifier}, true
}
func parseForecastDiscussionParenthesizedTerminalHeading(line string) (forecastDiscussionSectionHeading, bool) {
if len(line) < 4 || line[0] != '.' || !strings.HasSuffix(line, "...") {
return forecastDiscussionSectionHeading{}, false
}
content := strings.TrimRight(line[1:len(line)-3], " \t")
if !strings.HasSuffix(content, ")") {
return forecastDiscussionSectionHeading{}, false
}
separator := -1
for i := 1; i < len(content); i++ {
if content[i] == '(' && isForecastDiscussionHorizontalWhitespace(content[i-1]) {
separator = i
break
}
}
if separator < 0 {
return forecastDiscussionSectionHeading{}, false
}
section, ok := normalizeForecastDiscussionSectionIdentity(content[:separator])
if !ok {
return forecastDiscussionSectionHeading{}, false
}
qualifier := content[separator:]
if len(qualifier) <= 2 || strings.TrimSpace(qualifier[1:len(qualifier)-1]) == "" {
return forecastDiscussionSectionHeading{}, false
}
return forecastDiscussionSectionHeading{section: section, qualifier: qualifier}, true
}
func parseForecastDiscussionEllipsisHeading(line string) (forecastDiscussionSectionHeading, bool) {
content := line[1:]
delimiter := strings.Index(content, "...")
if delimiter < 0 {
return forecastDiscussionSectionHeading{}, false
}
section, ok := normalizeForecastDiscussionSectionIdentity(content[:delimiter])
if !ok {
return forecastDiscussionSectionHeading{}, false
}
return forecastDiscussionSectionHeading{
section: section,
qualifier: strings.TrimSpace(content[delimiter+3:]),
}, true
}
func normalizeForecastDiscussionSectionIdentity(raw string) (string, bool) {
var normalized strings.Builder
pendingSpace := false
hasLetterOrDigit := false
for i := 0; i < len(raw); i++ {
b := raw[i]
switch {
case isForecastDiscussionIdentityLetterOrDigit(b):
hasLetterOrDigit = true
case b == ' ' || b == '\t':
pendingSpace = normalized.Len() > 0
continue
case b == '/' && i > 0 && isForecastDiscussionHorizontalWhitespace(raw[i-1]):
return "", false
case b != '/' && b != '&' && b != '\'' && b != '-':
return "", false
}
if pendingSpace {
normalized.WriteByte(' ')
pendingSpace = false
}
normalized.WriteByte(b)
}
if !hasLetterOrDigit {
return "", false
}
return normalized.String(), true
}
func isForecastDiscussionIdentityLetterOrDigit(b byte) bool {
return b >= 'A' && b <= 'Z' || b >= '0' && b <= '9'
}
func isForecastDiscussionHorizontalWhitespace(b byte) bool {
return b == ' ' || b == '\t'
}
func parseForecastDiscussionSectionBlocks(lines []string) []forecastDiscussionSectionBlock {
var blocks []forecastDiscussionSectionBlock
var active *forecastDiscussionSectionBlock
embeddedHeadings := false
finish := func() {
if active == nil {
return
}
blocks = append(blocks, *active)
active = nil
}
for _, raw := range lines {
line := strings.TrimSpace(raw)
if !strings.HasPrefix(line, target) {
if line == "$$" {
finish()
break
}
if line == "&&" || strings.Contains(line, "WATCHES/WARNINGS/ADVISORIES") {
finish()
embeddedHeadings = false
continue
}
out := []string{line}
for j := i + 1; j < len(lines); j++ {
next := strings.TrimSpace(lines[j])
if next == "&&" || next == "$$" || strings.Contains(next, "WATCHES/WARNINGS/ADVISORIES") {
break
}
if j > i+1 && isForecastDiscussionSectionHeader(next) {
break
}
out = append(out, lines[j])
heading, ok := parseForecastDiscussionSectionHeading(raw)
if ok {
finish()
active = &forecastDiscussionSectionBlock{heading: heading}
embeddedHeadings = isForecastDiscussionEmbeddedSectionWrapper(heading.section)
continue
}
if embeddedHeadings {
heading, ok = parseForecastDiscussionEmbeddedSectionHeading(raw)
if ok {
finish()
active = &forecastDiscussionSectionBlock{heading: heading}
continue
}
}
if active != nil {
active.body = append(active.body, raw)
}
return out, true
}
return nil, false
finish()
return blocks
}
func isForecastDiscussionSectionHeader(line string) bool {
return forecastDiscussionHeaderRE.MatchString(strings.TrimSpace(line))
func isForecastDiscussionEmbeddedSectionWrapper(section string) bool {
return section == "PREV DISCUSSION"
}
func parseForecastDiscussionKeyMessages(block []string) []string {
if len(block) <= 1 {
func parseForecastDiscussionEmbeddedSectionHeading(line string) (forecastDiscussionSectionHeading, bool) {
line = strings.TrimSpace(line)
if line == "" || line[0] == '.' {
return forecastDiscussionSectionHeading{}, false
}
return parseForecastDiscussionSectionHeading("." + line)
}
func parseForecastDiscussionKeyMessages(body []string) []string {
body = removeForecastDiscussionPresentationMarkers(body)
body = trimBlankLines(body)
if len(body) > 0 && isForecastDiscussionKeyMessageMetadataLine(body[0]) {
body = trimBlankLines(body[1:])
}
if len(body) == 0 {
return nil
}
hasMarkers := false
for _, raw := range body {
line := strings.TrimSpace(raw)
if line == "" {
continue
}
if _, ok := stripForecastDiscussionKeyMessageMarker(line); ok {
hasMarkers = true
break
}
}
body := trimBlankLines(block[1:])
var messages []string
var current strings.Builder
@@ -431,15 +640,21 @@ func parseForecastDiscussionKeyMessages(block []string) []string {
current.Reset()
}
seenMarker := false
for _, raw := range body {
line := strings.TrimSpace(raw)
if line == "" {
if !hasMarkers || !seenMarker {
flush()
}
continue
}
if strings.HasPrefix(line, "-") {
if stripped, ok := stripForecastDiscussionKeyMessageMarker(line); ok {
flush()
line = strings.TrimSpace(strings.TrimPrefix(line, "-"))
current.WriteString(line)
seenMarker = true
line = stripped
}
if line == "" {
continue
}
if current.Len() > 0 {
@@ -452,25 +667,24 @@ func parseForecastDiscussionKeyMessages(block []string) []string {
return messages
}
func parseForecastDiscussionTextSection(block []string) (ForecastDiscussionSection, error) {
if len(block) == 0 {
return ForecastDiscussionSection{}, fmt.Errorf("empty section")
}
func parseForecastDiscussionTextSection(block forecastDiscussionSectionBlock) (ForecastDiscussionSection, error) {
section := ForecastDiscussionSection{
Qualifier: parseForecastDiscussionQualifier(strings.TrimSpace(block[0])),
Qualifier: block.heading.qualifier,
}
body := trimBlankLines(block[1:])
body := trimBlankLines(removeForecastDiscussionPresentationMarkers(block.body))
if section.Qualifier == "" && len(body) > 0 && isForecastDiscussionStandaloneParenthetical(body[0]) {
section.Qualifier = strings.TrimSpace(body[0])
body = trimBlankLines(body[1:])
}
if len(body) == 0 {
return section, nil
}
first := strings.TrimSpace(body[0])
if strings.HasPrefix(first, "Issued at ") {
issuedAt, err := parseForecastDiscussionIssueTime(first)
if isForecastDiscussionIssuedAtLine(body[0]) {
issuedAt, err := parseForecastDiscussionIssueTime(body[0])
if err != nil {
return ForecastDiscussionSection{}, fmt.Errorf("parse section issuedAt %q: %w", first, err)
return ForecastDiscussionSection{}, fmt.Errorf("parse section issuedAt %q: %w", strings.TrimSpace(body[0]), err)
}
tt := issuedAt.UTC()
section.IssuedAt = &tt
@@ -482,12 +696,147 @@ func parseForecastDiscussionTextSection(block []string) (ForecastDiscussionSecti
return section, nil
}
func parseForecastDiscussionQualifier(header string) string {
m := forecastDiscussionHeaderRE.FindStringSubmatch(header)
if len(m) != 3 {
return ""
func isForecastDiscussionPresentationMarker(line string) bool {
switch {
case strings.EqualFold(strings.TrimSpace(line), "-- Changed Discussion --"):
return true
case strings.EqualFold(strings.TrimSpace(line), "-- End Changed Discussion --"):
return true
default:
return false
}
return strings.TrimSpace(m[2])
}
func removeForecastDiscussionPresentationMarkers(lines []string) []string {
body := make([]string, 0, len(lines))
for _, line := range lines {
if !isForecastDiscussionPresentationMarker(line) {
body = append(body, line)
}
}
return body
}
func isForecastDiscussionStandaloneParenthetical(line string) bool {
line = strings.TrimSpace(line)
return len(line) > 2 && line[0] == '(' && line[len(line)-1] == ')' && strings.TrimSpace(line[1:len(line)-1]) != ""
}
func isForecastDiscussionIssuedAtLine(line string) bool {
line = strings.TrimSpace(line)
return len(line) > len("Issued at") &&
strings.EqualFold(line[:len("Issued at")], "Issued at") &&
isForecastDiscussionHorizontalWhitespace(line[len("Issued at")])
}
func isForecastDiscussionKeyMessageMetadataLine(line string) bool {
line = strings.TrimSpace(line)
for _, label := range []string{"Issued at", "Updated at"} {
if !hasForecastDiscussionASCIIPrefix(line, label) || len(line) == len(label) || !isForecastDiscussionHorizontalWhitespace(line[len(label)]) {
continue
}
if _, err := parseForecastDiscussionIssueTime(strings.TrimSpace(line[len(label):])); err == nil {
return true
}
}
return isForecastDiscussionKeyMessageAsOfLine(line)
}
func isForecastDiscussionKeyMessageAsOfLine(line string) bool {
const label = "As of"
if !hasForecastDiscussionASCIIPrefix(line, label) || len(line) == len(label) || !isForecastDiscussionHorizontalWhitespace(line[len(label)]) {
return false
}
remainder := strings.TrimSpace(line[len(label):])
if !strings.HasSuffix(remainder, "...") {
return false
}
fields := strings.Fields(strings.TrimSpace(strings.TrimSuffix(remainder, "...")))
if len(fields) != 3 {
return false
}
if _, _, err := parseForecastDiscussionClock(fields[0], fields[1]); err != nil {
return false
}
switch strings.ToLower(fields[2]) {
case "monday", "tuesday", "wednesday", "thursday", "friday", "saturday", "sunday":
return true
default:
return false
}
}
func stripForecastDiscussionKeyMessageMarker(line string) (string, bool) {
if line == "" {
return "", false
}
if line[0] == '-' || line[0] == '*' {
content := line[1:]
hadWhitespace := len(content) > 0 && isForecastDiscussionHorizontalWhitespace(content[0])
content = strings.TrimLeft(content, " \t")
if hadWhitespace {
if stripped, ok := stripForecastDiscussionKeyMessageNumericMarker(content); ok {
content = stripped
}
}
return content, true
}
return stripForecastDiscussionKeyMessageNumericMarker(line)
}
func stripForecastDiscussionKeyMessageNumericMarker(line string) (string, bool) {
digitStart := 0
digitEnd := 0
parenthesized := len(line) > 0 && line[0] == '('
if parenthesized {
digitStart = 1
digitEnd = 1
}
for digitEnd < len(line) && line[digitEnd] >= '0' && line[digitEnd] <= '9' {
digitEnd++
}
if digitEnd == digitStart || digitEnd == len(line) {
return "", false
}
if parenthesized && line[digitEnd] != ')' {
return "", false
}
if !parenthesized && line[digitEnd] != ')' && line[digitEnd] != '.' {
return "", false
}
markerEnd := digitEnd + 1
if markerEnd < len(line) && !isForecastDiscussionHorizontalWhitespace(line[markerEnd]) {
return "", false
}
value, err := strconv.ParseUint(line[digitStart:digitEnd], 10, 0)
if err != nil || value == 0 {
return "", false
}
return strings.TrimLeft(line[markerEnd:], " \t"), true
}
func hasForecastDiscussionASCIIPrefix(line, prefix string) bool {
if len(line) < len(prefix) {
return false
}
for i := range prefix {
actual := line[i]
if actual >= 'A' && actual <= 'Z' {
actual += 'a' - 'A'
}
expected := prefix[i]
if expected >= 'A' && expected <= 'Z' {
expected += 'a' - 'A'
}
if actual != expected {
return false
}
}
return true
}
func trimBlankLines(lines []string) []string {

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,26 @@
<!DOCTYPE html>
<!-- Representative BGM/CTP-style layout; prose is concise edited test data, not an archived product. -->
<html>
<body>
<pre class="glossaryProduct">
FXUS61 KBGM 101730
AFDBGM
Area Forecast Discussion
National Weather Service Binghamton NY
130 PM EDT Fri Apr 10 2026
.KEY MESSAGES...
1) Periods of rain are expected through Saturday,
with locally heavier amounts possible.
2. Cooler temperatures return late this weekend.
.DISCUSSION...
Discussion details remain boundary-only content.
$$
WFO BGM
</pre>
</body>
</html>

View File

@@ -0,0 +1,48 @@
<!DOCTYPE html>
<html>
<body>
<pre class="glossaryProduct">
FXUS65 KBOU 071900
AFDBOU
Area Forecast Discussion
National Weather Service Denver CO
100 PM MDT Tue Apr 7 2026
.KEY MESSAGES...
-- Changed Discussion --
Updated at 100 PM MDT Tue Apr 7 2026
- Strong winds are expected along the Front Range this evening.
- Cooler temperatures arrive on Wednesday.
-- End Changed Discussion --
&&
.SHORT TERM...
(Tonight through Wednesday)
Issued at 100 PM MDT Tue Apr 7 2026
Gusty west winds will continue through the evening before decreasing overnight.
&&
.LONG TERM...
(Thursday through Saturday)
ISSUED AT 100 PM MDT Tue Apr 7 2026
Warmer and drier conditions return Thursday, followed by a chance of showers Friday.
&&
.AVIATION...
VFR conditions are expected at Denver-area terminals through Wednesday morning.
&&
$$
WFO BOU
</pre>
</body>
</html>

View File

@@ -0,0 +1,27 @@
<!DOCTYPE html>
<!-- Representative LWX-style layout; prose is concise edited test data, not an archived product. -->
<html>
<body>
<pre class="glossaryProduct">
FXUS61 KLWX 021800
AFDLWX
Area Forecast Discussion
National Weather Service Baltimore MD/Washington DC
200 PM EDT Sun Aug 2 2026
.KEY MESSAGES...
- (1) Thunderstorms remain possible near the Blue Ridge this evening.
- (2) Seasonably warm conditions continue Monday.
&&
.AVIATION...
Aviation details remain boundary-only content.
$$
WFO LWX
</pre>
</body>
</html>

View File

@@ -0,0 +1,25 @@
<!DOCTYPE html>
<!-- Representative MFR-style layout; prose is concise edited test data, not an archived product. -->
<html>
<body>
<pre class="glossaryProduct">
FXUS66 KMFR 101945
AFDMFR
Area Forecast Discussion
National Weather Service Medford OR
1245 PM PDT Fri Apr 10 2026
.KEY POINTS...
* Gusty winds will develop over exposed ridges,
especially during the afternoon.
* Inland valleys remain dry through Saturday.
.DISCUSSION (Today through Thursday)...
Discussion details must not be included with key points.
$$
WFO MFR
</pre>
</body>
</html>

View File

@@ -0,0 +1,33 @@
<!DOCTYPE html>
<!-- Representative current MFR previous-discussion wrapper layout; prose is concise edited test data, not an archived product. -->
<html>
<body>
<pre class="glossaryProduct">
FXUS66 KMFR 022219
AFDMFR
Area Forecast Discussion
National Weather Service Medford OR
319 PM PDT Sun Aug 2 2026
.PREV DISCUSSION... /Issued 319 PM PDT Sun Aug 2 2026/
KEY MESSAGES...
* Heat returns to inland valleys Monday.
* Gusty afternoon winds develop east of the Cascades.
DISCUSSION...
Discussion details must not be included with key messages.
&&
.MFR WATCHES/WARNINGS/ADVISORIES...
None.
$$
WFO MFR
</pre>
</body>
</html>

View File

@@ -0,0 +1,29 @@
<!DOCTYPE html>
<!-- Representative RAH-style layout; prose is concise edited test data, not an archived product. -->
<html>
<body>
<pre class="glossaryProduct">
FXUS62 KRAH 021635
AFDRAH
Area Forecast Discussion
National Weather Service Raleigh NC
1235 PM EDT Sun Aug 2 2026
.KEY MESSAGES...
As of 1235 PM Sunday...
1) Scattered storms may produce locally heavy rain this afternoon.
2) Drier weather arrives Monday.
&&
.DISCUSSION...
Discussion details remain boundary-only content.
$$
WFO RAH
</pre>
</body>
</html>

View File

@@ -13,6 +13,8 @@ var (
preBlockRE = regexp.MustCompile(`(?is)<pre\b[^>]*>(.*?)</pre>`)
tagRE = regexp.MustCompile(`(?is)<[^>]+>`)
updatedRE = regexp.MustCompile(`(?im)^\s*Updated:\s*(.+?)\s*$`)
pageUpdatedRE = regexp.MustCompile(`(?i)\bUpdated:\s*((?:\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z)|(?:[A-Z][a-z]{2}\s+[A-Z][a-z]{2}\s+\d{1,2}\s+\d{2}:\d{2}:\d{2}\s+UTC\s+\d{4})|(?:\d{4}\s+UTC\s+[A-Z][a-z]{2}\s+[A-Z][a-z]{2}\s+\d{1,2}\s+\d{4})|(?:\d{4}Z\s+[A-Z][a-z]{2}\s+[A-Z][a-z]{2}\s+\d{1,2}\s+\d{4}))`)
productCodeRE = regexp.MustCompile(`(?i)^SPC\s+AC\s+\d+\s*$`)
sectionRE = regexp.MustCompile(`^\s*\.\.\.[A-Z0-9 /-]+\.{3}\s*$`)
)
@@ -41,6 +43,20 @@ func ExtractProductText(rawHTML string) (string, error) {
return "", fmt.Errorf("no useful pre block found")
}
// ParseDiscussionHTML extracts SPC product text and page-level metadata from a
// print-page HTML document.
func ParseDiscussionHTML(rawHTML string) (DiscussionText, error) {
text, err := ExtractProductText(rawHTML)
if err != nil {
return DiscussionText{}, err
}
parsed := ParseDiscussionText(text)
if updatedAt := ParsePageUpdatedTimestamp(rawHTML); updatedAt != nil {
parsed.UpdatedAt = updatedAt
}
return parsed, nil
}
// ParseDiscussionText extracts common SPC narrative metadata from cleaned
// product text.
func ParseDiscussionText(text string) DiscussionText {
@@ -55,6 +71,19 @@ func ParseDiscussionText(text string) DiscussionText {
}
}
// ParsePageUpdatedTimestamp parses the page-level Updated row from an SPC print
// page. SPC currently places this outside the product <pre> block.
func ParsePageUpdatedTimestamp(rawHTML string) *time.Time {
text := cleanHTMLText(rawHTML)
text = strings.ReplaceAll(text, "\u00a0", " ")
text = strings.Join(strings.Fields(text), " ")
match := pageUpdatedRE.FindStringSubmatch(text)
if len(match) != 2 {
return nil
}
return parseUpdatedValue(match[1])
}
// ParseUpdatedTimestamp parses an SPC print-page Updated line when present.
func ParseUpdatedTimestamp(text string) *time.Time {
match := updatedRE.FindStringSubmatch(normalizeNewlines(text))
@@ -68,7 +97,7 @@ func ParseUpdatedTimestamp(text string) *time.Time {
func ParseProductTitle(text string) string {
for _, line := range strings.Split(normalizeNewlines(text), "\n") {
line = strings.TrimSpace(line)
if line == "" || strings.HasPrefix(line, "Updated:") {
if line == "" || strings.HasPrefix(line, "Updated:") || productCodeRE.MatchString(line) {
continue
}
return line
@@ -142,6 +171,7 @@ func parseUpdatedValue(value string) *time.Time {
return t
}
for _, layout := range []string{
"Mon Jan 2 15:04:05 UTC 2006",
"1504 UTC Mon Jan 2 2006",
"1504Z Mon Jan 2 2006",
"3:04 PM UTC Mon Jan 2 2006",

View File

@@ -22,18 +22,19 @@ func TestExtractProductTextCleansPreBlock(t *testing.T) {
if !strings.Contains(got, "Day 1 Convective Outlook") {
t.Fatalf("ExtractProductText() missing headline: %q", got)
}
if !strings.HasPrefix(got, "SPC AC 111234") {
t.Fatalf("ExtractProductText() = %q, want product code prefix", got)
}
if strings.HasPrefix(got, "\n") || strings.HasSuffix(got, "\n") {
t.Fatalf("ExtractProductText() retained surrounding blank lines: %q", got)
}
}
func TestParseDiscussionTextExtractsHeadlineSummaryAndUpdated(t *testing.T) {
text, err := ExtractProductText(string(readTestFile(t, "day1_prt.html")))
func TestParseDiscussionHTMLExtractsHeadlineSummaryAndUpdated(t *testing.T) {
got, err := ParseDiscussionHTML(string(readTestFile(t, "day1_prt.html")))
if err != nil {
t.Fatalf("ExtractProductText() error = %v", err)
t.Fatalf("ParseDiscussionHTML() error = %v", err)
}
got := ParseDiscussionText(text)
if got.ProductTitle != "Day 1 Convective Outlook" {
t.Fatalf("ProductTitle = %q", got.ProductTitle)
}
@@ -47,27 +48,37 @@ func TestParseDiscussionTextExtractsHeadlineSummaryAndUpdated(t *testing.T) {
if !strings.Contains(got.Discussion, "...DISCUSSION...") {
t.Fatalf("Discussion missing full text: %q", got.Discussion)
}
if !strings.HasPrefix(got.Discussion, "SPC AC 111234") {
t.Fatalf("Discussion = %q, want product code prefix", got.Discussion)
}
wantUpdated := time.Date(2026, 6, 11, 12, 45, 0, 0, time.UTC)
if got.UpdatedAt == nil || !got.UpdatedAt.Equal(wantUpdated) {
t.Fatalf("UpdatedAt = %v, want %s", got.UpdatedAt, wantUpdated)
}
}
func TestParseProductTitleSkipsSPCProductCode(t *testing.T) {
got := ParseProductTitle("SPC AC 101959\nDay 1 Convective Outlook\nNWS Storm Prediction Center Norman OK")
if got != "Day 1 Convective Outlook" {
t.Fatalf("ParseProductTitle() = %q, want Day 1 Convective Outlook", got)
}
}
func TestParseDiscussionTextPreservesCorrectionMarker(t *testing.T) {
text, err := ExtractProductText(string(readTestFile(t, "day2_prt_corr.html")))
got, err := ParseDiscussionHTML(string(readTestFile(t, "day2_prt_corr.html")))
if err != nil {
t.Fatalf("ExtractProductText() error = %v", err)
t.Fatalf("ParseDiscussionHTML() error = %v", err)
}
got := ParseDiscussionText(text)
if !strings.Contains(got.Headline, "CORR 1") {
t.Fatalf("Headline = %q, want correction marker", got.Headline)
}
if !strings.Contains(got.Discussion, "CORR 1") {
t.Fatalf("Discussion = %q, want correction marker", got.Discussion)
}
if got.UpdatedAt != nil {
t.Fatalf("UpdatedAt = %v, want nil", got.UpdatedAt)
wantUpdated := time.Date(2026, 6, 11, 17, 30, 0, 0, time.UTC)
if got.UpdatedAt == nil || !got.UpdatedAt.Equal(wantUpdated) {
t.Fatalf("UpdatedAt = %v, want %s", got.UpdatedAt, wantUpdated)
}
}
@@ -88,3 +99,11 @@ func TestParseUpdatedTimestampAcceptsSPCUTCFormat(t *testing.T) {
t.Fatalf("ParseUpdatedTimestamp() = %v, want %s", got, want)
}
}
func TestParsePageUpdatedTimestampAcceptsLiveSPCShape(t *testing.T) {
got := ParsePageUpdatedTimestamp(string(readTestFile(t, "day3_prt.html")))
want := time.Date(2026, 6, 11, 20, 0, 0, 0, time.UTC)
if got == nil || !got.Equal(want) {
t.Fatalf("ParsePageUpdatedTimestamp() = %v, want %s", got, want)
}
}

View File

@@ -0,0 +1,17 @@
package spc
import (
"os"
"path/filepath"
"testing"
)
func readTestFile(t *testing.T, name string) []byte {
t.Helper()
path := filepath.Join("testdata", name)
raw, err := os.ReadFile(path)
if err != nil {
t.Fatalf("read %s: %v", path, err)
}
return raw
}

View File

@@ -33,6 +33,11 @@ type GeoJSONProperties struct {
DN *int `json:"DN"`
}
type geometryMetadata struct {
Type string `json:"type"`
Geometries []json.RawMessage `json:"geometries"`
}
// DecodeGeoJSON decodes an SPC GeoJSON outlook product and compacts feature
// geometry JSON for stable downstream storage.
func DecodeGeoJSON(raw []byte) (GeoJSONFeatureCollection, error) {
@@ -50,6 +55,16 @@ func DecodeGeoJSON(raw []byte) (GeoJSONFeatureCollection, error) {
return collection, nil
}
// IsEmptyGeometryCollection reports whether raw is SPC's no-polygon placeholder
// geometry shape: a GeometryCollection with no child geometries.
func IsEmptyGeometryCollection(raw json.RawMessage) bool {
var meta geometryMetadata
if err := json.Unmarshal(raw, &meta); err != nil {
return false
}
return meta.Type == "GeometryCollection" && len(meta.Geometries) == 0
}
func (p *GeoJSONProperties) UnmarshalJSON(raw []byte) error {
type alias GeoJSONProperties
var aux struct {

View File

@@ -1,8 +1,6 @@
package spc
import (
"os"
"path/filepath"
"strings"
"testing"
"time"
@@ -68,6 +66,43 @@ func TestDecodeGeoJSONParsesSeverityRankString(t *testing.T) {
}
}
func TestIsEmptyGeometryCollection(t *testing.T) {
tests := []struct {
name string
raw string
want bool
}{
{
name: "empty geometry collection",
raw: `{"type":"GeometryCollection","geometries":[]}`,
want: true,
},
{
name: "non-empty geometry collection",
raw: `{"type":"GeometryCollection","geometries":[{"type":"Polygon","coordinates":[]} ]}`,
want: false,
},
{
name: "polygon",
raw: `{"type":"Polygon","coordinates":[]}`,
want: false,
},
{
name: "invalid json",
raw: `{`,
want: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := IsEmptyGeometryCollection([]byte(tt.raw)); got != tt.want {
t.Fatalf("IsEmptyGeometryCollection() = %v, want %v", got, tt.want)
}
})
}
}
func TestParseISOTimestampTrimsAndReturnsUTC(t *testing.T) {
got, err := ParseISOTimestamp(" 2026-06-11T12:34:56Z ")
if err != nil {
@@ -78,13 +113,3 @@ func TestParseISOTimestampTrimsAndReturnsUTC(t *testing.T) {
t.Fatalf("ParseISOTimestamp() = %s, want %s", got, want)
}
}
func readTestFile(t *testing.T, name string) []byte {
t.Helper()
path := filepath.Join("testdata", name)
raw, err := os.ReadFile(path)
if err != nil {
t.Fatalf("read %s: %v", path, err)
}
return raw
}

View File

@@ -34,9 +34,6 @@ var geoJSONProducts = []GeoJSONProduct{
{Key: "day2_hail", Day: 2, OutlookType: OutlookTypeHail, URL: "https://www.spc.noaa.gov/products/outlook/day2otlk_hail.nolyr.geojson"},
{Key: "day2_wind", Day: 2, OutlookType: OutlookTypeWind, URL: "https://www.spc.noaa.gov/products/outlook/day2otlk_wind.nolyr.geojson"},
{Key: "day3_categorical", Day: 3, OutlookType: OutlookTypeCategorical, URL: "https://www.spc.noaa.gov/products/outlook/day3otlk_cat.nolyr.geojson"},
{Key: "day3_tornado", Day: 3, OutlookType: OutlookTypeTornado, URL: "https://www.spc.noaa.gov/products/outlook/day3otlk_torn.nolyr.geojson"},
{Key: "day3_hail", Day: 3, OutlookType: OutlookTypeHail, URL: "https://www.spc.noaa.gov/products/outlook/day3otlk_hail.nolyr.geojson"},
{Key: "day3_wind", Day: 3, OutlookType: OutlookTypeWind, URL: "https://www.spc.noaa.gov/products/outlook/day3otlk_wind.nolyr.geojson"},
}
var discussionProducts = []DiscussionProduct{

View File

@@ -4,8 +4,8 @@ import "testing"
func TestGeoJSONProductsStableOrder(t *testing.T) {
got := GeoJSONProducts()
if len(got) != 12 {
t.Fatalf("GeoJSONProducts() length = %d, want 12", len(got))
if len(got) != 9 {
t.Fatalf("GeoJSONProducts() length = %d, want 9", len(got))
}
wantKeys := []string{
@@ -18,9 +18,6 @@ func TestGeoJSONProductsStableOrder(t *testing.T) {
"day2_hail",
"day2_wind",
"day3_categorical",
"day3_tornado",
"day3_hail",
"day3_wind",
}
for i, want := range wantKeys {
if got[i].Key != want {

View File

@@ -2,11 +2,14 @@
<html>
<head><title>Day 1 Convective Outlook</title></head>
<body>
<table>
<tr><td align="center" class="rpttext" nowrap>Updated:&nbsp;Thu Jun 11 12:45:00 UTC 2026&nbsp;(<a href="archive/day1-geojson.zip">geojson</a>)</td></tr>
</table>
<pre>
<script>window.bad = "<b>ignore me</b>";</script>
SPC AC 111234
Day 1 Convective Outlook
NWS Storm Prediction Center Norman OK
Updated: 2026-06-11T12:45:00Z
...SUMMARY...
Severe thunderstorms are possible across parts of the central Plains

View File

@@ -1,7 +1,11 @@
<!doctype html>
<html>
<body>
<table>
<tr><td class="rpttext">Updated:&nbsp;Thu Jun 11 17:30:00 UTC 2026&nbsp;</td></tr>
</table>
<pre>
SPC AC 111730
Day 2 Convective Outlook CORR 1
NWS Storm Prediction Center Norman OK

View File

@@ -8,9 +8,9 @@
"EXPIRE_ISO": "2026-06-14T12:00:00Z",
"ISSUE_ISO": "2026-06-11T19:45:00Z",
"FORECASTER": "LEE",
"LABEL": "15",
"LABEL2": "15% Wind",
"DN": 15
"LABEL": "MRGL",
"LABEL2": "Marginal Risk",
"DN": 2
},
"geometry": {
"type": "MultiPolygon",

View File

@@ -1,10 +1,13 @@
<!doctype html>
<html>
<body>
<table>
<tr><td class="rpttext">Updated:&nbsp;Thu Jun 11 20:00:00 UTC 2026&nbsp;</td></tr>
</table>
<pre>
SPC AC 112000
Day 3 Convective Outlook
NWS Storm Prediction Center Norman OK
Updated: 2026-06-11T20:00:00Z
...SUMMARY...
A corridor of strong to severe storms may develop near a frontal zone.

View File

@@ -11,7 +11,7 @@
// - weather.forecast_discussion.v1 -> model.WeatherForecastDiscussion
// - weather.weather_story.v1 -> model.WeatherStoryRun
// - weather.alert.v1 -> model.WeatherAlertRun
// - weather.outlook.v1 -> model.WeatherOutlookRun
// - weather.outlook.v2 -> model.WeatherOutlookRun
//
// Parent/child relationships:
// - observations.event_id -> observation_present_weather.event_id
@@ -21,6 +21,7 @@
// - alert_runs.event_id -> alerts.run_event_id
// - alerts.(run_event_id, alert_index) -> alert_references.(run_event_id, alert_index)
// - outlook_runs.event_id -> outlooks.run_event_id
// - outlook_runs.event_id -> outlook_discussions.run_event_id
//
// Dedupe and retention behavior:
// - Parent primary keys (event_id): observations, forecasts, alert_runs, outlook_runs.
@@ -39,6 +40,7 @@
// - alert_references.as_of
// - outlook_runs.as_of
// - outlooks.as_of
// - outlook_discussions.as_of
//
// Envelope field mapping (shared parent columns)
//
@@ -196,6 +198,7 @@
// - sent TIMESTAMPTZ NULL -> payload.alerts[i].sent
// - effective TIMESTAMPTZ NULL -> payload.alerts[i].effective
// - onset TIMESTAMPTZ NULL -> payload.alerts[i].onset
// - ends TIMESTAMPTZ NULL -> payload.alerts[i].ends
// - expires TIMESTAMPTZ NULL -> payload.alerts[i].expires
// - area_description TEXT NULL -> payload.alerts[i].areaDescription
// - sender_name TEXT NULL -> payload.alerts[i].senderName
@@ -227,12 +230,15 @@
// - as_of TIMESTAMPTZ -> payload.asOf
// - issued_at TIMESTAMPTZ NULL -> payload.issuedAt
// - outlook_count INTEGER -> len(payload.outlooks)
// - discussion_count INTEGER -> len(payload.discussions)
//
// 11. outlooks (PK: run_event_id, outlook_index)
//
// - run_event_id TEXT -> outlook_runs.event_id / payload.outlooks[i]
// - outlook_index INTEGER -> i (array position in payload.outlooks)
// - as_of TIMESTAMPTZ -> payload.asOf (copied from parent)
// - outlook_id TEXT -> payload.outlooks[i].id
// - provider TEXT -> payload.outlooks[i].provider
// - product TEXT -> payload.outlooks[i].product
// - day INTEGER -> payload.outlooks[i].day
// - outlook_type TEXT -> payload.outlooks[i].outlookType
@@ -244,14 +250,22 @@
// - issued_at TIMESTAMPTZ -> payload.outlooks[i].issuedAt
// - expires_at TIMESTAMPTZ -> payload.outlooks[i].expiresAt
// - forecaster TEXT NULL -> payload.outlooks[i].forecaster
// - headline TEXT NULL -> payload.outlooks[i].headline
// - summary TEXT NULL -> payload.outlooks[i].summary
// - discussion TEXT NULL -> payload.outlooks[i].discussion
// - source_url TEXT NULL -> payload.outlooks[i].sourceUrl
// - image_url TEXT NULL -> payload.outlooks[i].imageUrl
// - contains_location BOOLEAN -> payload.outlooks[i].containsLocation
// - geometry_json TEXT -> compact JSON payload.outlooks[i].geometry
//
// 12. outlook_discussions (PK: run_event_id, discussion_index)
//
// - run_event_id TEXT -> outlook_runs.event_id / payload.discussions[i]
// - discussion_index INTEGER -> i (array position in payload.discussions)
// - as_of TIMESTAMPTZ -> payload.asOf (copied from parent)
// - day INTEGER -> payload.discussions[i].day
// - headline TEXT NULL -> payload.discussions[i].headline
// - summary TEXT NULL -> payload.discussions[i].summary
// - discussion TEXT NULL -> payload.discussions[i].discussion
// - updated_at TIMESTAMPTZ NULL -> payload.discussions[i].updatedAt
//
// Reconstructing canonical JSON payloads
//
// - WeatherObservation:
@@ -272,6 +286,7 @@
// ordered by reference_index to rebuild references per alert.
//
// - WeatherOutlookRun:
// read one row from outlook_runs, then join outlooks by run_event_id ordered
// by outlook_index to rebuild outlooks.
// read one row from outlook_runs, join outlooks by run_event_id ordered by
// outlook_index to rebuild outlooks, then join outlook_discussions by
// run_event_id ordered by discussion_index to rebuild discussions.
package postgres

View File

@@ -0,0 +1,33 @@
package postgres
import (
"os"
"strings"
"testing"
)
func TestDocumentedOutlookDiscussionStorage(t *testing.T) {
for _, path := range []string{
"../../../docs/integrations/postgres.md",
"../../../docs/internal/postgres-sink.md",
} {
t.Run(path, func(t *testing.T) {
raw, err := os.ReadFile(path)
if err != nil {
t.Fatalf("ReadFile(%s) error = %v", path, err)
}
doc := string(raw)
for _, want := range []string{
tableOutlookDiscussions,
"discussion_count",
"discussion_index",
"weather.outlook.v2",
} {
if !strings.Contains(doc, want) {
t.Fatalf("%s missing %q", path, want)
}
}
})
}
}

View File

@@ -27,7 +27,7 @@ func mapPostgresEvent(_ context.Context, e fkevent.Event) ([]fksinks.PostgresWri
return mapWeatherStoryEvent(e)
case standards.SchemaWeatherAlertV1:
return mapAlertEvent(e)
case standards.SchemaWeatherOutlookV1:
case standards.SchemaWeatherOutlookV2:
return mapOutlookEvent(e)
default:
return nil, nil
@@ -48,13 +48,7 @@ func mapObservationEvent(e fkevent.Event) ([]fksinks.PostgresWrite, error) {
writes = append(writes, fksinks.PostgresWrite{
Table: tableObservations,
Values: map[string]any{
"event_id": e.ID,
"event_kind": string(e.Kind),
"event_source": e.Source,
"event_schema": e.Schema,
"event_emitted_at": e.EmittedAt.UTC(),
"event_effective_at": nullableTime(e.EffectiveAt),
Values: parentEventValues(e, map[string]any{
"station_id": nullableString(obs.StationID),
"station_name": nullableString(obs.StationName),
"observed_at": observedAt,
@@ -70,7 +64,7 @@ func mapObservationEvent(e fkevent.Event) ([]fksinks.PostgresWrite, error) {
"visibility_meters": nullableFloat64(obs.VisibilityMeters),
"relative_humidity_percent": nullableFloat64(obs.RelativeHumidityPercent),
"apparent_temperature_c": nullableFloat64(obs.ApparentTemperatureC),
},
}),
})
for i, pw := range obs.PresentWeather {
@@ -109,23 +103,17 @@ func mapForecastEvent(e fkevent.Event) ([]fksinks.PostgresWrite, error) {
writes = append(writes, fksinks.PostgresWrite{
Table: tableForecasts,
Values: map[string]any{
"event_id": e.ID,
"event_kind": string(e.Kind),
"event_source": e.Source,
"event_schema": e.Schema,
"event_emitted_at": e.EmittedAt.UTC(),
"event_effective_at": nullableTime(e.EffectiveAt),
"location_id": nullableString(run.LocationID),
"location_name": nullableString(run.LocationName),
"issued_at": issuedAt,
"updated_at": nullableTime(run.UpdatedAt),
"product": string(run.Product),
"latitude": nullableFloat64(run.Latitude),
"longitude": nullableFloat64(run.Longitude),
"elevation_meters": nullableFloat64(run.ElevationMeters),
"period_count": len(run.Periods),
},
Values: parentEventValues(e, map[string]any{
"location_id": nullableString(run.LocationID),
"location_name": nullableString(run.LocationName),
"issued_at": issuedAt,
"updated_at": nullableTime(run.UpdatedAt),
"product": string(run.Product),
"latitude": nullableFloat64(run.Latitude),
"longitude": nullableFloat64(run.Longitude),
"elevation_meters": nullableFloat64(run.ElevationMeters),
"period_count": len(run.Periods),
}),
})
for i, p := range run.Periods {
@@ -186,13 +174,7 @@ func mapForecastDiscussionEvent(e fkevent.Event) ([]fksinks.PostgresWrite, error
writes := make([]fksinks.PostgresWrite, 0, 1+len(run.KeyMessages))
writes = append(writes, fksinks.PostgresWrite{
Table: tableForecastDiscussions,
Values: map[string]any{
"event_id": e.ID,
"event_kind": string(e.Kind),
"event_source": e.Source,
"event_schema": e.Schema,
"event_emitted_at": e.EmittedAt.UTC(),
"event_effective_at": nullableTime(e.EffectiveAt),
Values: parentEventValues(e, map[string]any{
"office_id": nullableString(run.OfficeID),
"office_name": nullableString(run.OfficeName),
"issued_at": issuedAt,
@@ -205,7 +187,7 @@ func mapForecastDiscussionEvent(e fkevent.Event) ([]fksinks.PostgresWrite, error
"long_term_issued_at": longTermIssuedAt,
"long_term_text": longTermText,
"key_message_count": len(run.KeyMessages),
},
}),
})
for i, msg := range run.KeyMessages {
@@ -236,17 +218,11 @@ func mapWeatherStoryEvent(e fkevent.Event) ([]fksinks.PostgresWrite, error) {
writes := make([]fksinks.PostgresWrite, 0, 1+len(run.Stories))
writes = append(writes, fksinks.PostgresWrite{
Table: tableWeatherStoryRuns,
Values: map[string]any{
"event_id": e.ID,
"event_kind": string(e.Kind),
"event_source": e.Source,
"event_schema": e.Schema,
"event_emitted_at": e.EmittedAt.UTC(),
"event_effective_at": nullableTime(e.EffectiveAt),
"office_id": nullableString(run.OfficeID),
"as_of": asOf,
"story_count": len(run.Stories),
},
Values: parentEventValues(e, map[string]any{
"office_id": nullableString(run.OfficeID),
"as_of": asOf,
"story_count": len(run.Stories),
}),
})
for i, story := range run.Stories {
@@ -290,20 +266,14 @@ func mapAlertEvent(e fkevent.Event) ([]fksinks.PostgresWrite, error) {
writes = append(writes, fksinks.PostgresWrite{
Table: tableAlertRuns,
Values: map[string]any{
"event_id": e.ID,
"event_kind": string(e.Kind),
"event_source": e.Source,
"event_schema": e.Schema,
"event_emitted_at": e.EmittedAt.UTC(),
"event_effective_at": nullableTime(e.EffectiveAt),
"location_id": nullableString(run.LocationID),
"location_name": nullableString(run.LocationName),
"as_of": asOf,
"latitude": nullableFloat64(run.Latitude),
"longitude": nullableFloat64(run.Longitude),
"alert_count": len(run.Alerts),
},
Values: parentEventValues(e, map[string]any{
"location_id": nullableString(run.LocationID),
"location_name": nullableString(run.LocationName),
"as_of": asOf,
"latitude": nullableFloat64(run.Latitude),
"longitude": nullableFloat64(run.Longitude),
"alert_count": len(run.Alerts),
}),
})
for i, a := range run.Alerts {
@@ -332,6 +302,7 @@ func mapAlertEvent(e fkevent.Event) ([]fksinks.PostgresWrite, error) {
"sent": nullableTime(a.Sent),
"effective": nullableTime(a.Effective),
"onset": nullableTime(a.Onset),
"ends": nullableTime(a.Ends),
"expires": nullableTime(a.Expires),
"area_description": nullableString(a.AreaDescription),
"sender_name": nullableString(a.SenderName),
@@ -369,24 +340,23 @@ func mapOutlookEvent(e fkevent.Event) ([]fksinks.PostgresWrite, error) {
}
asOf := run.AsOf.UTC()
writes := make([]fksinks.PostgresWrite, 0, 1+len(run.Outlooks))
if err := validateOutlookDiscussions(run.Discussions); err != nil {
return nil, err
}
writes := make([]fksinks.PostgresWrite, 0, 1+len(run.Outlooks)+len(run.Discussions))
writes = append(writes, fksinks.PostgresWrite{
Table: tableOutlookRuns,
Values: map[string]any{
"event_id": e.ID,
"event_kind": string(e.Kind),
"event_source": e.Source,
"event_schema": e.Schema,
"event_emitted_at": e.EmittedAt.UTC(),
"event_effective_at": nullableTime(e.EffectiveAt),
"location_id": nullableString(run.LocationID),
"location_name": nullableString(run.LocationName),
"latitude": nullableFloat64(run.Latitude),
"longitude": nullableFloat64(run.Longitude),
"as_of": asOf,
"issued_at": nullableTime(run.IssuedAt),
"outlook_count": len(run.Outlooks),
},
Values: parentEventValues(e, map[string]any{
"location_id": nullableString(run.LocationID),
"location_name": nullableString(run.LocationName),
"latitude": nullableFloat64(run.Latitude),
"longitude": nullableFloat64(run.Longitude),
"as_of": asOf,
"issued_at": nullableTime(run.IssuedAt),
"outlook_count": len(run.Outlooks),
"discussion_count": len(run.Discussions),
}),
})
for i, outlook := range run.Outlooks {
@@ -404,6 +374,8 @@ func mapOutlookEvent(e fkevent.Event) ([]fksinks.PostgresWrite, error) {
"run_event_id": e.ID,
"outlook_index": i,
"as_of": asOf,
"outlook_id": outlook.ID,
"provider": outlook.Provider,
"product": outlook.Product,
"day": outlook.Day,
"outlook_type": outlook.OutlookType,
@@ -415,9 +387,6 @@ func mapOutlookEvent(e fkevent.Event) ([]fksinks.PostgresWrite, error) {
"issued_at": outlook.IssuedAt.UTC(),
"expires_at": outlook.ExpiresAt.UTC(),
"forecaster": nullableString(outlook.Forecaster),
"headline": nullableString(outlook.Headline),
"summary": nullableString(outlook.Summary),
"discussion": nullableString(outlook.Discussion),
"source_url": nullableString(outlook.SourceURL),
"image_url": nullableString(outlook.ImageURL),
"contains_location": outlook.ContainsLocation,
@@ -426,6 +395,22 @@ func mapOutlookEvent(e fkevent.Event) ([]fksinks.PostgresWrite, error) {
})
}
for i, discussion := range run.Discussions {
writes = append(writes, fksinks.PostgresWrite{
Table: tableOutlookDiscussions,
Values: map[string]any{
"run_event_id": e.ID,
"discussion_index": i,
"as_of": asOf,
"day": discussion.Day,
"headline": nullableString(discussion.Headline),
"summary": nullableString(discussion.Summary),
"discussion": nullableString(discussion.Discussion),
"updated_at": nullableTime(discussion.UpdatedAt),
},
})
}
return writes, nil
}
@@ -457,6 +442,28 @@ func validateOutlook(outlook model.WeatherOutlook, index int) error {
if len(outlook.Geometry) == 0 {
return fmt.Errorf("decode outlook payload: outlooks[%d].geometry is required", index)
}
if !outlook.ContainsLocation {
return fmt.Errorf("decode outlook payload: outlooks[%d].containsLocation must be true", index)
}
return nil
}
func validateOutlookDiscussions(discussions []model.WeatherOutlookDiscussion) error {
seenDays := map[int]int{}
for i, discussion := range discussions {
if discussion.Day < 1 || discussion.Day > 3 {
return fmt.Errorf("decode outlook payload: discussions[%d].day must be 1, 2, or 3", i)
}
if strings.TrimSpace(discussion.Headline) == "" &&
strings.TrimSpace(discussion.Summary) == "" &&
strings.TrimSpace(discussion.Discussion) == "" {
return fmt.Errorf("decode outlook payload: discussions[%d] headline, summary, or discussion is required", i)
}
if first, ok := seenDays[discussion.Day]; ok {
return fmt.Errorf("decode outlook payload: discussions[%d].day duplicates discussions[%d].day %d", i, first, discussion.Day)
}
seenDays[discussion.Day] = i
}
return nil
}
@@ -486,6 +493,21 @@ func decodePayload[T any](payload any) (T, error) {
return out, nil
}
func parentEventValues(e fkevent.Event, values map[string]any) map[string]any {
out := map[string]any{
"event_id": e.ID,
"event_kind": string(e.Kind),
"event_source": e.Source,
"event_schema": e.Schema,
"event_emitted_at": e.EmittedAt.UTC(),
"event_effective_at": nullableTime(e.EffectiveAt),
}
for k, v := range values {
out[k] = v
}
return out
}
func nullableDiscussionSection(section *model.WeatherForecastDiscussionSection) (any, any, any) {
if section == nil {
return nil, nil, nil

View File

@@ -27,7 +27,7 @@ func TestMapPostgresEventObservationStructPayload(t *testing.T) {
PresentWeather: []model.PresentWeather{{Raw: map[string]any{"a": 1, "b": "x"}}},
}
writes, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherObservationV1, "observation", obs))
writes, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherObservationV1, standards.KindObservation, obs))
if err != nil {
t.Fatalf("mapPostgresEvent() error = %v", err)
}
@@ -74,7 +74,7 @@ func TestMapPostgresEventForecastStructPayload(t *testing.T) {
},
}
writes, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherForecastV1, "forecast", run))
writes, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherForecastV1, standards.KindForecast, run))
if err != nil {
t.Fatalf("mapPostgresEvent() error = %v", err)
}
@@ -103,6 +103,8 @@ func TestMapPostgresEventForecastStructPayload(t *testing.T) {
func TestMapPostgresEventAlertStructPayload(t *testing.T) {
sent := time.Date(2026, 3, 16, 17, 0, 0, 0, time.UTC)
ends := time.Date(2026, 3, 16, 20, 0, 0, 0, time.UTC)
expires := time.Date(2026, 3, 16, 18, 30, 0, 0, time.UTC)
run := model.WeatherAlertRun{
AsOf: time.Date(2026, 3, 16, 18, 0, 0, 0, time.UTC),
Alerts: []model.WeatherAlert{
@@ -110,6 +112,8 @@ func TestMapPostgresEventAlertStructPayload(t *testing.T) {
ID: "urn:alert:1",
Headline: "Winter Weather Advisory",
Severity: "Moderate",
Ends: &ends,
Expires: &expires,
References: []model.AlertReference{
{ID: "urn:ref:1", Sent: &sent},
{Identifier: "ref-two"},
@@ -122,7 +126,7 @@ func TestMapPostgresEventAlertStructPayload(t *testing.T) {
},
}
writes, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherAlertV1, "alert", run))
writes, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherAlertV1, standards.KindAlert, run))
if err != nil {
t.Fatalf("mapPostgresEvent() error = %v", err)
}
@@ -145,6 +149,20 @@ func TestMapPostgresEventAlertStructPayload(t *testing.T) {
if got := firstAlert.Values["reference_count"]; got != 2 {
t.Fatalf("alerts reference_count = %#v, want 2", got)
}
if got := firstAlert.Values["ends"]; got != ends {
t.Fatalf("alerts ends = %#v, want %#v", got, ends)
}
if got := firstAlert.Values["expires"]; got != expires {
t.Fatalf("alerts expires = %#v, want %#v", got, expires)
}
alertWrites := writesForTable(writes, tableAlerts)
if len(alertWrites) != 2 {
t.Fatalf("alert writes len = %d, want 2", len(alertWrites))
}
if got := alertWrites[1].Values["ends"]; got != nil {
t.Fatalf("second alert ends = %#v, want nil", got)
}
assertAllWritesIncludeAllColumns(t, writes)
}
@@ -163,7 +181,7 @@ func TestMapPostgresEventForecastDiscussionStructPayload(t *testing.T) {
LongTerm: &model.WeatherForecastDiscussionSection{Text: "Long term text"},
}
writes, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherForecastDiscussionV1, "forecast_discussion", run))
writes, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherForecastDiscussionV1, standards.KindForecastDiscussion, run))
if err != nil {
t.Fatalf("mapPostgresEvent() error = %v", err)
}
@@ -213,7 +231,7 @@ func TestMapPostgresEventWeatherStoryStructPayload(t *testing.T) {
},
}
writes, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherStoryV1, "weather_story", run))
writes, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherStoryV1, standards.KindWeatherStory, run))
if err != nil {
t.Fatalf("mapPostgresEvent() error = %v", err)
}
@@ -243,6 +261,7 @@ func TestMapPostgresEventOutlookStructPayload(t *testing.T) {
lat := 38.6239
lon := -90.3571
issuedAt := time.Date(2026, 6, 11, 19, 45, 0, 0, time.FixedZone("UTC-5", -5*60*60))
updatedAt := time.Date(2026, 6, 11, 21, 15, 0, 0, time.FixedZone("UTC-5", -5*60*60))
severity := 3
run := model.WeatherOutlookRun{
LocationID: "stl",
@@ -266,9 +285,6 @@ func TestMapPostgresEventOutlookStructPayload(t *testing.T) {
IssuedAt: issuedAt,
ExpiresAt: time.Date(2026, 6, 12, 12, 0, 0, 0, time.UTC),
Forecaster: "SMITH",
Headline: "Day 1 Convective Outlook",
Summary: "Severe thunderstorms are possible.",
Discussion: "Full discussion text.",
SourceURL: "https://example.invalid/day1.geojson",
ContainsLocation: true,
Geometry: json.RawMessage(`{ "type" : "Polygon", "coordinates" : [ [ [ -91.0, 38.0 ], [ -90.0, 38.0 ], [ -90.0, 39.0 ], [ -91.0, 39.0 ], [ -91.0, 38.0 ] ] ] }`),
@@ -284,18 +300,27 @@ func TestMapPostgresEventOutlookStructPayload(t *testing.T) {
ValidTo: time.Date(2026, 6, 12, 12, 0, 0, 0, time.UTC),
IssuedAt: time.Date(2026, 6, 11, 19, 45, 0, 0, time.UTC),
ExpiresAt: time.Date(2026, 6, 12, 12, 0, 0, 0, time.UTC),
ContainsLocation: false,
ContainsLocation: true,
Geometry: json.RawMessage(`{"type":"Polygon","coordinates":[[[-100,35],[-98,35],[-98,37],[-100,37],[-100,35]]]}`),
},
},
Discussions: []model.WeatherOutlookDiscussion{
{
Day: 1,
Headline: "Day 1 Convective Outlook",
Summary: "Severe thunderstorms are possible.",
Discussion: "Full discussion text.",
UpdatedAt: &updatedAt,
},
},
}
writes, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherOutlookV1, "outlook", run))
writes, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherOutlookV2, standards.KindOutlook, run))
if err != nil {
t.Fatalf("mapPostgresEvent() error = %v", err)
}
if len(writes) != 3 {
t.Fatalf("mapPostgresEvent() writes len = %d, want 3", len(writes))
if len(writes) != 4 {
t.Fatalf("mapPostgresEvent() writes len = %d, want 4", len(writes))
}
if writes[0].Table != tableOutlookRuns {
t.Fatalf("writes[0].Table = %q, want %q", writes[0].Table, tableOutlookRuns)
@@ -303,6 +328,9 @@ func TestMapPostgresEventOutlookStructPayload(t *testing.T) {
if got := writes[0].Values["outlook_count"]; got != 2 {
t.Fatalf("outlook_runs outlook_count = %#v, want 2", got)
}
if got := writes[0].Values["discussion_count"]; got != 1 {
t.Fatalf("outlook_runs discussion_count = %#v, want 1", got)
}
if got := writes[0].Values["issued_at"]; got != issuedAt.UTC() {
t.Fatalf("outlook_runs issued_at = %#v, want UTC %s", got, issuedAt.UTC())
}
@@ -312,21 +340,76 @@ func TestMapPostgresEventOutlookStructPayload(t *testing.T) {
if got := writes[1].Values["outlook_index"]; got != 0 {
t.Fatalf("first outlook index = %#v, want 0", got)
}
if got := writes[1].Values["outlook_id"]; got != "outlook-1" {
t.Fatalf("first outlook_id = %#v, want outlook-1", got)
}
if got := writes[1].Values["provider"]; got != "spc" {
t.Fatalf("first provider = %#v, want spc", got)
}
if got := writes[1].Values["valid_from"]; got != run.Outlooks[0].ValidFrom.UTC() {
t.Fatalf("first valid_from = %#v, want UTC %s", got, run.Outlooks[0].ValidFrom.UTC())
}
if got := writes[1].Values["geometry_json"]; got != `{"type":"Polygon","coordinates":[[[-91.0,38.0],[-90.0,38.0],[-90.0,39.0],[-91.0,39.0],[-91.0,38.0]]]}` {
t.Fatalf("first geometry_json = %#v", got)
}
if got := writes[2].Values["contains_location"]; got != false {
t.Fatalf("second contains_location = %#v, want false", got)
if got := writes[2].Values["contains_location"]; got != true {
t.Fatalf("second contains_location = %#v, want true", got)
}
if writes[3].Table != tableOutlookDiscussions {
t.Fatalf("writes[3].Table = %q, want %q", writes[3].Table, tableOutlookDiscussions)
}
if got := writes[3].Values["discussion_index"]; got != 0 {
t.Fatalf("discussion_index = %#v, want 0", got)
}
if got := writes[3].Values["as_of"]; got != run.AsOf.UTC() {
t.Fatalf("discussion as_of = %#v, want %s", got, run.AsOf.UTC())
}
if got := writes[3].Values["day"]; got != 1 {
t.Fatalf("discussion day = %#v, want 1", got)
}
if got := writes[3].Values["headline"]; got != "Day 1 Convective Outlook" {
t.Fatalf("discussion headline = %#v", got)
}
if got := writes[3].Values["summary"]; got != "Severe thunderstorms are possible." {
t.Fatalf("discussion summary = %#v", got)
}
if got := writes[3].Values["discussion"]; got != "Full discussion text." {
t.Fatalf("discussion text = %#v", got)
}
if got := writes[3].Values["updated_at"]; got != updatedAt.UTC() {
t.Fatalf("discussion updated_at = %#v, want UTC %s", got, updatedAt.UTC())
}
assertAllWritesIncludeAllColumns(t, writes)
}
func TestMapPostgresEventOutlookEmptyLocalRun(t *testing.T) {
run := model.WeatherOutlookRun{
AsOf: time.Date(2026, 6, 11, 19, 45, 0, 0, time.UTC),
}
writes, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherOutlookV2, standards.KindOutlook, run))
if err != nil {
t.Fatalf("mapPostgresEvent() error = %v", err)
}
if len(writes) != 1 {
t.Fatalf("mapPostgresEvent() writes len = %d, want 1", len(writes))
}
if writes[0].Table != tableOutlookRuns {
t.Fatalf("writes[0].Table = %q, want %q", writes[0].Table, tableOutlookRuns)
}
if got := writes[0].Values["outlook_count"]; got != 0 {
t.Fatalf("outlook_runs outlook_count = %#v, want 0", got)
}
if got := writes[0].Values["discussion_count"]; got != 0 {
t.Fatalf("outlook_runs discussion_count = %#v, want 0", got)
}
assertAllWritesIncludeAllColumns(t, writes)
}
func TestMapPostgresEventOutlookRejectsMissingAsOf(t *testing.T) {
_, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherOutlookV1, "outlook", model.WeatherOutlookRun{}))
_, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherOutlookV2, standards.KindOutlook, model.WeatherOutlookRun{}))
if err == nil {
t.Fatalf("mapPostgresEvent() error = nil, want missing asOf error")
}
@@ -335,20 +418,73 @@ func TestMapPostgresEventOutlookRejectsMissingAsOf(t *testing.T) {
}
}
func TestMapPostgresEventOutlookRejectsMissingIDAndProvider(t *testing.T) {
base := model.WeatherOutlook{
ID: "outlook-1",
Provider: "spc",
Product: "convective",
Day: 1,
OutlookType: "categorical",
Label: "SLGT",
ValidFrom: time.Date(2026, 6, 11, 13, 0, 0, 0, time.UTC),
ValidTo: time.Date(2026, 6, 12, 12, 0, 0, 0, time.UTC),
IssuedAt: time.Date(2026, 6, 11, 19, 45, 0, 0, time.UTC),
ExpiresAt: time.Date(2026, 6, 12, 12, 0, 0, 0, time.UTC),
ContainsLocation: true,
Geometry: json.RawMessage(`{"type":"Polygon","coordinates":[[[-91,38],[-90,38],[-90,39],[-91,39],[-91,38]]]}`),
}
tests := []struct {
name string
mutate func(*model.WeatherOutlook)
wantErr string
}{
{
name: "missing id",
mutate: func(outlook *model.WeatherOutlook) { outlook.ID = "" },
wantErr: "outlooks[0].id is required",
},
{
name: "missing provider",
mutate: func(outlook *model.WeatherOutlook) { outlook.Provider = "" },
wantErr: "outlooks[0].provider is required",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
outlook := base
tt.mutate(&outlook)
run := model.WeatherOutlookRun{
AsOf: time.Date(2026, 6, 11, 19, 45, 0, 0, time.UTC),
Outlooks: []model.WeatherOutlook{outlook},
}
_, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherOutlookV2, standards.KindOutlook, run))
if err == nil {
t.Fatalf("mapPostgresEvent() error = nil, want %q", tt.wantErr)
}
if !strings.Contains(err.Error(), tt.wantErr) {
t.Fatalf("error = %q, want %q", err, tt.wantErr)
}
})
}
}
func TestMapPostgresEventOutlookRejectsMissingRequiredTimes(t *testing.T) {
run := model.WeatherOutlookRun{
AsOf: time.Date(2026, 6, 11, 19, 45, 0, 0, time.UTC),
Outlooks: []model.WeatherOutlook{{
ID: "outlook-1",
Provider: "spc",
Product: "convective",
Day: 1,
OutlookType: "categorical",
Label: "SLGT",
Geometry: json.RawMessage(`{"type":"Polygon","coordinates":[[[-91,38],[-90,38],[-90,39],[-91,39],[-91,38]]]}`),
ID: "outlook-1",
Provider: "spc",
Product: "convective",
Day: 1,
OutlookType: "categorical",
Label: "SLGT",
ContainsLocation: true,
Geometry: json.RawMessage(`{"type":"Polygon","coordinates":[[[-91,38],[-90,38],[-90,39],[-91,39],[-91,38]]]}`),
}},
}
_, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherOutlookV1, "outlook", run))
_, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherOutlookV2, standards.KindOutlook, run))
if err == nil {
t.Fatalf("mapPostgresEvent() error = nil, want missing time error")
}
@@ -361,19 +497,20 @@ func TestMapPostgresEventOutlookRejectsEmptyGeometry(t *testing.T) {
run := model.WeatherOutlookRun{
AsOf: time.Date(2026, 6, 11, 19, 45, 0, 0, time.UTC),
Outlooks: []model.WeatherOutlook{{
ID: "outlook-1",
Provider: "spc",
Product: "convective",
Day: 1,
OutlookType: "categorical",
Label: "SLGT",
ValidFrom: time.Date(2026, 6, 11, 13, 0, 0, 0, time.UTC),
ValidTo: time.Date(2026, 6, 12, 12, 0, 0, 0, time.UTC),
IssuedAt: time.Date(2026, 6, 11, 19, 45, 0, 0, time.UTC),
ExpiresAt: time.Date(2026, 6, 12, 12, 0, 0, 0, time.UTC),
ID: "outlook-1",
Provider: "spc",
Product: "convective",
Day: 1,
OutlookType: "categorical",
Label: "SLGT",
ValidFrom: time.Date(2026, 6, 11, 13, 0, 0, 0, time.UTC),
ValidTo: time.Date(2026, 6, 12, 12, 0, 0, 0, time.UTC),
IssuedAt: time.Date(2026, 6, 11, 19, 45, 0, 0, time.UTC),
ExpiresAt: time.Date(2026, 6, 12, 12, 0, 0, 0, time.UTC),
ContainsLocation: true,
}},
}
_, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherOutlookV1, "outlook", run))
_, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherOutlookV2, standards.KindOutlook, run))
if err == nil {
t.Fatalf("mapPostgresEvent() error = nil, want geometry error")
}
@@ -382,8 +519,69 @@ func TestMapPostgresEventOutlookRejectsEmptyGeometry(t *testing.T) {
}
}
func TestMapPostgresEventOutlookRejectsDuplicateDiscussionDay(t *testing.T) {
run := model.WeatherOutlookRun{
AsOf: time.Date(2026, 6, 11, 19, 45, 0, 0, time.UTC),
Discussions: []model.WeatherOutlookDiscussion{
{Day: 1, Discussion: "First day one discussion."},
{Day: 1, Discussion: "Duplicate day one discussion."},
},
}
_, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherOutlookV2, standards.KindOutlook, run))
if err == nil {
t.Fatalf("mapPostgresEvent() error = nil, want duplicate discussion day error")
}
if !strings.Contains(err.Error(), "discussions[1].day duplicates discussions[0].day 1") {
t.Fatalf("error = %q, want duplicate discussion day context", err)
}
}
func TestMapPostgresEventOutlookRejectsInvalidDiscussionDay(t *testing.T) {
run := model.WeatherOutlookRun{
AsOf: time.Date(2026, 6, 11, 19, 45, 0, 0, time.UTC),
Discussions: []model.WeatherOutlookDiscussion{{Day: 4, Discussion: "Invalid day."}},
}
_, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherOutlookV2, standards.KindOutlook, run))
if err == nil {
t.Fatalf("mapPostgresEvent() error = nil, want invalid discussion day error")
}
if !strings.Contains(err.Error(), "discussions[0].day must be 1, 2, or 3") {
t.Fatalf("error = %q, want invalid discussion day context", err)
}
}
func TestMapPostgresEventOutlookRejectsEmptyDiscussionContent(t *testing.T) {
run := model.WeatherOutlookRun{
AsOf: time.Date(2026, 6, 11, 19, 45, 0, 0, time.UTC),
Discussions: []model.WeatherOutlookDiscussion{{Day: 1}},
}
_, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherOutlookV2, standards.KindOutlook, run))
if err == nil {
t.Fatalf("mapPostgresEvent() error = nil, want empty discussion content error")
}
if !strings.Contains(err.Error(), "discussions[0] headline, summary, or discussion is required") {
t.Fatalf("error = %q, want empty discussion content context", err)
}
}
func TestMapPostgresEventOutlookRejectsContainsLocationFalse(t *testing.T) {
run := model.WeatherOutlookRun{
AsOf: time.Date(2026, 6, 11, 19, 45, 0, 0, time.UTC),
Outlooks: []model.WeatherOutlook{validTestOutlook()},
}
run.Outlooks[0].ContainsLocation = false
_, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherOutlookV2, standards.KindOutlook, run))
if err == nil {
t.Fatalf("mapPostgresEvent() error = nil, want containsLocation error")
}
if !strings.Contains(err.Error(), "outlooks[0].containsLocation must be true") {
t.Fatalf("error = %q, want containsLocation context", err)
}
}
func TestMapPostgresEventWeatherStoryRejectsMissingAsOf(t *testing.T) {
_, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherStoryV1, "weather_story", model.WeatherStoryRun{}))
_, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherStoryV1, standards.KindWeatherStory, model.WeatherStoryRun{}))
if err == nil {
t.Fatalf("mapPostgresEvent() error = nil, want missing asOf error")
}
@@ -397,7 +595,7 @@ func TestMapPostgresEventWeatherStoryRejectsMissingStoryTimes(t *testing.T) {
AsOf: time.Date(2026, 5, 30, 9, 0, 34, 0, time.UTC),
Stories: []model.WeatherStory{{Title: "missing times"}},
}
_, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherStoryV1, "weather_story", run))
_, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherStoryV1, standards.KindWeatherStory, run))
if err == nil {
t.Fatalf("mapPostgresEvent() error = nil, want missing story times error")
}
@@ -427,7 +625,7 @@ func TestMapPostgresEventMapPayload(t *testing.T) {
t.Fatalf("json.Unmarshal() error = %v", err)
}
writes, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherForecastV1, "forecast", payload))
writes, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherForecastV1, standards.KindForecast, payload))
if err != nil {
t.Fatalf("mapPostgresEvent() error = %v", err)
}
@@ -442,7 +640,18 @@ func TestMapPostgresEventMapPayload(t *testing.T) {
}
func TestMapPostgresEventUnknownSchemaNoOp(t *testing.T) {
writes, err := mapPostgresEvent(context.Background(), testEvent("weather.unknown.v1", "observation", map[string]any{"x": 1}))
writes, err := mapPostgresEvent(context.Background(), testEvent("weather.unknown.v1", standards.KindObservation, map[string]any{"x": 1}))
if err != nil {
t.Fatalf("mapPostgresEvent() error = %v", err)
}
if len(writes) != 0 {
t.Fatalf("mapPostgresEvent() writes len = %d, want 0", len(writes))
}
}
func TestMapPostgresEventLegacyOutlookSchemaNoOp(t *testing.T) {
run := model.WeatherOutlookRun{AsOf: time.Date(2026, 6, 11, 19, 45, 0, 0, time.UTC)}
writes, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherOutlookV1, standards.KindOutlook, run))
if err != nil {
t.Fatalf("mapPostgresEvent() error = %v", err)
}
@@ -452,7 +661,7 @@ func TestMapPostgresEventUnknownSchemaNoOp(t *testing.T) {
}
func TestMapPostgresEventMalformedPayload(t *testing.T) {
_, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherForecastV1, "forecast", "bad"))
_, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherForecastV1, standards.KindForecast, "bad"))
if err == nil {
t.Fatalf("mapPostgresEvent() expected error for malformed payload")
}
@@ -462,7 +671,7 @@ func TestMapPostgresEventMalformedPayload(t *testing.T) {
}
func TestMapPostgresEventForecastDiscussionMalformedPayload(t *testing.T) {
_, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherForecastDiscussionV1, "forecast_discussion", "bad"))
_, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherForecastDiscussionV1, standards.KindForecastDiscussion, "bad"))
if err == nil {
t.Fatalf("mapPostgresEvent() expected error for malformed payload")
}
@@ -471,6 +680,56 @@ func TestMapPostgresEventForecastDiscussionMalformedPayload(t *testing.T) {
}
}
func TestParentEventValuesAddsEnvelopeAndPreservesProductValues(t *testing.T) {
emittedAt := time.Date(2026, 3, 16, 13, 31, 0, 0, time.FixedZone("CDT", -5*60*60))
effectiveAt := time.Date(2026, 3, 16, 13, 30, 0, 0, time.FixedZone("CDT", -5*60*60))
event := fkevent.Event{
ID: "evt-envelope",
Kind: fkevent.Kind(standards.KindForecast),
Source: "test-source",
Schema: standards.SchemaWeatherForecastV1,
EmittedAt: emittedAt,
EffectiveAt: &effectiveAt,
}
got := parentEventValues(event, map[string]any{"product_col": "product-value"})
assertParentEnvelopeValues(t, got, event)
if got["product_col"] != "product-value" {
t.Fatalf("product_col = %#v, want product-value", got["product_col"])
}
}
func TestParentEventValuesNullEffectiveAt(t *testing.T) {
base := fkevent.Event{
ID: "evt-envelope",
Kind: fkevent.Kind(standards.KindObservation),
Source: "test-source",
Schema: standards.SchemaWeatherObservationV1,
EmittedAt: time.Date(2026, 3, 16, 18, 31, 0, 0, time.UTC),
}
for _, tt := range []struct {
name string
mut func(*fkevent.Event)
}{
{name: "nil", mut: func(*fkevent.Event) {}},
{name: "zero", mut: func(event *fkevent.Event) {
zero := time.Time{}
event.EffectiveAt = &zero
}},
} {
t.Run(tt.name, func(t *testing.T) {
event := base
tt.mut(&event)
got := parentEventValues(event, nil)
if got["event_effective_at"] != nil {
t.Fatalf("event_effective_at = %#v, want nil", got["event_effective_at"])
}
})
}
}
func testEvent(schema string, kind fkevent.Kind, payload any) fkevent.Event {
effectiveAt := time.Date(2026, 3, 16, 18, 30, 0, 0, time.UTC)
return fkevent.Event{
@@ -484,6 +743,30 @@ func testEvent(schema string, kind fkevent.Kind, payload any) fkevent.Event {
}
}
func assertParentEnvelopeValues(t *testing.T, values map[string]any, event fkevent.Event) {
t.Helper()
if got := values["event_id"]; got != event.ID {
t.Fatalf("event_id = %#v, want %q", got, event.ID)
}
if got := values["event_kind"]; got != string(event.Kind) {
t.Fatalf("event_kind = %#v, want %q", got, event.Kind)
}
if got := values["event_source"]; got != event.Source {
t.Fatalf("event_source = %#v, want %q", got, event.Source)
}
if got := values["event_schema"]; got != event.Schema {
t.Fatalf("event_schema = %#v, want %q", got, event.Schema)
}
if got := values["event_emitted_at"]; got != event.EmittedAt.UTC() {
t.Fatalf("event_emitted_at = %#v, want %s", got, event.EmittedAt.UTC())
}
wantEffective := nullableTime(event.EffectiveAt)
if got := values["event_effective_at"]; got != wantEffective {
t.Fatalf("event_effective_at = %#v, want %#v", got, wantEffective)
}
}
func firstWriteForTable(writes []fksinks.PostgresWrite, table string) (fksinks.PostgresWrite, bool) {
for _, w := range writes {
if w.Table == table {
@@ -493,6 +776,16 @@ func firstWriteForTable(writes []fksinks.PostgresWrite, table string) (fksinks.P
return fksinks.PostgresWrite{}, false
}
func writesForTable(writes []fksinks.PostgresWrite, table string) []fksinks.PostgresWrite {
out := make([]fksinks.PostgresWrite, 0)
for _, w := range writes {
if w.Table == table {
out = append(out, w)
}
}
return out
}
func assertAllWritesIncludeAllColumns(t *testing.T, writes []fksinks.PostgresWrite) {
t.Helper()
colCounts := tableColumnCounts()
@@ -516,6 +809,23 @@ func tableColumnCounts() map[string]int {
return m
}
func validTestOutlook() model.WeatherOutlook {
return model.WeatherOutlook{
ID: "outlook-1",
Provider: "spc",
Product: "convective",
Day: 1,
OutlookType: "categorical",
Label: "SLGT",
ValidFrom: time.Date(2026, 6, 11, 13, 0, 0, 0, time.UTC),
ValidTo: time.Date(2026, 6, 12, 12, 0, 0, 0, time.UTC),
IssuedAt: time.Date(2026, 6, 11, 19, 45, 0, 0, time.UTC),
ExpiresAt: time.Date(2026, 6, 12, 12, 0, 0, 0, time.UTC),
ContainsLocation: true,
Geometry: json.RawMessage(`{"type":"Polygon","coordinates":[[[-91,38],[-90,38],[-90,39],[-91,39],[-91,38]]]}`),
}
}
func wmoCodePtr(v model.WMOCode) *model.WMOCode {
out := v
return &out

View File

@@ -18,6 +18,7 @@ const (
tableAlertReferences = "alert_references"
tableOutlookRuns = "outlook_runs"
tableOutlooks = "outlooks"
tableOutlookDiscussions = "outlook_discussions"
)
// PostgresSchema returns weatherfeeder's Postgres schema definition.
@@ -26,13 +27,7 @@ func PostgresSchema() fksinks.PostgresSchema {
Tables: []fksinks.PostgresTable{
{
Name: tableObservations,
Columns: []fksinks.PostgresColumn{
{Name: "event_id", Type: "TEXT", Nullable: false},
{Name: "event_kind", Type: "TEXT", Nullable: false},
{Name: "event_source", Type: "TEXT", Nullable: false},
{Name: "event_schema", Type: "TEXT", Nullable: false},
{Name: "event_emitted_at", Type: "TIMESTAMPTZ", Nullable: false},
{Name: "event_effective_at", Type: "TIMESTAMPTZ", Nullable: true},
Columns: parentEnvelopeColumns([]fksinks.PostgresColumn{
{Name: "station_id", Type: "TEXT", Nullable: true},
{Name: "station_name", Type: "TEXT", Nullable: true},
{Name: "observed_at", Type: "TIMESTAMPTZ", Nullable: false},
@@ -48,7 +43,7 @@ func PostgresSchema() fksinks.PostgresSchema {
{Name: "visibility_meters", Type: "DOUBLE PRECISION", Nullable: true},
{Name: "relative_humidity_percent", Type: "DOUBLE PRECISION", Nullable: true},
{Name: "apparent_temperature_c", Type: "DOUBLE PRECISION", Nullable: true},
},
}...),
PrimaryKey: []string{"event_id"},
PruneColumn: "observed_at",
Indexes: []fksinks.PostgresIndex{
@@ -73,13 +68,7 @@ func PostgresSchema() fksinks.PostgresSchema {
},
{
Name: tableForecasts,
Columns: []fksinks.PostgresColumn{
{Name: "event_id", Type: "TEXT", Nullable: false},
{Name: "event_kind", Type: "TEXT", Nullable: false},
{Name: "event_source", Type: "TEXT", Nullable: false},
{Name: "event_schema", Type: "TEXT", Nullable: false},
{Name: "event_emitted_at", Type: "TIMESTAMPTZ", Nullable: false},
{Name: "event_effective_at", Type: "TIMESTAMPTZ", Nullable: true},
Columns: parentEnvelopeColumns([]fksinks.PostgresColumn{
{Name: "location_id", Type: "TEXT", Nullable: true},
{Name: "location_name", Type: "TEXT", Nullable: true},
{Name: "issued_at", Type: "TIMESTAMPTZ", Nullable: false},
@@ -89,7 +78,7 @@ func PostgresSchema() fksinks.PostgresSchema {
{Name: "longitude", Type: "DOUBLE PRECISION", Nullable: true},
{Name: "elevation_meters", Type: "DOUBLE PRECISION", Nullable: true},
{Name: "period_count", Type: "INTEGER", Nullable: false},
},
}...),
PrimaryKey: []string{"event_id"},
PruneColumn: "issued_at",
Indexes: []fksinks.PostgresIndex{
@@ -137,13 +126,7 @@ func PostgresSchema() fksinks.PostgresSchema {
},
{
Name: tableForecastDiscussions,
Columns: []fksinks.PostgresColumn{
{Name: "event_id", Type: "TEXT", Nullable: false},
{Name: "event_kind", Type: "TEXT", Nullable: false},
{Name: "event_source", Type: "TEXT", Nullable: false},
{Name: "event_schema", Type: "TEXT", Nullable: false},
{Name: "event_emitted_at", Type: "TIMESTAMPTZ", Nullable: false},
{Name: "event_effective_at", Type: "TIMESTAMPTZ", Nullable: true},
Columns: parentEnvelopeColumns([]fksinks.PostgresColumn{
{Name: "office_id", Type: "TEXT", Nullable: true},
{Name: "office_name", Type: "TEXT", Nullable: true},
{Name: "issued_at", Type: "TIMESTAMPTZ", Nullable: false},
@@ -156,7 +139,7 @@ func PostgresSchema() fksinks.PostgresSchema {
{Name: "long_term_issued_at", Type: "TIMESTAMPTZ", Nullable: true},
{Name: "long_term_text", Type: "TEXT", Nullable: true},
{Name: "key_message_count", Type: "INTEGER", Nullable: false},
},
}...),
PrimaryKey: []string{"event_id"},
PruneColumn: "issued_at",
Indexes: []fksinks.PostgresIndex{
@@ -180,17 +163,11 @@ func PostgresSchema() fksinks.PostgresSchema {
},
{
Name: tableWeatherStoryRuns,
Columns: []fksinks.PostgresColumn{
{Name: "event_id", Type: "TEXT", Nullable: false},
{Name: "event_kind", Type: "TEXT", Nullable: false},
{Name: "event_source", Type: "TEXT", Nullable: false},
{Name: "event_schema", Type: "TEXT", Nullable: false},
{Name: "event_emitted_at", Type: "TIMESTAMPTZ", Nullable: false},
{Name: "event_effective_at", Type: "TIMESTAMPTZ", Nullable: true},
Columns: parentEnvelopeColumns([]fksinks.PostgresColumn{
{Name: "office_id", Type: "TEXT", Nullable: true},
{Name: "as_of", Type: "TIMESTAMPTZ", Nullable: false},
{Name: "story_count", Type: "INTEGER", Nullable: false},
},
}...),
PrimaryKey: []string{"event_id"},
PruneColumn: "as_of",
Indexes: []fksinks.PostgresIndex{
@@ -225,20 +202,14 @@ func PostgresSchema() fksinks.PostgresSchema {
},
{
Name: tableAlertRuns,
Columns: []fksinks.PostgresColumn{
{Name: "event_id", Type: "TEXT", Nullable: false},
{Name: "event_kind", Type: "TEXT", Nullable: false},
{Name: "event_source", Type: "TEXT", Nullable: false},
{Name: "event_schema", Type: "TEXT", Nullable: false},
{Name: "event_emitted_at", Type: "TIMESTAMPTZ", Nullable: false},
{Name: "event_effective_at", Type: "TIMESTAMPTZ", Nullable: true},
Columns: parentEnvelopeColumns([]fksinks.PostgresColumn{
{Name: "location_id", Type: "TEXT", Nullable: true},
{Name: "location_name", Type: "TEXT", Nullable: true},
{Name: "as_of", Type: "TIMESTAMPTZ", Nullable: false},
{Name: "latitude", Type: "DOUBLE PRECISION", Nullable: true},
{Name: "longitude", Type: "DOUBLE PRECISION", Nullable: true},
{Name: "alert_count", Type: "INTEGER", Nullable: false},
},
}...),
PrimaryKey: []string{"event_id"},
PruneColumn: "as_of",
Indexes: []fksinks.PostgresIndex{
@@ -267,6 +238,7 @@ func PostgresSchema() fksinks.PostgresSchema {
{Name: "sent", Type: "TIMESTAMPTZ", Nullable: true},
{Name: "effective", Type: "TIMESTAMPTZ", Nullable: true},
{Name: "onset", Type: "TIMESTAMPTZ", Nullable: true},
{Name: "ends", Type: "TIMESTAMPTZ", Nullable: true},
{Name: "expires", Type: "TIMESTAMPTZ", Nullable: true},
{Name: "area_description", Type: "TEXT", Nullable: true},
{Name: "sender_name", Type: "TEXT", Nullable: true},
@@ -301,13 +273,7 @@ func PostgresSchema() fksinks.PostgresSchema {
},
{
Name: tableOutlookRuns,
Columns: []fksinks.PostgresColumn{
{Name: "event_id", Type: "TEXT", Nullable: false},
{Name: "event_kind", Type: "TEXT", Nullable: false},
{Name: "event_source", Type: "TEXT", Nullable: false},
{Name: "event_schema", Type: "TEXT", Nullable: false},
{Name: "event_emitted_at", Type: "TIMESTAMPTZ", Nullable: false},
{Name: "event_effective_at", Type: "TIMESTAMPTZ", Nullable: true},
Columns: parentEnvelopeColumns([]fksinks.PostgresColumn{
{Name: "location_id", Type: "TEXT", Nullable: true},
{Name: "location_name", Type: "TEXT", Nullable: true},
{Name: "latitude", Type: "DOUBLE PRECISION", Nullable: true},
@@ -315,7 +281,8 @@ func PostgresSchema() fksinks.PostgresSchema {
{Name: "as_of", Type: "TIMESTAMPTZ", Nullable: false},
{Name: "issued_at", Type: "TIMESTAMPTZ", Nullable: true},
{Name: "outlook_count", Type: "INTEGER", Nullable: false},
},
{Name: "discussion_count", Type: "INTEGER", Nullable: false},
}...),
PrimaryKey: []string{"event_id"},
PruneColumn: "as_of",
Indexes: []fksinks.PostgresIndex{
@@ -329,6 +296,8 @@ func PostgresSchema() fksinks.PostgresSchema {
{Name: "run_event_id", Type: "TEXT REFERENCES outlook_runs(event_id) ON DELETE CASCADE", Nullable: false},
{Name: "outlook_index", Type: "INTEGER", Nullable: false},
{Name: "as_of", Type: "TIMESTAMPTZ", Nullable: false},
{Name: "outlook_id", Type: "TEXT", Nullable: false},
{Name: "provider", Type: "TEXT", Nullable: false},
{Name: "product", Type: "TEXT", Nullable: false},
{Name: "day", Type: "INTEGER", Nullable: false},
{Name: "outlook_type", Type: "TEXT", Nullable: false},
@@ -340,9 +309,6 @@ func PostgresSchema() fksinks.PostgresSchema {
{Name: "issued_at", Type: "TIMESTAMPTZ", Nullable: false},
{Name: "expires_at", Type: "TIMESTAMPTZ", Nullable: false},
{Name: "forecaster", Type: "TEXT", Nullable: true},
{Name: "headline", Type: "TEXT", Nullable: true},
{Name: "summary", Type: "TEXT", Nullable: true},
{Name: "discussion", Type: "TEXT", Nullable: true},
{Name: "source_url", Type: "TEXT", Nullable: true},
{Name: "image_url", Type: "TEXT", Nullable: true},
{Name: "contains_location", Type: "BOOLEAN", Nullable: false},
@@ -356,7 +322,38 @@ func PostgresSchema() fksinks.PostgresSchema {
{Name: "idx_wf_outlooks_valid", Columns: []string{"valid_from", "valid_to"}},
},
},
{
Name: tableOutlookDiscussions,
Columns: []fksinks.PostgresColumn{
{Name: "run_event_id", Type: "TEXT REFERENCES outlook_runs(event_id) ON DELETE CASCADE", Nullable: false},
{Name: "discussion_index", Type: "INTEGER", Nullable: false},
{Name: "as_of", Type: "TIMESTAMPTZ", Nullable: false},
{Name: "day", Type: "INTEGER", Nullable: false},
{Name: "headline", Type: "TEXT", Nullable: true},
{Name: "summary", Type: "TEXT", Nullable: true},
{Name: "discussion", Type: "TEXT", Nullable: true},
{Name: "updated_at", Type: "TIMESTAMPTZ", Nullable: true},
},
PrimaryKey: []string{"run_event_id", "discussion_index"},
PruneColumn: "as_of",
Indexes: []fksinks.PostgresIndex{
{Name: "idx_wf_outlook_discussions_day_as_of", Columns: []string{"day", "as_of"}},
{Name: "idx_wf_outlook_discussions_run_day", Columns: []string{"run_event_id", "day"}, Unique: true},
},
},
},
MapEvent: mapPostgresEvent,
}
}
func parentEnvelopeColumns(extra ...fksinks.PostgresColumn) []fksinks.PostgresColumn {
columns := []fksinks.PostgresColumn{
{Name: "event_id", Type: "TEXT", Nullable: false},
{Name: "event_kind", Type: "TEXT", Nullable: false},
{Name: "event_source", Type: "TEXT", Nullable: false},
{Name: "event_schema", Type: "TEXT", Nullable: false},
{Name: "event_emitted_at", Type: "TIMESTAMPTZ", Nullable: false},
{Name: "event_effective_at", Type: "TIMESTAMPTZ", Nullable: true},
}
return append(columns, extra...)
}

View File

@@ -1,8 +1,11 @@
package postgres
import (
"reflect"
"strings"
"testing"
fksinks "gitea.maximumdirect.net/ejr/feedkit/sinks"
)
func TestWeatherPostgresSchemaShape(t *testing.T) {
@@ -25,6 +28,7 @@ func TestWeatherPostgresSchemaShape(t *testing.T) {
tableAlertReferences: true,
tableOutlookRuns: true,
tableOutlooks: true,
tableOutlookDiscussions: true,
}
if len(s.Tables) != len(wantTables) {
@@ -50,7 +54,7 @@ func TestWeatherPostgresSchemaShape(t *testing.T) {
func TestWeatherPostgresSchemaIncludesOutlookTables(t *testing.T) {
runColumns := columnsForTable(t, tableOutlookRuns)
for _, col := range []string{"event_id", "event_kind", "event_source", "event_schema", "event_emitted_at", "event_effective_at", "location_id", "location_name", "latitude", "longitude", "as_of", "issued_at", "outlook_count"} {
for _, col := range []string{"event_id", "event_kind", "event_source", "event_schema", "event_emitted_at", "event_effective_at", "location_id", "location_name", "latitude", "longitude", "as_of", "issued_at", "outlook_count", "discussion_count"} {
if !runColumns[col] {
t.Fatalf("%s missing %s column", tableOutlookRuns, col)
}
@@ -60,15 +64,40 @@ func TestWeatherPostgresSchemaIncludesOutlookTables(t *testing.T) {
assertTableIndex(t, tableOutlookRuns, "idx_wf_outlook_run_as_of", []string{"as_of"})
outlookColumns := columnsForTable(t, tableOutlooks)
for _, col := range []string{"run_event_id", "outlook_index", "as_of", "product", "day", "outlook_type", "label", "label_text", "severity_rank", "valid_from", "valid_to", "issued_at", "expires_at", "forecaster", "headline", "summary", "discussion", "source_url", "image_url", "contains_location", "geometry_json"} {
for _, col := range []string{"run_event_id", "outlook_index", "as_of", "outlook_id", "provider", "product", "day", "outlook_type", "label", "label_text", "severity_rank", "valid_from", "valid_to", "issued_at", "expires_at", "forecaster", "source_url", "image_url", "contains_location", "geometry_json"} {
if !outlookColumns[col] {
t.Fatalf("%s missing %s column", tableOutlooks, col)
}
}
for _, col := range []string{"headline", "summary", "discussion"} {
if outlookColumns[col] {
t.Fatalf("%s still includes legacy %s column", tableOutlooks, col)
}
}
assertTablePrimaryKey(t, tableOutlooks, []string{"run_event_id", "outlook_index"})
assertTableIndex(t, tableOutlooks, "idx_wf_outlooks_contains_valid", []string{"contains_location", "valid_from", "valid_to"})
assertTableIndex(t, tableOutlooks, "idx_wf_outlooks_day_type_label", []string{"day", "outlook_type", "label"})
assertTableIndex(t, tableOutlooks, "idx_wf_outlooks_valid", []string{"valid_from", "valid_to"})
discussionColumns := columnsForTable(t, tableOutlookDiscussions)
for _, col := range []string{"run_event_id", "discussion_index", "as_of", "day", "headline", "summary", "discussion", "updated_at"} {
if !discussionColumns[col] {
t.Fatalf("%s missing %s column", tableOutlookDiscussions, col)
}
}
assertTablePrimaryKey(t, tableOutlookDiscussions, []string{"run_event_id", "discussion_index"})
assertTablePruneColumn(t, tableOutlookDiscussions, "as_of")
assertTableIndex(t, tableOutlookDiscussions, "idx_wf_outlook_discussions_day_as_of", []string{"day", "as_of"})
assertTableUniqueIndex(t, tableOutlookDiscussions, "idx_wf_outlook_discussions_run_day", []string{"run_event_id", "day"})
}
func TestWeatherPostgresSchemaIncludesAlertEndsColumn(t *testing.T) {
alertColumns := columnsForTable(t, tableAlerts)
for _, col := range []string{"run_event_id", "alert_index", "as_of", "alert_id", "onset", "ends", "expires"} {
if !alertColumns[col] {
t.Fatalf("%s missing %s column", tableAlerts, col)
}
}
}
func TestWeatherPostgresSchemaIncludesWeatherStoryColumns(t *testing.T) {
@@ -88,53 +117,102 @@ func TestWeatherPostgresSchemaIncludesWeatherStoryColumns(t *testing.T) {
}
}
func TestWeatherPostgresSchemaParentTablesStartWithEnvelopeColumns(t *testing.T) {
for _, table := range []string{
tableObservations,
tableForecasts,
tableForecastDiscussions,
tableWeatherStoryRuns,
tableAlertRuns,
tableOutlookRuns,
} {
t.Run(table, func(t *testing.T) {
columns := orderedColumnsForTable(t, table)
want := parentEnvelopeColumns()
if len(columns) < len(want) {
t.Fatalf("%s has %d columns, want at least %d", table, len(columns), len(want))
}
if !reflect.DeepEqual(columns[:len(want)], want) {
t.Fatalf("%s envelope prefix = %#v, want %#v", table, columns[:len(want)], want)
}
})
}
}
func assertTablePrimaryKey(t *testing.T, table string, want []string) {
t.Helper()
for _, tbl := range PostgresSchema().Tables {
if tbl.Name != table {
continue
}
if strings.Join(tbl.PrimaryKey, ",") != strings.Join(want, ",") {
t.Fatalf("%s primary key = %#v, want %#v", table, tbl.PrimaryKey, want)
}
return
tbl := tableByName(t, table)
if strings.Join(tbl.PrimaryKey, ",") != strings.Join(want, ",") {
t.Fatalf("%s primary key = %#v, want %#v", table, tbl.PrimaryKey, want)
}
}
func assertTablePruneColumn(t *testing.T, table string, want string) {
t.Helper()
tbl := tableByName(t, table)
if tbl.PruneColumn != want {
t.Fatalf("%s prune column = %q, want %q", table, tbl.PruneColumn, want)
}
t.Fatalf("missing table %q", table)
}
func assertTableIndex(t *testing.T, table string, name string, want []string) {
t.Helper()
for _, tbl := range PostgresSchema().Tables {
if tbl.Name != table {
continue
}
for _, idx := range tbl.Indexes {
if idx.Name == name {
if strings.Join(idx.Columns, ",") != strings.Join(want, ",") {
t.Fatalf("%s index %s columns = %#v, want %#v", table, name, idx.Columns, want)
}
return
assertTableIndexWithUnique(t, table, name, want, false)
}
func assertTableUniqueIndex(t *testing.T, table string, name string, want []string) {
t.Helper()
assertTableIndexWithUnique(t, table, name, want, true)
}
func assertTableIndexWithUnique(t *testing.T, table string, name string, want []string, unique bool) {
t.Helper()
tbl := tableByName(t, table)
for _, idx := range tbl.Indexes {
if idx.Name == name {
if strings.Join(idx.Columns, ",") != strings.Join(want, ",") {
t.Fatalf("%s index %s columns = %#v, want %#v", table, name, idx.Columns, want)
}
if idx.Unique != unique {
t.Fatalf("%s index %s unique = %v, want %v", table, name, idx.Unique, unique)
}
return
}
}
t.Fatalf("%s missing index %s", table, name)
}
func tableByName(t *testing.T, table string) fksinks.PostgresTable {
t.Helper()
for _, tbl := range PostgresSchema().Tables {
if tbl.Name == table {
return tbl
}
t.Fatalf("%s missing index %s", table, name)
}
t.Fatalf("missing table %q", table)
return fksinks.PostgresTable{}
}
func orderedColumnsForTable(t *testing.T, table string) []fksinks.PostgresColumn {
t.Helper()
schema := PostgresSchema()
for _, tbl := range schema.Tables {
if tbl.Name == table {
return tbl.Columns
}
}
t.Fatalf("missing table %q", table)
return nil
}
func columnsForTable(t *testing.T, table string) map[string]bool {
t.Helper()
schema := PostgresSchema()
for _, tbl := range schema.Tables {
if tbl.Name != table {
continue
}
cols := make(map[string]bool, len(tbl.Columns))
for _, col := range tbl.Columns {
cols[col.Name] = true
}
return cols
ordered := orderedColumnsForTable(t, table)
cols := make(map[string]bool, len(ordered))
for _, col := range ordered {
cols[col.Name] = true
}
t.Fatalf("missing table %q", table)
return nil
return cols
}

View File

@@ -16,20 +16,20 @@ type pollDriverRegistration struct {
}
var pollDriverRegistrations = []pollDriverRegistration{
{driver: "nws_observation", factory: func(cfg config.SourceConfig) (fksource.PollSource, error) { return nws.NewObservationSource(cfg) }},
{driver: "nws_alerts", factory: func(cfg config.SourceConfig) (fksource.PollSource, error) { return nws.NewAlertsSource(cfg) }},
{driver: "nws_forecast_hourly", factory: func(cfg config.SourceConfig) (fksource.PollSource, error) { return nws.NewHourlyForecastSource(cfg) }},
{driver: "nws_forecast_narrative", factory: func(cfg config.SourceConfig) (fksource.PollSource, error) { return nws.NewNarrativeForecastSource(cfg) }},
{driver: "nws_forecast_discussion", factory: func(cfg config.SourceConfig) (fksource.PollSource, error) {
{driver: nws.DriverObservation, factory: func(cfg config.SourceConfig) (fksource.PollSource, error) { return nws.NewObservationSource(cfg) }},
{driver: nws.DriverAlerts, factory: func(cfg config.SourceConfig) (fksource.PollSource, error) { return nws.NewAlertsSource(cfg) }},
{driver: nws.DriverForecastHourly, factory: func(cfg config.SourceConfig) (fksource.PollSource, error) { return nws.NewHourlyForecastSource(cfg) }},
{driver: nws.DriverForecastNarrative, factory: func(cfg config.SourceConfig) (fksource.PollSource, error) { return nws.NewNarrativeForecastSource(cfg) }},
{driver: nws.DriverForecastDiscussion, factory: func(cfg config.SourceConfig) (fksource.PollSource, error) {
return nws.NewForecastDiscussionSource(cfg)
}},
{driver: "nws_weatherstories", factory: func(cfg config.SourceConfig) (fksource.PollSource, error) { return nws.NewWeatherStoriesSource(cfg) }},
{driver: "openmeteo_observation", factory: func(cfg config.SourceConfig) (fksource.PollSource, error) { return openmeteo.NewObservationSource(cfg) }},
{driver: "openmeteo_forecast", factory: func(cfg config.SourceConfig) (fksource.PollSource, error) { return openmeteo.NewForecastSource(cfg) }},
{driver: "openweather_observation", factory: func(cfg config.SourceConfig) (fksource.PollSource, error) {
{driver: nws.DriverWeatherStories, factory: func(cfg config.SourceConfig) (fksource.PollSource, error) { return nws.NewWeatherStoriesSource(cfg) }},
{driver: openmeteo.DriverObservation, factory: func(cfg config.SourceConfig) (fksource.PollSource, error) { return openmeteo.NewObservationSource(cfg) }},
{driver: openmeteo.DriverForecast, factory: func(cfg config.SourceConfig) (fksource.PollSource, error) { return openmeteo.NewForecastSource(cfg) }},
{driver: openweather.DriverObservation, factory: func(cfg config.SourceConfig) (fksource.PollSource, error) {
return openweather.NewObservationSource(cfg)
}},
{driver: "spc_convective_outlook", factory: func(cfg config.SourceConfig) (fksource.PollSource, error) {
{driver: spc.DriverConvectiveOutlook, factory: func(cfg config.SourceConfig) (fksource.PollSource, error) {
return spc.NewConvectiveOutlookSource(cfg)
}},
}

View File

@@ -6,57 +6,29 @@ import (
"gitea.maximumdirect.net/ejr/feedkit/config"
fksource "gitea.maximumdirect.net/ejr/feedkit/sources"
"gitea.maximumdirect.net/ejr/weatherfeeder/internal/sources/openweather"
"gitea.maximumdirect.net/ejr/weatherfeeder/internal/sources/spc"
)
func TestRegisterBuiltinsRegistersNWSHourlyForecastDriver(t *testing.T) {
func TestRegisterBuiltinsRegistersCurrentPollDrivers(t *testing.T) {
reg := fksource.NewRegistry()
RegisterBuiltins(reg)
in, err := reg.BuildInput(sourceConfigForDriver("nws_forecast_hourly"))
if err != nil {
t.Fatalf("BuildInput(nws_forecast_hourly) error = %v", err)
if len(pollDriverRegistrations) == 0 {
t.Fatalf("pollDriverRegistrations is empty")
}
if _, ok := in.(fksource.PollSource); !ok {
t.Fatalf("BuildInput(nws_forecast_hourly) type = %T, want PollSource", in)
}
}
func TestRegisterBuiltinsRegistersNWSNarrativeForecastDriver(t *testing.T) {
reg := fksource.NewRegistry()
RegisterBuiltins(reg)
in, err := reg.BuildInput(sourceConfigForDriver("nws_forecast_narrative"))
if err != nil {
t.Fatalf("BuildInput(nws_forecast_narrative) error = %v", err)
}
if _, ok := in.(fksource.PollSource); !ok {
t.Fatalf("BuildInput(nws_forecast_narrative) type = %T, want PollSource", in)
}
}
func TestRegisterBuiltinsRegistersNWSForecastDiscussionDriver(t *testing.T) {
reg := fksource.NewRegistry()
RegisterBuiltins(reg)
in, err := reg.BuildInput(sourceConfigForDriver("nws_forecast_discussion"))
if err != nil {
t.Fatalf("BuildInput(nws_forecast_discussion) error = %v", err)
}
if _, ok := in.(fksource.PollSource); !ok {
t.Fatalf("BuildInput(nws_forecast_discussion) type = %T, want PollSource", in)
}
}
func TestRegisterBuiltinsRegistersNWSWeatherStoriesDriver(t *testing.T) {
reg := fksource.NewRegistry()
RegisterBuiltins(reg)
in, err := reg.BuildInput(sourceConfigForDriver("nws_weatherstories"))
if err != nil {
t.Fatalf("BuildInput(nws_weatherstories) error = %v", err)
}
if _, ok := in.(fksource.PollSource); !ok {
t.Fatalf("BuildInput(nws_weatherstories) type = %T, want PollSource", in)
for _, tt := range pollDriverRegistrations {
tt := tt
t.Run(tt.driver, func(t *testing.T) {
in, err := reg.BuildInput(sourceConfigForDriver(tt.driver))
if err != nil {
t.Fatalf("BuildInput(%q) error = %v", tt.driver, err)
}
if _, ok := in.(fksource.PollSource); !ok {
t.Fatalf("BuildInput(%q) type = %T, want PollSource", tt.driver, in)
}
})
}
}
@@ -73,44 +45,16 @@ func TestRegisterBuiltinsDoesNotRegisterLegacyNWSForecastDriver(t *testing.T) {
}
}
func TestRegisterBuiltinsRegistersAllCurrentDrivers(t *testing.T) {
reg := fksource.NewRegistry()
RegisterBuiltins(reg)
drivers := []string{
"nws_observation",
"nws_alerts",
"nws_forecast_hourly",
"nws_forecast_narrative",
"nws_forecast_discussion",
"nws_weatherstories",
"openmeteo_observation",
"openmeteo_forecast",
"openweather_observation",
"spc_convective_outlook",
}
for _, driver := range drivers {
in, err := reg.BuildInput(sourceConfigForDriver(driver))
if err != nil {
t.Fatalf("BuildInput(%s) error = %v", driver, err)
}
if _, ok := in.(fksource.PollSource); !ok {
t.Fatalf("BuildInput(%s) type = %T, want PollSource", driver, in)
}
}
}
func sourceConfigForDriver(driver string) config.SourceConfig {
url := "https://example.invalid"
if driver == "openweather_observation" {
if driver == openweather.DriverObservation {
url = "https://example.invalid?units=metric"
}
params := map[string]any{
"url": url,
"user_agent": "test-agent",
}
if driver == "spc_convective_outlook" {
if driver == spc.DriverConvectiveOutlook {
params["latitude"] = 38.6239
params["longitude"] = -90.3571
}

View File

@@ -0,0 +1,32 @@
package sources
import (
"os"
"strings"
"testing"
)
func TestDocumentedRegisteredSourceDrivers(t *testing.T) {
docs := map[string]string{
"docs/config.md": readDoc(t, "../../docs/config.md"),
"docs/internal/sources.md": readDoc(t, "../../docs/internal/sources.md"),
}
for _, reg := range pollDriverRegistrations {
for path, doc := range docs {
if !strings.Contains(doc, reg.driver) {
t.Fatalf("%s missing source driver %q", path, reg.driver)
}
}
}
}
func readDoc(t *testing.T, path string) string {
t.Helper()
raw, err := os.ReadFile(path)
if err != nil {
t.Fatalf("ReadFile(%s) error = %v", path, err)
}
return string(raw)
}

View File

@@ -0,0 +1,58 @@
package httpconfig
import (
"fmt"
"strings"
"time"
"gitea.maximumdirect.net/ejr/feedkit/config"
"gitea.maximumdirect.net/ejr/feedkit/transport"
)
// Settings contains common HTTP client config for sources that fetch multiple documents.
type Settings struct {
Name string
UserAgent string
Timeout time.Duration
BodyLimitBytes int64
}
func Parse(driver string, cfg config.SourceConfig) (Settings, error) {
name := strings.TrimSpace(cfg.Name)
if name == "" {
return Settings{}, fmt.Errorf("%s: name is required", driver)
}
if cfg.Params == nil {
return Settings{}, fmt.Errorf("%s %q: params are required", driver, name)
}
userAgent, ok := cfg.ParamString("user_agent", "userAgent")
if !ok {
return Settings{}, fmt.Errorf("%s %q: params.user_agent is required", driver, name)
}
timeout := transport.DefaultHTTPTimeout
if _, exists := cfg.Params["http_timeout"]; exists {
var ok bool
timeout, ok = cfg.ParamDuration("http_timeout")
if !ok || timeout <= 0 {
return Settings{}, fmt.Errorf("source %q: params.http_timeout must be a positive duration", name)
}
}
bodyLimit := transport.DefaultHTTPResponseBodyLimitBytes
if _, exists := cfg.Params["http_response_body_limit_bytes"]; exists {
rawLimit, ok := cfg.ParamInt("http_response_body_limit_bytes")
if !ok || rawLimit <= 0 {
return Settings{}, fmt.Errorf("source %q: params.http_response_body_limit_bytes must be a positive integer", name)
}
bodyLimit = int64(rawLimit)
}
return Settings{
Name: name,
UserAgent: userAgent,
Timeout: timeout,
BodyLimitBytes: bodyLimit,
}, nil
}

View File

@@ -0,0 +1,115 @@
package httpconfig
import (
"strings"
"testing"
"time"
"gitea.maximumdirect.net/ejr/feedkit/config"
"gitea.maximumdirect.net/ejr/feedkit/transport"
)
func TestParseUsesRequiredValuesAndDefaults(t *testing.T) {
got, err := Parse("test_driver", config.SourceConfig{
Name: " test-source ",
Params: map[string]any{
"user_agent": "test-agent",
},
})
if err != nil {
t.Fatalf("Parse() error = %v", err)
}
if got.Name != "test-source" {
t.Fatalf("Name = %q, want test-source", got.Name)
}
if got.UserAgent != "test-agent" {
t.Fatalf("UserAgent = %q, want test-agent", got.UserAgent)
}
if got.Timeout != transport.DefaultHTTPTimeout {
t.Fatalf("Timeout = %s, want %s", got.Timeout, transport.DefaultHTTPTimeout)
}
if got.BodyLimitBytes != transport.DefaultHTTPResponseBodyLimitBytes {
t.Fatalf("BodyLimitBytes = %d, want %d", got.BodyLimitBytes, transport.DefaultHTTPResponseBodyLimitBytes)
}
}
func TestParseUsesAliasesAndOverrides(t *testing.T) {
got, err := Parse("test_driver", config.SourceConfig{
Name: "test-source",
Params: map[string]any{
"userAgent": "test-agent",
"http_timeout": "2s",
"http_response_body_limit_bytes": 2048,
},
})
if err != nil {
t.Fatalf("Parse() error = %v", err)
}
if got.UserAgent != "test-agent" {
t.Fatalf("UserAgent = %q, want test-agent", got.UserAgent)
}
if got.Timeout != 2*time.Second {
t.Fatalf("Timeout = %s, want 2s", got.Timeout)
}
if got.BodyLimitBytes != 2048 {
t.Fatalf("BodyLimitBytes = %d, want 2048", got.BodyLimitBytes)
}
}
func TestParseRejectsInvalidConfig(t *testing.T) {
tests := []struct {
name string
cfg config.SourceConfig
wantErr string
}{
{
name: "missing name",
cfg: config.SourceConfig{Params: map[string]any{"user_agent": "test-agent"}},
wantErr: "test_driver: name is required",
},
{
name: "missing params",
cfg: config.SourceConfig{Name: "test-source"},
wantErr: `test_driver "test-source": params are required`,
},
{
name: "missing user agent",
cfg: config.SourceConfig{Name: "test-source", Params: map[string]any{}},
wantErr: `test_driver "test-source": params.user_agent is required`,
},
{
name: "invalid timeout",
cfg: config.SourceConfig{
Name: "test-source",
Params: map[string]any{
"user_agent": "test-agent",
"http_timeout": "0s",
},
},
wantErr: `source "test-source": params.http_timeout must be a positive duration`,
},
{
name: "invalid body limit",
cfg: config.SourceConfig{
Name: "test-source",
Params: map[string]any{
"user_agent": "test-agent",
"http_response_body_limit_bytes": 0,
},
},
wantErr: `source "test-source": params.http_response_body_limit_bytes must be a positive integer`,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
_, err := Parse("test_driver", tt.cfg)
if err == nil {
t.Fatalf("Parse() error = nil, want %q", tt.wantErr)
}
if !strings.Contains(err.Error(), tt.wantErr) {
t.Fatalf("Parse() error = %q, want %q", err, tt.wantErr)
}
})
}
}

View File

@@ -26,10 +26,8 @@ type AlertsSource struct {
}
func NewAlertsSource(cfg config.SourceConfig) (*AlertsSource, error) {
const driver = "nws_alerts"
// NWS alerts responses are GeoJSON-ish; allow fallback to plain JSON as well.
hs, err := fksources.NewHTTPSource(driver, cfg, "application/geo+json, application/json")
hs, err := fksources.NewHTTPSource(DriverAlerts, cfg, "application/geo+json, application/json")
if err != nil {
return nil, err
}
@@ -40,7 +38,7 @@ func NewAlertsSource(cfg config.SourceConfig) (*AlertsSource, error) {
func (s *AlertsSource) Name() string { return s.http.Name }
// Kinds is used for routing/policy.
func (s *AlertsSource) Kinds() []event.Kind { return []event.Kind{event.Kind("alert")} }
func (s *AlertsSource) Kinds() []event.Kind { return []event.Kind{event.Kind(standards.KindAlert)} }
func (s *AlertsSource) Poll(ctx context.Context) ([]event.Event, error) {
raw, meta, changed, err := s.fetchRaw(ctx)
@@ -71,7 +69,7 @@ func (s *AlertsSource) Poll(ctx context.Context) ([]event.Event, error) {
eventID := fksources.DefaultEventID("", s.http.Name, effectiveAt, emittedAt)
return fksources.SingleEvent(
event.Kind("alert"),
event.Kind(standards.KindAlert),
s.http.Name,
standards.SchemaRawNWSAlertsV1,
eventID,

View File

@@ -0,0 +1,11 @@
package nws
// Source driver strings registered by weatherfeeder for NWS sources.
const (
DriverObservation = "nws_observation"
DriverAlerts = "nws_alerts"
DriverForecastHourly = "nws_forecast_hourly"
DriverForecastNarrative = "nws_forecast_narrative"
DriverForecastDiscussion = "nws_forecast_discussion"
DriverWeatherStories = "nws_weatherstories"
)

View File

@@ -10,6 +10,7 @@ import (
"gitea.maximumdirect.net/ejr/feedkit/event"
fksources "gitea.maximumdirect.net/ejr/feedkit/sources"
nwscommon "gitea.maximumdirect.net/ejr/weatherfeeder/internal/providers/nws"
"gitea.maximumdirect.net/ejr/weatherfeeder/standards"
)
const nwsForecastAccept = "application/geo+json, application/json"
@@ -44,7 +45,9 @@ func newForecastSource(cfg config.SourceConfig, driver, rawSchema string) (*fore
func (s *forecastSource) Name() string { return s.http.Name }
func (s *forecastSource) Kinds() []event.Kind { return []event.Kind{event.Kind("forecast")} }
func (s *forecastSource) Kinds() []event.Kind {
return []event.Kind{event.Kind(standards.KindForecast)}
}
func (s *forecastSource) Poll(ctx context.Context) ([]event.Event, error) {
raw, meta, changed, err := s.fetchRaw(ctx)
@@ -69,7 +72,7 @@ func (s *forecastSource) Poll(ctx context.Context) ([]event.Event, error) {
eventID := fksources.DefaultEventID("", s.http.Name, effectiveAt, emittedAt)
return fksources.SingleEvent(
event.Kind("forecast"),
event.Kind(standards.KindForecast),
s.http.Name,
s.rawSchema,
eventID,

View File

@@ -20,9 +20,7 @@ type ForecastDiscussionSource struct {
}
func NewForecastDiscussionSource(cfg config.SourceConfig) (*ForecastDiscussionSource, error) {
const driver = "nws_forecast_discussion"
hs, err := fksources.NewHTTPSource(driver, cfg, "text/html, application/xhtml+xml")
hs, err := fksources.NewHTTPSource(DriverForecastDiscussion, cfg, "text/html, application/xhtml+xml")
if err != nil {
return nil, err
}
@@ -33,7 +31,7 @@ func NewForecastDiscussionSource(cfg config.SourceConfig) (*ForecastDiscussionSo
func (s *ForecastDiscussionSource) Name() string { return s.http.Name }
func (s *ForecastDiscussionSource) Kinds() []event.Kind {
return []event.Kind{event.Kind("forecast_discussion")}
return []event.Kind{event.Kind(standards.KindForecastDiscussion)}
}
func (s *ForecastDiscussionSource) Poll(ctx context.Context) ([]event.Event, error) {
@@ -57,7 +55,7 @@ func (s *ForecastDiscussionSource) Poll(ctx context.Context) ([]event.Event, err
eventID := fksources.DefaultEventID("", s.http.Name, effectiveAt, emittedAt)
return fksources.SingleEvent(
event.Kind("forecast_discussion"),
event.Kind(standards.KindForecastDiscussion),
s.http.Name,
standards.SchemaRawNWSForecastDiscussionV1,
eventID,

View File

@@ -27,7 +27,7 @@ func TestForecastDiscussionSourcePollEmitsExpectedEvent(t *testing.T) {
if err != nil {
t.Fatalf("NewForecastDiscussionSource() error = %v", err)
}
if got := src.Kinds(); len(got) != 1 || got[0] != event.Kind("forecast_discussion") {
if got := src.Kinds(); len(got) != 1 || got[0] != event.Kind(standards.KindForecastDiscussion) {
t.Fatalf("Kinds() = %#v, want [forecast_discussion]", got)
}
@@ -40,7 +40,7 @@ func TestForecastDiscussionSourcePollEmitsExpectedEvent(t *testing.T) {
}
got := events[0]
if got.Kind != event.Kind("forecast_discussion") {
if got.Kind != event.Kind(standards.KindForecastDiscussion) {
t.Fatalf("Kind = %q, want forecast_discussion", got.Kind)
}
if got.Schema != standards.SchemaRawNWSForecastDiscussionV1 {
@@ -117,7 +117,7 @@ func TestForecastDiscussionSourcePollRejectsInvalidHTML(t *testing.T) {
func forecastDiscussionSourceConfig(url string) config.SourceConfig {
return config.SourceConfig{
Name: "test-forecast-discussion-source",
Driver: "nws_forecast_discussion",
Driver: DriverForecastDiscussion,
Mode: config.SourceModePoll,
Params: map[string]any{
"url": url,

View File

@@ -18,8 +18,7 @@ type HourlyForecastSource struct {
}
func NewHourlyForecastSource(cfg config.SourceConfig) (*HourlyForecastSource, error) {
const driver = "nws_forecast_hourly"
src, err := newForecastSource(cfg, driver, standards.SchemaRawNWSHourlyForecastV1)
src, err := newForecastSource(cfg, DriverForecastHourly, standards.SchemaRawNWSHourlyForecastV1)
if err != nil {
return nil, err
}

View File

@@ -18,8 +18,7 @@ type NarrativeForecastSource struct {
}
func NewNarrativeForecastSource(cfg config.SourceConfig) (*NarrativeForecastSource, error) {
const driver = "nws_forecast_narrative"
src, err := newForecastSource(cfg, driver, standards.SchemaRawNWSNarrativeForecastV1)
src, err := newForecastSource(cfg, DriverForecastNarrative, standards.SchemaRawNWSNarrativeForecastV1)
if err != nil {
return nil, err
}

View File

@@ -26,7 +26,7 @@ func TestForecastSourcesEmitExpectedSchemaAndPreferGeneratedAt(t *testing.T) {
}{
{
name: "hourly",
driver: "nws_forecast_hourly",
driver: DriverForecastHourly,
wantSchema: standards.SchemaRawNWSHourlyForecastV1,
newSource: func(cfg config.SourceConfig) (forecastPoller, error) {
return NewHourlyForecastSource(cfg)
@@ -34,7 +34,7 @@ func TestForecastSourcesEmitExpectedSchemaAndPreferGeneratedAt(t *testing.T) {
},
{
name: "narrative",
driver: "nws_forecast_narrative",
driver: DriverForecastNarrative,
wantSchema: standards.SchemaRawNWSNarrativeForecastV1,
newSource: func(cfg config.SourceConfig) (forecastPoller, error) {
return NewNarrativeForecastSource(cfg)
@@ -55,7 +55,7 @@ func TestForecastSourcesEmitExpectedSchemaAndPreferGeneratedAt(t *testing.T) {
}
if ks, ok := src.(interface{ Kinds() []event.Kind }); !ok {
t.Fatalf("source does not implement Kinds()")
} else if gotKinds := ks.Kinds(); len(gotKinds) != 1 || gotKinds[0] != event.Kind("forecast") {
} else if gotKinds := ks.Kinds(); len(gotKinds) != 1 || gotKinds[0] != event.Kind(standards.KindForecast) {
t.Fatalf("Kinds() = %#v, want [forecast]", gotKinds)
}
@@ -69,7 +69,7 @@ func TestForecastSourcesEmitExpectedSchemaAndPreferGeneratedAt(t *testing.T) {
if got[0].Schema != tt.wantSchema {
t.Fatalf("Poll() schema = %q, want %q", got[0].Schema, tt.wantSchema)
}
if got[0].Kind != event.Kind("forecast") {
if got[0].Kind != event.Kind(standards.KindForecast) {
t.Fatalf("Poll() kind = %q, want forecast", got[0].Kind)
}
@@ -117,7 +117,7 @@ func TestForecastSourcePollEffectiveAtFallbackOrder(t *testing.T) {
}))
defer srv.Close()
src, err := NewHourlyForecastSource(forecastSourceConfig("nws_forecast_hourly", srv.URL))
src, err := NewHourlyForecastSource(forecastSourceConfig(DriverForecastHourly, srv.URL))
if err != nil {
t.Fatalf("NewHourlyForecastSource() error = %v", err)
}
@@ -148,7 +148,7 @@ func TestForecastSourcePollMetadataDecodeFailureStillEmitsRawEvent(t *testing.T)
}))
defer srv.Close()
src, err := NewNarrativeForecastSource(forecastSourceConfig("nws_forecast_narrative", srv.URL))
src, err := NewNarrativeForecastSource(forecastSourceConfig(DriverForecastNarrative, srv.URL))
if err != nil {
t.Fatalf("NewNarrativeForecastSource() error = %v", err)
}

View File

@@ -20,9 +20,7 @@ type ObservationSource struct {
}
func NewObservationSource(cfg config.SourceConfig) (*ObservationSource, error) {
const driver = "nws_observation"
hs, err := fksources.NewHTTPSource(driver, cfg, "application/geo+json, application/json")
hs, err := fksources.NewHTTPSource(DriverObservation, cfg, "application/geo+json, application/json")
if err != nil {
return nil, err
}
@@ -32,7 +30,9 @@ func NewObservationSource(cfg config.SourceConfig) (*ObservationSource, error) {
func (s *ObservationSource) Name() string { return s.http.Name }
func (s *ObservationSource) Kinds() []event.Kind { return []event.Kind{event.Kind("observation")} }
func (s *ObservationSource) Kinds() []event.Kind {
return []event.Kind{event.Kind(standards.KindObservation)}
}
func (s *ObservationSource) Poll(ctx context.Context) ([]event.Event, error) {
raw, meta, changed, err := s.fetchRaw(ctx)
@@ -54,7 +54,7 @@ func (s *ObservationSource) Poll(ctx context.Context) ([]event.Event, error) {
eventID := fksources.DefaultEventID(meta.ID, s.http.Name, effectiveAt, emittedAt)
return fksources.SingleEvent(
event.Kind("observation"),
event.Kind(standards.KindObservation),
s.http.Name,
standards.SchemaRawNWSObservationV1,
eventID,

View File

@@ -8,6 +8,7 @@ import (
"gitea.maximumdirect.net/ejr/feedkit/config"
"gitea.maximumdirect.net/ejr/feedkit/event"
"gitea.maximumdirect.net/ejr/weatherfeeder/standards"
)
func TestObservationSourcePollReturnsNoEventsOn304(t *testing.T) {
@@ -31,7 +32,7 @@ func TestObservationSourcePollReturnsNoEventsOn304(t *testing.T) {
src, err := NewObservationSource(config.SourceConfig{
Name: "NWSObservationTest",
Driver: "nws_observation",
Driver: DriverObservation,
Mode: config.SourceModePoll,
Params: map[string]any{
"url": srv.URL,
@@ -41,7 +42,7 @@ func TestObservationSourcePollReturnsNoEventsOn304(t *testing.T) {
if err != nil {
t.Fatalf("NewObservationSource() error = %v", err)
}
if got := src.Kinds(); len(got) != 1 || got[0] != event.Kind("observation") {
if got := src.Kinds(); len(got) != 1 || got[0] != event.Kind(standards.KindObservation) {
t.Fatalf("Kinds() = %#v, want [observation]", got)
}
@@ -52,7 +53,7 @@ func TestObservationSourcePollReturnsNoEventsOn304(t *testing.T) {
if len(first) != 1 {
t.Fatalf("first Poll() len = %d, want 1", len(first))
}
if first[0].Kind != event.Kind("observation") {
if first[0].Kind != event.Kind(standards.KindObservation) {
t.Fatalf("first Poll() kind = %q", first[0].Kind)
}

View File

@@ -22,9 +22,7 @@ type WeatherStoriesSource struct {
}
func NewWeatherStoriesSource(cfg config.SourceConfig) (*WeatherStoriesSource, error) {
const driver = "nws_weatherstories"
hs, err := fksources.NewHTTPSource(driver, cfg, "application/geo+json, application/json")
hs, err := fksources.NewHTTPSource(DriverWeatherStories, cfg, "application/geo+json, application/json")
if err != nil {
return nil, err
}
@@ -35,7 +33,7 @@ func NewWeatherStoriesSource(cfg config.SourceConfig) (*WeatherStoriesSource, er
func (s *WeatherStoriesSource) Name() string { return s.http.Name }
func (s *WeatherStoriesSource) Kinds() []event.Kind {
return []event.Kind{event.Kind("weather_story")}
return []event.Kind{event.Kind(standards.KindWeatherStory)}
}
func (s *WeatherStoriesSource) Poll(ctx context.Context) ([]event.Event, error) {
@@ -61,7 +59,7 @@ func (s *WeatherStoriesSource) Poll(ctx context.Context) ([]event.Event, error)
eventID := fksources.DefaultEventID("", s.http.Name, effectiveAt, emittedAt)
return fksources.SingleEvent(
event.Kind("weather_story"),
event.Kind(standards.KindWeatherStory),
s.http.Name,
standards.SchemaRawNWSWeatherStoriesV1,
eventID,

View File

@@ -28,7 +28,7 @@ func TestWeatherStoriesSourcePollEmitsExpectedEventAndPrefersLatestUpdateTime(t
if err != nil {
t.Fatalf("NewWeatherStoriesSource() error = %v", err)
}
if got := src.Kinds(); len(got) != 1 || got[0] != event.Kind("weather_story") {
if got := src.Kinds(); len(got) != 1 || got[0] != event.Kind(standards.KindWeatherStory) {
t.Fatalf("Kinds() = %#v, want [weather_story]", got)
}
@@ -41,7 +41,7 @@ func TestWeatherStoriesSourcePollEmitsExpectedEventAndPrefersLatestUpdateTime(t
}
got := events[0]
if got.Kind != event.Kind("weather_story") {
if got.Kind != event.Kind(standards.KindWeatherStory) {
t.Fatalf("Kind = %q, want weather_story", got.Kind)
}
if got.Schema != standards.SchemaRawNWSWeatherStoriesV1 {
@@ -148,7 +148,7 @@ func TestWeatherStoriesSourcePollMetadataDecodeFailureStillEmitsRawEvent(t *test
func weatherStoriesSourceConfig(url string) config.SourceConfig {
return config.SourceConfig{
Name: "test-weatherstories-source",
Driver: "nws_weatherstories",
Driver: DriverWeatherStories,
Mode: config.SourceModePoll,
Params: map[string]any{
"url": url,

View File

@@ -0,0 +1,7 @@
package openmeteo
// Source driver strings registered by weatherfeeder for Open-Meteo sources.
const (
DriverObservation = "openmeteo_observation"
DriverForecast = "openmeteo_forecast"
)

View File

@@ -19,9 +19,7 @@ type ForecastSource struct {
}
func NewForecastSource(cfg config.SourceConfig) (*ForecastSource, error) {
const driver = "openmeteo_forecast"
hs, err := fksources.NewHTTPSource(driver, cfg, "application/json")
hs, err := fksources.NewHTTPSource(DriverForecast, cfg, "application/json")
if err != nil {
return nil, err
}
@@ -31,7 +29,9 @@ func NewForecastSource(cfg config.SourceConfig) (*ForecastSource, error) {
func (s *ForecastSource) Name() string { return s.http.Name }
func (s *ForecastSource) Kinds() []event.Kind { return []event.Kind{event.Kind("forecast")} }
func (s *ForecastSource) Kinds() []event.Kind {
return []event.Kind{event.Kind(standards.KindForecast)}
}
func (s *ForecastSource) Poll(ctx context.Context) ([]event.Event, error) {
raw, meta, changed, err := s.fetchRaw(ctx)
@@ -55,7 +55,7 @@ func (s *ForecastSource) Poll(ctx context.Context) ([]event.Event, error) {
eventID := fksources.DefaultEventID("", s.http.Name, effectiveAt, emittedAt)
return fksources.SingleEvent(
event.Kind("forecast"),
event.Kind(standards.KindForecast),
s.http.Name,
standards.SchemaRawOpenMeteoHourlyForecastV1,
eventID,

View File

@@ -19,9 +19,7 @@ type ObservationSource struct {
}
func NewObservationSource(cfg config.SourceConfig) (*ObservationSource, error) {
const driver = "openmeteo_observation"
hs, err := fksources.NewHTTPSource(driver, cfg, "application/json")
hs, err := fksources.NewHTTPSource(DriverObservation, cfg, "application/json")
if err != nil {
return nil, err
}
@@ -31,7 +29,9 @@ func NewObservationSource(cfg config.SourceConfig) (*ObservationSource, error) {
func (s *ObservationSource) Name() string { return s.http.Name }
func (s *ObservationSource) Kinds() []event.Kind { return []event.Kind{event.Kind("observation")} }
func (s *ObservationSource) Kinds() []event.Kind {
return []event.Kind{event.Kind(standards.KindObservation)}
}
func (s *ObservationSource) Poll(ctx context.Context) ([]event.Event, error) {
raw, meta, changed, err := s.fetchRaw(ctx)
@@ -52,7 +52,7 @@ func (s *ObservationSource) Poll(ctx context.Context) ([]event.Event, error) {
eventID := fksources.DefaultEventID("", s.http.Name, effectiveAt, emittedAt)
return fksources.SingleEvent(
event.Kind("observation"),
event.Kind(standards.KindObservation),
s.http.Name,
standards.SchemaRawOpenMeteoCurrentV1,
eventID,

View File

@@ -5,12 +5,13 @@ import (
"gitea.maximumdirect.net/ejr/feedkit/config"
"gitea.maximumdirect.net/ejr/feedkit/event"
"gitea.maximumdirect.net/ejr/weatherfeeder/standards"
)
func TestObservationSourceAdvertisesKinds(t *testing.T) {
src, err := NewObservationSource(config.SourceConfig{
Name: "openmeteo-observation-test",
Driver: "openmeteo_observation",
Driver: DriverObservation,
Mode: config.SourceModePoll,
Params: map[string]any{
"url": "https://example.invalid",
@@ -20,7 +21,7 @@ func TestObservationSourceAdvertisesKinds(t *testing.T) {
if err != nil {
t.Fatalf("NewObservationSource() error = %v", err)
}
if got := src.Kinds(); len(got) != 1 || got[0] != event.Kind("observation") {
if got := src.Kinds(); len(got) != 1 || got[0] != event.Kind(standards.KindObservation) {
t.Fatalf("Kinds() = %#v, want [observation]", got)
}
}
@@ -28,7 +29,7 @@ func TestObservationSourceAdvertisesKinds(t *testing.T) {
func TestForecastSourceAdvertisesKinds(t *testing.T) {
src, err := NewForecastSource(config.SourceConfig{
Name: "openmeteo-forecast-test",
Driver: "openmeteo_forecast",
Driver: DriverForecast,
Mode: config.SourceModePoll,
Params: map[string]any{
"url": "https://example.invalid",
@@ -38,7 +39,7 @@ func TestForecastSourceAdvertisesKinds(t *testing.T) {
if err != nil {
t.Fatalf("NewForecastSource() error = %v", err)
}
if got := src.Kinds(); len(got) != 1 || got[0] != event.Kind("forecast") {
if got := src.Kinds(); len(got) != 1 || got[0] != event.Kind(standards.KindForecast) {
t.Fatalf("Kinds() = %#v, want [forecast]", got)
}
}

View File

@@ -0,0 +1,6 @@
package openweather
// Source driver strings registered by weatherfeeder for OpenWeather sources.
const (
DriverObservation = "openweather_observation"
)

View File

@@ -19,9 +19,7 @@ type ObservationSource struct {
}
func NewObservationSource(cfg config.SourceConfig) (*ObservationSource, error) {
const driver = "openweather_observation"
hs, err := fksources.NewHTTPSource(driver, cfg, "application/json")
hs, err := fksources.NewHTTPSource(DriverObservation, cfg, "application/json")
if err != nil {
return nil, err
}
@@ -35,7 +33,9 @@ func NewObservationSource(cfg config.SourceConfig) (*ObservationSource, error) {
func (s *ObservationSource) Name() string { return s.http.Name }
func (s *ObservationSource) Kinds() []event.Kind { return []event.Kind{event.Kind("observation")} }
func (s *ObservationSource) Kinds() []event.Kind {
return []event.Kind{event.Kind(standards.KindObservation)}
}
func (s *ObservationSource) Poll(ctx context.Context) ([]event.Event, error) {
if err := owcommon.RequireMetricUnits(s.http.URL); err != nil {
@@ -60,7 +60,7 @@ func (s *ObservationSource) Poll(ctx context.Context) ([]event.Event, error) {
eventID := fksources.DefaultEventID("", s.http.Name, effectiveAt, emittedAt)
return fksources.SingleEvent(
event.Kind("observation"),
event.Kind(standards.KindObservation),
s.http.Name,
standards.SchemaRawOpenWeatherCurrentV1,
eventID,

View File

@@ -5,12 +5,13 @@ import (
"gitea.maximumdirect.net/ejr/feedkit/config"
"gitea.maximumdirect.net/ejr/feedkit/event"
"gitea.maximumdirect.net/ejr/weatherfeeder/standards"
)
func TestObservationSourceAdvertisesKinds(t *testing.T) {
src, err := NewObservationSource(config.SourceConfig{
Name: "openweather-observation-test",
Driver: "openweather_observation",
Driver: DriverObservation,
Mode: config.SourceModePoll,
Params: map[string]any{
"url": "https://example.invalid?units=metric",
@@ -20,7 +21,7 @@ func TestObservationSourceAdvertisesKinds(t *testing.T) {
if err != nil {
t.Fatalf("NewObservationSource() error = %v", err)
}
if got := src.Kinds(); len(got) != 1 || got[0] != event.Kind("observation") {
if got := src.Kinds(); len(got) != 1 || got[0] != event.Kind(standards.KindObservation) {
t.Fatalf("Kinds() = %#v, want [observation]", got)
}
}

View File

@@ -16,12 +16,11 @@ import (
fksources "gitea.maximumdirect.net/ejr/feedkit/sources"
"gitea.maximumdirect.net/ejr/feedkit/transport"
spcprovider "gitea.maximumdirect.net/ejr/weatherfeeder/internal/providers/spc"
"gitea.maximumdirect.net/ejr/weatherfeeder/internal/sources/internal/httpconfig"
"gitea.maximumdirect.net/ejr/weatherfeeder/standards"
)
const (
driverConvectiveOutlook = "spc_convective_outlook"
acceptGeoJSON = "application/geo+json, application/json"
acceptDiscussion = "text/html, application/xhtml+xml"
acceptRSS = "application/rss+xml, application/xml, text/xml"
@@ -56,53 +55,27 @@ type ConvectiveOutlookSource struct {
}
func NewConvectiveOutlookSource(cfg config.SourceConfig) (*ConvectiveOutlookSource, error) {
name := strings.TrimSpace(cfg.Name)
if name == "" {
return nil, fmt.Errorf("%s: name is required", driverConvectiveOutlook)
}
if cfg.Params == nil {
return nil, fmt.Errorf("%s %q: params are required", driverConvectiveOutlook, name)
}
userAgent, ok := cfg.ParamString("user_agent", "userAgent")
if !ok {
return nil, fmt.Errorf("%s %q: params.user_agent is required", driverConvectiveOutlook, name)
httpSettings, err := httpconfig.Parse(DriverConvectiveOutlook, cfg)
if err != nil {
return nil, err
}
latitude, err := requireFloatParam(cfg, "latitude")
if err != nil {
return nil, fmt.Errorf("%s %q: %w", driverConvectiveOutlook, name, err)
return nil, fmt.Errorf("%s %q: %w", DriverConvectiveOutlook, httpSettings.Name, err)
}
longitude, err := requireFloatParam(cfg, "longitude")
if err != nil {
return nil, fmt.Errorf("%s %q: %w", driverConvectiveOutlook, name, err)
}
timeout := transport.DefaultHTTPTimeout
if _, exists := cfg.Params["http_timeout"]; exists {
var ok bool
timeout, ok = cfg.ParamDuration("http_timeout")
if !ok || timeout <= 0 {
return nil, fmt.Errorf("source %q: params.http_timeout must be a positive duration", name)
}
}
bodyLimit := transport.DefaultHTTPResponseBodyLimitBytes
if _, exists := cfg.Params["http_response_body_limit_bytes"]; exists {
rawLimit, ok := cfg.ParamInt("http_response_body_limit_bytes")
if !ok || rawLimit <= 0 {
return nil, fmt.Errorf("source %q: params.http_response_body_limit_bytes must be a positive integer", name)
}
bodyLimit = int64(rawLimit)
return nil, fmt.Errorf("%s %q: %w", DriverConvectiveOutlook, httpSettings.Name, err)
}
geoJSONProducts, err := configuredGeoJSONProducts(cfg)
if err != nil {
return nil, fmt.Errorf("%s %q: %w", driverConvectiveOutlook, name, err)
return nil, fmt.Errorf("%s %q: %w", DriverConvectiveOutlook, httpSettings.Name, err)
}
discussions, err := configuredDiscussionProducts(cfg)
if err != nil {
return nil, fmt.Errorf("%s %q: %w", driverConvectiveOutlook, name, err)
return nil, fmt.Errorf("%s %q: %w", DriverConvectiveOutlook, httpSettings.Name, err)
}
rssURL := ""
@@ -114,14 +87,14 @@ func NewConvectiveOutlookSource(cfg config.SourceConfig) (*ConvectiveOutlookSour
locationName, _ := cfg.ParamString("location_name", "locationName")
return &ConvectiveOutlookSource{
name: name,
userAgent: userAgent,
name: httpSettings.Name,
userAgent: httpSettings.UserAgent,
locationID: locationID,
locationName: locationName,
latitude: latitude,
longitude: longitude,
client: transport.NewHTTPClient(timeout),
bodyLimit: bodyLimit,
client: transport.NewHTTPClient(httpSettings.Timeout),
bodyLimit: httpSettings.BodyLimitBytes,
geoJSONProducts: geoJSONProducts,
discussions: discussions,
rssURL: rssURL,
@@ -131,7 +104,7 @@ func NewConvectiveOutlookSource(cfg config.SourceConfig) (*ConvectiveOutlookSour
func (s *ConvectiveOutlookSource) Name() string { return s.name }
func (s *ConvectiveOutlookSource) Kinds() []event.Kind {
return []event.Kind{event.Kind("outlook")}
return []event.Kind{event.Kind(standards.KindOutlook)}
}
func (s *ConvectiveOutlookSource) Poll(ctx context.Context) ([]event.Event, error) {
@@ -221,7 +194,7 @@ func (s *ConvectiveOutlookSource) Poll(ctx context.Context) ([]event.Event, erro
eventID := fksources.DefaultEventID("", s.name, &effectiveAt, emittedAt)
return fksources.SingleEvent(
event.Kind("outlook"),
event.Kind(standards.KindOutlook),
s.name,
standards.SchemaRawSPCConvectiveOutlookV1,
eventID,
@@ -377,11 +350,7 @@ func latestIssueTime(raw []byte) time.Time {
}
func discussionUpdatedTime(rawHTML string) time.Time {
text, err := spcprovider.ExtractProductText(rawHTML)
if err != nil {
return time.Time{}
}
t := spcprovider.ParseUpdatedTimestamp(text)
t := spcprovider.ParsePageUpdatedTimestamp(rawHTML)
if t == nil {
return time.Time{}
}

View File

@@ -4,8 +4,6 @@ import (
"encoding/json"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"testing"
"time"
@@ -22,7 +20,7 @@ func TestConvectiveOutlookSourceKinds(t *testing.T) {
t.Fatalf("NewConvectiveOutlookSource() error = %v", err)
}
got := src.Kinds()
if len(got) != 1 || got[0] != event.Kind("outlook") {
if len(got) != 1 || got[0] != event.Kind(standards.KindOutlook) {
t.Fatalf("Kinds() = %#v, want [outlook]", got)
}
}
@@ -57,7 +55,7 @@ func TestConvectiveOutlookSourcePollEmitsRawBundle(t *testing.T) {
t.Fatalf("Poll() returned %d events, want 1", len(events))
}
got := events[0]
if got.Kind != event.Kind("outlook") {
if got.Kind != event.Kind(standards.KindOutlook) {
t.Fatalf("Kind = %q, want outlook", got.Kind)
}
if got.Schema != standards.SchemaRawSPCConvectiveOutlookV1 {
@@ -78,8 +76,8 @@ func TestConvectiveOutlookSourcePollEmitsRawBundle(t *testing.T) {
if bundle.Latitude != 38.6239 || bundle.Longitude != -90.3571 {
t.Fatalf("coordinates = %v,%v", bundle.Latitude, bundle.Longitude)
}
if len(bundle.Products) != 12 {
t.Fatalf("Products length = %d, want 12", len(bundle.Products))
if len(bundle.Products) != 9 {
t.Fatalf("Products length = %d, want 9", len(bundle.Products))
}
if len(bundle.Discussions) != 3 {
t.Fatalf("Discussions length = %d, want 3", len(bundle.Discussions))
@@ -277,7 +275,7 @@ func convectiveOutlookConfig(extra map[string]any) config.SourceConfig {
}
return config.SourceConfig{
Name: "spc-test",
Driver: driverConvectiveOutlook,
Driver: DriverConvectiveOutlook,
Mode: config.SourceModePoll,
Params: params,
}
@@ -312,7 +310,7 @@ func geoJSONFixtureForProduct(t *testing.T, key string, blankIssueISO bool) []by
case strings.HasPrefix(key, "day2_"):
name = "day2_torn.geojson"
case strings.HasPrefix(key, "day3_"):
name = "day3_wind.geojson"
name = "day3_cat.geojson"
default:
t.Fatalf("unknown product key %q", key)
}
@@ -340,16 +338,6 @@ func discussionFixtureForProduct(t *testing.T, key string) []byte {
}
}
func readSPCTestFixture(t *testing.T, name string) []byte {
t.Helper()
path := filepath.Join("..", "..", "providers", "spc", "testdata", name)
raw, err := os.ReadFile(path)
if err != nil {
t.Fatalf("read fixture %s: %v", path, err)
}
return raw
}
const testRSS = `<?xml version="1.0"?>
<rss version="2.0">
<channel>

View File

@@ -0,0 +1,6 @@
package spc
// Source driver strings registered by weatherfeeder for SPC sources.
const (
DriverConvectiveOutlook = "spc_convective_outlook"
)

View File

@@ -0,0 +1,17 @@
package spc
import (
"os"
"path/filepath"
"testing"
)
func readSPCTestFixture(t *testing.T, name string) []byte {
t.Helper()
path := filepath.Join("..", "..", "providers", "spc", "testdata", name)
raw, err := os.ReadFile(path)
if err != nil {
t.Fatalf("read fixture %s: %v", path, err)
}
return raw
}

View File

@@ -55,9 +55,11 @@ type WeatherAlert struct {
Instruction string `json:"instruction,omitempty"`
// Timing (all optional; provider-dependent).
// Onset and Ends describe the alert period. Expires is provider expiration metadata.
Sent *time.Time `json:"sent,omitempty"`
Effective *time.Time `json:"effective,omitempty"`
Onset *time.Time `json:"onset,omitempty"`
Ends *time.Time `json:"ends,omitempty"`
Expires *time.Time `json:"expires,omitempty"`
// Scope / affected area.

38
model/docs_test.go Normal file
View File

@@ -0,0 +1,38 @@
package model
import (
"os"
"strings"
"testing"
)
func TestDocumentedConsumerModelTypes(t *testing.T) {
raw, err := os.ReadFile("../docs/consumers/pkg-model.md")
if err != nil {
t.Fatalf("ReadFile(pkg-model.md) error = %v", err)
}
doc := string(raw)
required := []string{
"gitea.maximumdirect.net/ejr/weatherfeeder/model",
"WeatherObservation",
"WeatherForecastRun",
"WeatherForecastPeriod",
"WeatherForecastDiscussion",
"WeatherForecastDiscussionSection",
"WeatherStoryRun",
"WeatherStory",
"WeatherAlertRun",
"WeatherAlert",
"WeatherAlertReference",
"WeatherOutlookRun",
"WeatherOutlookDiscussion",
"WeatherOutlook",
"WMOCode",
}
for _, want := range required {
if !strings.Contains(doc, want) {
t.Fatalf("docs/consumers/pkg-model.md missing %q", want)
}
}
}

View File

@@ -8,13 +8,23 @@ import (
// WeatherOutlookRun is a snapshot of convective outlook polygons for a
// configured location as-of a provider issue time.
type WeatherOutlookRun struct {
LocationID string `json:"locationId,omitempty"`
LocationName string `json:"locationName,omitempty"`
Latitude *float64 `json:"latitude,omitempty"`
Longitude *float64 `json:"longitude,omitempty"`
AsOf time.Time `json:"asOf"`
IssuedAt *time.Time `json:"issuedAt,omitempty"`
Outlooks []WeatherOutlook `json:"outlooks"`
LocationID string `json:"locationId,omitempty"`
LocationName string `json:"locationName,omitempty"`
Latitude *float64 `json:"latitude,omitempty"`
Longitude *float64 `json:"longitude,omitempty"`
AsOf time.Time `json:"asOf"`
IssuedAt *time.Time `json:"issuedAt,omitempty"`
Outlooks []WeatherOutlook `json:"outlooks"`
Discussions []WeatherOutlookDiscussion `json:"discussions"`
}
// WeatherOutlookDiscussion is run-level SPC outlook prose for one outlook day.
type WeatherOutlookDiscussion struct {
Day int `json:"day"`
Headline string `json:"headline,omitempty"`
Summary string `json:"summary,omitempty"`
Discussion string `json:"discussion,omitempty"`
UpdatedAt *time.Time `json:"updatedAt,omitempty"`
}
// WeatherOutlook is a canonical representation of one outlook polygon.
@@ -32,9 +42,6 @@ type WeatherOutlook struct {
IssuedAt time.Time `json:"issuedAt"`
ExpiresAt time.Time `json:"expiresAt"`
Forecaster string `json:"forecaster,omitempty"`
Headline string `json:"headline,omitempty"`
Summary string `json:"summary,omitempty"`
Discussion string `json:"discussion,omitempty"`
SourceURL string `json:"sourceUrl,omitempty"`
ImageURL string `json:"imageUrl,omitempty"`
ContainsLocation bool `json:"containsLocation"`

60
model/outlook_test.go Normal file
View File

@@ -0,0 +1,60 @@
package model
import (
"encoding/json"
"strings"
"testing"
"time"
)
func TestWeatherOutlookJSONShape(t *testing.T) {
updatedAt := time.Date(2026, 6, 11, 16, 30, 0, 0, time.UTC)
run := WeatherOutlookRun{
AsOf: time.Date(2026, 6, 11, 19, 45, 0, 0, time.UTC),
Outlooks: []WeatherOutlook{{
ID: "outlook-1",
Provider: "spc",
Product: "convective",
Day: 1,
OutlookType: "categorical",
Label: "SLGT",
ValidFrom: time.Date(2026, 6, 11, 13, 0, 0, 0, time.UTC),
ValidTo: time.Date(2026, 6, 12, 12, 0, 0, 0, time.UTC),
IssuedAt: time.Date(2026, 6, 11, 12, 34, 56, 0, time.UTC),
ExpiresAt: time.Date(2026, 6, 12, 12, 0, 0, 0, time.UTC),
ContainsLocation: true,
Geometry: json.RawMessage(`{"type":"Polygon","coordinates":[[[-91,38],[-90,38],[-90,39],[-91,39],[-91,38]]]}`),
}},
Discussions: []WeatherOutlookDiscussion{{
Day: 1,
Headline: "Day 1 Convective Outlook",
Summary: "Severe thunderstorms are possible.",
Discussion: "Full discussion text.",
UpdatedAt: &updatedAt,
}},
}
raw, err := json.Marshal(run)
if err != nil {
t.Fatalf("Marshal(WeatherOutlookRun) error = %v", err)
}
got := string(raw)
for _, want := range []string{`"outlooks"`, `"discussions"`, `"headline"`, `"summary"`, `"discussion"`, `"updatedAt"`} {
if !strings.Contains(got, want) {
t.Fatalf("WeatherOutlookRun JSON missing %s: %s", want, got)
}
}
outlookStart := strings.Index(got, `"outlooks"`)
discussionStart := strings.Index(got, `"discussions"`)
if outlookStart == -1 || discussionStart == -1 || discussionStart <= outlookStart {
t.Fatalf("WeatherOutlookRun JSON has unexpected outlook/discussion order: %s", got)
}
outlookJSON := got[outlookStart:discussionStart]
for _, unwanted := range []string{`"headline"`, `"summary"`, `"discussion"`} {
if strings.Contains(outlookJSON, unwanted) {
t.Fatalf("WeatherOutlook JSON contains polygon-level prose key %s: %s", unwanted, got)
}
}
}

106
standards/docs_test.go Normal file
View File

@@ -0,0 +1,106 @@
package standards
import (
"go/ast"
"go/parser"
"go/token"
"os"
"strconv"
"strings"
"testing"
)
func TestDocumentedEventSchemas(t *testing.T) {
raw, err := os.ReadFile("../docs/integrations/events.md")
if err != nil {
t.Fatalf("ReadFile(events.md) error = %v", err)
}
doc := string(raw)
for _, schema := range schemaConstants(t, false) {
if !strings.Contains(doc, schema) {
t.Fatalf("docs/integrations/events.md missing schema %q", schema)
}
}
}
func TestDocumentedConsumerStandardsConstants(t *testing.T) {
raw, err := os.ReadFile("../docs/consumers/pkg-standards.md")
if err != nil {
t.Fatalf("ReadFile(pkg-standards.md) error = %v", err)
}
doc := string(raw)
for _, schema := range schemaConstants(t, true) {
if !strings.Contains(doc, schema) {
t.Fatalf("docs/consumers/pkg-standards.md missing schema %q", schema)
}
}
for _, kind := range kindConstants(t) {
if !strings.Contains(doc, kind) {
t.Fatalf("docs/consumers/pkg-standards.md missing kind %q", kind)
}
}
}
func schemaConstants(t *testing.T, includeNonCurrent bool) []string {
t.Helper()
return stringConstantsFromFile(t, "schema.go", "Schema", func(name string) bool {
return !includeNonCurrent && schemaConstantNotInCurrentContract(name)
})
}
func kindConstants(t *testing.T) []string {
t.Helper()
return stringConstantsFromFile(t, "kind.go", "Kind", nil)
}
func stringConstantsFromFile(t *testing.T, path string, prefix string, skip func(string) bool) []string {
t.Helper()
file, err := parser.ParseFile(token.NewFileSet(), path, nil, 0)
if err != nil {
t.Fatalf("ParseFile(%s) error = %v", path, err)
}
var out []string
ast.Inspect(file, func(n ast.Node) bool {
valueSpec, ok := n.(*ast.ValueSpec)
if !ok {
return true
}
for i, name := range valueSpec.Names {
if !strings.HasPrefix(name.Name, prefix) || (skip != nil && skip(name.Name)) {
continue
}
if i >= len(valueSpec.Values) {
t.Fatalf("constant %s has no explicit value", name.Name)
}
lit, ok := valueSpec.Values[i].(*ast.BasicLit)
if !ok || lit.Kind != token.STRING {
t.Fatalf("constant %s is not a string literal", name.Name)
}
value, err := strconv.Unquote(lit.Value)
if err != nil {
t.Fatalf("constant %s value is not a quoted string: %v", name.Name, err)
}
out = append(out, value)
}
return true
})
if len(out) == 0 {
t.Fatalf("no %s constants found in %s", prefix, path)
}
return out
}
func schemaConstantNotInCurrentContract(name string) bool {
switch name {
case "SchemaRawOpenWeatherHourlyForecastV1":
return true
default:
return false
}
}

11
standards/kind.go Normal file
View File

@@ -0,0 +1,11 @@
package standards
// Event kind strings used by weatherfeeder events and routing policy.
const (
KindObservation = "observation"
KindForecast = "forecast"
KindForecastDiscussion = "forecast_discussion"
KindWeatherStory = "weather_story"
KindAlert = "alert"
KindOutlook = "outlook"
)

View File

@@ -33,4 +33,5 @@ const (
SchemaWeatherStoryV1 = "weather.weather_story.v1"
SchemaWeatherAlertV1 = "weather.alert.v1"
SchemaWeatherOutlookV1 = "weather.outlook.v1"
SchemaWeatherOutlookV2 = "weather.outlook.v2"
)